Home

Context Navigation

source: palm/trunk/SOURCE/pmc_parent_mod.f90 @ 4649

Last change on this file since 4649 was 4649, checked in by raasch, 4 years ago
files re-formatted to follow the PALM coding standard
Property svn:keywords set to `Id`
File size: 37.9 KB

Rev	Line
[4246]	1	MODULE pmc_parent
[4649]	2	!--------------------------------------------------------------------------------------------------!
[4246]	3	! This file is part of the PALM model system.
	4	!
[4649]	5	! PALM is free software: you can redistribute it and/or modify it under the terms of the GNU General
	6	! Public License as published by the Free Software Foundation, either version 3 of the License, or
	7	! (at your option) any later version.
[4246]	8	!
[4649]	9	! PALM is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the
	10	! implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General
	11	! Public License for more details.
[4246]	12	!
[4649]	13	! You should have received a copy of the GNU General Public License along with PALM. If not, see
	14	! <http://www.gnu.org/licenses/>.
[4246]	15	!
[4360]	16	! Copyright 1997-2020 Leibniz Universitaet Hannover
[4649]	17	!--------------------------------------------------------------------------------------------------!
[4246]	18	!
[4649]	19	!
[4246]	20	! Current revisions:
[4649]	21	! -----------------
[4629]	22	!
	23	!
[4246]	24	! Former revisions:
	25	! -----------------
	26	! $Id: pmc_parent_mod.f90 4649 2020-08-25 12:11:17Z raasch $
[4649]	27	! File re-formatted to follow the PALM coding standard
	28	!
	29	!
	30	! 4629 2020-07-29 09:37:56Z raasch
	31	! Support for MPI Fortran77 interface (mpif.h) removed
	32	!
[4629]	33	! 4360 2020-01-07 11:25:50Z suehring
[4649]	34	!
	35	!
[4245]	36	! 4213 2019-09-02 14:25:56Z suehring
[4649]	37	! Allocate array for index_list_2d also for zero-size arrays, in order to avoid errors when array
	38	! bound checks are enabled
	39	!
[4246]	40	! 4212 2019-09-02 14:23:05Z suehring
	41	! Corrected "Former revisions" section
[4649]	42	!
[4246]	43	! 3962 2019-05-08 19:40:33Z suehring
[4649]	44	! Bugfixes in initial settings of child and parent communication patterns.
	45	!
[4246]	46	! 3655 2019-01-07 16:51:22Z knoop
[4649]	47	! Explicit kind settings
[4246]	48	!
	49	! 1762 2016-02-25 12:31:13Z hellstea
	50	! Initial revision by K. Ketelsen
	51	!
[4649]	52	!--------------------------------------------------------------------------------------------------!
[4246]	53	! Description:
	54	! ------------
	55	!> Parent part of Palm Model Coupler
[4649]	56	!--------------------------------------------------------------------------------------------------!
[4246]	57
	58	#if defined( __parallel )
	59	USE, INTRINSIC :: ISO_C_BINDING
	60
	61	USE MPI
[4629]	62
[4246]	63	USE kinds
[4649]	64	USE pmc_general, &
	65	ONLY: arraydef, &
	66	childdef, &
	67	da_namedef, &
	68	da_namelen, &
	69	pedef, &
	70	pmc_g_setname, &
	71	pmc_max_array, &
	72	pmc_max_models, &
	73	pmc_sort
[4246]	74
[4649]	75	USE pmc_handle_communicator, &
	76	ONLY: m_model_comm, &
	77	m_model_rank, &
	78	m_model_npes, &
	79	m_to_child_comm, &
	80	m_world_rank, &
	81	pmc_parent_for_child
[4246]	82
[4649]	83	USE pmc_mpi_wrapper, &
	84	ONLY: pmc_alloc_mem, &
	85	pmc_bcast, &
	86	pmc_time
[4246]	87
	88	IMPLICIT NONE
	89
	90
	91	PRIVATE
	92	SAVE
	93
[4649]	94	INTEGER :: next_array_in_list = 0 !<
	95
[4246]	96	TYPE childindexdef
	97	INTEGER :: nrpoints !<
	98	INTEGER, DIMENSION(:,:), ALLOCATABLE :: index_list_2d !<
	99	END TYPE childindexdef
	100
[4649]	101	TYPE(childdef), DIMENSION(pmc_max_models),PUBLIC :: children !<
	102	TYPE(childindexdef), DIMENSION(pmc_max_models) :: indchildren !<
[4246]	103
	104
	105	PUBLIC pmc_parent_for_child
	106
	107
	108	INTERFACE pmc_parentinit
	109	MODULE PROCEDURE pmc_parentinit
	110	END INTERFACE pmc_parentinit
	111
	112	INTERFACE pmc_s_set_2d_index_list
	113	MODULE PROCEDURE pmc_s_set_2d_index_list
	114	END INTERFACE pmc_s_set_2d_index_list
	115
	116	INTERFACE pmc_s_clear_next_array_list
	117	MODULE PROCEDURE pmc_s_clear_next_array_list
	118	END INTERFACE pmc_s_clear_next_array_list
	119
	120	INTERFACE pmc_s_getnextarray
	121	MODULE PROCEDURE pmc_s_getnextarray
	122	END INTERFACE pmc_s_getnextarray
	123
	124	INTERFACE pmc_s_set_dataarray
	125	MODULE PROCEDURE pmc_s_set_dataarray_2d
	126	MODULE PROCEDURE pmc_s_set_dataarray_3d
	127	MODULE PROCEDURE pmc_s_set_dataarray_ip2d
	128	END INTERFACE pmc_s_set_dataarray
	129
	130	INTERFACE pmc_s_setind_and_allocmem
	131	MODULE PROCEDURE pmc_s_setind_and_allocmem
	132	END INTERFACE pmc_s_setind_and_allocmem
	133
	134	INTERFACE pmc_s_fillbuffer
	135	MODULE PROCEDURE pmc_s_fillbuffer
	136	END INTERFACE pmc_s_fillbuffer
	137
	138	INTERFACE pmc_s_getdata_from_buffer
	139	MODULE PROCEDURE pmc_s_getdata_from_buffer
	140	END INTERFACE pmc_s_getdata_from_buffer
	141
	142	INTERFACE pmc_s_set_active_data_array
	143	MODULE PROCEDURE pmc_s_set_active_data_array
	144	END INTERFACE pmc_s_set_active_data_array
	145
	146	INTERFACE pmc_s_get_child_npes
	147	MODULE PROCEDURE pmc_s_get_child_npes
	148	END INTERFACE pmc_s_get_child_npes
	149
[4649]	150	PUBLIC pmc_parentinit, &
	151	pmc_s_clear_next_array_list, &
	152	pmc_s_fillbuffer, &
	153	pmc_s_getdata_from_buffer, &
	154	pmc_s_getnextarray, &
	155	pmc_s_setind_and_allocmem, &
	156	pmc_s_set_active_data_array, &
	157	pmc_s_set_dataarray, &
	158	pmc_s_set_2d_index_list, &
[4246]	159	pmc_s_get_child_npes
	160
	161	CONTAINS
	162
	163
[4649]	164	!--------------------------------------------------------------------------------------------------!
	165	! Description:
	166	! ------------
	167	!
	168	!> @Todo: Missing subroutine description.
	169	!--------------------------------------------------------------------------------------------------!
[4246]	170	SUBROUTINE pmc_parentinit
	171
	172	IMPLICIT NONE
	173
[4649]	174	INTEGER(iwp) :: childid !<
	175	INTEGER(iwp) :: i !<
	176	INTEGER(iwp) :: istat !<
	177	INTEGER(iwp) :: j !<
[4246]	178
	179
[4649]	180	DO i = 1, SIZE( pmc_parent_for_child ) - 1
	181
[4246]	182	childid = pmc_parent_for_child( i )
	183
	184	children(childid)%model_comm = m_model_comm
	185	children(childid)%inter_comm = m_to_child_comm(childid)
	186
	187	!
	188	!-- Get rank and size
[4649]	189	CALL MPI_COMM_RANK( children(childid)%model_comm, children(childid)%model_rank, istat )
	190	CALL MPI_COMM_SIZE( children(childid)%model_comm, children(childid)%model_npes, istat )
	191	CALL MPI_COMM_REMOTE_SIZE( children(childid)%inter_comm, children(childid)%inter_npes, &
	192	istat )
[4246]	193	!
	194	!-- Intra communicator is used for MPI_GET
[4649]	195	CALL MPI_INTERCOMM_MERGE( children(childid)%inter_comm, .FALSE., &
[4246]	196	children(childid)%intra_comm, istat )
[4649]	197	CALL MPI_COMM_RANK( children(childid)%intra_comm, children(childid)%intra_rank, istat )
[4246]	198
[4649]	199	ALLOCATE( children(childid)%pes(children(childid)%inter_npes) )
[4246]	200	!
[4649]	201	!-- Allocate array of TYPE arraydef for all child PEs to store information of the transfer array
[4246]	202	DO j = 1, children(childid)%inter_npes
	203	ALLOCATE( children(childid)%pes(j)%array_list(pmc_max_array) )
	204	ENDDO
	205
[4649]	206	CALL get_da_names_from_child( childid )
[4246]	207
	208	ENDDO
	209
	210	END SUBROUTINE pmc_parentinit
	211
[4649]	212	!--------------------------------------------------------------------------------------------------!
	213	! Description:
	214	! ------------
	215	!
	216	!> @Todo: Missing subroutine description.
	217	!--------------------------------------------------------------------------------------------------!
[4246]	218	SUBROUTINE pmc_s_set_2d_index_list( childid, index_list )
	219
	220	IMPLICIT NONE
	221
	222	INTEGER(iwp) :: ian !<
	223	INTEGER(iwp) :: ie !<
	224	INTEGER(iwp) :: ip !<
	225	INTEGER(iwp) :: is !<
	226	INTEGER(iwp) :: istat !<
	227
[4649]	228	INTEGER(iwp), INTENT(IN) :: childid !<
[4246]	229
[4649]	230	INTEGER(iwp), DIMENSION(:,:), INTENT(INOUT) :: index_list !<
	231
	232
[4246]	233	IF ( m_model_rank == 0 ) THEN
	234	!
	235	!-- Sort to ascending parent process order
	236	CALL pmc_sort( index_list, 6 )
	237	is = 1
	238	DO ip = 0, m_model_npes-1
	239	!
	240	!-- Split into parent processes
	241	ie = is - 1
	242	!
	243	!-- There may be no entry for this process
[4649]	244	IF ( is <= SIZE( index_list, 2 ) .AND. ie >= 0 ) THEN
[4246]	245	DO WHILE ( index_list(6,ie+1 ) == ip )
	246	ie = ie + 1
[4649]	247	IF ( ie == SIZE( index_list, 2 ) ) EXIT
[4246]	248	ENDDO
	249	ian = ie - is + 1
	250	ELSE
	251	is = -1
	252	ie = -2
	253	ian = 0
	254	ENDIF
	255	!
	256	!-- Send data to other parent processes
	257	IF ( ip == 0 ) THEN
	258	indchildren(childid)%nrpoints = ian
	259	!
[4649]	260	!-- Allocate array for index_list_2d. Note, the array will also be allocated in case
	261	!-- ian = 0, in order to avoid errors when array bounds are checked.
[4246]	262	ALLOCATE( indchildren(childid)%index_list_2d(6,1:ian) )
	263	IF ( ian > 0) THEN
[4649]	264	indchildren(childid)%index_list_2d(:,1:ian) = index_list(:,is:ie)
[4246]	265	ENDIF
	266	ELSE
[4649]	267	CALL MPI_SEND( ian, 1, MPI_INTEGER, ip, 1000, m_model_comm, istat )
[4246]	268	IF ( ian > 0) THEN
[4649]	269	CALL MPI_SEND( index_list(1,is), 6*ian, MPI_INTEGER, ip, 1001, m_model_comm, &
	270	istat )
[4246]	271	ENDIF
	272	ENDIF
	273	is = ie + 1
	274	ENDDO
	275	ELSE
[4649]	276	CALL MPI_RECV( indchildren(childid)%nrpoints, 1, MPI_INTEGER, 0, 1000, m_model_comm, &
	277	MPI_STATUS_IGNORE, istat )
[4246]	278	ian = indchildren(childid)%nrpoints
	279	!
[4649]	280	!-- Allocate array for index_list_2d. Note, the array will also be allocated in case ian=0, in
	281	!-- order to avoid errors when array bounds are checked.
[4246]	282	ALLOCATE( indchildren(childid)%index_list_2d(6,1:ian) )
	283	IF ( ian > 0 ) THEN
[4649]	284	CALL MPI_RECV( indchildren(childid)%index_list_2d, 6*ian, MPI_INTEGER, 0, 1001, &
	285	m_model_comm, MPI_STATUS_IGNORE, istat)
[4246]	286	ENDIF
	287	ENDIF
[4649]	288	CALL set_pe_index_list( children(childid), indchildren(childid)%index_list_2d, &
[4246]	289	indchildren(childid)%nrpoints )
	290
	291	END SUBROUTINE pmc_s_set_2d_index_list
	292
	293
[4649]	294	!--------------------------------------------------------------------------------------------------!
	295	! Description:
	296	! ------------
	297	!
	298	!> @Todo: Missing subroutine description.
	299	!--------------------------------------------------------------------------------------------------!
[4246]	300	SUBROUTINE pmc_s_clear_next_array_list
	301
[4649]	302
[4246]	303	IMPLICIT NONE
	304
	305	next_array_in_list = 0
	306
	307	END SUBROUTINE pmc_s_clear_next_array_list
	308
	309
	310
	311	LOGICAL FUNCTION pmc_s_getnextarray( childid, myname )
	312
	313	!
	314	!-- Althoug there are no linked lists any more in PMC, this call still looks like working with a list
	315
[4649]	316	CHARACTER(LEN=*), INTENT(OUT) :: myname !<
[4246]	317
[4649]	318	INTEGER(iwp), INTENT(IN) :: childid !<
[4246]	319
[4649]	320	TYPE(pedef), POINTER :: ape !<
	321	TYPE(arraydef), POINTER :: ar !<
	322
[4246]	323	next_array_in_list = next_array_in_list + 1
	324	!
[4649]	325	!-- Array names are the same on all children processes, so take first process to get the name
[4246]	326	ape => children(childid)%pes(1)
	327
	328	IF ( next_array_in_list > ape%nr_arrays ) THEN
	329	!
	330	!-- All arrays are done
	331	pmc_s_getnextarray = .FALSE.
	332	RETURN
	333	ENDIF
	334
	335	ar => ape%array_list(next_array_in_list)
	336	myname = ar%name
	337	!
	338	!-- Return true if there is still an array in the list
	339
	340	pmc_s_getnextarray = .TRUE.
	341
	342	END FUNCTION pmc_s_getnextarray
	343
	344
	345
[4649]	346	!--------------------------------------------------------------------------------------------------!
	347	! Description:
	348	! ------------
	349	!
	350	!> @Todo: Missing subroutine description.
	351	!--------------------------------------------------------------------------------------------------!
[4246]	352	SUBROUTINE pmc_s_set_dataarray_2d( childid, array, array_2 )
	353
	354	IMPLICIT NONE
	355
[4649]	356	INTEGER(iwp) :: nrdims !<
[4246]	357
[4649]	358	INTEGER(iwp), INTENT(IN) :: childid !<
	359
	360	INTEGER(iwp), DIMENSION(4) :: dims !<
	361
	362	REAL(wp), INTENT(IN), DIMENSION(:,:), POINTER :: array !<
	363
[4246]	364	REAL(wp), INTENT(IN), DIMENSION(:,:), POINTER, OPTIONAL :: array_2 !<
	365
[4649]	366	TYPE(C_PTR) :: array_adr !<
	367	TYPE(C_PTR) :: second_adr !<
[4246]	368
	369
	370	dims = 1
	371	nrdims = 2
[4649]	372	dims(1) = SIZE( array, 1 )
	373	dims(2) = SIZE( array, 2 )
[4246]	374	array_adr = C_LOC( array )
	375
	376	IF ( PRESENT( array_2 ) ) THEN
[4649]	377	second_adr = C_LOC( array_2 )
	378	CALL pmc_s_setarray( childid, nrdims, dims, array_adr, second_adr = second_adr )
[4246]	379	ELSE
	380	CALL pmc_s_setarray( childid, nrdims, dims, array_adr )
	381	ENDIF
	382
	383	END SUBROUTINE pmc_s_set_dataarray_2d
	384
[4649]	385
	386	!--------------------------------------------------------------------------------------------------!
	387	! Description:
	388	! ------------
	389	!
	390	!> @Todo: Missing subroutine description.
	391	!--------------------------------------------------------------------------------------------------!
[4246]	392	SUBROUTINE pmc_s_set_dataarray_ip2d( childid, array )
	393
	394	IMPLICIT NONE
	395
[4649]	396	INTEGER(iwp) :: nrdims !<
[4246]	397
[4649]	398	INTEGER(iwp),INTENT(IN) :: childid !<
[4246]	399
[4649]	400	INTEGER(idp), INTENT(IN), DIMENSION(:,:), POINTER :: array !<
[4246]	401
[4649]	402	INTEGER(iwp), DIMENSION(4) :: dims !<
[4246]	403
[4649]	404	TYPE(C_PTR) :: array_adr !<
	405
	406
[4246]	407	dims = 1
	408	nrdims = 2
[4649]	409	dims(1) = SIZE( array, 1 )
	410	dims(2) = SIZE( array, 2 )
[4246]	411	array_adr = C_LOC( array )
	412
[4649]	413	CALL pmc_s_setarray( childid, nrdims, dims, array_adr , dimkey = 22 )
[4246]	414
	415	END SUBROUTINE pmc_s_set_dataarray_ip2d
	416
	417
[4649]	418	!--------------------------------------------------------------------------------------------------!
	419	! Description:
	420	! ------------
	421	!
	422	!> @Todo: Missing subroutine description.
	423	!--------------------------------------------------------------------------------------------------!
[4246]	424	SUBROUTINE pmc_s_set_dataarray_3d( childid, array, nz_cl, nz, array_2 )
	425
	426	IMPLICIT NONE
	427
[4649]	428	INTEGER(iwp) :: nrdims !<
[4246]	429
[4649]	430	INTEGER(iwp), INTENT(IN) :: childid !<
	431	INTEGER(iwp), INTENT(IN) :: nz !<
	432	INTEGER(iwp), INTENT(IN) :: nz_cl !<
	433
	434	INTEGER(iwp), DIMENSION(4) :: dims !<
	435
	436	REAL(wp), INTENT(IN), DIMENSION(:,:,:), POINTER :: array !<
	437
[4246]	438	REAL(wp), INTENT(IN), DIMENSION(:,:,:), POINTER, OPTIONAL :: array_2 !<
	439
[4649]	440	TYPE(C_PTR) :: array_adr !<
	441	TYPE(C_PTR) :: second_adr !<
[4246]	442
[4649]	443	nrdims = 3
	444	dims(1) = SIZE( array, 1 )
	445	dims(2) = SIZE( array, 2 )
	446	dims(3) = SIZE( array, 3 )
	447	dims(4) = nz_cl+dims(1)-nz ! Works for first dimension 1:nz and 0:nz+1
[4246]	448
[4649]	449	array_adr = C_LOC( array )
[4246]	450	!
	451	!-- In PALM's pointer version, two indices have to be stored internally.
	452	!-- The active address of the data array is set in swap_timelevel.
	453	IF ( PRESENT( array_2 ) ) THEN
	454	second_adr = C_LOC( array_2 )
[4649]	455	CALL pmc_s_setarray( childid, nrdims, dims, array_adr, second_adr = second_adr )
[4246]	456	ELSE
	457	CALL pmc_s_setarray( childid, nrdims, dims, array_adr )
	458	ENDIF
	459
	460	END SUBROUTINE pmc_s_set_dataarray_3d
	461
	462
[4649]	463	!--------------------------------------------------------------------------------------------------!
	464	! Description:
	465	! ------------
	466	!
	467	!> @Todo: Missing subroutine description.
	468	!--------------------------------------------------------------------------------------------------!
[4246]	469	SUBROUTINE pmc_s_setind_and_allocmem( childid )
	470
[4649]	471	USE control_parameters, &
[4246]	472	ONLY: message_string
	473
	474	IMPLICIT NONE
	475
	476	!
	477	!-- Naming convention for appendices: _pc -> parent to child transfer
	478	!-- _cp -> child to parent transfer
	479	!-- send -> parent to child transfer
	480	!-- recv -> child to parent transfer
	481
[4649]	482	INTEGER(iwp) :: arlen !<
	483	INTEGER(iwp) :: i !<
	484	INTEGER(iwp) :: ierr !<
	485	INTEGER(iwp) :: j !<
	486	INTEGER(iwp) :: lo_nr_arrays !< store number of arrays in local variiab le
	487	INTEGER(iwp) :: myindex !<
	488	INTEGER(iwp) :: total_npes !< Total Number of PEs Parent and Child
[4246]	489
[4649]	490	INTEGER(idp) :: bufsize !< size of MPI data window
	491
	492	INTEGER(iwp), INTENT(IN) :: childid !<
	493
[4246]	494	INTEGER(KIND=MPI_ADDRESS_KIND) :: winsize !<
	495
[4649]	496	INTEGER(iwp), DIMENSION(:,:), ALLOCATABLE :: myindex_s !<
	497	INTEGER(iwp), DIMENSION(:,:), ALLOCATABLE :: myindex_r !<
[4246]	498
[4649]	499	REAL(wp),DIMENSION(:), POINTER, SAVE :: base_array_cp !< base array for child to parent transfer
[4246]	500	REAL(wp),DIMENSION(:), POINTER, SAVE :: base_array_pc !< base array for parent to child transfer
	501
[4649]	502	TYPE(C_PTR) :: base_ptr !<
	503
	504	TYPE(pedef), POINTER :: ape !<
	505	TYPE(arraydef), POINTER :: ar !<
	506
	507
	508	call MPI_COMM_SIZE( children(childid)%intra_comm, total_npes, ierr )
[4246]	509	!
	510	!-- Parent to child direction
	511	myindex = 1
	512	bufsize = 8
	513	!
[4649]	514	!-- All Child processes get the same number of arrays.
[4246]	515	!-- Therfore the number of arrays form the first Child process can be used for Dimension.
	516	lo_nr_arrays = children(childid)%pes(1)%nr_arrays
	517
	518	ALLOCATE( myindex_s(lo_nr_arrays,0:total_npes-1) )
	519	ALLOCATE( myindex_r(lo_nr_arrays,0:total_npes-1) )
	520
	521	myindex_s = 0
	522
	523	!
	524	!-- First stride: compute size and set index
	525	DO i = 1, children(childid)%inter_npes
	526
	527	ape => children(childid)%pes(i)
	528
	529	DO j = 1, ape%nr_arrays
	530
	531	ar => ape%array_list(j)
	532	IF ( ar%nrdims == 2 ) THEN
	533	arlen = ape%nrele
	534	ELSEIF ( ar%nrdims == 3 ) THEN
	535	arlen = ape%nrele * ar%a_dim(4)
	536	ELSE
	537	arlen = -1
	538	ENDIF
	539	ar%sendindex = myindex
	540	!
[4649]	541	!-- Using intra communicator for MPU_Alltoall, the numbers of the child processes are after
	542	!-- the parent ones.
[4246]	543
	544	myindex_s(j,i-1+children(childid)%model_npes) = myindex
	545
	546	myindex = myindex + arlen
	547	bufsize = bufsize + arlen
	548	ar%sendsize = arlen
	549	ENDDO
	550
	551	ENDDO
	552	!
	553	!-- Using MPI_Alltoall to send indices from Parent to Child
	554	!-- The data comming back from the child processes are ignored.
	555
[4649]	556	CALL MPI_ALLTOALL( myindex_s, lo_nr_arrays, MPI_INTEGER, myindex_r, lo_nr_arrays, MPI_INTEGER, &
[4246]	557	children(childid)%intra_comm, ierr )
	558
	559	!
	560	!-- Using MPI_Alltoall to receive indices from Child
	561	myindex_s = 0
	562	myindex_r = 0
	563
[4649]	564	CALL MPI_ALLTOALL( myindex_s, lo_nr_arrays, MPI_INTEGER, myindex_r, lo_nr_arrays, MPI_INTEGER, &
[4246]	565	children(childid)%intra_comm, ierr )
	566	!
[4649]	567	!-- Create RMA (One Sided Communication) window for data buffer parent to child transfer.
	568	!-- The buffer of MPI_GET (counterpart of transfer) can be PE-local, i.e. it can but must not be
	569	!-- part of the MPI RMA window. Only one RMA window is required to prepare the data for:
	570	!-- parent -> child transfer on the parent side
	571	!-- and for:
	572	!-- child -> parent transfer on the child side
[4246]	573	CALL pmc_alloc_mem( base_array_pc, bufsize )
	574	children(childid)%totalbuffersize = bufsize * wp
	575
	576	winsize = bufsize * wp
[4649]	577	CALL MPI_WIN_CREATE( base_array_pc, winsize, wp, MPI_INFO_NULL, children(childid)%intra_comm, &
[4246]	578	children(childid)%win_parent_child, ierr )
	579	!
	580	!-- Open window to set data
	581	CALL MPI_WIN_FENCE( 0, children(childid)%win_parent_child, ierr )
	582	!
	583	!-- Second stride: set buffer pointer
	584	DO i = 1, children(childid)%inter_npes
	585
	586	ape => children(childid)%pes(i)
	587
	588	DO j = 1, ape%nr_arrays
	589
	590	ar => ape%array_list(j)
	591	ar%sendbuf = C_LOC( base_array_pc(ar%sendindex) )
	592
[4649]	593	IF ( ar%sendindex + ar%sendsize > bufsize ) THEN
	594	WRITE( message_string, '(a,i4,4i7,1x,a)' ) 'parent buffer too small ',i , &
	595	ar%sendindex, ar%sendsize, ar%sendindex + ar%sendsize, bufsize, TRIM( ar%name )
[4246]	596	CALL message( 'pmc_s_setind_and_allocmem', 'PA0429', 3, 2, 0, 6, 0 )
	597	ENDIF
	598	ENDDO
	599	ENDDO
	600	!
	601	!-- Child to parent direction
	602	bufsize = 8
	603	!
	604	!-- First stride: compute size and set index
	605	DO i = 1, children(childid)%inter_npes
	606	ape => children(childid)%pes(i)
	607	DO j = 1, ape%nr_arrays
	608	ar => ape%array_list(j)
	609	!
	610	!-- Receive index from child
	611	IF ( ar%nrdims == 3 ) THEN
[4649]	612	bufsize = MAX( bufsize, INT( ape%nrele * ar%a_dim(4), MPI_ADDRESS_KIND ) )
[4246]	613	ELSE
	614	bufsize = MAX( bufsize, INT( ape%nrele, MPI_ADDRESS_KIND ) )
	615	ENDIF
	616	ar%recvindex = myindex_r(j,i-1+children(childid)%model_npes)
	617	ENDDO
	618	ENDDO
	619
	620	DEALLOCATE( myindex_s )
	621	DEALLOCATE( myindex_r )
	622
	623	!
	624	!-- Create RMA (one sided communication, RMA = Remote Memory Access) data buffer.
[4649]	625	!-- The buffer for MPI_GET can be PE local, i.e. it can but must not be part of the MPI RMA window.
[4246]	626	CALL pmc_alloc_mem( base_array_cp, bufsize, base_ptr )
	627	children(childid)%totalbuffersize = bufsize * wp
	628
	629	CALL MPI_BARRIER( children(childid)%intra_comm, ierr )
	630	!
	631	!-- Second stride: set buffer pointer
	632	DO i = 1, children(childid)%inter_npes
	633	ape => children(childid)%pes(i)
	634	DO j = 1, ape%nr_arrays
	635	ar => ape%array_list(j)
	636	ar%recvbuf = base_ptr
	637	ENDDO
	638	ENDDO
	639
	640	END SUBROUTINE pmc_s_setind_and_allocmem
	641
	642
[4649]	643	!--------------------------------------------------------------------------------------------------!
	644	! Description:
	645	! ------------
	646	!
	647	!> @Todo: Missing subroutine description.
	648	!--------------------------------------------------------------------------------------------------!
[4246]	649	SUBROUTINE pmc_s_fillbuffer( childid, waittime, particle_transfer )
	650
	651	IMPLICIT NONE
	652
[4649]	653	INTEGER(iwp) :: ierr !<
	654	INTEGER(iwp) :: ij !<
	655	INTEGER(iwp) :: ip !<
	656	INTEGER(iwp) :: j !<
	657	INTEGER(iwp) :: myindex !<
[4246]	658
[4649]	659	INTEGER(iwp), INTENT(IN) :: childid !<
	660
	661	INTEGER(iwp), DIMENSION(1) :: buf_shape !<
	662
	663	INTEGER(idp), POINTER, DIMENSION(:) :: ibuf !<
	664
	665	INTEGER(idp), POINTER, DIMENSION(:,:) :: idata_2d !<
	666
	667	LOGICAL :: lo_ptrans !<
	668
[4246]	669	LOGICAL, INTENT(IN), OPTIONAL :: particle_transfer !<
	670
[4649]	671	REAL(wp) :: t1 !<
	672	REAL(wp) :: t2 !<
[4246]	673
[4649]	674	REAL(wp), INTENT(OUT), OPTIONAL :: waittime !<
[4246]	675
[4649]	676	REAL(wp), POINTER, DIMENSION(:) :: buf !<
[4246]	677
[4649]	678	REAL(wp), POINTER, DIMENSION(:,:) :: data_2d !<
	679
[4246]	680	REAL(wp), POINTER, DIMENSION(:,:,:) :: data_3d !<
	681
[4649]	682	TYPE(pedef), POINTER :: ape !<
[4246]	683	TYPE(arraydef), POINTER :: ar !<
	684
	685	!
[4649]	686	!-- Synchronization of the model is done in pmci_synchronize. Therefor the RMA window can be filled
	687	!-- without sychronization at this point and a barrier is not necessary.
	688	!-- Please note that waittime has to be set in pmc_s_fillbuffer AND pmc_c_getbuffer.
[4246]	689	IF ( PRESENT( waittime) ) THEN
	690	t1 = pmc_time()
	691	CALL MPI_BARRIER( children(childid)%intra_comm, ierr )
	692	t2 = pmc_time()
[4649]	693	waittime = t2 - t1
[4246]	694	ENDIF
	695
	696	lo_ptrans = .FALSE.
[4649]	697	IF ( PRESENT( particle_transfer ) ) lo_ptrans = particle_transfer
[4246]	698
	699	DO ip = 1, children(childid)%inter_npes
	700	ape => children(childid)%pes(ip)
	701	DO j = 1, ape%nr_arrays
	702	ar => ape%array_list(j)
	703	myindex = 1
	704
[4649]	705	IF ( ar%dimkey == 2 .AND. .NOT.lo_ptrans ) THEN ! PALM 2D REAL*8 Array
[4246]	706
	707	buf_shape(1) = ape%nrele
	708	CALL C_F_POINTER( ar%sendbuf, buf, buf_shape )
	709	CALL C_F_POINTER( ar%data, data_2d, ar%a_dim(1:2) )
	710	DO ij = 1, ape%nrele
	711	buf(myindex) = data_2d(ape%locind(ij)%j,ape%locind(ij)%i)
	712	myindex = myindex + 1
	713	ENDDO
	714
[4649]	715	ELSEIF ( ar%dimkey == 3 .AND. .NOT.lo_ptrans ) THEN ! PALM 3D REAL*8 Array
[4246]	716
	717	buf_shape(1) = ape%nrele*ar%a_dim(4)
	718	CALL C_F_POINTER( ar%sendbuf, buf, buf_shape )
	719	CALL C_F_POINTER( ar%data, data_3d, ar%a_dim(1:3) )
	720	DO ij = 1, ape%nrele
[4649]	721	buf(myindex:myindex+ar%a_dim(4)-1) = &
	722	data_3d(1:ar%a_dim(4),ape%locind(ij)%j,ape%locind(ij)%i)
[4246]	723	myindex = myindex + ar%a_dim(4)
	724	ENDDO
[4649]	725	ELSEIF ( ar%dimkey == 22 .AND. lo_ptrans ) THEN ! 2D INTEGER*8 Array for particle Transfer
[4246]	726
	727	buf_shape(1) = ape%nrele
	728	CALL C_F_POINTER( ar%sendbuf, ibuf, buf_shape )
	729	CALL C_F_POINTER( ar%data, idata_2d, ar%a_dim(1:2) )
	730	DO ij = 1, ape%nrele
	731	ibuf(myindex) = idata_2d(ape%locind(ij)%j,ape%locind(ij)%i)
	732	myindex = myindex + 1
	733	ENDDO
	734	ENDIF
	735	ENDDO
	736	ENDDO
	737	!
	738	!-- Buffer is filled
	739	CALL MPI_BARRIER( children(childid)%intra_comm, ierr )
	740
	741	END SUBROUTINE pmc_s_fillbuffer
	742
	743
	744
[4649]	745	!--------------------------------------------------------------------------------------------------!
	746	! Description:
	747	! ------------
	748	!
	749	!> @Todo: Missing subroutine description.
	750	!--------------------------------------------------------------------------------------------------!
[4246]	751	SUBROUTINE pmc_s_getdata_from_buffer( childid, waittime , particle_transfer, child_process_nr )
	752
	753	IMPLICIT NONE
	754
[4649]	755	INTEGER(iwp) :: ierr !<
	756	INTEGER(iwp) :: ij !<
	757	INTEGER(iwp) :: ip !<
	758	INTEGER(iwp) :: ip_start !<
	759	INTEGER(iwp) :: ip_end !<
	760	INTEGER(iwp) :: j !<
	761	INTEGER(iwp) :: myindex !<
	762	INTEGER(iwp) :: nr !<
	763	INTEGER(iwp) :: target_pe !<
[4246]	764
[4649]	765	INTEGER(iwp), INTENT(IN) :: childid !<
[4246]	766
[4649]	767	INTEGER(iwp), INTENT(IN), OPTIONAL :: child_process_nr !<
[4246]	768
[4649]	769	INTEGER(KIND=MPI_ADDRESS_KIND) :: target_disp !<
[4246]	770
[4649]	771	INTEGER(iwp), DIMENSION(1) :: buf_shape !<
[4246]	772
[4649]	773	INTEGER(idp), POINTER, DIMENSION(:) :: ibuf !<
[4246]	774
[4649]	775	INTEGER(idp), POINTER, DIMENSION(:,:) :: idata_2d !<
	776
	777	LOGICAL :: lo_ptrans !<
	778
	779	LOGICAL, INTENT(IN), OPTIONAL :: particle_transfer !<
	780
	781	REAL(wp) :: t1 !<
	782	REAL(wp) :: t2 !<
	783
	784	REAL(wp), INTENT(OUT), OPTIONAL :: waittime !<
	785
	786	REAL(wp), POINTER, DIMENSION(:) :: buf !<
	787
	788	REAL(wp), POINTER, DIMENSION(:,:) :: data_2d !<
	789
	790	REAL(wp), POINTER, DIMENSION(:,:,:) :: data_3d !<
	791
	792	TYPE(pedef), POINTER :: ape !<
	793	TYPE(arraydef), POINTER :: ar !<
	794
	795
[4246]	796	t1 = pmc_time()
	797
[4649]	798	IF( PRESENT( child_process_nr ) ) THEN
[4246]	799	ip_start = child_process_nr
	800	ip_end = child_process_nr
	801	ELSE
	802	ip_start = 1
	803	ip_end = children(childid)%inter_npes
	804	END IF
	805
	806	lo_ptrans = .FALSE.
[4649]	807	IF ( PRESENT( particle_transfer ) ) lo_ptrans = particle_transfer
[4246]	808
[4649]	809	IF(ip_start == 1) THEN
[4246]	810	!
	811	!-- Wait for child to fill buffer
	812	CALL MPI_BARRIER( children(childid)%intra_comm, ierr )
	813	t2 = pmc_time() - t1
	814	IF ( PRESENT( waittime ) ) waittime = t2
	815
	816	CALL MPI_BARRIER( children(childid)%intra_comm, ierr )
	817	ENDIF
	818
	819	DO ip = ip_start,ip_end
	820	ape => children(childid)%pes(ip)
	821	DO j = 1, ape%nr_arrays
	822	ar => ape%array_list(j)
[4649]	823
[4246]	824	IF ( ar%recvindex < 0 ) CYCLE
	825
[4649]	826	IF ( ar%dimkey == 2 .AND. .NOT. lo_ptrans ) THEN
[4246]	827	nr = ape%nrele
[4649]	828	ELSEIF ( ar%dimkey == 3 .AND. .NOT. lo_ptrans ) THEN
[4246]	829	nr = ape%nrele * ar%a_dim(4)
	830	ELSE IF ( ar%dimkey == 22 .AND. lo_ptrans) THEN
	831	nr = ape%nrele
	832	ELSE
[4649]	833	CYCLE ! Particle arrays are not transfered here
[4246]	834	ENDIF
	835	buf_shape(1) = nr
	836	IF(lo_ptrans) THEN
	837	CALL C_F_POINTER( ar%recvbuf, ibuf, buf_shape )
	838	ELSE
	839	CALL C_F_POINTER( ar%recvbuf, buf, buf_shape )
	840	ENDIF
	841
	842	!
	843	!-- MPI passive target RMA
	844	IF ( nr > 0 ) THEN
	845	target_disp = ar%recvindex - 1
	846	!
	847	!-- Child processes are located behind parent process
	848	target_pe = ip - 1 + m_model_npes
[4649]	849	CALL MPI_WIN_LOCK( MPI_LOCK_SHARED, target_pe, 0, children(childid)%win_parent_child, &
	850	ierr )
	851	IF ( lo_ptrans ) THEN
	852	CALL MPI_GET( ibuf, nr * 8, MPI_BYTE, target_pe, target_disp, nr * 8, &
	853	!There is no MPI_INTEGER8 datatype
[4246]	854	MPI_BYTE, children(childid)%win_parent_child, ierr )
	855	ELSE
[4649]	856	CALL MPI_GET( buf, nr, MPI_REAL, target_pe, target_disp, nr, MPI_REAL, &
	857	children(childid)%win_parent_child, ierr )
[4246]	858	ENDIF
[4649]	859	CALL MPI_WIN_UNLOCK( target_pe, children(childid)%win_parent_child, ierr )
[4246]	860	ENDIF
	861	myindex = 1
[4649]	862	IF ( ar%dimkey == 2 .AND. .NOT. lo_ptrans ) THEN
[4246]	863
	864	CALL C_F_POINTER( ar%data, data_2d, ar%a_dim(1:2) )
	865	DO ij = 1, ape%nrele
	866	data_2d(ape%locind(ij)%j,ape%locind(ij)%i) = buf(myindex)
	867	myindex = myindex + 1
	868	ENDDO
	869
[4649]	870	ELSE IF ( ar%dimkey == 3 .AND. .NOT. lo_ptrans ) THEN
[4246]	871
[4649]	872	CALL C_F_POINTER( ar%data, data_3d, ar%a_dim(1:3) )
[4246]	873	DO ij = 1, ape%nrele
[4649]	874	data_3d(1:ar%a_dim(4),ape%locind(ij)%j,ape%locind(ij)%i) = &
	875	buf(myindex:myindex+ar%a_dim(4)-1)
[4246]	876	myindex = myindex + ar%a_dim(4)
	877	ENDDO
	878
	879	ELSE IF ( ar%dimkey == 22 .AND. lo_ptrans) THEN
	880
	881	CALL C_F_POINTER( ar%data, idata_2d, ar%a_dim(1:2) )
	882	DO ij = 1, ape%nrele
	883	idata_2d(ape%locind(ij)%j,ape%locind(ij)%i) = ibuf(myindex)
	884	myindex = myindex + 1
	885	ENDDO
	886
	887	ENDIF
	888	ENDDO
	889	ENDDO
	890
	891	END SUBROUTINE pmc_s_getdata_from_buffer
	892
	893
[4649]	894	!--------------------------------------------------------------------------------------------------!
	895	! Description:
	896	! ------------
	897	!
	898	!> @Todo: Missing subroutine description.
	899	!--------------------------------------------------------------------------------------------------!
[4246]	900	SUBROUTINE get_da_names_from_child( childid )
	901
	902	!
	903	!-- Get data array description and name from child
	904	IMPLICIT NONE
	905
	906	INTEGER(iwp), INTENT(IN) :: childid !<
	907
	908	TYPE(da_namedef) :: myname !<
	909
	910	DO
	911	CALL pmc_bcast( myname%couple_index, 0, comm=m_to_child_comm(childid) )
	912
	913	IF ( myname%couple_index == -1 ) EXIT
	914
	915	CALL pmc_bcast( myname%parentdesc, 0, comm=m_to_child_comm(childid) )
	916	CALL pmc_bcast( myname%nameonparent, 0, comm=m_to_child_comm(childid) )
	917	CALL pmc_bcast( myname%childdesc, 0, comm=m_to_child_comm(childid) )
	918	CALL pmc_bcast( myname%nameonchild, 0, comm=m_to_child_comm(childid) )
	919
[4649]	920	CALL pmc_g_setname( children(childid), myname%couple_index, myname%nameonparent )
[4246]	921	ENDDO
	922
	923	END SUBROUTINE get_da_names_from_child
	924
	925
	926
[4649]	927	!--------------------------------------------------------------------------------------------------!
	928	! Description:
	929	! ------------
	930	!
	931	!> @Todo: Missing subroutine description.
	932	!--------------------------------------------------------------------------------------------------!
	933	SUBROUTINE pmc_s_setarray( childid, nrdims, dims, array_adr, second_adr, dimkey )
[4246]	934
	935	!
	936	!-- Set array for child inter process 0
	937	IMPLICIT NONE
	938
[4649]	939	INTEGER(iwp) :: i !< local counter
[4246]	940
[4649]	941	INTEGER(iwp), INTENT(IN) :: childid !<
	942	INTEGER(iwp), INTENT(IN) :: nrdims !<
[4246]	943
[4649]	944	INTEGER(iwp), INTENT(IN), OPTIONAL :: dimkey !<
[4246]	945
[4649]	946	INTEGER(iwp), INTENT(IN), DIMENSION(:) :: dims !<
	947
	948	TYPE(C_PTR), INTENT(IN) :: array_adr !<
	949
	950	TYPE(C_PTR), INTENT(IN), OPTIONAL :: second_adr !<
	951
	952	TYPE(pedef), POINTER :: ape !<
[4246]	953	TYPE(arraydef), POINTER :: ar !<
	954
	955
	956	DO i = 1, children(childid)%inter_npes
	957	ape => children(childid)%pes(i)
	958	ar => ape%array_list(next_array_in_list)
	959	ar%nrdims = nrdims
	960	ar%dimkey = nrdims
[4649]	961
	962	IF( PRESENT( dimkey ) ) ar%dimkey = dimkey
[4246]	963	ar%a_dim = dims
	964	ar%data = array_adr
	965	IF ( PRESENT( second_adr ) ) THEN
	966	ar%po_data(1) = array_adr
	967	ar%po_data(2) = second_adr
	968	ELSE
	969	ar%po_data(1) = C_NULL_PTR
	970	ar%po_data(2) = C_NULL_PTR
	971	ENDIF
[4649]	972
[4246]	973	ENDDO
	974
	975	END SUBROUTINE pmc_s_setarray
	976
	977
	978
[4649]	979	!--------------------------------------------------------------------------------------------------!
	980	! Description:
	981	! ------------
	982	!
	983	!> @Todo: Missing subroutine description.
	984	!--------------------------------------------------------------------------------------------------!
[4246]	985	SUBROUTINE pmc_s_set_active_data_array( childid, iactive )
	986
	987	IMPLICIT NONE
	988
	989	INTEGER(iwp) :: ip !<
	990	INTEGER(iwp) :: j !<
	991
[4649]	992	INTEGER(iwp), INTENT(IN) :: childid !<
	993	INTEGER(iwp), INTENT(IN) :: iactive !<
	994
	995	TYPE(pedef), POINTER :: ape !<
[4246]	996	TYPE(arraydef), POINTER :: ar !<
	997
	998	DO ip = 1, children(childid)%inter_npes
	999	ape => children(childid)%pes(ip)
	1000	DO j = 1, ape%nr_arrays
	1001	ar => ape%array_list(j)
[4649]	1002	IF ( MOD( ar%dimkey, 10 ) == 2 ) CYCLE !Not for 2D array
[4246]	1003	IF ( iactive == 1 .OR. iactive == 2 ) THEN
	1004	ar%data = ar%po_data(iactive)
	1005	ENDIF
	1006	ENDDO
	1007	ENDDO
	1008
	1009	END SUBROUTINE pmc_s_set_active_data_array
	1010
[4649]	1011	INTEGER FUNCTION pmc_s_get_child_npes( child_id )
[4246]	1012	IMPLICIT NONE
	1013
[4649]	1014	INTEGER(iwp),INTENT(IN) :: child_id !<
[4246]	1015
	1016	pmc_s_get_child_npes = children(child_id)%inter_npes
	1017
	1018	RETURN
	1019	END FUNCTION pmc_s_get_child_npes
	1020
	1021
[4649]	1022
	1023	!--------------------------------------------------------------------------------------------------!
	1024	! Description:
	1025	! ------------
	1026	!
	1027	!> @Todo: Missing subroutine description.
	1028	!--------------------------------------------------------------------------------------------------!
[4246]	1029	SUBROUTINE set_pe_index_list( mychild, index_list, nrp )
	1030
	1031	IMPLICIT NONE
	1032
[4649]	1033	INTEGER(iwp) :: i !<
	1034	INTEGER(iwp) :: ierr !<
	1035	INTEGER(iwp) :: ind !<
	1036	INTEGER(iwp) :: indwin !<
	1037	INTEGER(iwp) :: indwin2 !<
	1038	INTEGER(iwp) :: i2 !<
	1039	INTEGER(iwp) :: j !<
	1040	INTEGER(iwp) :: rempe !<
[4246]	1041
[4649]	1042	TYPE(childdef), INTENT(INOUT) :: mychild !<
[4246]	1043
[4649]	1044	INTEGER(iwp), DIMENSION(mychild%inter_npes) :: remind !<
[4246]	1045
[4649]	1046	INTEGER(iwp), INTENT(IN) :: nrp !<
[4246]	1047
[4649]	1048	INTEGER(iwp), INTENT(IN), DIMENSION(:,:) :: index_list !<
[4246]	1049
[4649]	1050	INTEGER(KIND=MPI_ADDRESS_KIND) :: winsize !<
[4246]	1051
[4649]	1052	INTEGER(iwp), DIMENSION(:), POINTER :: remindw !<
	1053	INTEGER(iwp), DIMENSION(:), POINTER :: rldef !<
	1054
	1055	TYPE(pedef), POINTER :: ape !<
	1056
[4246]	1057	!
	1058	!-- First, count entries for every remote child process
	1059	DO i = 1, mychild%inter_npes
	1060	ape => mychild%pes(i)
	1061	ape%nrele = 0
	1062	ENDDO
	1063	!
	1064	!-- Loop over number of coarse grid cells
	1065	DO j = 1, nrp
[4649]	1066	rempe = index_list(5,j) + 1 ! Process number on remote process
[4246]	1067	ape => mychild%pes(rempe)
	1068	ape%nrele = ape%nrele + 1 ! Increment number of elements for this child process
	1069	ENDDO
	1070
	1071	DO i = 1, mychild%inter_npes
	1072	ape => mychild%pes(i)
	1073	ALLOCATE( ape%locind(ape%nrele) )
	1074	ENDDO
	1075
	1076	remind = 0
	1077	!
	1078	!-- Second, create lists
	1079	!-- Loop over number of coarse grid cells
	1080	DO j = 1, nrp
	1081	rempe = index_list(5,j) + 1
	1082	ape => mychild%pes(rempe)
[4649]	1083	remind(rempe) = remind(rempe) + 1
[4246]	1084	ind = remind(rempe)
	1085	ape%locind(ind)%i = index_list(1,j)
	1086	ape%locind(ind)%j = index_list(2,j)
	1087	ENDDO
	1088	!
	1089	!-- Prepare number of elements for children processes
[4649]	1090	CALL pmc_alloc_mem( rldef, mychild%inter_npes * 2 )
[4246]	1091	!
	1092	!-- Number of child processes * size of INTEGER (i just arbitrary INTEGER)
[4649]	1093	winsize = mychild%inter_npes * STORAGE_SIZE( i ) / 8 * 2
[4246]	1094
[4649]	1095	CALL MPI_WIN_CREATE( rldef, winsize, iwp, MPI_INFO_NULL, mychild%intra_comm, indwin, ierr )
[4246]	1096	!
	1097	!-- Open window to set data
	1098	CALL MPI_WIN_FENCE( 0, indwin, ierr )
	1099
[4649]	1100	rldef(1) = 0 ! Index on remote process 0
	1101	rldef(2) = remind(1) ! Number of elements on remote process 0
[4246]	1102	!
	1103	!-- Reserve buffer for index array
	1104	DO i = 2, mychild%inter_npes
[4649]	1105	i2 = ( i - 1 ) * 2 + 1
	1106	rldef(i2) = rldef(i2-2) + rldef(i2-1) * 2 ! Index on remote process
	1107	rldef(i2+1) = remind(i) ! Number of elements on remote process
[4246]	1108	ENDDO
	1109	!
	1110	!-- Close window to allow child to access data
	1111	CALL MPI_WIN_FENCE( 0, indwin, ierr )
	1112	!
	1113	!-- Child has retrieved data
	1114	CALL MPI_WIN_FENCE( 0, indwin, ierr )
	1115
	1116	i2 = 2 * mychild%inter_npes - 1
	1117	winsize = ( rldef(i2) + rldef(i2+1) ) * 2
	1118	!
	1119	!-- Make sure, MPI_ALLOC_MEM works
	1120	winsize = MAX( winsize, INT( 1, MPI_ADDRESS_KIND ) )
	1121
	1122	CALL pmc_alloc_mem( remindw, INT( winsize ) )
	1123
	1124	CALL MPI_BARRIER( m_model_comm, ierr )
[4649]	1125	CALL MPI_WIN_CREATE( remindw, winsize * STORAGE_SIZE( i ) / 8, iwp, MPI_INFO_NULL, &
[4246]	1126	mychild%intra_comm, indwin2, ierr )
	1127	!
	1128	!-- Open window to set data
	1129	CALL MPI_WIN_FENCE( 0, indwin2, ierr )
	1130	!
	1131	!-- Create the 2D index list
	1132	DO j = 1, nrp
[4649]	1133	rempe = index_list(5,j) + 1 ! Process number on remote process
[4246]	1134	ape => mychild%pes(rempe)
	1135	i2 = rempe * 2 - 1
	1136	ind = rldef(i2) + 1
	1137	remindw(ind) = index_list(3,j)
	1138	remindw(ind+1) = index_list(4,j)
	1139	rldef(i2) = rldef(i2)+2
	1140	ENDDO
	1141	!
	1142	!-- All data are set
	1143	CALL MPI_WIN_FENCE( 0, indwin2, ierr )
	1144	!
	1145	!-- Don't know why, but this barrier is necessary before windows can be freed
	1146	!-- TODO: find out why this is required
	1147	CALL MPI_BARRIER( mychild%intra_comm, ierr )
	1148
	1149	CALL MPI_WIN_FREE( indwin, ierr )
	1150	CALL MPI_WIN_FREE( indwin2, ierr )
	1151
	1152	!
	1153	!-- TODO: check if the following idea needs to be done
	1154	!-- Sollte funktionieren, Problem mit MPI implementation
	1155	!-- https://www.lrz.de/services/software/parallel/mpi/onesided
	1156	!-- CALL MPI_Free_mem (remindw, ierr)
	1157
	1158	END SUBROUTINE set_pe_index_list
	1159
	1160	#endif
	1161	END MODULE pmc_parent

Note: See TracBrowser for help on using the repository browser.

Download in other formats:

| Impressum | ©Leibniz Universität Hannover |