Home

Context Navigation

source: palm/trunk/SOURCE/pmc_parent_mod.f90 @ 4829

Last change on this file since 4829 was 4828, checked in by Giersch, 4 years ago
Copyright updated to year 2021, interface pmc_sort removed to accelarate the nesting code
Property svn:keywords set to `Id`
File size: 38.6 KB

Rev	Line
[4246]	1	MODULE pmc_parent
[4649]	2	!--------------------------------------------------------------------------------------------------!
[4246]	3	! This file is part of the PALM model system.
	4	!
[4649]	5	! PALM is free software: you can redistribute it and/or modify it under the terms of the GNU General
	6	! Public License as published by the Free Software Foundation, either version 3 of the License, or
	7	! (at your option) any later version.
[4246]	8	!
[4649]	9	! PALM is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the
	10	! implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General
	11	! Public License for more details.
[4246]	12	!
[4649]	13	! You should have received a copy of the GNU General Public License along with PALM. If not, see
	14	! <http://www.gnu.org/licenses/>.
[4246]	15	!
[4828]	16	! Copyright 1997-2021 Leibniz Universitaet Hannover
[4649]	17	!--------------------------------------------------------------------------------------------------!
[4246]	18	!
[4649]	19	!
[4246]	20	! Current revisions:
[4649]	21	! -----------------
[4629]	22	!
	23	!
[4246]	24	! Former revisions:
	25	! -----------------
	26	! $Id: pmc_parent_mod.f90 4828 2021-01-05 11:21:41Z suehring $
[4828]	27	! pmc_s_set_2d_index_list revised for accelerating the code. Subroutine
	28	! description added.
	29	!
	30	! 4649 2020-08-25 12:11:17Z raasch
[4649]	31	! File re-formatted to follow the PALM coding standard
	32	!
	33	!
	34	! 4629 2020-07-29 09:37:56Z raasch
	35	! Support for MPI Fortran77 interface (mpif.h) removed
	36	!
[4629]	37	! 4360 2020-01-07 11:25:50Z suehring
[4649]	38	!
	39	!
[4245]	40	! 4213 2019-09-02 14:25:56Z suehring
[4649]	41	! Allocate array for index_list_2d also for zero-size arrays, in order to avoid errors when array
	42	! bound checks are enabled
	43	!
[4246]	44	! 4212 2019-09-02 14:23:05Z suehring
	45	! Corrected "Former revisions" section
[4649]	46	!
[4246]	47	! 3962 2019-05-08 19:40:33Z suehring
[4649]	48	! Bugfixes in initial settings of child and parent communication patterns.
	49	!
[4246]	50	! 3655 2019-01-07 16:51:22Z knoop
[4649]	51	! Explicit kind settings
[4246]	52	!
	53	! 1762 2016-02-25 12:31:13Z hellstea
	54	! Initial revision by K. Ketelsen
	55	!
[4649]	56	!--------------------------------------------------------------------------------------------------!
[4246]	57	! Description:
	58	! ------------
	59	!> Parent part of Palm Model Coupler
[4649]	60	!--------------------------------------------------------------------------------------------------!
[4246]	61
	62	#if defined( __parallel )
	63	USE, INTRINSIC :: ISO_C_BINDING
	64
	65	USE MPI
[4629]	66
[4246]	67	USE kinds
[4649]	68	USE pmc_general, &
	69	ONLY: arraydef, &
	70	childdef, &
	71	da_namedef, &
	72	da_namelen, &
	73	pedef, &
	74	pmc_g_setname, &
	75	pmc_max_array, &
[4828]	76	pmc_max_models
[4246]	77
[4649]	78	USE pmc_handle_communicator, &
	79	ONLY: m_model_comm, &
	80	m_model_rank, &
	81	m_model_npes, &
	82	m_to_child_comm, &
	83	m_world_rank, &
	84	pmc_parent_for_child
[4246]	85
[4649]	86	USE pmc_mpi_wrapper, &
	87	ONLY: pmc_alloc_mem, &
	88	pmc_bcast, &
	89	pmc_time
[4246]	90
	91	IMPLICIT NONE
	92
	93
	94	PRIVATE
	95	SAVE
	96
[4649]	97	INTEGER :: next_array_in_list = 0 !<
	98
[4246]	99	TYPE childindexdef
	100	INTEGER :: nrpoints !<
	101	INTEGER, DIMENSION(:,:), ALLOCATABLE :: index_list_2d !<
	102	END TYPE childindexdef
	103
[4649]	104	TYPE(childdef), DIMENSION(pmc_max_models),PUBLIC :: children !<
	105	TYPE(childindexdef), DIMENSION(pmc_max_models) :: indchildren !<
[4246]	106
	107
	108	PUBLIC pmc_parent_for_child
	109
	110
	111	INTERFACE pmc_parentinit
	112	MODULE PROCEDURE pmc_parentinit
	113	END INTERFACE pmc_parentinit
	114
	115	INTERFACE pmc_s_set_2d_index_list
	116	MODULE PROCEDURE pmc_s_set_2d_index_list
	117	END INTERFACE pmc_s_set_2d_index_list
	118
	119	INTERFACE pmc_s_clear_next_array_list
	120	MODULE PROCEDURE pmc_s_clear_next_array_list
	121	END INTERFACE pmc_s_clear_next_array_list
	122
	123	INTERFACE pmc_s_getnextarray
	124	MODULE PROCEDURE pmc_s_getnextarray
	125	END INTERFACE pmc_s_getnextarray
	126
	127	INTERFACE pmc_s_set_dataarray
	128	MODULE PROCEDURE pmc_s_set_dataarray_2d
	129	MODULE PROCEDURE pmc_s_set_dataarray_3d
	130	MODULE PROCEDURE pmc_s_set_dataarray_ip2d
	131	END INTERFACE pmc_s_set_dataarray
	132
	133	INTERFACE pmc_s_setind_and_allocmem
	134	MODULE PROCEDURE pmc_s_setind_and_allocmem
	135	END INTERFACE pmc_s_setind_and_allocmem
	136
	137	INTERFACE pmc_s_fillbuffer
	138	MODULE PROCEDURE pmc_s_fillbuffer
	139	END INTERFACE pmc_s_fillbuffer
	140
	141	INTERFACE pmc_s_getdata_from_buffer
	142	MODULE PROCEDURE pmc_s_getdata_from_buffer
	143	END INTERFACE pmc_s_getdata_from_buffer
	144
	145	INTERFACE pmc_s_set_active_data_array
	146	MODULE PROCEDURE pmc_s_set_active_data_array
	147	END INTERFACE pmc_s_set_active_data_array
	148
	149	INTERFACE pmc_s_get_child_npes
	150	MODULE PROCEDURE pmc_s_get_child_npes
	151	END INTERFACE pmc_s_get_child_npes
	152
[4649]	153	PUBLIC pmc_parentinit, &
	154	pmc_s_clear_next_array_list, &
	155	pmc_s_fillbuffer, &
	156	pmc_s_getdata_from_buffer, &
	157	pmc_s_getnextarray, &
	158	pmc_s_setind_and_allocmem, &
	159	pmc_s_set_active_data_array, &
	160	pmc_s_set_dataarray, &
	161	pmc_s_set_2d_index_list, &
[4246]	162	pmc_s_get_child_npes
	163
	164	CONTAINS
	165
	166
[4649]	167	!--------------------------------------------------------------------------------------------------!
	168	! Description:
	169	! ------------
	170	!
[4828]	171	!> If this thread is intended as parent, initialize parent part of parent-client data transfer
[4649]	172	!--------------------------------------------------------------------------------------------------!
[4246]	173	SUBROUTINE pmc_parentinit
	174
	175	IMPLICIT NONE
	176
[4649]	177	INTEGER(iwp) :: childid !<
	178	INTEGER(iwp) :: i !<
	179	INTEGER(iwp) :: istat !<
	180	INTEGER(iwp) :: j !<
[4246]	181
	182
[4649]	183	DO i = 1, SIZE( pmc_parent_for_child ) - 1
	184
[4246]	185	childid = pmc_parent_for_child( i )
	186
	187	children(childid)%model_comm = m_model_comm
	188	children(childid)%inter_comm = m_to_child_comm(childid)
	189
	190	!
	191	!-- Get rank and size
[4649]	192	CALL MPI_COMM_RANK( children(childid)%model_comm, children(childid)%model_rank, istat )
	193	CALL MPI_COMM_SIZE( children(childid)%model_comm, children(childid)%model_npes, istat )
	194	CALL MPI_COMM_REMOTE_SIZE( children(childid)%inter_comm, children(childid)%inter_npes, &
	195	istat )
[4246]	196	!
	197	!-- Intra communicator is used for MPI_GET
[4649]	198	CALL MPI_INTERCOMM_MERGE( children(childid)%inter_comm, .FALSE., &
[4246]	199	children(childid)%intra_comm, istat )
[4649]	200	CALL MPI_COMM_RANK( children(childid)%intra_comm, children(childid)%intra_rank, istat )
[4246]	201
[4649]	202	ALLOCATE( children(childid)%pes(children(childid)%inter_npes) )
[4246]	203	!
[4649]	204	!-- Allocate array of TYPE arraydef for all child PEs to store information of the transfer array
[4246]	205	DO j = 1, children(childid)%inter_npes
	206	ALLOCATE( children(childid)%pes(j)%array_list(pmc_max_array) )
	207	ENDDO
	208
[4649]	209	CALL get_da_names_from_child( childid )
[4246]	210
	211	ENDDO
	212
	213	END SUBROUTINE pmc_parentinit
	214
[4649]	215	!--------------------------------------------------------------------------------------------------!
	216	! Description:
	217	! ------------
	218	!
[4828]	219	!> thread 0 transfers the index list, which contains all parent grid cells involved in
	220	!> parent client data transfer to the thread, on which this grid cell is located
[4649]	221	!--------------------------------------------------------------------------------------------------!
[4246]	222	SUBROUTINE pmc_s_set_2d_index_list( childid, index_list )
	223
	224	IMPLICIT NONE
	225
[4828]	226	INTEGER(iwp), INTENT(IN) :: childid !<
	227	INTEGER(iwp), DIMENSION(:,:), INTENT(INOUT) :: index_list !<
	228
[4246]	229	INTEGER(iwp) :: ian !<
[4828]	230	INTEGER(iwp) :: i !<
[4246]	231	INTEGER(iwp) :: ip !<
	232	INTEGER(iwp) :: istat !<
[4828]	233	INTEGER(iwp) :: max_cells !<
[4246]	234
[4828]	235	INTEGER(iwp), DIMENSION(:), ALLOCATABLE :: cells_on_pe !<
	236	INTEGER(iwp), DIMENSION(:,:), ALLOCATABLE :: lo_ind_list !<
[4246]	237
[4649]	238
[4246]	239	IF ( m_model_rank == 0 ) THEN
	240	!
[4828]	241	!-- Compute maximum number of grid cells located on one parent thread
	242
	243	ALLOCATE(cells_on_pe(0:m_model_npes-1))
	244	cells_on_pe = 0
	245
	246	DO i=1,SIZE( index_list, 2 )
	247	cells_on_pe(index_list(6,i )) = cells_on_pe(index_list(6,i ))+1
	248	END DO
	249
	250	max_cells = MAXVAL(cells_on_pe)
	251	!
	252	!-- Allocate temp array for thread dependent transfer of index_list
	253
	254	ALLOCATE(lo_ind_list(SIZE(index_list,1),max_cells))
	255
[4246]	256	DO ip = 0, m_model_npes-1
	257	!
	258	!-- Split into parent processes
[4828]	259
	260	ian = 0
	261
	262	DO i=1,SIZE( index_list, 2 )
	263	IF(index_list(6,i ) == ip ) THEN
	264	ian = ian+1
	265	lo_ind_list(:,ian) = index_list(:,i)
	266	END IF
	267	END DO
[4246]	268	!
	269	!-- Send data to other parent processes
	270	IF ( ip == 0 ) THEN
	271	indchildren(childid)%nrpoints = ian
	272	!
[4649]	273	!-- Allocate array for index_list_2d. Note, the array will also be allocated in case
	274	!-- ian = 0, in order to avoid errors when array bounds are checked.
[4246]	275	ALLOCATE( indchildren(childid)%index_list_2d(6,1:ian) )
	276	IF ( ian > 0) THEN
[4828]	277	indchildren(childid)%index_list_2d(:,1:ian) = lo_ind_list(:,1:ian)
[4246]	278	ENDIF
	279	ELSE
[4649]	280	CALL MPI_SEND( ian, 1, MPI_INTEGER, ip, 1000, m_model_comm, istat )
[4246]	281	IF ( ian > 0) THEN
[4828]	282	CALL MPI_SEND( lo_ind_list, 6*ian, MPI_INTEGER, ip, 1001, m_model_comm, istat )
[4246]	283	ENDIF
	284	ENDIF
	285	ENDDO
[4828]	286
	287	DEALLOCATE(lo_ind_list)
	288	DEALLOCATE(cells_on_pe)
[4246]	289	ELSE
[4649]	290	CALL MPI_RECV( indchildren(childid)%nrpoints, 1, MPI_INTEGER, 0, 1000, m_model_comm, &
	291	MPI_STATUS_IGNORE, istat )
[4246]	292	ian = indchildren(childid)%nrpoints
	293	!
[4649]	294	!-- Allocate array for index_list_2d. Note, the array will also be allocated in case ian=0, in
	295	!-- order to avoid errors when array bounds are checked.
[4246]	296	ALLOCATE( indchildren(childid)%index_list_2d(6,1:ian) )
	297	IF ( ian > 0 ) THEN
[4649]	298	CALL MPI_RECV( indchildren(childid)%index_list_2d, 6*ian, MPI_INTEGER, 0, 1001, &
	299	m_model_comm, MPI_STATUS_IGNORE, istat)
[4246]	300	ENDIF
	301	ENDIF
[4649]	302	CALL set_pe_index_list( children(childid), indchildren(childid)%index_list_2d, &
[4246]	303	indchildren(childid)%nrpoints )
	304
	305	END SUBROUTINE pmc_s_set_2d_index_list
	306
	307
[4649]	308	!--------------------------------------------------------------------------------------------------!
	309	! Description:
	310	! ------------
	311	!
[4828]	312	!> Before creating an array list with arrays schedule for parent client transfer
	313	!> make sure that the list is empty
[4649]	314	!--------------------------------------------------------------------------------------------------!
[4828]	315
[4246]	316	SUBROUTINE pmc_s_clear_next_array_list
	317
[4649]	318
[4246]	319	IMPLICIT NONE
	320
	321	next_array_in_list = 0
	322
	323	END SUBROUTINE pmc_s_clear_next_array_list
	324
	325
	326	LOGICAL FUNCTION pmc_s_getnextarray( childid, myname )
	327
	328	!
	329	!-- Althoug there are no linked lists any more in PMC, this call still looks like working with a list
	330
[4649]	331	CHARACTER(LEN=*), INTENT(OUT) :: myname !<
[4246]	332
[4649]	333	INTEGER(iwp), INTENT(IN) :: childid !<
[4246]	334
[4649]	335	TYPE(pedef), POINTER :: ape !<
	336	TYPE(arraydef), POINTER :: ar !<
	337
[4246]	338	next_array_in_list = next_array_in_list + 1
	339	!
[4649]	340	!-- Array names are the same on all children processes, so take first process to get the name
[4246]	341	ape => children(childid)%pes(1)
	342
	343	IF ( next_array_in_list > ape%nr_arrays ) THEN
	344	!
	345	!-- All arrays are done
	346	pmc_s_getnextarray = .FALSE.
	347	RETURN
	348	ENDIF
	349
	350	ar => ape%array_list(next_array_in_list)
	351	myname = ar%name
	352	!
	353	!-- Return true if there is still an array in the list
	354
	355	pmc_s_getnextarray = .TRUE.
	356
	357	END FUNCTION pmc_s_getnextarray
	358
	359
	360
[4649]	361	!--------------------------------------------------------------------------------------------------!
	362	! Description:
	363	! ------------
	364	!
[4828]	365	!> add 2D real array to list of arrays scheduled for parent-client transfer
[4649]	366	!--------------------------------------------------------------------------------------------------!
[4246]	367	SUBROUTINE pmc_s_set_dataarray_2d( childid, array, array_2 )
	368
	369	IMPLICIT NONE
	370
[4649]	371	INTEGER(iwp) :: nrdims !<
[4246]	372
[4649]	373	INTEGER(iwp), INTENT(IN) :: childid !<
	374
	375	INTEGER(iwp), DIMENSION(4) :: dims !<
	376
	377	REAL(wp), INTENT(IN), DIMENSION(:,:), POINTER :: array !<
	378
[4246]	379	REAL(wp), INTENT(IN), DIMENSION(:,:), POINTER, OPTIONAL :: array_2 !<
	380
[4649]	381	TYPE(C_PTR) :: array_adr !<
	382	TYPE(C_PTR) :: second_adr !<
[4246]	383
	384
	385	dims = 1
	386	nrdims = 2
[4649]	387	dims(1) = SIZE( array, 1 )
	388	dims(2) = SIZE( array, 2 )
[4246]	389	array_adr = C_LOC( array )
	390
	391	IF ( PRESENT( array_2 ) ) THEN
[4649]	392	second_adr = C_LOC( array_2 )
	393	CALL pmc_s_setarray( childid, nrdims, dims, array_adr, second_adr = second_adr )
[4246]	394	ELSE
	395	CALL pmc_s_setarray( childid, nrdims, dims, array_adr )
	396	ENDIF
	397
	398	END SUBROUTINE pmc_s_set_dataarray_2d
	399
[4649]	400
	401	!--------------------------------------------------------------------------------------------------!
	402	! Description:
	403	! ------------
	404	!
[4828]	405	!> add 2D integer array to list of arrays scheduled for parent-client transfer
[4649]	406	!--------------------------------------------------------------------------------------------------!
[4246]	407	SUBROUTINE pmc_s_set_dataarray_ip2d( childid, array )
	408
	409	IMPLICIT NONE
	410
[4649]	411	INTEGER(iwp) :: nrdims !<
[4246]	412
[4649]	413	INTEGER(iwp),INTENT(IN) :: childid !<
[4246]	414
[4649]	415	INTEGER(idp), INTENT(IN), DIMENSION(:,:), POINTER :: array !<
[4246]	416
[4649]	417	INTEGER(iwp), DIMENSION(4) :: dims !<
[4246]	418
[4649]	419	TYPE(C_PTR) :: array_adr !<
	420
	421
[4246]	422	dims = 1
	423	nrdims = 2
[4649]	424	dims(1) = SIZE( array, 1 )
	425	dims(2) = SIZE( array, 2 )
[4246]	426	array_adr = C_LOC( array )
	427
[4649]	428	CALL pmc_s_setarray( childid, nrdims, dims, array_adr , dimkey = 22 )
[4246]	429
	430	END SUBROUTINE pmc_s_set_dataarray_ip2d
	431
	432
[4649]	433	!--------------------------------------------------------------------------------------------------!
	434	! Description:
	435	! ------------
	436	!
[4828]	437	!> add 3D real array to list of arrays scheduled for parent-client transfer
[4649]	438	!--------------------------------------------------------------------------------------------------!
[4246]	439	SUBROUTINE pmc_s_set_dataarray_3d( childid, array, nz_cl, nz, array_2 )
	440
	441	IMPLICIT NONE
	442
[4649]	443	INTEGER(iwp) :: nrdims !<
[4246]	444
[4649]	445	INTEGER(iwp), INTENT(IN) :: childid !<
	446	INTEGER(iwp), INTENT(IN) :: nz !<
	447	INTEGER(iwp), INTENT(IN) :: nz_cl !<
	448
	449	INTEGER(iwp), DIMENSION(4) :: dims !<
	450
	451	REAL(wp), INTENT(IN), DIMENSION(:,:,:), POINTER :: array !<
	452
[4246]	453	REAL(wp), INTENT(IN), DIMENSION(:,:,:), POINTER, OPTIONAL :: array_2 !<
	454
[4649]	455	TYPE(C_PTR) :: array_adr !<
	456	TYPE(C_PTR) :: second_adr !<
[4246]	457
[4649]	458	nrdims = 3
	459	dims(1) = SIZE( array, 1 )
	460	dims(2) = SIZE( array, 2 )
	461	dims(3) = SIZE( array, 3 )
	462	dims(4) = nz_cl+dims(1)-nz ! Works for first dimension 1:nz and 0:nz+1
[4246]	463
[4649]	464	array_adr = C_LOC( array )
[4246]	465	!
	466	!-- In PALM's pointer version, two indices have to be stored internally.
	467	!-- The active address of the data array is set in swap_timelevel.
	468	IF ( PRESENT( array_2 ) ) THEN
	469	second_adr = C_LOC( array_2 )
[4649]	470	CALL pmc_s_setarray( childid, nrdims, dims, array_adr, second_adr = second_adr )
[4246]	471	ELSE
	472	CALL pmc_s_setarray( childid, nrdims, dims, array_adr )
	473	ENDIF
	474
	475	END SUBROUTINE pmc_s_set_dataarray_3d
	476
	477
[4649]	478	!--------------------------------------------------------------------------------------------------!
	479	! Description:
	480	! ------------
	481	!
	482	!> @Todo: Missing subroutine description.
	483	!--------------------------------------------------------------------------------------------------!
[4246]	484	SUBROUTINE pmc_s_setind_and_allocmem( childid )
	485
[4649]	486	USE control_parameters, &
[4246]	487	ONLY: message_string
	488
	489	IMPLICIT NONE
	490
	491	!
	492	!-- Naming convention for appendices: _pc -> parent to child transfer
	493	!-- _cp -> child to parent transfer
	494	!-- send -> parent to child transfer
	495	!-- recv -> child to parent transfer
	496
[4649]	497	INTEGER(iwp) :: arlen !<
	498	INTEGER(iwp) :: i !<
	499	INTEGER(iwp) :: ierr !<
	500	INTEGER(iwp) :: j !<
	501	INTEGER(iwp) :: lo_nr_arrays !< store number of arrays in local variiab le
	502	INTEGER(iwp) :: myindex !<
	503	INTEGER(iwp) :: total_npes !< Total Number of PEs Parent and Child
[4246]	504
[4649]	505	INTEGER(idp) :: bufsize !< size of MPI data window
	506
	507	INTEGER(iwp), INTENT(IN) :: childid !<
	508
[4246]	509	INTEGER(KIND=MPI_ADDRESS_KIND) :: winsize !<
	510
[4649]	511	INTEGER(iwp), DIMENSION(:,:), ALLOCATABLE :: myindex_s !<
	512	INTEGER(iwp), DIMENSION(:,:), ALLOCATABLE :: myindex_r !<
[4246]	513
[4649]	514	REAL(wp),DIMENSION(:), POINTER, SAVE :: base_array_cp !< base array for child to parent transfer
[4246]	515	REAL(wp),DIMENSION(:), POINTER, SAVE :: base_array_pc !< base array for parent to child transfer
	516
[4649]	517	TYPE(C_PTR) :: base_ptr !<
	518
	519	TYPE(pedef), POINTER :: ape !<
	520	TYPE(arraydef), POINTER :: ar !<
	521
	522
	523	call MPI_COMM_SIZE( children(childid)%intra_comm, total_npes, ierr )
[4246]	524	!
	525	!-- Parent to child direction
	526	myindex = 1
	527	bufsize = 8
	528	!
[4649]	529	!-- All Child processes get the same number of arrays.
[4246]	530	!-- Therfore the number of arrays form the first Child process can be used for Dimension.
	531	lo_nr_arrays = children(childid)%pes(1)%nr_arrays
	532
	533	ALLOCATE( myindex_s(lo_nr_arrays,0:total_npes-1) )
	534	ALLOCATE( myindex_r(lo_nr_arrays,0:total_npes-1) )
	535
	536	myindex_s = 0
	537
	538	!
	539	!-- First stride: compute size and set index
	540	DO i = 1, children(childid)%inter_npes
	541
	542	ape => children(childid)%pes(i)
	543
	544	DO j = 1, ape%nr_arrays
	545
	546	ar => ape%array_list(j)
	547	IF ( ar%nrdims == 2 ) THEN
	548	arlen = ape%nrele
	549	ELSEIF ( ar%nrdims == 3 ) THEN
	550	arlen = ape%nrele * ar%a_dim(4)
	551	ELSE
	552	arlen = -1
	553	ENDIF
	554	ar%sendindex = myindex
	555	!
[4649]	556	!-- Using intra communicator for MPU_Alltoall, the numbers of the child processes are after
	557	!-- the parent ones.
[4246]	558
	559	myindex_s(j,i-1+children(childid)%model_npes) = myindex
	560
	561	myindex = myindex + arlen
	562	bufsize = bufsize + arlen
	563	ar%sendsize = arlen
	564	ENDDO
	565
	566	ENDDO
	567	!
	568	!-- Using MPI_Alltoall to send indices from Parent to Child
	569	!-- The data comming back from the child processes are ignored.
	570
[4649]	571	CALL MPI_ALLTOALL( myindex_s, lo_nr_arrays, MPI_INTEGER, myindex_r, lo_nr_arrays, MPI_INTEGER, &
[4246]	572	children(childid)%intra_comm, ierr )
	573
	574	!
	575	!-- Using MPI_Alltoall to receive indices from Child
	576	myindex_s = 0
	577	myindex_r = 0
	578
[4649]	579	CALL MPI_ALLTOALL( myindex_s, lo_nr_arrays, MPI_INTEGER, myindex_r, lo_nr_arrays, MPI_INTEGER, &
[4246]	580	children(childid)%intra_comm, ierr )
	581	!
[4649]	582	!-- Create RMA (One Sided Communication) window for data buffer parent to child transfer.
	583	!-- The buffer of MPI_GET (counterpart of transfer) can be PE-local, i.e. it can but must not be
	584	!-- part of the MPI RMA window. Only one RMA window is required to prepare the data for:
	585	!-- parent -> child transfer on the parent side
	586	!-- and for:
	587	!-- child -> parent transfer on the child side
[4246]	588	CALL pmc_alloc_mem( base_array_pc, bufsize )
	589	children(childid)%totalbuffersize = bufsize * wp
	590
	591	winsize = bufsize * wp
[4649]	592	CALL MPI_WIN_CREATE( base_array_pc, winsize, wp, MPI_INFO_NULL, children(childid)%intra_comm, &
[4246]	593	children(childid)%win_parent_child, ierr )
	594	!
	595	!-- Open window to set data
	596	CALL MPI_WIN_FENCE( 0, children(childid)%win_parent_child, ierr )
	597	!
	598	!-- Second stride: set buffer pointer
	599	DO i = 1, children(childid)%inter_npes
	600
	601	ape => children(childid)%pes(i)
	602
	603	DO j = 1, ape%nr_arrays
	604
	605	ar => ape%array_list(j)
	606	ar%sendbuf = C_LOC( base_array_pc(ar%sendindex) )
	607
[4649]	608	IF ( ar%sendindex + ar%sendsize > bufsize ) THEN
	609	WRITE( message_string, '(a,i4,4i7,1x,a)' ) 'parent buffer too small ',i , &
	610	ar%sendindex, ar%sendsize, ar%sendindex + ar%sendsize, bufsize, TRIM( ar%name )
[4246]	611	CALL message( 'pmc_s_setind_and_allocmem', 'PA0429', 3, 2, 0, 6, 0 )
	612	ENDIF
	613	ENDDO
	614	ENDDO
	615	!
	616	!-- Child to parent direction
	617	bufsize = 8
	618	!
	619	!-- First stride: compute size and set index
	620	DO i = 1, children(childid)%inter_npes
	621	ape => children(childid)%pes(i)
	622	DO j = 1, ape%nr_arrays
	623	ar => ape%array_list(j)
	624	!
	625	!-- Receive index from child
	626	IF ( ar%nrdims == 3 ) THEN
[4649]	627	bufsize = MAX( bufsize, INT( ape%nrele * ar%a_dim(4), MPI_ADDRESS_KIND ) )
[4246]	628	ELSE
	629	bufsize = MAX( bufsize, INT( ape%nrele, MPI_ADDRESS_KIND ) )
	630	ENDIF
	631	ar%recvindex = myindex_r(j,i-1+children(childid)%model_npes)
	632	ENDDO
	633	ENDDO
	634
	635	DEALLOCATE( myindex_s )
	636	DEALLOCATE( myindex_r )
	637
	638	!
	639	!-- Create RMA (one sided communication, RMA = Remote Memory Access) data buffer.
[4649]	640	!-- The buffer for MPI_GET can be PE local, i.e. it can but must not be part of the MPI RMA window.
[4246]	641	CALL pmc_alloc_mem( base_array_cp, bufsize, base_ptr )
	642	children(childid)%totalbuffersize = bufsize * wp
	643
	644	CALL MPI_BARRIER( children(childid)%intra_comm, ierr )
	645	!
	646	!-- Second stride: set buffer pointer
	647	DO i = 1, children(childid)%inter_npes
	648	ape => children(childid)%pes(i)
	649	DO j = 1, ape%nr_arrays
	650	ar => ape%array_list(j)
	651	ar%recvbuf = base_ptr
	652	ENDDO
	653	ENDDO
	654
	655	END SUBROUTINE pmc_s_setind_and_allocmem
	656
	657
[4649]	658	!--------------------------------------------------------------------------------------------------!
	659	! Description:
	660	! ------------
	661	!
[4828]	662	!> Fill buffer in RMA window to enable the client to fetch the dat with MPI_Get
[4649]	663	!--------------------------------------------------------------------------------------------------!
[4246]	664	SUBROUTINE pmc_s_fillbuffer( childid, waittime, particle_transfer )
	665
	666	IMPLICIT NONE
	667
[4649]	668	INTEGER(iwp) :: ierr !<
	669	INTEGER(iwp) :: ij !<
	670	INTEGER(iwp) :: ip !<
	671	INTEGER(iwp) :: j !<
	672	INTEGER(iwp) :: myindex !<
[4246]	673
[4649]	674	INTEGER(iwp), INTENT(IN) :: childid !<
	675
	676	INTEGER(iwp), DIMENSION(1) :: buf_shape !<
	677
	678	INTEGER(idp), POINTER, DIMENSION(:) :: ibuf !<
	679
	680	INTEGER(idp), POINTER, DIMENSION(:,:) :: idata_2d !<
	681
	682	LOGICAL :: lo_ptrans !<
	683
[4246]	684	LOGICAL, INTENT(IN), OPTIONAL :: particle_transfer !<
	685
[4649]	686	REAL(wp) :: t1 !<
	687	REAL(wp) :: t2 !<
[4246]	688
[4649]	689	REAL(wp), INTENT(OUT), OPTIONAL :: waittime !<
[4246]	690
[4649]	691	REAL(wp), POINTER, DIMENSION(:) :: buf !<
[4246]	692
[4649]	693	REAL(wp), POINTER, DIMENSION(:,:) :: data_2d !<
	694
[4246]	695	REAL(wp), POINTER, DIMENSION(:,:,:) :: data_3d !<
	696
[4649]	697	TYPE(pedef), POINTER :: ape !<
[4246]	698	TYPE(arraydef), POINTER :: ar !<
	699
	700	!
[4649]	701	!-- Synchronization of the model is done in pmci_synchronize. Therefor the RMA window can be filled
	702	!-- without sychronization at this point and a barrier is not necessary.
	703	!-- Please note that waittime has to be set in pmc_s_fillbuffer AND pmc_c_getbuffer.
[4246]	704	IF ( PRESENT( waittime) ) THEN
	705	t1 = pmc_time()
	706	CALL MPI_BARRIER( children(childid)%intra_comm, ierr )
	707	t2 = pmc_time()
[4649]	708	waittime = t2 - t1
[4246]	709	ENDIF
	710
	711	lo_ptrans = .FALSE.
[4649]	712	IF ( PRESENT( particle_transfer ) ) lo_ptrans = particle_transfer
[4246]	713
	714	DO ip = 1, children(childid)%inter_npes
	715	ape => children(childid)%pes(ip)
	716	DO j = 1, ape%nr_arrays
	717	ar => ape%array_list(j)
	718	myindex = 1
	719
[4649]	720	IF ( ar%dimkey == 2 .AND. .NOT.lo_ptrans ) THEN ! PALM 2D REAL*8 Array
[4246]	721
	722	buf_shape(1) = ape%nrele
	723	CALL C_F_POINTER( ar%sendbuf, buf, buf_shape )
	724	CALL C_F_POINTER( ar%data, data_2d, ar%a_dim(1:2) )
	725	DO ij = 1, ape%nrele
	726	buf(myindex) = data_2d(ape%locind(ij)%j,ape%locind(ij)%i)
	727	myindex = myindex + 1
	728	ENDDO
	729
[4649]	730	ELSEIF ( ar%dimkey == 3 .AND. .NOT.lo_ptrans ) THEN ! PALM 3D REAL*8 Array
[4246]	731
	732	buf_shape(1) = ape%nrele*ar%a_dim(4)
	733	CALL C_F_POINTER( ar%sendbuf, buf, buf_shape )
	734	CALL C_F_POINTER( ar%data, data_3d, ar%a_dim(1:3) )
	735	DO ij = 1, ape%nrele
[4649]	736	buf(myindex:myindex+ar%a_dim(4)-1) = &
	737	data_3d(1:ar%a_dim(4),ape%locind(ij)%j,ape%locind(ij)%i)
[4246]	738	myindex = myindex + ar%a_dim(4)
	739	ENDDO
[4649]	740	ELSEIF ( ar%dimkey == 22 .AND. lo_ptrans ) THEN ! 2D INTEGER*8 Array for particle Transfer
[4246]	741
	742	buf_shape(1) = ape%nrele
	743	CALL C_F_POINTER( ar%sendbuf, ibuf, buf_shape )
	744	CALL C_F_POINTER( ar%data, idata_2d, ar%a_dim(1:2) )
	745	DO ij = 1, ape%nrele
	746	ibuf(myindex) = idata_2d(ape%locind(ij)%j,ape%locind(ij)%i)
	747	myindex = myindex + 1
	748	ENDDO
	749	ENDIF
	750	ENDDO
	751	ENDDO
	752	!
	753	!-- Buffer is filled
	754	CALL MPI_BARRIER( children(childid)%intra_comm, ierr )
	755
	756	END SUBROUTINE pmc_s_fillbuffer
	757
	758
	759
[4649]	760	!--------------------------------------------------------------------------------------------------!
	761	! Description:
	762	! ------------
	763	!
[4828]	764	!> Get client data from RMM window
[4649]	765	!--------------------------------------------------------------------------------------------------!
[4246]	766	SUBROUTINE pmc_s_getdata_from_buffer( childid, waittime , particle_transfer, child_process_nr )
	767
	768	IMPLICIT NONE
	769
[4649]	770	INTEGER(iwp) :: ierr !<
	771	INTEGER(iwp) :: ij !<
	772	INTEGER(iwp) :: ip !<
	773	INTEGER(iwp) :: ip_start !<
	774	INTEGER(iwp) :: ip_end !<
	775	INTEGER(iwp) :: j !<
	776	INTEGER(iwp) :: myindex !<
	777	INTEGER(iwp) :: nr !<
	778	INTEGER(iwp) :: target_pe !<
[4246]	779
[4649]	780	INTEGER(iwp), INTENT(IN) :: childid !<
[4246]	781
[4649]	782	INTEGER(iwp), INTENT(IN), OPTIONAL :: child_process_nr !<
[4246]	783
[4649]	784	INTEGER(KIND=MPI_ADDRESS_KIND) :: target_disp !<
[4246]	785
[4649]	786	INTEGER(iwp), DIMENSION(1) :: buf_shape !<
[4246]	787
[4649]	788	INTEGER(idp), POINTER, DIMENSION(:) :: ibuf !<
[4246]	789
[4649]	790	INTEGER(idp), POINTER, DIMENSION(:,:) :: idata_2d !<
	791
	792	LOGICAL :: lo_ptrans !<
	793
	794	LOGICAL, INTENT(IN), OPTIONAL :: particle_transfer !<
	795
	796	REAL(wp) :: t1 !<
	797	REAL(wp) :: t2 !<
	798
	799	REAL(wp), INTENT(OUT), OPTIONAL :: waittime !<
	800
	801	REAL(wp), POINTER, DIMENSION(:) :: buf !<
	802
	803	REAL(wp), POINTER, DIMENSION(:,:) :: data_2d !<
	804
	805	REAL(wp), POINTER, DIMENSION(:,:,:) :: data_3d !<
	806
	807	TYPE(pedef), POINTER :: ape !<
	808	TYPE(arraydef), POINTER :: ar !<
	809
	810
[4246]	811	t1 = pmc_time()
	812
[4649]	813	IF( PRESENT( child_process_nr ) ) THEN
[4246]	814	ip_start = child_process_nr
	815	ip_end = child_process_nr
	816	ELSE
	817	ip_start = 1
	818	ip_end = children(childid)%inter_npes
	819	END IF
	820
	821	lo_ptrans = .FALSE.
[4649]	822	IF ( PRESENT( particle_transfer ) ) lo_ptrans = particle_transfer
[4246]	823
[4649]	824	IF(ip_start == 1) THEN
[4246]	825	!
	826	!-- Wait for child to fill buffer
	827	CALL MPI_BARRIER( children(childid)%intra_comm, ierr )
	828	t2 = pmc_time() - t1
	829	IF ( PRESENT( waittime ) ) waittime = t2
	830
	831	CALL MPI_BARRIER( children(childid)%intra_comm, ierr )
	832	ENDIF
	833
	834	DO ip = ip_start,ip_end
	835	ape => children(childid)%pes(ip)
	836	DO j = 1, ape%nr_arrays
	837	ar => ape%array_list(j)
[4649]	838
[4246]	839	IF ( ar%recvindex < 0 ) CYCLE
	840
[4649]	841	IF ( ar%dimkey == 2 .AND. .NOT. lo_ptrans ) THEN
[4246]	842	nr = ape%nrele
[4649]	843	ELSEIF ( ar%dimkey == 3 .AND. .NOT. lo_ptrans ) THEN
[4246]	844	nr = ape%nrele * ar%a_dim(4)
	845	ELSE IF ( ar%dimkey == 22 .AND. lo_ptrans) THEN
	846	nr = ape%nrele
	847	ELSE
[4649]	848	CYCLE ! Particle arrays are not transfered here
[4246]	849	ENDIF
	850	buf_shape(1) = nr
	851	IF(lo_ptrans) THEN
	852	CALL C_F_POINTER( ar%recvbuf, ibuf, buf_shape )
	853	ELSE
	854	CALL C_F_POINTER( ar%recvbuf, buf, buf_shape )
	855	ENDIF
	856
	857	!
	858	!-- MPI passive target RMA
	859	IF ( nr > 0 ) THEN
	860	target_disp = ar%recvindex - 1
	861	!
	862	!-- Child processes are located behind parent process
	863	target_pe = ip - 1 + m_model_npes
[4649]	864	CALL MPI_WIN_LOCK( MPI_LOCK_SHARED, target_pe, 0, children(childid)%win_parent_child, &
	865	ierr )
	866	IF ( lo_ptrans ) THEN
	867	CALL MPI_GET( ibuf, nr * 8, MPI_BYTE, target_pe, target_disp, nr * 8, &
	868	!There is no MPI_INTEGER8 datatype
[4246]	869	MPI_BYTE, children(childid)%win_parent_child, ierr )
	870	ELSE
[4649]	871	CALL MPI_GET( buf, nr, MPI_REAL, target_pe, target_disp, nr, MPI_REAL, &
	872	children(childid)%win_parent_child, ierr )
[4246]	873	ENDIF
[4649]	874	CALL MPI_WIN_UNLOCK( target_pe, children(childid)%win_parent_child, ierr )
[4246]	875	ENDIF
	876	myindex = 1
[4649]	877	IF ( ar%dimkey == 2 .AND. .NOT. lo_ptrans ) THEN
[4246]	878
	879	CALL C_F_POINTER( ar%data, data_2d, ar%a_dim(1:2) )
	880	DO ij = 1, ape%nrele
	881	data_2d(ape%locind(ij)%j,ape%locind(ij)%i) = buf(myindex)
	882	myindex = myindex + 1
	883	ENDDO
	884
[4649]	885	ELSE IF ( ar%dimkey == 3 .AND. .NOT. lo_ptrans ) THEN
[4246]	886
[4649]	887	CALL C_F_POINTER( ar%data, data_3d, ar%a_dim(1:3) )
[4246]	888	DO ij = 1, ape%nrele
[4649]	889	data_3d(1:ar%a_dim(4),ape%locind(ij)%j,ape%locind(ij)%i) = &
	890	buf(myindex:myindex+ar%a_dim(4)-1)
[4246]	891	myindex = myindex + ar%a_dim(4)
	892	ENDDO
	893
	894	ELSE IF ( ar%dimkey == 22 .AND. lo_ptrans) THEN
	895
	896	CALL C_F_POINTER( ar%data, idata_2d, ar%a_dim(1:2) )
	897	DO ij = 1, ape%nrele
	898	idata_2d(ape%locind(ij)%j,ape%locind(ij)%i) = ibuf(myindex)
	899	myindex = myindex + 1
	900	ENDDO
	901
	902	ENDIF
	903	ENDDO
	904	ENDDO
	905
	906	END SUBROUTINE pmc_s_getdata_from_buffer
	907
	908
[4649]	909	!--------------------------------------------------------------------------------------------------!
	910	! Description:
	911	! ------------
	912	!
[4828]	913	!> broadcast name of transfer arrays from child thread 0 to parent threads
[4649]	914	!--------------------------------------------------------------------------------------------------!
[4246]	915	SUBROUTINE get_da_names_from_child( childid )
	916
	917	!
	918	!-- Get data array description and name from child
	919	IMPLICIT NONE
	920
	921	INTEGER(iwp), INTENT(IN) :: childid !<
	922
	923	TYPE(da_namedef) :: myname !<
	924
	925	DO
	926	CALL pmc_bcast( myname%couple_index, 0, comm=m_to_child_comm(childid) )
	927
	928	IF ( myname%couple_index == -1 ) EXIT
	929
	930	CALL pmc_bcast( myname%parentdesc, 0, comm=m_to_child_comm(childid) )
	931	CALL pmc_bcast( myname%nameonparent, 0, comm=m_to_child_comm(childid) )
	932	CALL pmc_bcast( myname%childdesc, 0, comm=m_to_child_comm(childid) )
	933	CALL pmc_bcast( myname%nameonchild, 0, comm=m_to_child_comm(childid) )
	934
[4649]	935	CALL pmc_g_setname( children(childid), myname%couple_index, myname%nameonparent )
[4246]	936	ENDDO
	937
	938	END SUBROUTINE get_da_names_from_child
	939
	940
	941
[4649]	942	!--------------------------------------------------------------------------------------------------!
	943	! Description:
	944	! ------------
	945	!
	946	!> @Todo: Missing subroutine description.
	947	!--------------------------------------------------------------------------------------------------!
	948	SUBROUTINE pmc_s_setarray( childid, nrdims, dims, array_adr, second_adr, dimkey )
[4246]	949
	950	!
	951	!-- Set array for child inter process 0
	952	IMPLICIT NONE
	953
[4649]	954	INTEGER(iwp) :: i !< local counter
[4246]	955
[4649]	956	INTEGER(iwp), INTENT(IN) :: childid !<
	957	INTEGER(iwp), INTENT(IN) :: nrdims !<
[4246]	958
[4649]	959	INTEGER(iwp), INTENT(IN), OPTIONAL :: dimkey !<
[4246]	960
[4649]	961	INTEGER(iwp), INTENT(IN), DIMENSION(:) :: dims !<
	962
	963	TYPE(C_PTR), INTENT(IN) :: array_adr !<
	964
	965	TYPE(C_PTR), INTENT(IN), OPTIONAL :: second_adr !<
	966
	967	TYPE(pedef), POINTER :: ape !<
[4246]	968	TYPE(arraydef), POINTER :: ar !<
	969
	970
	971	DO i = 1, children(childid)%inter_npes
	972	ape => children(childid)%pes(i)
	973	ar => ape%array_list(next_array_in_list)
	974	ar%nrdims = nrdims
	975	ar%dimkey = nrdims
[4649]	976
	977	IF( PRESENT( dimkey ) ) ar%dimkey = dimkey
[4246]	978	ar%a_dim = dims
	979	ar%data = array_adr
	980	IF ( PRESENT( second_adr ) ) THEN
	981	ar%po_data(1) = array_adr
	982	ar%po_data(2) = second_adr
	983	ELSE
	984	ar%po_data(1) = C_NULL_PTR
	985	ar%po_data(2) = C_NULL_PTR
	986	ENDIF
[4649]	987
[4246]	988	ENDDO
	989
	990	END SUBROUTINE pmc_s_setarray
	991
	992
	993
[4649]	994	!--------------------------------------------------------------------------------------------------!
	995	! Description:
	996	! ------------
	997	!
	998	!> @Todo: Missing subroutine description.
	999	!--------------------------------------------------------------------------------------------------!
[4246]	1000	SUBROUTINE pmc_s_set_active_data_array( childid, iactive )
	1001
	1002	IMPLICIT NONE
	1003
	1004	INTEGER(iwp) :: ip !<
	1005	INTEGER(iwp) :: j !<
	1006
[4649]	1007	INTEGER(iwp), INTENT(IN) :: childid !<
	1008	INTEGER(iwp), INTENT(IN) :: iactive !<
	1009
	1010	TYPE(pedef), POINTER :: ape !<
[4246]	1011	TYPE(arraydef), POINTER :: ar !<
	1012
	1013	DO ip = 1, children(childid)%inter_npes
	1014	ape => children(childid)%pes(ip)
	1015	DO j = 1, ape%nr_arrays
	1016	ar => ape%array_list(j)
[4649]	1017	IF ( MOD( ar%dimkey, 10 ) == 2 ) CYCLE !Not for 2D array
[4246]	1018	IF ( iactive == 1 .OR. iactive == 2 ) THEN
	1019	ar%data = ar%po_data(iactive)
	1020	ENDIF
	1021	ENDDO
	1022	ENDDO
	1023
	1024	END SUBROUTINE pmc_s_set_active_data_array
	1025
[4649]	1026	INTEGER FUNCTION pmc_s_get_child_npes( child_id )
[4246]	1027	IMPLICIT NONE
	1028
[4649]	1029	INTEGER(iwp),INTENT(IN) :: child_id !<
[4246]	1030
	1031	pmc_s_get_child_npes = children(child_id)%inter_npes
	1032
	1033	RETURN
	1034	END FUNCTION pmc_s_get_child_npes
	1035
	1036
[4649]	1037
	1038	!--------------------------------------------------------------------------------------------------!
	1039	! Description:
	1040	! ------------
	1041	!
	1042	!> @Todo: Missing subroutine description.
	1043	!--------------------------------------------------------------------------------------------------!
[4246]	1044	SUBROUTINE set_pe_index_list( mychild, index_list, nrp )
	1045
	1046	IMPLICIT NONE
	1047
[4649]	1048	INTEGER(iwp) :: i !<
	1049	INTEGER(iwp) :: ierr !<
	1050	INTEGER(iwp) :: ind !<
	1051	INTEGER(iwp) :: indwin !<
	1052	INTEGER(iwp) :: indwin2 !<
	1053	INTEGER(iwp) :: i2 !<
	1054	INTEGER(iwp) :: j !<
	1055	INTEGER(iwp) :: rempe !<
[4246]	1056
[4649]	1057	TYPE(childdef), INTENT(INOUT) :: mychild !<
[4246]	1058
[4649]	1059	INTEGER(iwp), DIMENSION(mychild%inter_npes) :: remind !<
[4246]	1060
[4649]	1061	INTEGER(iwp), INTENT(IN) :: nrp !<
[4246]	1062
[4649]	1063	INTEGER(iwp), INTENT(IN), DIMENSION(:,:) :: index_list !<
[4246]	1064
[4649]	1065	INTEGER(KIND=MPI_ADDRESS_KIND) :: winsize !<
[4246]	1066
[4649]	1067	INTEGER(iwp), DIMENSION(:), POINTER :: remindw !<
	1068	INTEGER(iwp), DIMENSION(:), POINTER :: rldef !<
	1069
	1070	TYPE(pedef), POINTER :: ape !<
	1071
[4246]	1072	!
	1073	!-- First, count entries for every remote child process
	1074	DO i = 1, mychild%inter_npes
	1075	ape => mychild%pes(i)
	1076	ape%nrele = 0
	1077	ENDDO
	1078	!
	1079	!-- Loop over number of coarse grid cells
	1080	DO j = 1, nrp
[4649]	1081	rempe = index_list(5,j) + 1 ! Process number on remote process
[4246]	1082	ape => mychild%pes(rempe)
	1083	ape%nrele = ape%nrele + 1 ! Increment number of elements for this child process
	1084	ENDDO
	1085
	1086	DO i = 1, mychild%inter_npes
	1087	ape => mychild%pes(i)
	1088	ALLOCATE( ape%locind(ape%nrele) )
	1089	ENDDO
	1090
	1091	remind = 0
	1092	!
	1093	!-- Second, create lists
	1094	!-- Loop over number of coarse grid cells
	1095	DO j = 1, nrp
	1096	rempe = index_list(5,j) + 1
	1097	ape => mychild%pes(rempe)
[4649]	1098	remind(rempe) = remind(rempe) + 1
[4246]	1099	ind = remind(rempe)
	1100	ape%locind(ind)%i = index_list(1,j)
	1101	ape%locind(ind)%j = index_list(2,j)
	1102	ENDDO
	1103	!
	1104	!-- Prepare number of elements for children processes
[4649]	1105	CALL pmc_alloc_mem( rldef, mychild%inter_npes * 2 )
[4246]	1106	!
	1107	!-- Number of child processes * size of INTEGER (i just arbitrary INTEGER)
[4649]	1108	winsize = mychild%inter_npes * STORAGE_SIZE( i ) / 8 * 2
[4246]	1109
[4649]	1110	CALL MPI_WIN_CREATE( rldef, winsize, iwp, MPI_INFO_NULL, mychild%intra_comm, indwin, ierr )
[4246]	1111	!
	1112	!-- Open window to set data
	1113	CALL MPI_WIN_FENCE( 0, indwin, ierr )
	1114
[4649]	1115	rldef(1) = 0 ! Index on remote process 0
	1116	rldef(2) = remind(1) ! Number of elements on remote process 0
[4246]	1117	!
	1118	!-- Reserve buffer for index array
	1119	DO i = 2, mychild%inter_npes
[4649]	1120	i2 = ( i - 1 ) * 2 + 1
	1121	rldef(i2) = rldef(i2-2) + rldef(i2-1) * 2 ! Index on remote process
	1122	rldef(i2+1) = remind(i) ! Number of elements on remote process
[4246]	1123	ENDDO
	1124	!
	1125	!-- Close window to allow child to access data
	1126	CALL MPI_WIN_FENCE( 0, indwin, ierr )
	1127	!
	1128	!-- Child has retrieved data
	1129	CALL MPI_WIN_FENCE( 0, indwin, ierr )
	1130
	1131	i2 = 2 * mychild%inter_npes - 1
	1132	winsize = ( rldef(i2) + rldef(i2+1) ) * 2
	1133	!
	1134	!-- Make sure, MPI_ALLOC_MEM works
	1135	winsize = MAX( winsize, INT( 1, MPI_ADDRESS_KIND ) )
	1136
	1137	CALL pmc_alloc_mem( remindw, INT( winsize ) )
	1138
	1139	CALL MPI_BARRIER( m_model_comm, ierr )
[4649]	1140	CALL MPI_WIN_CREATE( remindw, winsize * STORAGE_SIZE( i ) / 8, iwp, MPI_INFO_NULL, &
[4246]	1141	mychild%intra_comm, indwin2, ierr )
	1142	!
	1143	!-- Open window to set data
	1144	CALL MPI_WIN_FENCE( 0, indwin2, ierr )
	1145	!
	1146	!-- Create the 2D index list
	1147	DO j = 1, nrp
[4649]	1148	rempe = index_list(5,j) + 1 ! Process number on remote process
[4246]	1149	ape => mychild%pes(rempe)
	1150	i2 = rempe * 2 - 1
	1151	ind = rldef(i2) + 1
	1152	remindw(ind) = index_list(3,j)
	1153	remindw(ind+1) = index_list(4,j)
	1154	rldef(i2) = rldef(i2)+2
	1155	ENDDO
	1156	!
	1157	!-- All data are set
	1158	CALL MPI_WIN_FENCE( 0, indwin2, ierr )
	1159	!
	1160	!-- Don't know why, but this barrier is necessary before windows can be freed
	1161	!-- TODO: find out why this is required
	1162	CALL MPI_BARRIER( mychild%intra_comm, ierr )
	1163
	1164	CALL MPI_WIN_FREE( indwin, ierr )
	1165	CALL MPI_WIN_FREE( indwin2, ierr )
	1166
	1167	!
	1168	!-- TODO: check if the following idea needs to be done
	1169	!-- Sollte funktionieren, Problem mit MPI implementation
	1170	!-- https://www.lrz.de/services/software/parallel/mpi/onesided
	1171	!-- CALL MPI_Free_mem (remindw, ierr)
	1172
	1173	END SUBROUTINE set_pe_index_list
	1174
	1175	#endif
	1176	END MODULE pmc_parent

Note: See TracBrowser for help on using the repository browser.

Download in other formats:

| Impressum | ©Leibniz Universität Hannover |