Home

Context Navigation

pmc_parent_mod.f90 @ 4881

Last change on this file since 4881 was 4830, checked in by Giersch, 4 years ago
Reformatted to follow PALM coding standard
Property svn:keywords set to `Id`
File size: 38.8 KB

Rev	Line
[4830]	1	!> @file pmc_parent_mod.f90
[4649]	2	!--------------------------------------------------------------------------------------------------!
[4246]	3	! This file is part of the PALM model system.
	4	!
[4649]	5	! PALM is free software: you can redistribute it and/or modify it under the terms of the GNU General
	6	! Public License as published by the Free Software Foundation, either version 3 of the License, or
	7	! (at your option) any later version.
[4246]	8	!
[4649]	9	! PALM is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the
	10	! implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General
	11	! Public License for more details.
[4246]	12	!
[4649]	13	! You should have received a copy of the GNU General Public License along with PALM. If not, see
	14	! <http://www.gnu.org/licenses/>.
[4246]	15	!
[4828]	16	! Copyright 1997-2021 Leibniz Universitaet Hannover
[4649]	17	!--------------------------------------------------------------------------------------------------!
[4246]	18	!
[4649]	19	!
[4246]	20	! Current revisions:
[4649]	21	! -----------------
[4629]	22	!
	23	!
[4246]	24	! Former revisions:
	25	! -----------------
	26	! $Id: pmc_parent_mod.f90 4830 2021-01-06 11:25:45Z forkel $
[4830]	27	! Reformatted to follow PALM coding standard
	28	!
	29	! 4828 2021-01-05 11:21:41Z Giersch
	30	! pmc_s_set_2d_index_list revised for accelerating the code. Subroutine
[4828]	31	! description added.
	32	!
	33	! 4649 2020-08-25 12:11:17Z raasch
[4649]	34	! File re-formatted to follow the PALM coding standard
	35	!
	36	!
	37	! 4629 2020-07-29 09:37:56Z raasch
	38	! Support for MPI Fortran77 interface (mpif.h) removed
	39	!
[4629]	40	! 4360 2020-01-07 11:25:50Z suehring
[4649]	41	!
	42	!
[4245]	43	! 4213 2019-09-02 14:25:56Z suehring
[4649]	44	! Allocate array for index_list_2d also for zero-size arrays, in order to avoid errors when array
	45	! bound checks are enabled
	46	!
[4246]	47	! 4212 2019-09-02 14:23:05Z suehring
	48	! Corrected "Former revisions" section
[4649]	49	!
[4246]	50	! 3962 2019-05-08 19:40:33Z suehring
[4649]	51	! Bugfixes in initial settings of child and parent communication patterns.
	52	!
[4246]	53	! 3655 2019-01-07 16:51:22Z knoop
[4649]	54	! Explicit kind settings
[4246]	55	!
	56	! 1762 2016-02-25 12:31:13Z hellstea
	57	! Initial revision by K. Ketelsen
	58	!
[4830]	59	! Authors:
	60	! --------
	61	!> @author Klaus Ketelsen (no affiliation)
	62	!
[4246]	63	! Description:
	64	! ------------
	65	!> Parent part of Palm Model Coupler
[4649]	66	!--------------------------------------------------------------------------------------------------!
[4830]	67	MODULE pmc_parent
[4246]	68
	69	#if defined( __parallel )
	70	USE, INTRINSIC :: ISO_C_BINDING
	71
	72	USE MPI
[4629]	73
[4246]	74	USE kinds
[4830]	75
[4649]	76	USE pmc_general, &
	77	ONLY: arraydef, &
	78	childdef, &
	79	da_namedef, &
	80	da_namelen, &
	81	pedef, &
	82	pmc_g_setname, &
	83	pmc_max_array, &
[4828]	84	pmc_max_models
[4246]	85
[4649]	86	USE pmc_handle_communicator, &
	87	ONLY: m_model_comm, &
	88	m_model_rank, &
	89	m_model_npes, &
	90	m_to_child_comm, &
	91	m_world_rank, &
	92	pmc_parent_for_child
[4246]	93
[4649]	94	USE pmc_mpi_wrapper, &
	95	ONLY: pmc_alloc_mem, &
	96	pmc_bcast, &
	97	pmc_time
[4246]	98
	99
[4830]	100	IMPLICIT NONE
[4246]	101
[4830]	102	INTEGER :: next_array_in_list = 0 !<
[4246]	103
[4830]	104	TYPE childindexdef
	105	INTEGER :: nrpoints !<
[4649]	106
[4830]	107	INTEGER, DIMENSION(:,:), ALLOCATABLE :: index_list_2d !<
	108	END TYPE childindexdef
[4246]	109
[4830]	110	TYPE(childdef), DIMENSION(pmc_max_models) :: children !<
[4246]	111
[4830]	112	TYPE(childindexdef), DIMENSION(pmc_max_models) :: indchildren !<
[4246]	113
[4830]	114	SAVE
[4246]	115
[4830]	116	PRIVATE
[4246]	117
[4830]	118	!
	119	!-- Public functions
	120	PUBLIC pmc_parent_for_child
[4246]	121
[4830]	122	!
	123	!-- Public variables, constants and types
	124	PUBLIC children, &
	125	pmc_parentinit, &
	126	pmc_s_clear_next_array_list, &
	127	pmc_s_fillbuffer, &
	128	pmc_s_getdata_from_buffer, &
	129	pmc_s_getnextarray, &
	130	pmc_s_setind_and_allocmem, &
	131	pmc_s_set_active_data_array, &
	132	pmc_s_set_dataarray, &
	133	pmc_s_set_2d_index_list, &
	134	pmc_s_get_child_npes
	135
	136	INTERFACE pmc_parentinit
	137	MODULE PROCEDURE pmc_parentinit
	138	END INTERFACE pmc_parentinit
	139
[4246]	140	INTERFACE pmc_s_set_2d_index_list
[4830]	141	MODULE PROCEDURE pmc_s_set_2d_index_list
[4246]	142	END INTERFACE pmc_s_set_2d_index_list
	143
	144	INTERFACE pmc_s_clear_next_array_list
[4830]	145	MODULE PROCEDURE pmc_s_clear_next_array_list
[4246]	146	END INTERFACE pmc_s_clear_next_array_list
	147
	148	INTERFACE pmc_s_getnextarray
[4830]	149	MODULE PROCEDURE pmc_s_getnextarray
[4246]	150	END INTERFACE pmc_s_getnextarray
	151
	152	INTERFACE pmc_s_set_dataarray
[4830]	153	MODULE PROCEDURE pmc_s_set_dataarray_2d
	154	MODULE PROCEDURE pmc_s_set_dataarray_3d
	155	MODULE PROCEDURE pmc_s_set_dataarray_ip2d
[4246]	156	END INTERFACE pmc_s_set_dataarray
	157
	158	INTERFACE pmc_s_setind_and_allocmem
[4830]	159	MODULE PROCEDURE pmc_s_setind_and_allocmem
[4246]	160	END INTERFACE pmc_s_setind_and_allocmem
	161
	162	INTERFACE pmc_s_fillbuffer
[4830]	163	MODULE PROCEDURE pmc_s_fillbuffer
[4246]	164	END INTERFACE pmc_s_fillbuffer
	165
	166	INTERFACE pmc_s_getdata_from_buffer
[4830]	167	MODULE PROCEDURE pmc_s_getdata_from_buffer
[4246]	168	END INTERFACE pmc_s_getdata_from_buffer
	169
	170	INTERFACE pmc_s_set_active_data_array
[4830]	171	MODULE PROCEDURE pmc_s_set_active_data_array
[4246]	172	END INTERFACE pmc_s_set_active_data_array
	173
	174	INTERFACE pmc_s_get_child_npes
[4830]	175	MODULE PROCEDURE pmc_s_get_child_npes
[4246]	176	END INTERFACE pmc_s_get_child_npes
	177
	178
	179	CONTAINS
	180
	181
[4649]	182	!--------------------------------------------------------------------------------------------------!
	183	! Description:
	184	! ------------
[4828]	185	!> If this thread is intended as parent, initialize parent part of parent-client data transfer
[4649]	186	!--------------------------------------------------------------------------------------------------!
[4246]	187	SUBROUTINE pmc_parentinit
	188
[4649]	189	INTEGER(iwp) :: childid !<
	190	INTEGER(iwp) :: i !<
	191	INTEGER(iwp) :: istat !<
	192	INTEGER(iwp) :: j !<
[4246]	193
	194
[4649]	195	DO i = 1, SIZE( pmc_parent_for_child ) - 1
[4246]	196	childid = pmc_parent_for_child( i )
	197
	198	children(childid)%model_comm = m_model_comm
	199	children(childid)%inter_comm = m_to_child_comm(childid)
	200
	201	!
	202	!-- Get rank and size
[4649]	203	CALL MPI_COMM_RANK( children(childid)%model_comm, children(childid)%model_rank, istat )
	204	CALL MPI_COMM_SIZE( children(childid)%model_comm, children(childid)%model_npes, istat )
	205	CALL MPI_COMM_REMOTE_SIZE( children(childid)%inter_comm, children(childid)%inter_npes, &
	206	istat )
[4830]	207
[4246]	208	!
	209	!-- Intra communicator is used for MPI_GET
[4649]	210	CALL MPI_INTERCOMM_MERGE( children(childid)%inter_comm, .FALSE., &
[4246]	211	children(childid)%intra_comm, istat )
[4649]	212	CALL MPI_COMM_RANK( children(childid)%intra_comm, children(childid)%intra_rank, istat )
[4246]	213
[4649]	214	ALLOCATE( children(childid)%pes(children(childid)%inter_npes) )
[4830]	215
[4246]	216	!
[4649]	217	!-- Allocate array of TYPE arraydef for all child PEs to store information of the transfer array
[4246]	218	DO j = 1, children(childid)%inter_npes
	219	ALLOCATE( children(childid)%pes(j)%array_list(pmc_max_array) )
	220	ENDDO
	221
[4649]	222	CALL get_da_names_from_child( childid )
[4246]	223	ENDDO
	224
	225	END SUBROUTINE pmc_parentinit
	226
[4830]	227
[4649]	228	!--------------------------------------------------------------------------------------------------!
	229	! Description:
	230	! ------------
[4828]	231	!> thread 0 transfers the index list, which contains all parent grid cells involved in
	232	!> parent client data transfer to the thread, on which this grid cell is located
[4649]	233	!--------------------------------------------------------------------------------------------------!
[4246]	234	SUBROUTINE pmc_s_set_2d_index_list( childid, index_list )
	235
[4830]	236	INTEGER(iwp) :: ian !<
	237	INTEGER(iwp) :: i !<
	238	INTEGER(iwp) :: ip !<
	239	INTEGER(iwp) :: istat !<
	240	INTEGER(iwp) :: max_cells !<
[4246]	241
[4828]	242	INTEGER(iwp), INTENT(IN) :: childid !<
	243
[4830]	244	INTEGER(iwp), DIMENSION(:,:), INTENT(INOUT) :: index_list !<
[4246]	245
[4830]	246	INTEGER(iwp), DIMENSION(:), ALLOCATABLE :: cells_on_pe !<
[4246]	247
[4830]	248	INTEGER(iwp), DIMENSION(:,:), ALLOCATABLE :: lo_ind_list !<
[4649]	249
[4830]	250
[4246]	251	IF ( m_model_rank == 0 ) THEN
	252	!
[4828]	253	!-- Compute maximum number of grid cells located on one parent thread
[4830]	254	ALLOCATE( cells_on_pe(0:m_model_npes-1) )
[4828]	255	cells_on_pe = 0
	256
[4830]	257	DO i = 1, SIZE( index_list, 2 )
[4828]	258	cells_on_pe(index_list(6,i )) = cells_on_pe(index_list(6,i ))+1
[4830]	259	ENDDO
[4828]	260
[4830]	261	max_cells = MAXVAL( cells_on_pe )
	262
[4828]	263	!
	264	!-- Allocate temp array for thread dependent transfer of index_list
[4830]	265	ALLOCATE( lo_ind_list(SIZE( index_list,1 ),max_cells) )
[4828]	266
[4246]	267	DO ip = 0, m_model_npes-1
	268	!
	269	!-- Split into parent processes
[4828]	270	ian = 0
	271
[4830]	272	DO i = 1, SIZE( index_list, 2 )
	273	IF ( index_list(6,i) == ip ) THEN
[4828]	274	ian = ian+1
	275	lo_ind_list(:,ian) = index_list(:,i)
[4830]	276	ENDIF
	277	ENDDO
	278
[4246]	279	!
	280	!-- Send data to other parent processes
	281	IF ( ip == 0 ) THEN
	282	indchildren(childid)%nrpoints = ian
	283	!
[4649]	284	!-- Allocate array for index_list_2d. Note, the array will also be allocated in case
	285	!-- ian = 0, in order to avoid errors when array bounds are checked.
[4246]	286	ALLOCATE( indchildren(childid)%index_list_2d(6,1:ian) )
[4830]	287	IF ( ian > 0 ) THEN
[4828]	288	indchildren(childid)%index_list_2d(:,1:ian) = lo_ind_list(:,1:ian)
[4246]	289	ENDIF
	290	ELSE
[4649]	291	CALL MPI_SEND( ian, 1, MPI_INTEGER, ip, 1000, m_model_comm, istat )
[4830]	292	IF ( ian > 0 ) THEN
[4828]	293	CALL MPI_SEND( lo_ind_list, 6*ian, MPI_INTEGER, ip, 1001, m_model_comm, istat )
[4246]	294	ENDIF
	295	ENDIF
	296	ENDDO
[4828]	297
[4830]	298	DEALLOCATE( lo_ind_list )
	299	DEALLOCATE( cells_on_pe )
[4246]	300	ELSE
[4649]	301	CALL MPI_RECV( indchildren(childid)%nrpoints, 1, MPI_INTEGER, 0, 1000, m_model_comm, &
	302	MPI_STATUS_IGNORE, istat )
[4246]	303	ian = indchildren(childid)%nrpoints
	304	!
[4649]	305	!-- Allocate array for index_list_2d. Note, the array will also be allocated in case ian=0, in
	306	!-- order to avoid errors when array bounds are checked.
[4246]	307	ALLOCATE( indchildren(childid)%index_list_2d(6,1:ian) )
	308	IF ( ian > 0 ) THEN
[4649]	309	CALL MPI_RECV( indchildren(childid)%index_list_2d, 6*ian, MPI_INTEGER, 0, 1001, &
	310	m_model_comm, MPI_STATUS_IGNORE, istat)
[4246]	311	ENDIF
	312	ENDIF
[4830]	313
[4649]	314	CALL set_pe_index_list( children(childid), indchildren(childid)%index_list_2d, &
[4246]	315	indchildren(childid)%nrpoints )
	316
	317	END SUBROUTINE pmc_s_set_2d_index_list
	318
	319
[4649]	320	!--------------------------------------------------------------------------------------------------!
	321	! Description:
	322	! ------------
[4828]	323	!> Before creating an array list with arrays schedule for parent client transfer
	324	!> make sure that the list is empty
[4649]	325	!--------------------------------------------------------------------------------------------------!
[4246]	326	SUBROUTINE pmc_s_clear_next_array_list
	327
	328	next_array_in_list = 0
	329
	330	END SUBROUTINE pmc_s_clear_next_array_list
	331
	332
	333	LOGICAL FUNCTION pmc_s_getnextarray( childid, myname )
	334
	335	!
	336	!-- Althoug there are no linked lists any more in PMC, this call still looks like working with a list
[4649]	337	CHARACTER(LEN=*), INTENT(OUT) :: myname !<
[4246]	338
[4649]	339	INTEGER(iwp), INTENT(IN) :: childid !<
[4246]	340
[4649]	341	TYPE(pedef), POINTER :: ape !<
	342
[4830]	343	TYPE(arraydef), POINTER :: ar !<
	344
	345
[4246]	346	next_array_in_list = next_array_in_list + 1
[4830]	347
[4246]	348	!
[4649]	349	!-- Array names are the same on all children processes, so take first process to get the name
[4246]	350	ape => children(childid)%pes(1)
	351
	352	IF ( next_array_in_list > ape%nr_arrays ) THEN
	353	!
	354	!-- All arrays are done
	355	pmc_s_getnextarray = .FALSE.
	356	RETURN
	357	ENDIF
	358
	359	ar => ape%array_list(next_array_in_list)
	360	myname = ar%name
[4830]	361
[4246]	362	!
	363	!-- Return true if there is still an array in the list
	364	pmc_s_getnextarray = .TRUE.
	365
	366	END FUNCTION pmc_s_getnextarray
	367
	368
[4649]	369	!--------------------------------------------------------------------------------------------------!
	370	! Description:
	371	! ------------
[4828]	372	!> add 2D real array to list of arrays scheduled for parent-client transfer
[4649]	373	!--------------------------------------------------------------------------------------------------!
[4246]	374	SUBROUTINE pmc_s_set_dataarray_2d( childid, array, array_2 )
	375
[4649]	376	INTEGER(iwp) :: nrdims !<
[4246]	377
[4649]	378	INTEGER(iwp), INTENT(IN) :: childid !<
	379
[4830]	380	INTEGER(iwp), DIMENSION(4) :: dims !<
[4649]	381
	382	REAL(wp), INTENT(IN), DIMENSION(:,:), POINTER :: array !<
	383
[4246]	384	REAL(wp), INTENT(IN), DIMENSION(:,:), POINTER, OPTIONAL :: array_2 !<
	385
[4649]	386	TYPE(C_PTR) :: array_adr !<
	387	TYPE(C_PTR) :: second_adr !<
[4246]	388
	389
	390	dims = 1
	391	nrdims = 2
[4649]	392	dims(1) = SIZE( array, 1 )
	393	dims(2) = SIZE( array, 2 )
[4246]	394	array_adr = C_LOC( array )
	395
	396	IF ( PRESENT( array_2 ) ) THEN
[4649]	397	second_adr = C_LOC( array_2 )
	398	CALL pmc_s_setarray( childid, nrdims, dims, array_adr, second_adr = second_adr )
[4246]	399	ELSE
	400	CALL pmc_s_setarray( childid, nrdims, dims, array_adr )
	401	ENDIF
	402
	403	END SUBROUTINE pmc_s_set_dataarray_2d
	404
[4649]	405
	406	!--------------------------------------------------------------------------------------------------!
	407	! Description:
	408	! ------------
[4828]	409	!> add 2D integer array to list of arrays scheduled for parent-client transfer
[4649]	410	!--------------------------------------------------------------------------------------------------!
[4246]	411	SUBROUTINE pmc_s_set_dataarray_ip2d( childid, array )
	412
[4649]	413	INTEGER(iwp) :: nrdims !<
[4246]	414
[4830]	415	INTEGER(iwp), DIMENSION(4) :: dims !<
[4246]	416
[4830]	417	INTEGER(iwp), INTENT(IN) :: childid !<
	418
[4649]	419	INTEGER(idp), INTENT(IN), DIMENSION(:,:), POINTER :: array !<
[4246]	420
[4649]	421	TYPE(C_PTR) :: array_adr !<
	422
	423
[4246]	424	dims = 1
	425	nrdims = 2
[4649]	426	dims(1) = SIZE( array, 1 )
	427	dims(2) = SIZE( array, 2 )
[4246]	428	array_adr = C_LOC( array )
	429
[4649]	430	CALL pmc_s_setarray( childid, nrdims, dims, array_adr , dimkey = 22 )
[4246]	431
	432	END SUBROUTINE pmc_s_set_dataarray_ip2d
	433
	434
[4649]	435	!--------------------------------------------------------------------------------------------------!
	436	! Description:
	437	! ------------
[4828]	438	!> add 3D real array to list of arrays scheduled for parent-client transfer
[4649]	439	!--------------------------------------------------------------------------------------------------!
[4246]	440	SUBROUTINE pmc_s_set_dataarray_3d( childid, array, nz_cl, nz, array_2 )
	441
[4649]	442	INTEGER(iwp) :: nrdims !<
[4246]	443
[4649]	444	INTEGER(iwp), INTENT(IN) :: childid !<
	445	INTEGER(iwp), INTENT(IN) :: nz !<
	446	INTEGER(iwp), INTENT(IN) :: nz_cl !<
	447
	448	INTEGER(iwp), DIMENSION(4) :: dims !<
	449
	450	REAL(wp), INTENT(IN), DIMENSION(:,:,:), POINTER :: array !<
	451
[4246]	452	REAL(wp), INTENT(IN), DIMENSION(:,:,:), POINTER, OPTIONAL :: array_2 !<
	453
[4649]	454	TYPE(C_PTR) :: array_adr !<
	455	TYPE(C_PTR) :: second_adr !<
[4246]	456
[4830]	457
[4649]	458	nrdims = 3
	459	dims(1) = SIZE( array, 1 )
	460	dims(2) = SIZE( array, 2 )
	461	dims(3) = SIZE( array, 3 )
	462	dims(4) = nz_cl+dims(1)-nz ! Works for first dimension 1:nz and 0:nz+1
[4246]	463
[4649]	464	array_adr = C_LOC( array )
[4246]	465	!
	466	!-- In PALM's pointer version, two indices have to be stored internally.
	467	!-- The active address of the data array is set in swap_timelevel.
	468	IF ( PRESENT( array_2 ) ) THEN
	469	second_adr = C_LOC( array_2 )
[4649]	470	CALL pmc_s_setarray( childid, nrdims, dims, array_adr, second_adr = second_adr )
[4246]	471	ELSE
	472	CALL pmc_s_setarray( childid, nrdims, dims, array_adr )
	473	ENDIF
	474
	475	END SUBROUTINE pmc_s_set_dataarray_3d
	476
	477
[4649]	478	!--------------------------------------------------------------------------------------------------!
	479	! Description:
	480	! ------------
[4830]	481	!> Naming convention for appendices: _pc -> parent to child transfer
	482	!> _cp -> child to parent transfer
	483	!> send -> parent to child transfer
	484	!> recv -> child to parent transfer
	485	!>
	486	!> @todo: Missing subroutine description.
[4649]	487	!--------------------------------------------------------------------------------------------------!
[4246]	488	SUBROUTINE pmc_s_setind_and_allocmem( childid )
	489
[4649]	490	USE control_parameters, &
[4246]	491	ONLY: message_string
	492
[4649]	493	INTEGER(iwp) :: arlen !<
	494	INTEGER(iwp) :: i !<
	495	INTEGER(iwp) :: ierr !<
	496	INTEGER(iwp) :: j !<
	497	INTEGER(iwp) :: lo_nr_arrays !< store number of arrays in local variiab le
	498	INTEGER(iwp) :: myindex !<
	499	INTEGER(iwp) :: total_npes !< Total Number of PEs Parent and Child
	500	INTEGER(idp) :: bufsize !< size of MPI data window
	501
	502	INTEGER(iwp), INTENT(IN) :: childid !<
	503
[4246]	504	INTEGER(KIND=MPI_ADDRESS_KIND) :: winsize !<
	505
[4649]	506	INTEGER(iwp), DIMENSION(:,:), ALLOCATABLE :: myindex_s !<
	507	INTEGER(iwp), DIMENSION(:,:), ALLOCATABLE :: myindex_r !<
[4246]	508
[4649]	509	REAL(wp),DIMENSION(:), POINTER, SAVE :: base_array_cp !< base array for child to parent transfer
[4246]	510	REAL(wp),DIMENSION(:), POINTER, SAVE :: base_array_pc !< base array for parent to child transfer
	511
[4649]	512	TYPE(C_PTR) :: base_ptr !<
	513
[4830]	514	TYPE(pedef), POINTER :: ape !<
	515
[4649]	516	TYPE(arraydef), POINTER :: ar !<
	517
	518
	519	call MPI_COMM_SIZE( children(childid)%intra_comm, total_npes, ierr )
[4246]	520	!
	521	!-- Parent to child direction
	522	myindex = 1
	523	bufsize = 8
[4830]	524
[4246]	525	!
[4649]	526	!-- All Child processes get the same number of arrays.
[4246]	527	!-- Therfore the number of arrays form the first Child process can be used for Dimension.
	528	lo_nr_arrays = children(childid)%pes(1)%nr_arrays
	529
	530	ALLOCATE( myindex_s(lo_nr_arrays,0:total_npes-1) )
	531	ALLOCATE( myindex_r(lo_nr_arrays,0:total_npes-1) )
	532
	533	myindex_s = 0
	534
	535	!
	536	!-- First stride: compute size and set index
	537	DO i = 1, children(childid)%inter_npes
	538	ape => children(childid)%pes(i)
	539	DO j = 1, ape%nr_arrays
	540	ar => ape%array_list(j)
	541	IF ( ar%nrdims == 2 ) THEN
	542	arlen = ape%nrele
	543	ELSEIF ( ar%nrdims == 3 ) THEN
	544	arlen = ape%nrele * ar%a_dim(4)
	545	ELSE
	546	arlen = -1
	547	ENDIF
	548	ar%sendindex = myindex
[4830]	549
[4246]	550	!
[4649]	551	!-- Using intra communicator for MPU_Alltoall, the numbers of the child processes are after
	552	!-- the parent ones.
[4246]	553	myindex_s(j,i-1+children(childid)%model_npes) = myindex
	554
	555	myindex = myindex + arlen
	556	bufsize = bufsize + arlen
	557	ar%sendsize = arlen
	558	ENDDO
[4830]	559	ENDDO
[4246]	560
	561	!
	562	!-- Using MPI_Alltoall to send indices from Parent to Child
	563	!-- The data comming back from the child processes are ignored.
[4649]	564	CALL MPI_ALLTOALL( myindex_s, lo_nr_arrays, MPI_INTEGER, myindex_r, lo_nr_arrays, MPI_INTEGER, &
[4246]	565	children(childid)%intra_comm, ierr )
	566
	567	!
	568	!-- Using MPI_Alltoall to receive indices from Child
	569	myindex_s = 0
	570	myindex_r = 0
	571
[4649]	572	CALL MPI_ALLTOALL( myindex_s, lo_nr_arrays, MPI_INTEGER, myindex_r, lo_nr_arrays, MPI_INTEGER, &
[4246]	573	children(childid)%intra_comm, ierr )
[4830]	574
[4246]	575	!
[4649]	576	!-- Create RMA (One Sided Communication) window for data buffer parent to child transfer.
	577	!-- The buffer of MPI_GET (counterpart of transfer) can be PE-local, i.e. it can but must not be
	578	!-- part of the MPI RMA window. Only one RMA window is required to prepare the data for:
	579	!-- parent -> child transfer on the parent side
	580	!-- and for:
	581	!-- child -> parent transfer on the child side
[4246]	582	CALL pmc_alloc_mem( base_array_pc, bufsize )
	583	children(childid)%totalbuffersize = bufsize * wp
	584
	585	winsize = bufsize * wp
[4649]	586	CALL MPI_WIN_CREATE( base_array_pc, winsize, wp, MPI_INFO_NULL, children(childid)%intra_comm, &
[4246]	587	children(childid)%win_parent_child, ierr )
[4830]	588
[4246]	589	!
	590	!-- Open window to set data
	591	CALL MPI_WIN_FENCE( 0, children(childid)%win_parent_child, ierr )
[4830]	592
[4246]	593	!
	594	!-- Second stride: set buffer pointer
	595	DO i = 1, children(childid)%inter_npes
	596	ape => children(childid)%pes(i)
	597	DO j = 1, ape%nr_arrays
	598	ar => ape%array_list(j)
	599	ar%sendbuf = C_LOC( base_array_pc(ar%sendindex) )
[4649]	600	IF ( ar%sendindex + ar%sendsize > bufsize ) THEN
	601	WRITE( message_string, '(a,i4,4i7,1x,a)' ) 'parent buffer too small ',i , &
	602	ar%sendindex, ar%sendsize, ar%sendindex + ar%sendsize, bufsize, TRIM( ar%name )
[4246]	603	CALL message( 'pmc_s_setind_and_allocmem', 'PA0429', 3, 2, 0, 6, 0 )
	604	ENDIF
	605	ENDDO
	606	ENDDO
[4830]	607
[4246]	608	!
	609	!-- Child to parent direction
	610	bufsize = 8
[4830]	611
[4246]	612	!
	613	!-- First stride: compute size and set index
	614	DO i = 1, children(childid)%inter_npes
	615	ape => children(childid)%pes(i)
	616	DO j = 1, ape%nr_arrays
	617	ar => ape%array_list(j)
	618	!
	619	!-- Receive index from child
	620	IF ( ar%nrdims == 3 ) THEN
[4649]	621	bufsize = MAX( bufsize, INT( ape%nrele * ar%a_dim(4), MPI_ADDRESS_KIND ) )
[4246]	622	ELSE
	623	bufsize = MAX( bufsize, INT( ape%nrele, MPI_ADDRESS_KIND ) )
	624	ENDIF
	625	ar%recvindex = myindex_r(j,i-1+children(childid)%model_npes)
	626	ENDDO
	627	ENDDO
	628
	629	DEALLOCATE( myindex_s )
	630	DEALLOCATE( myindex_r )
	631
	632	!
	633	!-- Create RMA (one sided communication, RMA = Remote Memory Access) data buffer.
[4649]	634	!-- The buffer for MPI_GET can be PE local, i.e. it can but must not be part of the MPI RMA window.
[4246]	635	CALL pmc_alloc_mem( base_array_cp, bufsize, base_ptr )
	636	children(childid)%totalbuffersize = bufsize * wp
	637
	638	CALL MPI_BARRIER( children(childid)%intra_comm, ierr )
[4830]	639
[4246]	640	!
	641	!-- Second stride: set buffer pointer
	642	DO i = 1, children(childid)%inter_npes
	643	ape => children(childid)%pes(i)
	644	DO j = 1, ape%nr_arrays
	645	ar => ape%array_list(j)
	646	ar%recvbuf = base_ptr
	647	ENDDO
	648	ENDDO
	649
	650	END SUBROUTINE pmc_s_setind_and_allocmem
	651
	652
[4649]	653	!--------------------------------------------------------------------------------------------------!
	654	! Description:
	655	! ------------
[4828]	656	!> Fill buffer in RMA window to enable the client to fetch the dat with MPI_Get
[4649]	657	!--------------------------------------------------------------------------------------------------!
[4246]	658	SUBROUTINE pmc_s_fillbuffer( childid, waittime, particle_transfer )
	659
[4649]	660	INTEGER(iwp) :: ierr !<
	661	INTEGER(iwp) :: ij !<
	662	INTEGER(iwp) :: ip !<
	663	INTEGER(iwp) :: j !<
	664	INTEGER(iwp) :: myindex !<
[4246]	665
[4649]	666	INTEGER(iwp), INTENT(IN) :: childid !<
	667
	668	INTEGER(iwp), DIMENSION(1) :: buf_shape !<
	669
	670	INTEGER(idp), POINTER, DIMENSION(:) :: ibuf !<
	671
	672	INTEGER(idp), POINTER, DIMENSION(:,:) :: idata_2d !<
	673
	674	LOGICAL :: lo_ptrans !<
	675
[4246]	676	LOGICAL, INTENT(IN), OPTIONAL :: particle_transfer !<
	677
[4649]	678	REAL(wp) :: t1 !<
	679	REAL(wp) :: t2 !<
[4246]	680
[4830]	681	REAL(wp), INTENT(OUT), OPTIONAL :: waittime !<
[4246]	682
[4649]	683	REAL(wp), POINTER, DIMENSION(:) :: buf !<
[4246]	684
[4649]	685	REAL(wp), POINTER, DIMENSION(:,:) :: data_2d !<
	686
[4246]	687	REAL(wp), POINTER, DIMENSION(:,:,:) :: data_3d !<
	688
[4830]	689	TYPE(pedef), POINTER :: ape !<
	690
[4246]	691	TYPE(arraydef), POINTER :: ar !<
	692
	693	!
[4649]	694	!-- Synchronization of the model is done in pmci_synchronize. Therefor the RMA window can be filled
	695	!-- without sychronization at this point and a barrier is not necessary.
	696	!-- Please note that waittime has to be set in pmc_s_fillbuffer AND pmc_c_getbuffer.
[4830]	697	IF ( PRESENT( waittime ) ) THEN
[4246]	698	t1 = pmc_time()
	699	CALL MPI_BARRIER( children(childid)%intra_comm, ierr )
	700	t2 = pmc_time()
[4649]	701	waittime = t2 - t1
[4246]	702	ENDIF
	703
	704	lo_ptrans = .FALSE.
[4649]	705	IF ( PRESENT( particle_transfer ) ) lo_ptrans = particle_transfer
[4246]	706
	707	DO ip = 1, children(childid)%inter_npes
	708	ape => children(childid)%pes(ip)
	709	DO j = 1, ape%nr_arrays
	710	ar => ape%array_list(j)
	711	myindex = 1
[4649]	712	IF ( ar%dimkey == 2 .AND. .NOT.lo_ptrans ) THEN ! PALM 2D REAL*8 Array
[4246]	713	buf_shape(1) = ape%nrele
	714	CALL C_F_POINTER( ar%sendbuf, buf, buf_shape )
	715	CALL C_F_POINTER( ar%data, data_2d, ar%a_dim(1:2) )
	716	DO ij = 1, ape%nrele
	717	buf(myindex) = data_2d(ape%locind(ij)%j,ape%locind(ij)%i)
	718	myindex = myindex + 1
	719	ENDDO
[4649]	720	ELSEIF ( ar%dimkey == 3 .AND. .NOT.lo_ptrans ) THEN ! PALM 3D REAL*8 Array
[4246]	721	buf_shape(1) = ape%nrele*ar%a_dim(4)
	722	CALL C_F_POINTER( ar%sendbuf, buf, buf_shape )
	723	CALL C_F_POINTER( ar%data, data_3d, ar%a_dim(1:3) )
	724	DO ij = 1, ape%nrele
[4649]	725	buf(myindex:myindex+ar%a_dim(4)-1) = &
	726	data_3d(1:ar%a_dim(4),ape%locind(ij)%j,ape%locind(ij)%i)
[4246]	727	myindex = myindex + ar%a_dim(4)
	728	ENDDO
[4649]	729	ELSEIF ( ar%dimkey == 22 .AND. lo_ptrans ) THEN ! 2D INTEGER*8 Array for particle Transfer
[4246]	730	buf_shape(1) = ape%nrele
	731	CALL C_F_POINTER( ar%sendbuf, ibuf, buf_shape )
	732	CALL C_F_POINTER( ar%data, idata_2d, ar%a_dim(1:2) )
	733	DO ij = 1, ape%nrele
	734	ibuf(myindex) = idata_2d(ape%locind(ij)%j,ape%locind(ij)%i)
	735	myindex = myindex + 1
	736	ENDDO
	737	ENDIF
	738	ENDDO
	739	ENDDO
[4830]	740
[4246]	741	!
	742	!-- Buffer is filled
	743	CALL MPI_BARRIER( children(childid)%intra_comm, ierr )
	744
	745	END SUBROUTINE pmc_s_fillbuffer
	746
	747
[4649]	748	!--------------------------------------------------------------------------------------------------!
	749	! Description:
	750	! ------------
[4828]	751	!> Get client data from RMM window
[4649]	752	!--------------------------------------------------------------------------------------------------!
[4246]	753	SUBROUTINE pmc_s_getdata_from_buffer( childid, waittime , particle_transfer, child_process_nr )
	754
[4649]	755	INTEGER(iwp) :: ierr !<
	756	INTEGER(iwp) :: ij !<
	757	INTEGER(iwp) :: ip !<
	758	INTEGER(iwp) :: ip_start !<
	759	INTEGER(iwp) :: ip_end !<
	760	INTEGER(iwp) :: j !<
	761	INTEGER(iwp) :: myindex !<
	762	INTEGER(iwp) :: nr !<
	763	INTEGER(iwp) :: target_pe !<
[4246]	764
[4649]	765	INTEGER(iwp), INTENT(IN) :: childid !<
[4246]	766
[4649]	767	INTEGER(iwp), INTENT(IN), OPTIONAL :: child_process_nr !<
[4246]	768
[4649]	769	INTEGER(KIND=MPI_ADDRESS_KIND) :: target_disp !<
[4246]	770
[4649]	771	INTEGER(iwp), DIMENSION(1) :: buf_shape !<
[4246]	772
[4649]	773	INTEGER(idp), POINTER, DIMENSION(:) :: ibuf !<
[4246]	774
[4649]	775	INTEGER(idp), POINTER, DIMENSION(:,:) :: idata_2d !<
	776
	777	LOGICAL :: lo_ptrans !<
	778
	779	LOGICAL, INTENT(IN), OPTIONAL :: particle_transfer !<
	780
	781	REAL(wp) :: t1 !<
	782	REAL(wp) :: t2 !<
	783
	784	REAL(wp), INTENT(OUT), OPTIONAL :: waittime !<
	785
	786	REAL(wp), POINTER, DIMENSION(:) :: buf !<
	787
	788	REAL(wp), POINTER, DIMENSION(:,:) :: data_2d !<
	789
	790	REAL(wp), POINTER, DIMENSION(:,:,:) :: data_3d !<
	791
[4830]	792	TYPE(pedef), POINTER :: ape !<
	793
[4649]	794	TYPE(arraydef), POINTER :: ar !<
	795
	796
[4246]	797	t1 = pmc_time()
	798
[4649]	799	IF( PRESENT( child_process_nr ) ) THEN
[4246]	800	ip_start = child_process_nr
	801	ip_end = child_process_nr
	802	ELSE
	803	ip_start = 1
	804	ip_end = children(childid)%inter_npes
	805	END IF
	806
	807	lo_ptrans = .FALSE.
[4649]	808	IF ( PRESENT( particle_transfer ) ) lo_ptrans = particle_transfer
[4246]	809
[4649]	810	IF(ip_start == 1) THEN
[4246]	811	!
	812	!-- Wait for child to fill buffer
	813	CALL MPI_BARRIER( children(childid)%intra_comm, ierr )
	814	t2 = pmc_time() - t1
	815	IF ( PRESENT( waittime ) ) waittime = t2
	816
	817	CALL MPI_BARRIER( children(childid)%intra_comm, ierr )
	818	ENDIF
	819
[4830]	820	DO ip = ip_start, ip_end
[4246]	821	ape => children(childid)%pes(ip)
	822	DO j = 1, ape%nr_arrays
	823	ar => ape%array_list(j)
[4649]	824
[4246]	825	IF ( ar%recvindex < 0 ) CYCLE
	826
[4649]	827	IF ( ar%dimkey == 2 .AND. .NOT. lo_ptrans ) THEN
[4246]	828	nr = ape%nrele
[4649]	829	ELSEIF ( ar%dimkey == 3 .AND. .NOT. lo_ptrans ) THEN
[4246]	830	nr = ape%nrele * ar%a_dim(4)
	831	ELSE IF ( ar%dimkey == 22 .AND. lo_ptrans) THEN
	832	nr = ape%nrele
	833	ELSE
[4649]	834	CYCLE ! Particle arrays are not transfered here
[4246]	835	ENDIF
[4830]	836
[4246]	837	buf_shape(1) = nr
[4830]	838
[4246]	839	IF(lo_ptrans) THEN
	840	CALL C_F_POINTER( ar%recvbuf, ibuf, buf_shape )
	841	ELSE
	842	CALL C_F_POINTER( ar%recvbuf, buf, buf_shape )
	843	ENDIF
	844
	845	!
	846	!-- MPI passive target RMA
	847	IF ( nr > 0 ) THEN
	848	target_disp = ar%recvindex - 1
	849	!
	850	!-- Child processes are located behind parent process
	851	target_pe = ip - 1 + m_model_npes
[4649]	852	CALL MPI_WIN_LOCK( MPI_LOCK_SHARED, target_pe, 0, children(childid)%win_parent_child, &
	853	ierr )
	854	IF ( lo_ptrans ) THEN
	855	CALL MPI_GET( ibuf, nr * 8, MPI_BYTE, target_pe, target_disp, nr * 8, &
	856	!There is no MPI_INTEGER8 datatype
[4246]	857	MPI_BYTE, children(childid)%win_parent_child, ierr )
	858	ELSE
[4649]	859	CALL MPI_GET( buf, nr, MPI_REAL, target_pe, target_disp, nr, MPI_REAL, &
	860	children(childid)%win_parent_child, ierr )
[4246]	861	ENDIF
[4649]	862	CALL MPI_WIN_UNLOCK( target_pe, children(childid)%win_parent_child, ierr )
[4246]	863	ENDIF
[4830]	864
[4246]	865	myindex = 1
[4830]	866
[4649]	867	IF ( ar%dimkey == 2 .AND. .NOT. lo_ptrans ) THEN
[4246]	868	CALL C_F_POINTER( ar%data, data_2d, ar%a_dim(1:2) )
	869	DO ij = 1, ape%nrele
	870	data_2d(ape%locind(ij)%j,ape%locind(ij)%i) = buf(myindex)
	871	myindex = myindex + 1
	872	ENDDO
[4649]	873	ELSE IF ( ar%dimkey == 3 .AND. .NOT. lo_ptrans ) THEN
	874	CALL C_F_POINTER( ar%data, data_3d, ar%a_dim(1:3) )
[4246]	875	DO ij = 1, ape%nrele
[4649]	876	data_3d(1:ar%a_dim(4),ape%locind(ij)%j,ape%locind(ij)%i) = &
	877	buf(myindex:myindex+ar%a_dim(4)-1)
[4246]	878	myindex = myindex + ar%a_dim(4)
	879	ENDDO
	880	ELSE IF ( ar%dimkey == 22 .AND. lo_ptrans) THEN
	881	CALL C_F_POINTER( ar%data, idata_2d, ar%a_dim(1:2) )
	882	DO ij = 1, ape%nrele
	883	idata_2d(ape%locind(ij)%j,ape%locind(ij)%i) = ibuf(myindex)
	884	myindex = myindex + 1
	885	ENDDO
	886	ENDIF
	887	ENDDO
	888	ENDDO
	889
	890	END SUBROUTINE pmc_s_getdata_from_buffer
	891
	892
[4649]	893	!--------------------------------------------------------------------------------------------------!
	894	! Description:
	895	! ------------
[4828]	896	!> broadcast name of transfer arrays from child thread 0 to parent threads
[4649]	897	!--------------------------------------------------------------------------------------------------!
[4246]	898	SUBROUTINE get_da_names_from_child( childid )
	899
	900	!
	901	!-- Get data array description and name from child
	902	INTEGER(iwp), INTENT(IN) :: childid !<
	903
	904	TYPE(da_namedef) :: myname !<
	905
[4830]	906
[4246]	907	DO
	908	CALL pmc_bcast( myname%couple_index, 0, comm=m_to_child_comm(childid) )
	909
	910	IF ( myname%couple_index == -1 ) EXIT
	911
	912	CALL pmc_bcast( myname%parentdesc, 0, comm=m_to_child_comm(childid) )
	913	CALL pmc_bcast( myname%nameonparent, 0, comm=m_to_child_comm(childid) )
	914	CALL pmc_bcast( myname%childdesc, 0, comm=m_to_child_comm(childid) )
	915	CALL pmc_bcast( myname%nameonchild, 0, comm=m_to_child_comm(childid) )
	916
[4649]	917	CALL pmc_g_setname( children(childid), myname%couple_index, myname%nameonparent )
[4246]	918	ENDDO
	919
	920	END SUBROUTINE get_da_names_from_child
	921
	922
[4649]	923	!--------------------------------------------------------------------------------------------------!
	924	! Description:
	925	! ------------
[4830]	926	!> @todo: Missing subroutine description.
[4649]	927	!--------------------------------------------------------------------------------------------------!
	928	SUBROUTINE pmc_s_setarray( childid, nrdims, dims, array_adr, second_adr, dimkey )
[4246]	929
	930	!
	931	!-- Set array for child inter process 0
[4649]	932	INTEGER(iwp) :: i !< local counter
[4246]	933
[4649]	934	INTEGER(iwp), INTENT(IN) :: childid !<
	935	INTEGER(iwp), INTENT(IN) :: nrdims !<
[4246]	936
[4649]	937	INTEGER(iwp), INTENT(IN), OPTIONAL :: dimkey !<
[4246]	938
[4649]	939	INTEGER(iwp), INTENT(IN), DIMENSION(:) :: dims !<
	940
	941	TYPE(C_PTR), INTENT(IN) :: array_adr !<
	942
	943	TYPE(C_PTR), INTENT(IN), OPTIONAL :: second_adr !<
	944
[4830]	945	TYPE(pedef), POINTER :: ape !<
	946
[4246]	947	TYPE(arraydef), POINTER :: ar !<
	948
	949
	950	DO i = 1, children(childid)%inter_npes
	951	ape => children(childid)%pes(i)
	952	ar => ape%array_list(next_array_in_list)
	953	ar%nrdims = nrdims
	954	ar%dimkey = nrdims
[4649]	955
	956	IF( PRESENT( dimkey ) ) ar%dimkey = dimkey
[4246]	957	ar%a_dim = dims
	958	ar%data = array_adr
	959	IF ( PRESENT( second_adr ) ) THEN
	960	ar%po_data(1) = array_adr
	961	ar%po_data(2) = second_adr
	962	ELSE
	963	ar%po_data(1) = C_NULL_PTR
	964	ar%po_data(2) = C_NULL_PTR
	965	ENDIF
	966	ENDDO
	967
	968	END SUBROUTINE pmc_s_setarray
	969
	970
[4649]	971	!--------------------------------------------------------------------------------------------------!
	972	! Description:
	973	! ------------
[4830]	974	!> @todo: Missing subroutine description.
[4649]	975	!--------------------------------------------------------------------------------------------------!
[4246]	976	SUBROUTINE pmc_s_set_active_data_array( childid, iactive )
	977
	978	INTEGER(iwp) :: ip !<
	979	INTEGER(iwp) :: j !<
	980
[4649]	981	INTEGER(iwp), INTENT(IN) :: childid !<
	982	INTEGER(iwp), INTENT(IN) :: iactive !<
	983
[4830]	984	TYPE(pedef), POINTER :: ape !<
	985
[4246]	986	TYPE(arraydef), POINTER :: ar !<
	987
	988	DO ip = 1, children(childid)%inter_npes
	989	ape => children(childid)%pes(ip)
	990	DO j = 1, ape%nr_arrays
	991	ar => ape%array_list(j)
[4649]	992	IF ( MOD( ar%dimkey, 10 ) == 2 ) CYCLE !Not for 2D array
[4246]	993	IF ( iactive == 1 .OR. iactive == 2 ) THEN
	994	ar%data = ar%po_data(iactive)
	995	ENDIF
	996	ENDDO
	997	ENDDO
	998
	999	END SUBROUTINE pmc_s_set_active_data_array
	1000
[4830]	1001
	1002	!--------------------------------------------------------------------------------------------------!
	1003	! Description:
	1004	! ------------
	1005	!> @todo: Missing function description.
	1006	!--------------------------------------------------------------------------------------------------!
[4649]	1007	INTEGER FUNCTION pmc_s_get_child_npes( child_id )
[4246]	1008
[4649]	1009	INTEGER(iwp),INTENT(IN) :: child_id !<
[4246]	1010
	1011	pmc_s_get_child_npes = children(child_id)%inter_npes
	1012
	1013	RETURN
[4830]	1014
[4246]	1015	END FUNCTION pmc_s_get_child_npes
	1016
	1017
[4649]	1018	!--------------------------------------------------------------------------------------------------!
	1019	! Description:
	1020	! ------------
[4830]	1021	!> @todo: Missing subroutine description.
[4649]	1022	!--------------------------------------------------------------------------------------------------!
[4246]	1023	SUBROUTINE set_pe_index_list( mychild, index_list, nrp )
	1024
[4649]	1025	INTEGER(iwp) :: i !<
	1026	INTEGER(iwp) :: ierr !<
	1027	INTEGER(iwp) :: ind !<
	1028	INTEGER(iwp) :: indwin !<
	1029	INTEGER(iwp) :: indwin2 !<
	1030	INTEGER(iwp) :: i2 !<
	1031	INTEGER(iwp) :: j !<
	1032	INTEGER(iwp) :: rempe !<
[4246]	1033
[4649]	1034	TYPE(childdef), INTENT(INOUT) :: mychild !<
[4246]	1035
[4649]	1036	INTEGER(iwp), DIMENSION(mychild%inter_npes) :: remind !<
[4246]	1037
[4649]	1038	INTEGER(iwp), INTENT(IN) :: nrp !<
[4246]	1039
[4649]	1040	INTEGER(iwp), INTENT(IN), DIMENSION(:,:) :: index_list !<
[4246]	1041
[4649]	1042	INTEGER(KIND=MPI_ADDRESS_KIND) :: winsize !<
[4246]	1043
[4649]	1044	INTEGER(iwp), DIMENSION(:), POINTER :: remindw !<
	1045	INTEGER(iwp), DIMENSION(:), POINTER :: rldef !<
	1046
	1047	TYPE(pedef), POINTER :: ape !<
	1048
[4830]	1049
[4246]	1050	!
	1051	!-- First, count entries for every remote child process
	1052	DO i = 1, mychild%inter_npes
	1053	ape => mychild%pes(i)
	1054	ape%nrele = 0
	1055	ENDDO
[4830]	1056
[4246]	1057	!
	1058	!-- Loop over number of coarse grid cells
	1059	DO j = 1, nrp
[4649]	1060	rempe = index_list(5,j) + 1 ! Process number on remote process
[4246]	1061	ape => mychild%pes(rempe)
	1062	ape%nrele = ape%nrele + 1 ! Increment number of elements for this child process
	1063	ENDDO
	1064
	1065	DO i = 1, mychild%inter_npes
	1066	ape => mychild%pes(i)
	1067	ALLOCATE( ape%locind(ape%nrele) )
	1068	ENDDO
	1069
	1070	remind = 0
[4830]	1071
[4246]	1072	!
	1073	!-- Second, create lists
	1074	!-- Loop over number of coarse grid cells
	1075	DO j = 1, nrp
	1076	rempe = index_list(5,j) + 1
	1077	ape => mychild%pes(rempe)
[4649]	1078	remind(rempe) = remind(rempe) + 1
[4246]	1079	ind = remind(rempe)
	1080	ape%locind(ind)%i = index_list(1,j)
	1081	ape%locind(ind)%j = index_list(2,j)
	1082	ENDDO
[4830]	1083
[4246]	1084	!
	1085	!-- Prepare number of elements for children processes
[4649]	1086	CALL pmc_alloc_mem( rldef, mychild%inter_npes * 2 )
[4830]	1087
[4246]	1088	!
	1089	!-- Number of child processes * size of INTEGER (i just arbitrary INTEGER)
[4649]	1090	winsize = mychild%inter_npes * STORAGE_SIZE( i ) / 8 * 2
[4246]	1091
[4649]	1092	CALL MPI_WIN_CREATE( rldef, winsize, iwp, MPI_INFO_NULL, mychild%intra_comm, indwin, ierr )
[4830]	1093
[4246]	1094	!
	1095	!-- Open window to set data
	1096	CALL MPI_WIN_FENCE( 0, indwin, ierr )
	1097
[4649]	1098	rldef(1) = 0 ! Index on remote process 0
	1099	rldef(2) = remind(1) ! Number of elements on remote process 0
[4830]	1100
[4246]	1101	!
	1102	!-- Reserve buffer for index array
	1103	DO i = 2, mychild%inter_npes
[4649]	1104	i2 = ( i - 1 ) * 2 + 1
	1105	rldef(i2) = rldef(i2-2) + rldef(i2-1) * 2 ! Index on remote process
	1106	rldef(i2+1) = remind(i) ! Number of elements on remote process
[4246]	1107	ENDDO
[4830]	1108
[4246]	1109	!
	1110	!-- Close window to allow child to access data
	1111	CALL MPI_WIN_FENCE( 0, indwin, ierr )
[4830]	1112
[4246]	1113	!
	1114	!-- Child has retrieved data
	1115	CALL MPI_WIN_FENCE( 0, indwin, ierr )
	1116
	1117	i2 = 2 * mychild%inter_npes - 1
	1118	winsize = ( rldef(i2) + rldef(i2+1) ) * 2
[4830]	1119
[4246]	1120	!
	1121	!-- Make sure, MPI_ALLOC_MEM works
	1122	winsize = MAX( winsize, INT( 1, MPI_ADDRESS_KIND ) )
	1123
	1124	CALL pmc_alloc_mem( remindw, INT( winsize ) )
	1125
	1126	CALL MPI_BARRIER( m_model_comm, ierr )
[4649]	1127	CALL MPI_WIN_CREATE( remindw, winsize * STORAGE_SIZE( i ) / 8, iwp, MPI_INFO_NULL, &
[4246]	1128	mychild%intra_comm, indwin2, ierr )
[4830]	1129
[4246]	1130	!
	1131	!-- Open window to set data
	1132	CALL MPI_WIN_FENCE( 0, indwin2, ierr )
[4830]	1133
[4246]	1134	!
	1135	!-- Create the 2D index list
	1136	DO j = 1, nrp
[4649]	1137	rempe = index_list(5,j) + 1 ! Process number on remote process
[4246]	1138	ape => mychild%pes(rempe)
	1139	i2 = rempe * 2 - 1
	1140	ind = rldef(i2) + 1
	1141	remindw(ind) = index_list(3,j)
	1142	remindw(ind+1) = index_list(4,j)
	1143	rldef(i2) = rldef(i2)+2
	1144	ENDDO
[4830]	1145
[4246]	1146	!
	1147	!-- All data are set
	1148	CALL MPI_WIN_FENCE( 0, indwin2, ierr )
[4830]	1149
[4246]	1150	!
	1151	!-- Don't know why, but this barrier is necessary before windows can be freed
	1152	!-- TODO: find out why this is required
	1153	CALL MPI_BARRIER( mychild%intra_comm, ierr )
	1154
	1155	CALL MPI_WIN_FREE( indwin, ierr )
	1156	CALL MPI_WIN_FREE( indwin2, ierr )
	1157
	1158	!
	1159	!-- TODO: check if the following idea needs to be done
[4830]	1160	!-- Should work, Problem with MPI implementation
[4246]	1161	!-- https://www.lrz.de/services/software/parallel/mpi/onesided
	1162	!-- CALL MPI_Free_mem (remindw, ierr)
	1163
	1164	END SUBROUTINE set_pe_index_list
[4830]	1165	#endif
[4246]	1166
[4830]	1167
[4246]	1168	END MODULE pmc_parent

Note: See TracBrowser for help on using the repository browser.

Context Navigation

source: palm/trunk/SOURCE/pmc_parent_mod.f90 @ 4881

Download in other formats: