Home

Context Navigation

source: palm/trunk/SOURCE/pmc_client.f90 @ 1763

Last change on this file since 1763 was 1763, checked in by hellstea, 8 years ago
last commit documented
Property svn:keywords set to `Id`
File size: 20.6 KB

Line
1	MODULE pmc_client
2
3	!--------------------------------------------------------------------------------!
4	! This file is part of PALM.
5	!
6	! PALM is free software: you can redistribute it and/or modify it under the terms
7	! of the GNU General Public License as published by the Free Software Foundation,
8	! either version 3 of the License, or (at your option) any later version.
9	!
10	! PALM is distributed in the hope that it will be useful, but WITHOUT ANY
11	! WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR
12	! A PARTICULAR PURPOSE. See the GNU General Public License for more details.
13	!
14	! You should have received a copy of the GNU General Public License along with
15	! PALM. If not, see <http://www.gnu.org/licenses/>.
16	!
17	! Copyright 1997-2015 Leibniz Universitaet Hannover
18	!--------------------------------------------------------------------------------!
19	!
20	! Current revisions:
21	! ------------------
22	!
23	!
24	! Former revisions:
25	! -----------------
26	! $Id: pmc_client.f90 1763 2016-02-25 13:00:19Z hellstea $
27	!
28	! 1762 2016-02-25 12:31:13Z hellstea
29	! Initial revision by K. Ketelsen
30	!
31	! Description:
32	! ------------
33	!
34	! Client part of Palm Model Coupler
35	!------------------------------------------------------------------------------!
36
37
38	use, intrinsic :: iso_c_binding
39
40	USE mpi
41	USE kinds, ONLY: wp, iwp
42	USE PMC_general, ONLY: ClientDef, DA_NameDef, DA_Namelen, PMC_STATUS_OK, PMC_DA_NAME_ERR, PeDef, ArrayDef, &
43	DA_Desclen, DA_Namelen, PMC_G_SetName, PMC_G_GetName
44	USE PMC_handle_communicator, ONLY: m_model_comm,m_model_rank,m_model_npes, m_to_server_comm
45	USE PMC_MPI_wrapper, ONLY: PMC_Send_to_Server, PMC_Recv_from_Server, PMC_Time, &
46	PMC_Bcast, PMC_Inter_Bcast, PMC_Alloc_mem
47	IMPLICIT none
48	PRIVATE
49	SAVE
50
51	! data local to this MODULE
52	Type(ClientDef) :: me
53	INTEGER, PARAMETER :: dp = wp
54
55	INTEGER, save :: myIndex = 0 !Counter and unique number for Data Arrays
56
57	! INTERFACE section
58
59	INTERFACE PMC_ClientInit
60	MODULE procedure PMC_ClientInit
61	END INTERFACE PMC_ClientInit
62
63	INTERFACE PMC_Set_DataArray_Name
64	MODULE procedure PMC_Set_DataArray_Name
65	MODULE procedure PMC_Set_DataArray_Name_LastEntry
66	END INTERFACE PMC_Set_DataArray_Name
67
68	INTERFACE PMC_C_Get_2D_index_list
69	MODULE procedure PMC_C_Get_2D_index_list
70	END INTERFACE PMC_C_Get_2D_index_list
71
72	INTERFACE PMC_C_GetNextArray
73	MODULE procedure PMC_C_GetNextArray
74	END INTERFACE PMC_C_GetNextArray
75
76	INTERFACE PMC_C_Set_DataArray
77	MODULE procedure PMC_C_Set_DataArray_2d
78	MODULE procedure PMC_C_Set_DataArray_3d
79	END INTERFACE PMC_C_Set_DataArray
80
81	INTERFACE PMC_C_setInd_and_AllocMem
82	MODULE procedure PMC_C_setInd_and_AllocMem
83	END INTERFACE PMC_C_setInd_and_AllocMem
84
85	INTERFACE PMC_C_GetBuffer
86	MODULE procedure PMC_C_GetBuffer
87	END INTERFACE PMC_C_GetBuffer
88
89	INTERFACE PMC_C_PutBuffer
90	MODULE procedure PMC_C_PutBuffer
91	END INTERFACE PMC_C_PutBuffer
92
93	! Public section
94
95	PUBLIC PMC_ClientInit , PMC_Set_DataArray_Name, PMC_C_Get_2D_index_list
96	PUBLIC PMC_C_GetNextArray, PMC_C_Set_DataArray
97	PUBLIC PMC_C_setInd_and_AllocMem , PMC_C_GetBuffer, PMC_C_PutBuffer ! ,PMC_C_GetServerType
98
99	CONTAINS
100
101	SUBROUTINE PMC_ClientInit
102	IMPLICIT none
103
104	INTEGER :: i
105	INTEGER :: istat
106
107
108	! Tailor MPI environment
109
110	me%model_comm = m_model_comm
111	me%inter_comm = m_to_server_comm
112
113	! Get rank and size
114	CALL MPI_Comm_rank (me%model_comm, me%model_rank, istat);
115	CALL MPI_Comm_size (me%model_comm, me%model_npes, istat);
116	CALL MPI_Comm_remote_size (me%inter_comm, me%inter_npes, istat);
117
118	! intra communicater is used for MPI_Get
119	CALL MPI_Intercomm_merge (me%inter_comm, .true., me%intra_comm, istat);
120	CALL MPI_Comm_rank (me%intra_comm, me%intra_rank, istat);
121
122	ALLOCATE (me%PEs(me%inter_npes))
123
124	do i=1,me%inter_npes
125	NULLIFY(me%PEs(i)%Arrays)
126	end do
127
128	if(me%model_rank == 0) write(0,'(a,5i6)') 'PMC_ClientInit ',me%model_rank,me%model_npes,me%inter_npes,me%intra_rank
129
130	return
131	END SUBROUTINE PMC_ClientInit
132
133	SUBROUTINE PMC_Set_DataArray_Name (ServerArrayDesc, ServerArrayName, ClientArrayDesc, ClientArrayName, istat, LastEntry)
134	IMPLICIT none
135	character(len=*),INTENT(IN) :: ServerArrayName
136	character(len=*),INTENT(IN) :: ServerArrayDesc
137	character(len=*),INTENT(IN) :: ClientArrayName
138	character(len=*),INTENT(IN) :: ClientArrayDesc
139	INTEGER,INTENT(OUT) :: istat
140	LOGICAL,INTENT(IN),optional :: LastEntry
141
142	!-- local variables
143	type(DA_NameDef) :: myName
144	INTEGER :: myPe
145	INTEGER :: my_AddiArray=0
146
147	istat = PMC_STATUS_OK
148	if(len(trim(ServerArrayName)) > DA_Namelen .or. &
149	len(trim(ClientArrayName)) > DA_Namelen) then !Name too long
150	istat = PMC_DA_NAME_ERR
151	end if
152
153	if(m_model_rank == 0) then
154	myIndex = myIndex+1
155	myName%couple_index = myIndex
156	myName%ServerDesc = trim(ServerArrayDesc)
157	myName%NameOnServer = trim(ServerArrayName)
158	myName%ClientDesc = trim(ClientArrayDesc)
159	myName%NameOnClient = trim(ClientArrayName)
160	end if
161
162	! Broadcat to all Client PEs
163
164	CALL PMC_Bcast ( myName%couple_index, 0, comm=m_model_comm)
165	CALL PMC_Bcast ( myName%ServerDesc, 0, comm=m_model_comm)
166	CALL PMC_Bcast ( myName%NameOnServer, 0, comm=m_model_comm)
167	CALL PMC_Bcast ( myName%ClientDesc, 0, comm=m_model_comm)
168	CALL PMC_Bcast ( myName%NameOnClient, 0, comm=m_model_comm)
169
170	! Broadcat to all Server PEs
171
172	if(m_model_rank == 0) then
173	myPE = MPI_ROOT
174	else
175	myPE = MPI_PROC_NULL
176	endif
177	CALL PMC_Bcast ( myName%couple_index, myPE, comm=m_to_server_comm)
178	CALL PMC_Bcast ( myName%ServerDesc, myPE, comm=m_to_server_comm)
179	CALL PMC_Bcast ( myName%NameOnServer, myPE, comm=m_to_server_comm)
180	CALL PMC_Bcast ( myName%ClientDesc, myPE, comm=m_to_server_comm)
181	CALL PMC_Bcast ( myName%NameOnClient, myPE, comm=m_to_server_comm)
182
183	if(present (LastEntry)) then
184	CALL PMC_Set_DataArray_Name_LastEntry ( LastEntry = LastEntry)
185	end if
186
187	CALL PMC_G_SetName (me, myName%couple_index, myName%NameOnClient)
188
189	return
190	END SUBROUTINE PMC_Set_DataArray_Name
191
192	SUBROUTINE PMC_Set_DataArray_Name_LastEntry (LastEntry)
193	IMPLICIT none
194	LOGICAL,INTENT(IN),optional :: LastEntry
195
196	!-- local variables
197	type(DA_NameDef) :: myName
198	INTEGER :: myPe
199
200	myName%couple_index = -1
201
202	if(m_model_rank == 0) then
203	myPE = MPI_ROOT
204	else
205	myPE = MPI_PROC_NULL
206	endif
207	CALL PMC_Bcast ( myName%couple_index, myPE, comm=m_to_server_comm)
208
209	return
210	END SUBROUTINE PMC_Set_DataArray_Name_LastEntry
211
212	SUBROUTINE PMC_C_Get_2D_index_list
213	IMPLICIT none
214
215	INTEGER :: i,j,i2,nr,ierr
216	INTEGER :: dummy
217	INTEGER :: indWin !: MPI window object
218	INTEGER :: indWin2 !: MPI window object
219	INTEGER(KIND=MPI_ADDRESS_KIND) :: win_size !: Size of MPI window 1 (in bytes)
220	INTEGER(KIND=MPI_ADDRESS_KIND) :: disp !: Displacement Unit (Integer = 4, floating poit = 8
221	INTEGER,DIMENSION(me%inter_npes*2) :: NrEle !: Number of Elements of a horizontal slice
222	TYPE(PeDef),POINTER :: aPE !: Pointer to PeDef strzcture
223	INTEGER(KIND=MPI_ADDRESS_KIND) :: WinSize !: Size of MPI window 2 (in bytes)
224	INTEGER,DIMENSION(:),POINTER :: myInd
225
226	! CALL PMC_C_CGet_Rem_index_list
227
228	win_size = c_sizeof(dummy)
229	CALL MPI_Win_create (dummy, win_size, iwp, MPI_INFO_NULL, me%intra_comm, indWin, ierr);
230	CALL MPI_Win_fence (0, indWin, ierr) ! Open Window on Server side
231	CALL MPI_Win_fence (0, indWin, ierr) ! Close Window on Server Side and opem on Client side
232
233	do i=1,me%inter_npes
234	disp = me%model_rank*2
235	CALL MPI_Get (NrEle((i-1)*2+1),2,MPI_INTEGER,i-1,disp,2,MPI_INTEGER,indWin, ierr)
236	end do
237	CALL MPI_Win_fence (0, indWin, ierr) ! MPI_get is Non-blocking -> data in NrEle is not available until MPI_fence CALL
238
239	WinSize = 0
240	do i=1,me%inter_npes !Allocate memory for index array
241	aPE => me%PEs(i)
242	i2 = (i-1)*2+1
243	nr = NrEle(i2+1)
244	if(nr > 0) then
245	ALLOCATE(aPE%locInd(nr))
246	else
247	NULLIFY (aPE%locInd)
248	endif
249	WinSize = max(nr,WinSize) !Maximum window size
250	end do
251
252	ALLOCATE(myInd(2*WinSize))
253	WinSize = WinSize*c_sizeof(i)
254
255	! local Buffer used in MPI_Get can but must not be inside the MPI Window
256	! Here, we use a dummy for MPI Window because the server PEs do not access the RMA window via MPI_get or MPI_Put
257
258	CALL MPI_Win_create (dummy, 1, iwp, MPI_INFO_NULL, me%intra_comm, indWin2, ierr);
259
260	CALL MPI_Win_fence (0, indWin2, ierr) ! MPI_get is Non-blocking -> data in NrEle is not available until MPI_fence CALL
261	CALL MPI_Win_fence (0, indWin2, ierr) ! MPI_get is Non-blocking -> data in NrEle is not available until MPI_fence CALL
262
263	do i=1,me%inter_npes
264	aPE => me%PEs(i)
265	nr = NrEle(i*2)
266	if(nr > 0 ) then
267	disp = NrEle(2*(i-1)+1)
268	CALL MPI_Win_lock (MPI_LOCK_SHARED , i-1, 0, indWin2, ierr)
269	CALL MPI_Get (myInd,2nr,MPI_INTEGER,i-1,disp,2nr,MPI_INTEGER,indWin2, ierr)
270	CALL MPI_Win_unlock (i-1, indWin2, ierr)
271	do j=1,nr
272	aPE%locInd(j)%i = myInd(2*j-1)
273	aPE%locInd(j)%j = myInd(2*j)
274	end do
275	aPE%NrEle = nr
276	else
277	aPE%NrEle = -1
278	end if
279	end do
280
281	CALL MPI_Barrier(me%intra_comm, ierr) ! Dont know why, but this barrier is necessary before we can free the windows
282
283	CALL MPI_Win_free(indWin, ierr);
284	CALL MPI_Win_free(indWin2, ierr);
285	DEALLOCATE (myInd)
286
287	return
288	END SUBROUTINE PMC_C_Get_2D_index_list
289
290	LOGICAL function PMC_C_GetNextArray (myName)
291	character(len=*),INTENT(OUT) :: myName
292
293	!-- local variables
294	INTEGER :: MyCoupleIndex
295	LOGICAL :: MyLast !Last Array in List
296	character(len=DA_Namelen) :: loName
297
298	loName = 'NoName '
299	MyLast = .true.
300
301	CALL PMC_G_GetName (me, MyCoupleIndex, loName, MyLast)
302
303	myName = trim(loName)
304
305	PMC_C_GetNextArray = .NOT. MyLast ! Return true if valid array
306
307	return
308	END function PMC_C_GetNextArray
309
310	SUBROUTINE PMC_C_Set_DataArray_2d (array)
311	IMPLICIT none
312	REAL(kind=dp),INTENT(IN),DIMENSION(:,:) :: array
313	!-- local variables
314	INTEGER :: NrDims
315	INTEGER,DIMENSION (4) :: dims
316	INTEGER :: dim_order
317	TYPE(c_ptr) :: array_adr
318	INTEGER :: i
319	TYPE(PeDef),POINTER :: aPE
320	TYPE(ArrayDef),POINTER :: ar
321
322
323	dims = 1
324
325	NrDims = 2
326	dims(1) = size(array,1)
327	dims(2) = size(array,2)
328	dim_order = 2
329
330	array_adr = c_loc(array)
331
332	do i=1,me%inter_npes
333	aPE => me%PEs(i)
334	ar => aPE%Arrays
335	ar%NrDims = NrDims
336	ar%A_dim = dims
337	ar%dim_order = dim_order
338	ar%data = array_adr
339	end do
340
341	return
342	END SUBROUTINE PMC_C_Set_DataArray_2d
343
344	SUBROUTINE PMC_C_Set_DataArray_3d (array)
345	IMPLICIT none
346	REAL(kind=dp),INTENT(IN),DIMENSION(:,:,:) :: array
347	!-- local variables
348	INTEGER :: NrDims
349	INTEGER,DIMENSION (4) :: dims
350	INTEGER :: dim_order
351	TYPE(c_ptr) :: array_adr
352	INTEGER :: i
353	TYPE(PeDef),POINTER :: aPE
354	TYPE(ArrayDef),POINTER :: ar
355
356	dims = 1
357
358	NrDims = 3
359	dims(1) = size(array,1)
360	dims(2) = size(array,2)
361	dims(3) = size(array,3)
362	dim_order =33
363
364	array_adr = c_loc(array)
365
366	do i=1,me%inter_npes
367	aPE => me%PEs(i)
368	ar => aPE%Arrays
369	ar%NrDims = NrDims
370	ar%A_dim = dims
371	ar%dim_order = dim_order
372	ar%data = array_adr
373	end do
374
375	return
376	END SUBROUTINE PMC_C_Set_DataArray_3d
377
378	SUBROUTINE PMC_C_setInd_and_AllocMem
379	IMPLICIT none
380
381	INTEGER :: i, ierr
382	INTEGER :: arlen, myIndex, tag
383	INTEGER(kind=8) :: bufsize ! Size of MPI data Window
384	TYPE(PeDef),POINTER :: aPE
385	TYPE(ArrayDef),POINTER :: ar
386	character(len=DA_Namelen) :: myName
387	Type(c_ptr) :: base_ptr
388	REAL(kind=wp),DIMENSION(:),POINTER :: base_array
389	INTEGER(KIND=MPI_ADDRESS_KIND) :: WinSize
390
391	myIndex = 0
392	bufsize = 8
393
394	! First stride, Compute size and set index
395
396	do i=1,me%inter_npes
397	aPE => me%PEs(i)
398	tag = 200
399
400	do while (PMC_C_GetNextArray (myName))
401	ar => aPE%Arrays
402
403	! Receive Index from client
404	tag = tag+1
405	CALL MPI_Recv (myIndex, 1, MPI_INTEGER, i-1, tag, me%inter_comm, MPI_STATUS_IGNORE, ierr)
406
407	if(ar%dim_order == 33) then ! PALM has k in first dimension
408	bufsize = max(bufsize,ar%A_dim(1)ar%A_dim(2)ar%A_dim(3)) ! determine max, because client buffer is allocated only once
409	else
410	bufsize = max(bufsize,ar%A_dim(1)*ar%A_dim(2))
411	end if
412	ar%BufIndex = myIndex
413	end do
414	end do
415
416	! Create RMA (One Sided Communication) window for data buffer
417
418	CALL PMC_Alloc_mem (base_array, bufsize, base_ptr)
419	me%TotalBufferSize = bufsize*wp !Total buffer size in Byte
420
421	WinSize = me%TotalBufferSize
422	! write(9,'(a,8i7)') 'PMC_S_SetInd_and_Mem ',m_model_rank,me%inter_npes,WinSize,ar%A_dim
423	CALL MPI_Win_create (base_array, WinSize, wp, MPI_INFO_NULL, me%intra_comm, me%BufWin, ierr);
424	CALL MPI_Win_fence (0, me%BufWin, ierr); ! Open Window to set data
425	CALL MPI_Barrier(me%intra_comm, ierr)
426
427	do i=1,me%inter_npes
428	aPE => me%PEs(i)
429
430	do while (PMC_C_GetNextArray (myName))
431	ar => aPE%Arrays
432	ar%SendBuf = base_ptr
433	end do
434	end do
435
436	return
437	END SUBROUTINE PMC_C_setInd_and_AllocMem
438
439	SUBROUTINE PMC_C_GetBuffer (WaitTime)
440	IMPLICIT none
441	REAL(kind=dp),INTENT(OUT),optional :: WaitTime
442
443	!-- local variables
444	INTEGER :: ip, ij, ierr
445	INTEGER :: nr ! Number of Elements to getb from server
446	INTEGER :: myIndex
447	REAL(kind=dp) :: t1,t2
448	TYPE(PeDef),POINTER :: aPE
449	TYPE(ArrayDef),POINTER :: ar
450	INTEGER,DIMENSION(1) :: buf_shape
451	REAL(kind=wp),POINTER,DIMENSION(:) :: buf
452	REAL(kind=wp),POINTER,DIMENSION(:,:) :: data_2d
453	REAL(kind=wp),POINTER,DIMENSION(:,:,:) :: data_3d
454	character(len=DA_Namelen) :: myName
455	INTEGER(kind=MPI_ADDRESS_KIND) :: target_disp
456
457	t1 = PMC_Time()
458	CALL MPI_Barrier(me%intra_comm, ierr) ! Wait for server to fill buffer
459	t2 = PMC_Time()
460	if(present(WaitTime)) WaitTime = t2-t1
461
462	CALL MPI_Barrier(me%intra_comm, ierr) ! Wait for buffer is filled
463
464	do ip=1,me%inter_npes
465	aPE => me%PEs(ip)
466
467	do while (PMC_C_GetNextArray (myName))
468	ar => aPE%Arrays
469	if(ar%dim_order == 2) then
470	nr = aPE%NrEle
471	else if(ar%dim_order == 33) then
472	nr = aPE%NrEle*ar%A_dim(1)
473	end if
474
475	buf_shape(1) = nr
476	CALL c_f_pointer(ar%SendBuf, buf, buf_shape)
477
478	if(nr > 0) then
479	target_disp = (ar%BufIndex-1)
480	CALL MPI_Win_lock (MPI_LOCK_SHARED , ip-1, 0, me%BufWin, ierr)
481	CALL MPI_Get (buf, nr, MPI_REAL, ip-1, target_disp, nr, MPI_REAL, me%BufWin, ierr)
482	CALL MPI_Win_unlock (ip-1, me%BufWin, ierr)
483	end if
484
485	myIndex = 1
486	if(ar%dim_order == 2) then
487
488	CALL c_f_pointer(ar%data, data_2d, ar%A_dim(1:2))
489	do ij=1,aPE%NrEle
490	data_2d(aPE%locInd(ij)%j,aPE%locInd(ij)%i) = buf(myIndex)
491	myIndex = myIndex+1
492	end do
493	else if(ar%dim_order == 33) then
494	CALL c_f_pointer(ar%data, data_3d, ar%A_dim(1:3))
495	do ij=1,aPE%NrEle
496	data_3d(:,aPE%locInd(ij)%j,aPE%locInd(ij)%i) = buf(myIndex:myIndex+ar%A_dim(1)-1)
497	myIndex = myIndex+ar%A_dim(1)
498	end do
499	end if
500
501	end do
502	end do
503	return
504	END SUBROUTINE PMC_C_GetBuffer
505
506	SUBROUTINE PMC_C_PutBuffer (WaitTime)
507	IMPLICIT none
508	REAL(kind=dp),INTENT(OUT),optional :: WaitTime
509
510	!-- local variables
511	INTEGER :: ip, ij, ierr
512	INTEGER :: nr ! Number of Elements to getb from server
513	INTEGER :: myIndex
514	REAL(kind=dp) :: t1,t2
515	TYPE(PeDef),POINTER :: aPE
516	TYPE(ArrayDef),POINTER :: ar
517	INTEGER,DIMENSION(1) :: buf_shape
518	REAL(kind=wp),POINTER,DIMENSION(:) :: buf
519	REAL(kind=wp),POINTER,DIMENSION(:,:) :: data_2d
520	REAL(kind=wp),POINTER,DIMENSION(:,:,:) :: data_3d
521	character(len=DA_Namelen) :: myName
522	INTEGER(kind=MPI_ADDRESS_KIND) :: target_disp
523
524
525	do ip=1,me%inter_npes
526	aPE => me%PEs(ip)
527
528	do while (PMC_C_GetNextArray (myName))
529	ar => aPE%Arrays
530	if(ar%dim_order == 2) then
531	nr = aPE%NrEle
532	else if(ar%dim_order == 33) then
533	nr = aPE%NrEle*ar%A_dim(1)
534	end if
535
536	buf_shape(1) = nr
537	CALL c_f_pointer(ar%SendBuf, buf, buf_shape)
538
539	myIndex = 1
540	if(ar%dim_order == 2) then
541	CALL c_f_pointer(ar%data, data_2d, ar%A_dim(1:2))
542	do ij=1,aPE%NrEle
543	buf(myIndex) = data_2d(aPE%locInd(ij)%j,aPE%locInd(ij)%i)
544	myIndex = myIndex+1
545	end do
546	else if(ar%dim_order == 33) then
547	CALL c_f_pointer(ar%data, data_3d, ar%A_dim(1:3))
548	do ij=1,aPE%NrEle
549	buf(myIndex:myIndex+ar%A_dim(1)-1) = data_3d(:,aPE%locInd(ij)%j,aPE%locInd(ij)%i)
550	myIndex = myIndex+ar%A_dim(1)
551	end do
552	end if
553
554	if(nr > 0) then
555	target_disp = (ar%BufIndex-1)
556	CALL MPI_Win_lock (MPI_LOCK_SHARED , ip-1, 0, me%BufWin, ierr)
557	CALL MPI_Put (buf, nr, MPI_REAL, ip-1, target_disp, nr, MPI_REAL, me%BufWin, ierr)
558	CALL MPI_Win_unlock (ip-1, me%BufWin, ierr)
559	end if
560
561	end do
562	end do
563
564
565	t1 = PMC_Time()
566	CALL MPI_Barrier(me%intra_comm, ierr) ! Wait for server to fill buffer
567	t2 = PMC_Time()
568	if(present(WaitTime)) WaitTime = t2-t1
569
570	CALL MPI_Barrier(me%intra_comm, ierr) ! Wait for buffer is filled
571
572
573	return
574	END SUBROUTINE PMC_C_PutBuffer
575
576
577	! Private SUBROUTINEs
578
579	END MODULE pmc_client

Note: See TracBrowser for help on using the repository browser.

Download in other formats:

| Impressum | ©Leibniz Universität Hannover |