Changeset 1482

Timestamp:

Oct 18, 2014 12:34:45 PM (10 years ago)

Author:

raasch

Message:

adjustments for using CUDA-aware MPI

Location:

palm/trunk/SOURCE

Files:

• fft_xy.f90 (modified) (7 diffs)
• flow_statistics.f90 (modified) (2 diffs)
• header.f90 (modified) (2 diffs)
• palm.f90 (modified) (4 diffs)
• poisfft.f90 (modified) (3 diffs)

palm/trunk/SOURCE/fft_xy.f90

@@ -20 +20 @@
 ! Current revisions:
 ! -----------------
-! 
+! cudafft workaround for data declaration of ar_tmp because of PGI 14.1 bug
 ! 
 ! Former revisions:
@@ -404 +404 @@
        COMPLEX(dp), DIMENSION(0:(nx+1)/2,nys_x:nyn_x,nzb_x:nzt_x) ::           &
           ar_tmp  !:
-       !$acc declare create( ar_tmp )
+       ! following does not work for PGI 14.1 -> to be removed later
+       ! !$acc declare create( ar_tmp )
 #endif
 
@@ -711 +712 @@
 #elif defined( __cuda_fft )
 
+          !$acc data create( ar_tmp )
           IF ( forward_fft )  THEN
 
@@ -757 +759 @@
 
           ENDIF
+          !$acc end data
 
 #else
@@ -1054 +1057 @@
        COMPLEX(dp), DIMENSION(0:(ny+1)/2,nxl_y:nxr_y,nzb_y:nzt_y) ::           &
           ar_tmp  !:
+       ! following does not work for PGI 14.1 -> to be removed later
        !$acc declare create( ar_tmp )
 #endif
@@ -1334 +1338 @@
 #elif defined( __cuda_fft )
 
+          !$acc data create( ar_tmp )
           IF ( forward_fft )  THEN
 
@@ -1380 +1385 @@
 
           ENDIF
+          !$acc end data
 
 #else

palm/trunk/SOURCE/flow_statistics.f90

@@ -21 +21 @@
 ! Current revisions:
 ! -----------------
-! 
+! missing ngp_sums_ls added in accelerator version
 ! 
 ! Former revisions:
@@ -1433 +1433 @@
         
     USE indices,                                                               &
-        ONLY:  ngp_2dh, ngp_2dh_s_inner, ngp_3d, ngp_3d_inner, ngp_sums, nxl,  &
-               nxr, nyn, nys, nzb, nzb_diff_s_inner, nzb_s_inner, nzt,         &
-               nzt_diff, rflags_invers
+        ONLY:  ngp_2dh, ngp_2dh_s_inner, ngp_3d, ngp_3d_inner, ngp_sums,       &
+               ngp_sums_ls, nxl, nxr, nyn, nys, nzb, nzb_diff_s_inner,         &
+               nzb_s_inner, nzt, nzt_diff, rflags_invers
         
     USE kinds

palm/trunk/SOURCE/header.f90

@@ -20 +20 @@
 ! Current revisions:
 ! -----------------
-! 
+! information about calculated or predefined virtual processor topology adjusted
 ! 
 ! Former revisions:
@@ -341 +341 @@
     ENDIF
 #if defined( __parallel )
-    IF ( npex == -1  .AND.  pdims(2) /= 1 )  THEN
+    IF ( npex == -1  .AND.  npey == -1 )  THEN
        char1 = 'calculated'
-    ELSEIF ( ( host(1:3) == 'ibm'  .OR.  host(1:3) == 'nec'  .OR.  &
-               host(1:2) == 'lc' )  .AND.                          &
-             npex == -1  .AND.  pdims(2) == 1 )  THEN
-       char1 = 'forced'
     ELSE
        char1 = 'predefined'

palm/trunk/SOURCE/palm.f90

@@ -20 +20 @@
 ! Current revisions:
 ! -----------------
-! 
+! adjustments for using CUDA-aware OpenMPI
 ! 
 ! Former revisions:
@@ -136 +136 @@
 !
 !-- Local variables
-    CHARACTER(LEN=9) ::  time_to_string  !:
-    INTEGER(iwp)     ::  i               !:
+    CHARACTER(LEN=9)  ::  time_to_string  !:
+    CHARACTER(LEN=10) ::  env_string      !: to store string of environment var
+    INTEGER(iwp)      ::  env_stat        !: to hold status of GET_ENV
+    INTEGER(iwp)      ::  i               !:
+    INTEGER(iwp)      ::  myid_openmpi    !: OpenMPI local rank for CUDA aware MPI
 #if defined( __openacc )
     REAL(wp), DIMENSION(100) ::  acc_dum     !:
@@ -161 +164 @@
 #if defined( __openacc )
 !
+!-- Get the local MPI rank in case of CUDA aware OpenMPI. Important, if there
+!-- is more than one accelerator board on the node
+    CALL GET_ENVIRONMENT_VARIABLE('OMPI_COMM_WORLD_LOCAL_RANK',                &
+         VALUE=env_string, STATUS=env_stat )
+    READ( env_string, '(I1)' )  myid_openmpi
+    PRINT*, '### local_rank = ', myid_openmpi, '  status=',env_stat
+!
 !-- Get the number of accelerator boards per node and assign the MPI processes
 !-- to these boards
@@ -166 +176 @@
     num_acc_per_node  = ACC_GET_NUM_DEVICES( ACC_DEVICE_NVIDIA )
     IF ( numprocs == 1  .AND.  num_acc_per_node > 0 )  num_acc_per_node = 1
-    PRINT*, '*** myid = ', myid, ' num_acc_per_node = ', num_acc_per_node
-    acc_rank = MOD( myid, num_acc_per_node )
-!    STOP '****'
+    PRINT*, '*** myid = ', myid_openmpi, ' num_acc_per_node = ', num_acc_per_node
+    acc_rank = MOD( myid_openmpi, num_acc_per_node )
     CALL ACC_SET_DEVICE_NUM ( acc_rank, ACC_DEVICE_NVIDIA )
 !
 !-- Test output (to be removed later)
-    WRITE (*,'(A,I6,A,I3,A,I3,A,I3)') '*** Connect MPI-Task ', myid,' to CPU ',&
-                                      acc_rank, ' Devices: ', num_acc_per_node,&
-                                      ' connected to:',                        &
+    WRITE (*,'(A,I6,A,I3,A,I3,A,I3)') '*** Connect MPI-Task ', myid_openmpi,   &
+                                      ' to CPU ', acc_rank, ' Devices: ',      &
+                                      num_acc_per_node, ' connected to:',      &
                                       ACC_GET_DEVICE_NUM( ACC_DEVICE_NVIDIA )
 #endif

palm/trunk/SOURCE/poisfft.f90

@@ -20 +20 @@
 ! Current revisions:
 ! -----------------
-! 
+! use 2d-decomposition, if accelerator boards are used
 ! 
 ! Former revisions:
@@ -239 +239 @@
 !
 !--    Two-dimensional Fourier Transformation in x- and y-direction.
-       IF ( pdims(2) == 1  .AND.  pdims(1) > 1 )  THEN
+       IF ( pdims(2) == 1  .AND.  pdims(1) > 1  .AND.  num_acc_per_node == 0 ) &
+       THEN
 
 !
@@ -254 +255 @@
           CALL tr_xy_ffty( ar, ar )
 
-       ELSEIF ( pdims(1) == 1  .AND.  pdims(2) > 1 )  THEN
+       ELSEIF ( pdims(1) == 1 .AND. pdims(2) > 1 .AND. num_acc_per_node == 0 ) &
+       THEN
 
 !