Changeset 1482 for palm/trunk/SOURCE/palm.f90

Timestamp:

Oct 18, 2014 12:34:45 PM (10 years ago)

Author:

raasch

Message:

adjustments for using CUDA-aware MPI

File:

• palm/trunk/SOURCE/palm.f90 (modified) (4 diffs)

palm/trunk/SOURCE/palm.f90

@@ -20 +20 @@
 ! Current revisions:
 ! -----------------
-! 
+! adjustments for using CUDA-aware OpenMPI
 ! 
 ! Former revisions:
@@ -136 +136 @@
 !
 !-- Local variables
-    CHARACTER(LEN=9) ::  time_to_string  !:
-    INTEGER(iwp)     ::  i               !:
+    CHARACTER(LEN=9)  ::  time_to_string  !:
+    CHARACTER(LEN=10) ::  env_string      !: to store string of environment var
+    INTEGER(iwp)      ::  env_stat        !: to hold status of GET_ENV
+    INTEGER(iwp)      ::  i               !:
+    INTEGER(iwp)      ::  myid_openmpi    !: OpenMPI local rank for CUDA aware MPI
 #if defined( __openacc )
     REAL(wp), DIMENSION(100) ::  acc_dum     !:
@@ -161 +164 @@
 #if defined( __openacc )
 !
+!-- Get the local MPI rank in case of CUDA aware OpenMPI. Important, if there
+!-- is more than one accelerator board on the node
+    CALL GET_ENVIRONMENT_VARIABLE('OMPI_COMM_WORLD_LOCAL_RANK',                &
+         VALUE=env_string, STATUS=env_stat )
+    READ( env_string, '(I1)' )  myid_openmpi
+    PRINT*, '### local_rank = ', myid_openmpi, '  status=',env_stat
+!
 !-- Get the number of accelerator boards per node and assign the MPI processes
 !-- to these boards
@@ -166 +176 @@
     num_acc_per_node  = ACC_GET_NUM_DEVICES( ACC_DEVICE_NVIDIA )
     IF ( numprocs == 1  .AND.  num_acc_per_node > 0 )  num_acc_per_node = 1
-    PRINT*, '*** myid = ', myid, ' num_acc_per_node = ', num_acc_per_node
-    acc_rank = MOD( myid, num_acc_per_node )
-!    STOP '****'
+    PRINT*, '*** myid = ', myid_openmpi, ' num_acc_per_node = ', num_acc_per_node
+    acc_rank = MOD( myid_openmpi, num_acc_per_node )
     CALL ACC_SET_DEVICE_NUM ( acc_rank, ACC_DEVICE_NVIDIA )
 !
 !-- Test output (to be removed later)
-    WRITE (*,'(A,I6,A,I3,A,I3,A,I3)') '*** Connect MPI-Task ', myid,' to CPU ',&
-                                      acc_rank, ' Devices: ', num_acc_per_node,&
-                                      ' connected to:',                        &
+    WRITE (*,'(A,I6,A,I3,A,I3,A,I3)') '*** Connect MPI-Task ', myid_openmpi,   &
+                                      ' to CPU ', acc_rank, ' Devices: ',      &
+                                      num_acc_per_node, ' connected to:',      &
                                       ACC_GET_DEVICE_NUM( ACC_DEVICE_NVIDIA )
 #endif