Changeset 3689

Timestamp:

Jan 22, 2019 7:34:35 PM (5 years ago)

Author:

knoop

Message:

Added multi-GPU capability when using OpenACC.

File:

• palm/trunk/SOURCE/palm.f90 (modified) (3 diffs)

palm/trunk/SOURCE/palm.f90

@@ -324 +324 @@
         ONLY:  wrd_global, wrd_local
 
+#if defined( __parallel) && defined( _OPENACC )
+    USE openacc
+#endif
+
 
     IMPLICIT NONE
@@ -331 +335 @@
     CHARACTER(LEN=9) ::  time_to_string  !<
     INTEGER(iwp)     ::  i               !< loop counter for blocked I/O
+#if defined( __parallel) && defined( _OPENACC )
+    INTEGER(iwp)     :: local_comm       !< local communicator (shared memory)
+    INTEGER(iwp)     :: local_num_procs  !< local number of processes
+    INTEGER(iwp)     :: local_id         !< local id
+    INTEGER(acc_device_kind) :: device_type !< device type for OpenACC
+    INTEGER(iwp)     ::  num_devices     !< number of devices visible to OpenACC
+    INTEGER(iwp)     ::  my_device       !< device used by this process
+#endif
 
     version = 'PALM 6.0'
@@ -368 +380 @@
        CALL init_coupling
     ENDIF
+
+#ifdef _OPENACC
+!
+!-- Select OpenACC device to use in this process. For this find out how many
+!-- neighbors there are running on the same node and which id this process is.
+    IF ( nested_run )  THEN
+       CALL MPI_COMM_SPLIT_TYPE( comm_palm, MPI_COMM_TYPE_SHARED, 0,           &
+                                 MPI_INFO_NULL, local_comm, ierr )
+    ELSE
+       CALL MPI_COMM_SPLIT_TYPE( MPI_COMM_WORLD, MPI_COMM_TYPE_SHARED, 0,      &
+                                 MPI_INFO_NULL, local_comm, ierr )
+    ENDIF
+    CALL MPI_COMM_SIZE( local_comm, local_num_procs, ierr )
+    CALL MPI_COMM_RANK( local_comm, local_id, ierr )
+
+!
+!-- This loop including the barrier is a workaround for PGI compiler versions
+!-- up to and including 18.4. Later releases are able to select their GPUs in
+!-- parallel, without running into spurious errors.
+    DO i = 0, local_num_procs-1
+       CALL MPI_BARRIER( local_comm, ierr )
+
+       IF ( i == local_id )  THEN
+          device_type = acc_get_device_type()
+          num_devices = acc_get_num_devices( device_type )
+          my_device = MOD( local_id, num_devices )
+          CALL acc_set_device_num( my_device, device_type )
+       ENDIF
+    ENDDO
+
+    CALL MPI_COMM_FREE( local_comm, ierr )
+#endif
 #endif