Index: palm/trunk/SOURCE/transpose.f90
===================================================================
--- palm/trunk/SOURCE/transpose.f90	(revision 3655)
+++ palm/trunk/SOURCE/transpose.f90	(revision 3657)
@@ -180,10 +180,18 @@
 !--    Transpose array
        CALL cpu_log( log_point_s(32), 'mpi_alltoall', 'start', cpu_log_nowait )
+#ifndef __cuda_aware_mpi
        !$ACC UPDATE HOST(f_inv)
+#else
+       !$ACC HOST_DATA USE_DEVICE(work, f_inv)
+#endif
        IF ( collective_wait )  CALL MPI_BARRIER( comm2d, ierr )
        CALL MPI_ALLTOALL( f_inv(nys_x,nzb_x,0),  sendrecvcount_xy, MPI_REAL, &
                           work(1,nzb_y,nxl_y,0), sendrecvcount_xy, MPI_REAL, &
                           comm1dy, ierr )
+#ifndef __cuda_aware_mpi
        !$ACC UPDATE DEVICE(work)
+#else
+       !$ACC END HOST_DATA
+#endif
        CALL cpu_log( log_point_s(32), 'mpi_alltoall', 'stop' )
 
@@ -337,10 +345,18 @@
 !--    Transpose array
        CALL cpu_log( log_point_s(32), 'mpi_alltoall', 'start', cpu_log_nowait )
+#ifndef __cuda_aware_mpi
        !$ACC UPDATE HOST(work)
+#else
+       !$ACC HOST_DATA USE_DEVICE(work, f_inv)
+#endif
        IF ( collective_wait )  CALL MPI_BARRIER( comm2d, ierr )
        CALL MPI_ALLTOALL( work(nys_x,1,nzb_x,0), sendrecvcount_zx, MPI_REAL, &
                           f_inv(nys,nxl,1),      sendrecvcount_zx, MPI_REAL, &
                           comm1dx, ierr )
+#ifndef __cuda_aware_mpi
        !$ACC UPDATE DEVICE(f_inv)
+#else
+       !$ACC END HOST_DATA
+#endif
        CALL cpu_log( log_point_s(32), 'mpi_alltoall', 'stop' )
 #endif
@@ -475,10 +491,18 @@
 !--    Transpose array
        CALL cpu_log( log_point_s(32), 'mpi_alltoall', 'start', cpu_log_nowait )
+#ifndef __cuda_aware_mpi
        !$ACC UPDATE HOST(work)
+#else
+       !$ACC HOST_DATA USE_DEVICE(work, f_inv)
+#endif
        IF ( collective_wait )  CALL MPI_BARRIER( comm2d, ierr )
        CALL MPI_ALLTOALL( work(1,nzb_y,nxl_y,0), sendrecvcount_xy, MPI_REAL, &
                           f_inv(nys_x,nzb_x,0),  sendrecvcount_xy, MPI_REAL, &
                           comm1dy, ierr )
+#ifndef __cuda_aware_mpi
        !$ACC UPDATE DEVICE(f_inv)
+#else
+       !$ACC END HOST_DATA
+#endif
        CALL cpu_log( log_point_s(32), 'mpi_alltoall', 'stop' )
 #endif
@@ -693,10 +717,18 @@
 !--    Transpose array
        CALL cpu_log( log_point_s(32), 'mpi_alltoall', 'start', cpu_log_nowait )
+#ifndef __cuda_aware_mpi
        !$ACC UPDATE HOST(f_inv)
+#else
+       !$ACC HOST_DATA USE_DEVICE(work, f_inv)
+#endif
        IF ( collective_wait )  CALL MPI_BARRIER( comm2d, ierr )
        CALL MPI_ALLTOALL( f_inv(nxl_y,nzb_y,0),  sendrecvcount_yz, MPI_REAL, &
                           work(nxl_z,1,nys_z,0), sendrecvcount_yz, MPI_REAL, &
                           comm1dx, ierr )
+#ifndef __cuda_aware_mpi
        !$ACC UPDATE DEVICE(work)
+#else
+       !$ACC END HOST_DATA
+#endif
        CALL cpu_log( log_point_s(32), 'mpi_alltoall', 'stop' )
 
@@ -829,10 +861,18 @@
 !--    Transpose array
        CALL cpu_log( log_point_s(32), 'mpi_alltoall', 'start', cpu_log_nowait )
+#ifndef __cuda_aware_mpi
        !$ACC UPDATE HOST(f_inv)
+#else
+       !$ACC HOST_DATA USE_DEVICE(work, f_inv)
+#endif
        IF ( collective_wait )  CALL MPI_BARRIER( comm2d, ierr )
        CALL MPI_ALLTOALL( f_inv(nys,nxl,1),      sendrecvcount_zx, MPI_REAL, &
                           work(nys_x,1,nzb_x,0), sendrecvcount_zx, MPI_REAL, &
                           comm1dx, ierr )
+#ifndef __cuda_aware_mpi
        !$ACC UPDATE DEVICE(work)
+#else
+       !$ACC END HOST_DATA
+#endif
        CALL cpu_log( log_point_s(32), 'mpi_alltoall', 'stop' )
 
@@ -971,10 +1011,18 @@
 !--    Transpose array
        CALL cpu_log( log_point_s(32), 'mpi_alltoall', 'start', cpu_log_nowait )
+#ifndef __cuda_aware_mpi
        !$ACC UPDATE HOST(work)
+#else
+       !$ACC HOST_DATA USE_DEVICE(work, f_inv)
+#endif
        IF ( collective_wait )  CALL MPI_BARRIER( comm2d, ierr )
        CALL MPI_ALLTOALL( work(nxl_z,1,nys_z,0), sendrecvcount_yz, MPI_REAL, &
                           f_inv(nxl_y,nzb_y,0),  sendrecvcount_yz, MPI_REAL, &
                           comm1dx, ierr )
+#ifndef __cuda_aware_mpi
        !$ACC UPDATE DEVICE(f_inv)
+#else
+       !$ACC END HOST_DATA
+#endif
        CALL cpu_log( log_point_s(32), 'mpi_alltoall', 'stop' )
 #endif
