Index: palm/trunk/SOURCE/transpose.f90
===================================================================
--- palm/trunk/SOURCE/transpose.f90	(revision 3241)
+++ palm/trunk/SOURCE/transpose.f90	(revision 3634)
@@ -25,4 +25,7 @@
 ! -----------------
 ! $Id$
+! OpenACC port for SPEC
+! 
+! 3241 2018-09-12 15:02:00Z raasch
 ! unused variables removed
 ! 
@@ -119,4 +122,6 @@
     !$OMP  PARALLEL PRIVATE ( i, j, k )
     !$OMP  DO
+     !$ACC PARALLEL LOOP COLLAPSE(3) PRIVATE(i,j,k) &
+     !$ACC PRESENT(f_inv, f_in)
      DO  i = 0, nx
          DO  k = nzb_x, nzt_x
@@ -166,4 +171,5 @@
 
     REAL(wp), DIMENSION(nyn_x-nys_x+1,nzb_y:nzt_y,nxl_y:nxr_y,0:pdims(2)-1) ::  work !< 
+    !$ACC DECLARE CREATE(work)
 
 
@@ -174,8 +180,10 @@
 !--    Transpose array
        CALL cpu_log( log_point_s(32), 'mpi_alltoall', 'start', cpu_log_nowait )
+       !$ACC UPDATE HOST(f_inv)
        IF ( collective_wait )  CALL MPI_BARRIER( comm2d, ierr )
        CALL MPI_ALLTOALL( f_inv(nys_x,nzb_x,0),  sendrecvcount_xy, MPI_REAL, &
                           work(1,nzb_y,nxl_y,0), sendrecvcount_xy, MPI_REAL, &
                           comm1dy, ierr )
+       !$ACC UPDATE DEVICE(work)
        CALL cpu_log( log_point_s(32), 'mpi_alltoall', 'stop' )
 
@@ -186,4 +194,6 @@
        DO  l = 0, pdims(2) - 1
           ys = 0 + l * ( nyn_x - nys_x + 1 )
+          !$ACC PARALLEL LOOP COLLAPSE(3) PRIVATE(i,j,k) &
+          !$ACC PRESENT(f_out, work)
           DO  i = nxl_y, nxr_y
              DO  k = nzb_y, nzt_y
@@ -203,4 +213,6 @@
 !$OMP  PARALLEL PRIVATE ( i, j, k )
 !$OMP  DO
+       !$ACC PARALLEL LOOP COLLAPSE(3) PRIVATE(i,j,k) &
+       !$ACC PRESENT(f_out, f_inv)
        DO  k = nzb_y, nzt_y
           DO  i = nxl_y, nxr_y
@@ -246,4 +258,6 @@
     !$OMP  PARALLEL PRIVATE ( i, j, k )
     !$OMP  DO
+     !$ACC PARALLEL LOOP COLLAPSE(3) PRIVATE(i,j,k) &
+     !$ACC PRESENT(f_out, f_inv)
      DO  k = 1, nz
          DO  i = nxl, nxr
@@ -293,4 +307,5 @@
 
     REAL(wp), DIMENSION(nys_x:nyn_x,nnx,nzb_x:nzt_x,0:pdims(1)-1) ::  work !< 
+    !$ACC DECLARE CREATE(work)
 
 
@@ -307,4 +322,6 @@
        DO  l = 0, pdims(1) - 1
           xs = 0 + l * nnx
+          !$ACC PARALLEL LOOP COLLAPSE(3) PRIVATE(i,j,k) &
+          !$ACC PRESENT(work, f_in)
           DO  k = nzb_x, nzt_x
              DO  i = xs, xs + nnx - 1
@@ -320,8 +337,10 @@
 !--    Transpose array
        CALL cpu_log( log_point_s(32), 'mpi_alltoall', 'start', cpu_log_nowait )
+       !$ACC UPDATE HOST(work)
        IF ( collective_wait )  CALL MPI_BARRIER( comm2d, ierr )
        CALL MPI_ALLTOALL( work(nys_x,1,nzb_x,0), sendrecvcount_zx, MPI_REAL, &
                           f_inv(nys,nxl,1),      sendrecvcount_zx, MPI_REAL, &
                           comm1dx, ierr )
+       !$ACC UPDATE DEVICE(f_inv)
        CALL cpu_log( log_point_s(32), 'mpi_alltoall', 'stop' )
 #endif
@@ -333,4 +352,6 @@
 !$OMP  PARALLEL PRIVATE ( i, j, k )
 !$OMP  DO
+       !$ACC PARALLEL LOOP COLLAPSE(3) PRIVATE(i,j,k) &
+       !$ACC PRESENT(f_inv, f_in)
        DO  i = nxl, nxr
           DO  j = nys, nyn
@@ -378,4 +399,6 @@
     !$OMP  PARALLEL PRIVATE ( i, j, k )
     !$OMP  DO
+     !$ACC PARALLEL LOOP COLLAPSE(3) PRIVATE(i,j,k) &
+     !$ACC PRESENT(f_out, f_inv)
      DO  i = 0, nx
          DO  k = nzb_x, nzt_x
@@ -425,4 +448,5 @@
 
     REAL(wp), DIMENSION(nyn_x-nys_x+1,nzb_y:nzt_y,nxl_y:nxr_y,0:pdims(2)-1) ::  work !< 
+    !$ACC DECLARE CREATE(work)
 
 
@@ -436,4 +460,6 @@
        DO  l = 0, pdims(2) - 1
           ys = 0 + l * ( nyn_x - nys_x + 1 )
+          !$ACC PARALLEL LOOP COLLAPSE(3) PRIVATE(i,j,k) &
+          !$ACC PRESENT(work, f_in)
           DO  i = nxl_y, nxr_y
              DO  k = nzb_y, nzt_y
@@ -449,8 +475,10 @@
 !--    Transpose array
        CALL cpu_log( log_point_s(32), 'mpi_alltoall', 'start', cpu_log_nowait )
+       !$ACC UPDATE HOST(work)
        IF ( collective_wait )  CALL MPI_BARRIER( comm2d, ierr )
        CALL MPI_ALLTOALL( work(1,nzb_y,nxl_y,0), sendrecvcount_xy, MPI_REAL, &
                           f_inv(nys_x,nzb_x,0),  sendrecvcount_xy, MPI_REAL, &
                           comm1dy, ierr )
+       !$ACC UPDATE DEVICE(f_inv)
        CALL cpu_log( log_point_s(32), 'mpi_alltoall', 'stop' )
 #endif
@@ -462,4 +490,6 @@
 !$OMP  PARALLEL PRIVATE ( i, j, k )
 !$OMP  DO
+       !$ACC PARALLEL LOOP COLLAPSE(3) PRIVATE(i,j,k) &
+       !$ACC PRESENT(f_inv, f_in)
        DO  i = nxl_y, nxr_y
           DO  k = nzb_y, nzt_y
@@ -587,4 +617,6 @@
     !$OMP  PARALLEL PRIVATE ( i, j, k )
     !$OMP  DO
+     !$ACC PARALLEL LOOP COLLAPSE(3) PRIVATE(i,j,k) &
+     !$ACC PRESENT(f_inv, f_in)
      DO  j = 0, ny
          DO  k = nzb_y, nzt_y
@@ -634,4 +666,5 @@
 
     REAL(wp), DIMENSION(nxl_z:nxr_z,nzt_y-nzb_y+1,nys_z:nyn_z,0:pdims(1)-1) ::  work !< 
+    !$ACC DECLARE CREATE(work)
 
 
@@ -643,4 +676,6 @@
 !$OMP  PARALLEL PRIVATE ( i, j, k )
 !$OMP  DO
+       !$ACC PARALLEL LOOP COLLAPSE(3) PRIVATE(i,j,k) &
+       !$ACC PRESENT(f_out, f_inv)
        DO  j = 0, ny
           DO  k = nzb_y, nzt_y
@@ -658,8 +693,10 @@
 !--    Transpose array
        CALL cpu_log( log_point_s(32), 'mpi_alltoall', 'start', cpu_log_nowait )
+       !$ACC UPDATE HOST(f_inv)
        IF ( collective_wait )  CALL MPI_BARRIER( comm2d, ierr )
        CALL MPI_ALLTOALL( f_inv(nxl_y,nzb_y,0),  sendrecvcount_yz, MPI_REAL, &
                           work(nxl_z,1,nys_z,0), sendrecvcount_yz, MPI_REAL, &
                           comm1dx, ierr )
+       !$ACC UPDATE DEVICE(work)
        CALL cpu_log( log_point_s(32), 'mpi_alltoall', 'stop' )
 
@@ -670,4 +707,6 @@
        DO  l = 0, pdims(1) - 1
           zs = 1 + l * ( nzt_y - nzb_y + 1 )
+          !$ACC PARALLEL LOOP COLLAPSE(3) PRIVATE(i,j,k) &
+          !$ACC PRESENT(f_out, work)
           DO  j = nys_z, nyn_z
              DO  k = zs, zs + nzt_y - nzb_y
@@ -714,4 +753,6 @@
     !$OMP  PARALLEL PRIVATE ( i, j, k )
     !$OMP  DO
+    !$ACC PARALLEL LOOP COLLAPSE(3) PRIVATE(i,j,k) &
+    !$ACC PRESENT(f_in, f_inv)
      DO  k = 1,nz
          DO  i = nxl, nxr
@@ -761,4 +802,5 @@
 
     REAL(wp), DIMENSION(nys_x:nyn_x,nnx,nzb_x:nzt_x,0:pdims(1)-1) ::  work !< 
+    !$ACC DECLARE CREATE(work)
 
 
@@ -770,4 +812,6 @@
 !$OMP  PARALLEL PRIVATE ( i, j, k )
 !$OMP  DO
+       !$ACC PARALLEL LOOP COLLAPSE(3) PRIVATE(i,j,k) &
+       !$ACC PRESENT(f_out, f_inv)
        DO  k = 1, nz
           DO  i = nxl, nxr
@@ -785,8 +829,10 @@
 !--    Transpose array
        CALL cpu_log( log_point_s(32), 'mpi_alltoall', 'start', cpu_log_nowait )
+       !$ACC UPDATE HOST(f_inv)
        IF ( collective_wait )  CALL MPI_BARRIER( comm2d, ierr )
        CALL MPI_ALLTOALL( f_inv(nys,nxl,1),      sendrecvcount_zx, MPI_REAL, &
                           work(nys_x,1,nzb_x,0), sendrecvcount_zx, MPI_REAL, &
                           comm1dx, ierr )
+       !$ACC UPDATE DEVICE(work)
        CALL cpu_log( log_point_s(32), 'mpi_alltoall', 'stop' )
 
@@ -797,4 +843,6 @@
        DO  l = 0, pdims(1) - 1
           xs = 0 + l * nnx
+          !$ACC PARALLEL LOOP COLLAPSE(3) PRIVATE(i,j,k) &
+          !$ACC PRESENT(f_out, work)
           DO  k = nzb_x, nzt_x
              DO  i = xs, xs + nnx - 1
@@ -845,4 +893,6 @@
     !$OMP  PARALLEL PRIVATE ( i, j, k )
     !$OMP  DO
+    !$ACC PARALLEL LOOP COLLAPSE(3) PRIVATE(i,j,k) &
+    !$ACC PRESENT(f_out, f_inv)
      DO  k = nzb_y, nzt_y
          DO  j = 0, ny
@@ -892,4 +942,5 @@
 
     REAL(wp), DIMENSION(nxl_z:nxr_z,nzt_y-nzb_y+1,nys_z:nyn_z,0:pdims(1)-1) ::  work !<
+    !$ACC DECLARE CREATE(work)
 
 !
@@ -905,4 +956,6 @@
        DO  l = 0, pdims(1) - 1
           zs = 1 + l * ( nzt_y - nzb_y + 1 )
+          !$ACC PARALLEL LOOP COLLAPSE(3) PRIVATE(i,j,k) &
+          !$ACC PRESENT(work, f_in)
           DO  j = nys_z, nyn_z
              DO  k = zs, zs + nzt_y - nzb_y
@@ -918,8 +971,10 @@
 !--    Transpose array
        CALL cpu_log( log_point_s(32), 'mpi_alltoall', 'start', cpu_log_nowait )
+       !$ACC UPDATE HOST(work)
        IF ( collective_wait )  CALL MPI_BARRIER( comm2d, ierr )
        CALL MPI_ALLTOALL( work(nxl_z,1,nys_z,0), sendrecvcount_yz, MPI_REAL, &
                           f_inv(nxl_y,nzb_y,0),  sendrecvcount_yz, MPI_REAL, &
                           comm1dx, ierr )
+       !$ACC UPDATE DEVICE(f_inv)
        CALL cpu_log( log_point_s(32), 'mpi_alltoall', 'stop' )
 #endif
@@ -930,4 +985,6 @@
 !$OMP  PARALLEL PRIVATE ( i, j, k )
 !$OMP  DO
+       !$ACC PARALLEL LOOP COLLAPSE(3) PRIVATE(i,j,k) &
+       !$ACC PRESENT(f_inv, f_in)
        DO  k = nzb_y, nzt_y
           DO  j = 0, ny
