Index: palm/trunk/SOURCE/transpose.f90
===================================================================
--- palm/trunk/SOURCE/transpose.f90	(revision 1107)
+++ palm/trunk/SOURCE/transpose.f90	(revision 1111)
@@ -20,5 +20,6 @@
 ! Current revisions:
 ! -----------------
-! 
+! openACC directives added,
+! resorting data from/to work changed, work got 4 dimensions instead of 1
 !
 ! Former revisions:
@@ -81,10 +82,13 @@
     IMPLICIT NONE
 
-    INTEGER ::  i, j, k, l, m, ys
+    INTEGER ::  i, j, k, l, ys
     
-    REAL ::  f_in(0:nx,nys_x:nyn_x,nzb_x:nzt_x),   &
-             f_inv(nys_x:nyn_x,nzb_x:nzt_x,0:nx),  &
-             f_out(0:ny,nxl_y:nxr_y,nzb_y:nzt_y),  &
-             work(nnx*nny*nnz)
+    REAL ::  f_in(0:nx,nys_x:nyn_x,nzb_x:nzt_x), f_out(0:ny,nxl_y:nxr_y,nzb_y:nzt_y)
+
+    REAL, DIMENSION(nyn_x-nys_x+1,nzb_y:nzt_y,nxl_y:nxr_y,0:pdims(2)-1) ::  work
+
+    !$acc declare create( f_inv )
+    REAL ::  f_inv(nys_x:nyn_x,nzb_x:nzt_x,0:nx)
+
 
 !
@@ -93,6 +97,9 @@
 !$OMP  PARALLEL PRIVATE ( i, j, k )
 !$OMP  DO
+    !$acc kernels present( f_in )
+    !$acc loop
     DO  i = 0, nx
        DO  k = nzb_x, nzt_x
+          !$acc loop vector( 32 )
           DO  j = nys_x, nyn_x
              f_inv(j,k,i) = f_in(i,j,k)
@@ -100,4 +107,5 @@
        ENDDO
     ENDDO
+    !$acc end kernels
 !$OMP  END PARALLEL
 
@@ -109,25 +117,28 @@
        CALL cpu_log( log_point_s(32), 'mpi_alltoall', 'start' )
        IF ( collective_wait )  CALL MPI_BARRIER( comm2d, ierr )
-       CALL MPI_ALLTOALL( f_inv(nys_x,nzb_x,0), sendrecvcount_xy, MPI_REAL, &
-                          work(1),              sendrecvcount_xy, MPI_REAL, &
+       !$acc update host( f_inv )
+       CALL MPI_ALLTOALL( f_inv(nys_x,nzb_x,0),  sendrecvcount_xy, MPI_REAL, &
+                          work(1,nzb_y,nxl_y,0), sendrecvcount_xy, MPI_REAL, &
                           comm1dy, ierr )
+       !$acc update device( work )
        CALL cpu_log( log_point_s(32), 'mpi_alltoall', 'stop' )
 
 !
 !--    Reorder transposed array
-!$OMP  PARALLEL PRIVATE ( i, j, k, l, m, ys )
+!$OMP  PARALLEL PRIVATE ( i, j, k, l, ys )
 !$OMP  DO
        DO  l = 0, pdims(2) - 1
-          m  = l * ( nxr_y - nxl_y + 1 ) * ( nzt_y - nzb_y + 1 ) * &
-                   ( nyn_x - nys_x + 1 )
           ys = 0 + l * ( nyn_x - nys_x + 1 )
+          !$acc kernels present( f_out, work )
+          !$acc loop
           DO  i = nxl_y, nxr_y
              DO  k = nzb_y, nzt_y
+                !$acc loop vector( 32 )
                 DO  j = ys, ys + nyn_x - nys_x
-                   m = m + 1
-                   f_out(j,i,k) = work(m)
+                   f_out(j,i,k) = work(j-ys+1,k,i,l)
                 ENDDO
              ENDDO
           ENDDO
+          !$acc end kernels
        ENDDO
 !$OMP  END PARALLEL
@@ -140,6 +151,9 @@
 !$OMP  PARALLEL PRIVATE ( i, j, k )
 !$OMP  DO
+       !$acc kernels present( f_out )
+       !$acc loop
        DO  k = nzb_y, nzt_y
           DO  i = nxl_y, nxr_y
+             !$acc loop vector( 32 )
              DO  j = 0, ny
                 f_out(j,i,k) = f_inv(j,k,i)
@@ -147,4 +161,5 @@
           ENDDO
        ENDDO
+       !$acc end kernels
 !$OMP  END PARALLEL
 
@@ -172,10 +187,12 @@
     IMPLICIT NONE
 
-    INTEGER ::  i, j, k, l, m, xs
+    INTEGER ::  i, j, k, l, xs
     
-    REAL ::  f_in(0:nx,nys_x:nyn_x,nzb_x:nzt_x),  &
-             f_inv(nys:nyn,nxl:nxr,1:nz),         &
-             f_out(1:nz,nys:nyn,nxl:nxr),         &
-             work(nnx*nny*nnz)
+    REAL ::  f_in(0:nx,nys_x:nyn_x,nzb_x:nzt_x), f_out(1:nz,nys:nyn,nxl:nxr)
+
+    REAL, DIMENSION(nys_x:nyn_x,nnx,nzb_x:nzt_x,0:pdims(1)-1) ::  work
+
+    !$acc declare create( f_inv )
+    REAL ::  f_inv(nys:nyn,nxl:nxr,1:nz)
 
 
@@ -188,17 +205,19 @@
 !
 !--    Reorder input array for transposition
-!$OMP  PARALLEL PRIVATE ( i, j, k, l, m, xs )
+!$OMP  PARALLEL PRIVATE ( i, j, k, l, xs )
 !$OMP  DO
        DO  l = 0, pdims(1) - 1
-          m  = l * ( nzt_x - nzb_x + 1 ) * nnx * ( nyn_x - nys_x + 1 )
           xs = 0 + l * nnx
+          !$acc kernels present( f_in, work )
+          !$acc loop
           DO  k = nzb_x, nzt_x
              DO  i = xs, xs + nnx - 1
+                !$acc loop vector( 32 )
                 DO  j = nys_x, nyn_x
-                   m = m + 1
-                   work(m) = f_in(i,j,k)
+                   work(j,i-xs+1,k,l) = f_in(i,j,k)
                 ENDDO
              ENDDO
           ENDDO
+          !$acc end kernels
        ENDDO
 !$OMP  END PARALLEL
@@ -208,7 +227,9 @@
        CALL cpu_log( log_point_s(32), 'mpi_alltoall', 'start' )
        IF ( collective_wait )  CALL MPI_BARRIER( comm2d, ierr )
-       CALL MPI_ALLTOALL( work(1),          sendrecvcount_zx, MPI_REAL, &
-                          f_inv(nys,nxl,1), sendrecvcount_zx, MPI_REAL, &
+       !$acc update host( work )
+       CALL MPI_ALLTOALL( work(nys_x,1,nzb_x,0), sendrecvcount_zx, MPI_REAL, &
+                          f_inv(nys,nxl,1),      sendrecvcount_zx, MPI_REAL, &
                           comm1dx, ierr )
+       !$acc update device( f_inv )
        CALL cpu_log( log_point_s(32), 'mpi_alltoall', 'stop' )
 
@@ -217,6 +238,9 @@
 !$OMP  PARALLEL PRIVATE ( i, j, k )
 !$OMP  DO
+       !$acc kernels present( f_out )
+       !$acc loop
        DO  k = 1, nz
           DO  i = nxl, nxr
+             !$acc loop vector( 32 )
              DO  j = nys, nyn
                 f_out(k,j,i) = f_inv(j,i,k)
@@ -224,4 +248,5 @@
           ENDDO
        ENDDO
+       !$acc end kernels
 !$OMP  END PARALLEL
 #endif
@@ -233,6 +258,9 @@
 !$OMP  PARALLEL PRIVATE ( i, j, k )
 !$OMP  DO
+       !$acc kernels present( f_in )
+       !$acc loop
        DO  i = nxl, nxr
           DO  j = nys, nyn
+             !$acc loop vector( 32 )
              DO  k = 1, nz
                 f_inv(j,i,k) = f_in(i,j,k)
@@ -240,10 +268,14 @@
           ENDDO
        ENDDO
-!$OMP  END PARALLEL
-
-!$OMP  PARALLEL PRIVATE ( i, j, k )
-!$OMP  DO
+       !$acc end kernels
+!$OMP  END PARALLEL
+
+!$OMP  PARALLEL PRIVATE ( i, j, k )
+!$OMP  DO
+       !$acc kernels present( f_out )
+       !$acc loop
        DO  k = 1, nz
           DO  i = nxl, nxr
+             !$acc loop vector( 32 )
              DO  j = nys, nyn
                 f_out(k,j,i) = f_inv(j,i,k)
@@ -251,4 +283,5 @@
           ENDDO
        ENDDO
+       !$acc end kernels
 !$OMP  END PARALLEL
 
@@ -276,10 +309,13 @@
     IMPLICIT NONE
 
-    INTEGER ::  i, j, k, l, m, ys
+    INTEGER ::  i, j, k, l, ys
     
-    REAL ::  f_in(0:ny,nxl_y:nxr_y,nzb_y:nzt_y),  &
-             f_inv(nys_x:nyn_x,nzb_x:nzt_x,0:nx), &
-             f_out(0:nx,nys_x:nyn_x,nzb_x:nzt_x), &
-             work(nnx*nny*nnz)
+    REAL ::  f_in(0:ny,nxl_y:nxr_y,nzb_y:nzt_y), f_out(0:nx,nys_x:nyn_x,nzb_x:nzt_x)
+
+    REAL, DIMENSION(nyn_x-nys_x+1,nzb_y:nzt_y,nxl_y:nxr_y,0:pdims(2)-1) ::  work
+
+    !$acc declare create( f_inv )
+    REAL ::  f_inv(nys_x:nyn_x,nzb_x:nzt_x,0:nx)
+
 
     IF ( numprocs /= 1 )  THEN
@@ -288,18 +324,19 @@
 !
 !--    Reorder input array for transposition
-!$OMP  PARALLEL PRIVATE ( i, j, k, l, m, ys )
+!$OMP  PARALLEL PRIVATE ( i, j, k, l, ys )
 !$OMP  DO
        DO  l = 0, pdims(2) - 1
-          m  = l * ( nxr_y - nxl_y + 1 ) * ( nzt_y - nzb_y + 1 ) * &
-                   ( nyn_x - nys_x + 1 )
           ys = 0 + l * ( nyn_x - nys_x + 1 )
+          !$acc kernels present( f_in, work )
+          !$acc loop
           DO  i = nxl_y, nxr_y
              DO  k = nzb_y, nzt_y
+                !$acc loop vector( 32 )
                 DO  j = ys, ys + nyn_x - nys_x
-                   m = m + 1
-                   work(m) = f_in(j,i,k)
+                   work(j-ys+1,k,i,l) = f_in(j,i,k)
                 ENDDO
              ENDDO
           ENDDO
+          !$acc end kernels
        ENDDO
 !$OMP  END PARALLEL
@@ -309,7 +346,9 @@
        CALL cpu_log( log_point_s(32), 'mpi_alltoall', 'start' )
        IF ( collective_wait )  CALL MPI_BARRIER( comm2d, ierr )
-       CALL MPI_ALLTOALL( work(1),              sendrecvcount_xy, MPI_REAL, &
-                          f_inv(nys_x,nzb_x,0), sendrecvcount_xy, MPI_REAL, &
+       !$acc update host( work )
+       CALL MPI_ALLTOALL( work(1,nzb_y,nxl_y,0), sendrecvcount_xy, MPI_REAL, &
+                          f_inv(nys_x,nzb_x,0),  sendrecvcount_xy, MPI_REAL, &
                           comm1dy, ierr )
+       !$acc update device( f_inv )
        CALL cpu_log( log_point_s(32), 'mpi_alltoall', 'stop' )
 #endif
@@ -321,6 +360,9 @@
 !$OMP  PARALLEL PRIVATE ( i, j, k )
 !$OMP  DO
+       !$acc kernels present( f_in )
+       !$acc loop
        DO  i = nxl_y, nxr_y
           DO  k = nzb_y, nzt_y
+             !$acc loop vector( 32 )
              DO  j = 0, ny
                 f_inv(j,k,i) = f_in(j,i,k)
@@ -328,4 +370,5 @@
           ENDDO
        ENDDO
+       !$acc end kernels
 !$OMP  END PARALLEL
 
@@ -336,6 +379,9 @@
 !$OMP  PARALLEL PRIVATE ( i, j, k )
 !$OMP  DO
+    !$acc kernels present( f_out )
+    !$acc loop
     DO  i = 0, nx
        DO  k = nzb_x, nzt_x
+          !$acc loop vector( 32 )
           DO  j = nys_x, nyn_x
              f_out(i,j,k) = f_inv(j,k,i)
@@ -343,4 +389,5 @@
        ENDDO
     ENDDO
+    !$acc end kernels
 !$OMP  END PARALLEL
 
@@ -434,10 +481,13 @@
     IMPLICIT NONE
 
-    INTEGER ::  i, j, k, l, m, zs
+    INTEGER ::  i, j, k, l, zs
     
-    REAL ::  f_in(0:ny,nxl_y:nxr_y,nzb_y:nzt_y),  &
-             f_inv(nxl_y:nxr_y,nzb_y:nzt_y,0:ny), &
-             f_out(nxl_z:nxr_z,nys_z:nyn_z,1:nz), &
-             work(nnx*nny*nnz)
+    REAL ::  f_in(0:ny,nxl_y:nxr_y,nzb_y:nzt_y), f_out(nxl_z:nxr_z,nys_z:nyn_z,1:nz)
+
+    REAL, DIMENSION(nxl_z:nxr_z,nzt_y-nzb_y+1,nys_z:nyn_z,0:pdims(1)-1) ::  work
+
+    !$acc declare create( f_inv )
+    REAL ::  f_inv(nxl_y:nxr_y,nzb_y:nzt_y,0:ny)
+
 
 !
@@ -446,6 +496,9 @@
 !$OMP  PARALLEL PRIVATE ( i, j, k )
 !$OMP  DO
+    !$acc kernels present( f_in )
+    !$acc loop
     DO  j = 0, ny
        DO  k = nzb_y, nzt_y
+          !$acc loop vector( 32 )
           DO  i = nxl_y, nxr_y
              f_inv(i,k,j) = f_in(j,i,k)
@@ -453,4 +506,5 @@
        ENDDO
     ENDDO
+    !$acc end kernels
 !$OMP  END PARALLEL
 
@@ -464,6 +518,9 @@
 !$OMP  PARALLEL PRIVATE ( i, j, k )
 !$OMP  DO
+       !$acc kernels present( f_out )
+       !$acc loop
        DO  j = 0, ny
           DO  k = nzb_y, nzt_y
+             !$acc loop vector( 32 )
              DO  i = nxl_y, nxr_y
                 f_out(i,j,k) = f_inv(i,k,j)
@@ -471,4 +528,5 @@
           ENDDO
        ENDDO
+       !$acc end kernels
 !$OMP  END PARALLEL
 
@@ -480,25 +538,28 @@
        CALL cpu_log( log_point_s(32), 'mpi_alltoall', 'start' )
        IF ( collective_wait )  CALL MPI_BARRIER( comm2d, ierr )
-       CALL MPI_ALLTOALL( f_inv(nxl_y,nzb_y,0), sendrecvcount_yz, MPI_REAL, &
-                          work(1),              sendrecvcount_yz, MPI_REAL, &
+       !$acc update host( f_inv )
+       CALL MPI_ALLTOALL( f_inv(nxl_y,nzb_y,0),  sendrecvcount_yz, MPI_REAL, &
+                          work(nxl_z,1,nys_z,0), sendrecvcount_yz, MPI_REAL, &
                           comm1dx, ierr )
+       !$acc update device( work )
        CALL cpu_log( log_point_s(32), 'mpi_alltoall', 'stop' )
 
 !
 !--    Reorder transposed array
-!$OMP  PARALLEL PRIVATE ( i, j, k, l, m, zs )
+!$OMP  PARALLEL PRIVATE ( i, j, k, l, zs )
 !$OMP  DO
        DO  l = 0, pdims(1) - 1
-          m  = l * ( nyn_z - nys_z + 1 ) * ( nzt_y - nzb_y + 1 ) * &
-                   ( nxr_z - nxl_z + 1 )
           zs = 1 + l * ( nzt_y - nzb_y + 1 )
+          !$acc kernels present( f_out, work )
+          !$acc loop
           DO  j = nys_z, nyn_z
              DO  k = zs, zs + nzt_y - nzb_y
+                !$acc loop vector( 32 )
                 DO  i = nxl_z, nxr_z
-                   m = m + 1
-                   f_out(i,j,k) = work(m)
+                   f_out(i,j,k) = work(i,k-zs+1,j,l)
                 ENDDO
              ENDDO
           ENDDO
+          !$acc end kernels
        ENDDO
 !$OMP  END PARALLEL
@@ -528,10 +589,11 @@
     IMPLICIT NONE
 
-    INTEGER ::  i, j, k, l, m, xs
+    INTEGER ::  i, j, k, l, xs
     
-    REAL ::  f_in(1:nz,nys:nyn,nxl:nxr), f_out(0:nx,nys_x:nyn_x,nzb_x:nzt_x), &
-             work(nnx*nny*nnz)
-
-    !$acc declare create ( f_inv )
+    REAL ::  f_in(1:nz,nys:nyn,nxl:nxr), f_out(0:nx,nys_x:nyn_x,nzb_x:nzt_x)
+
+    REAL, DIMENSION(nys_x:nyn_x,nnx,nzb_x:nzt_x,0:pdims(1)-1) ::  work
+
+    !$acc declare create( f_inv )
     REAL ::  f_inv(nys:nyn,nxl:nxr,1:nz)
 
@@ -552,4 +614,5 @@
        ENDDO
     ENDDO
+    !$acc end kernels
 !$OMP  END PARALLEL
 
@@ -573,4 +636,5 @@
           ENDDO
        ENDDO
+       !$acc end kernels
 !$OMP  END PARALLEL
 
@@ -582,24 +646,28 @@
        CALL cpu_log( log_point_s(32), 'mpi_alltoall', 'start' )
        IF ( collective_wait )  CALL MPI_BARRIER( comm2d, ierr )
-       CALL MPI_ALLTOALL( f_inv(nys,nxl,1), sendrecvcount_zx, MPI_REAL, &
-                          work(1),          sendrecvcount_zx, MPI_REAL, &
+       !$acc update host( f_inv )
+       CALL MPI_ALLTOALL( f_inv(nys,nxl,1),      sendrecvcount_zx, MPI_REAL, &
+                          work(nys_x,1,nzb_x,0), sendrecvcount_zx, MPI_REAL, &
                           comm1dx, ierr )
+       !$acc update device( work )
        CALL cpu_log( log_point_s(32), 'mpi_alltoall', 'stop' )
 
 !
 !--    Reorder transposed array
-!$OMP  PARALLEL PRIVATE ( i, j, k, l, m, xs )
+!$OMP  PARALLEL PRIVATE ( i, j, k, l, xs )
 !$OMP  DO
        DO  l = 0, pdims(1) - 1
-          m  = l * ( nzt_x - nzb_x + 1 ) * nnx * ( nyn_x - nys_x + 1 )
           xs = 0 + l * nnx
+          !$acc kernels present( f_out, work )
+          !$acc loop
           DO  k = nzb_x, nzt_x
              DO  i = xs, xs + nnx - 1
+                !$acc loop vector( 32 )
                 DO  j = nys_x, nyn_x
-                   m = m + 1
-                   f_out(i,j,k) = work(m)
+                   f_out(i,j,k) = work(j,i-xs+1,k,l)
                 ENDDO
              ENDDO
           ENDDO
+          !$acc end kernels
        ENDDO
 !$OMP  END PARALLEL
@@ -629,10 +697,13 @@
     IMPLICIT NONE
 
-    INTEGER ::  i, j, k, l, m, zs
+    INTEGER ::  i, j, k, l, zs
     
-    REAL ::  f_in(nxl_z:nxr_z,nys_z:nyn_z,1:nz),  &
-             f_inv(nxl_y:nxr_y,nzb_y:nzt_y,0:ny), &
-             f_out(0:ny,nxl_y:nxr_y,nzb_y:nzt_y), &
-             work(nnx*nny*nnz)
+    REAL ::  f_in(nxl_z:nxr_z,nys_z:nyn_z,1:nz), f_out(0:ny,nxl_y:nxr_y,nzb_y:nzt_y)
+
+    REAL, DIMENSION(nxl_z:nxr_z,nzt_y-nzb_y+1,nys_z:nyn_z,0:pdims(1)-1) ::  work
+
+    !$acc declare create( f_inv )
+    REAL ::  f_inv(nxl_y:nxr_y,nzb_y:nzt_y,0:ny)
+
 
 !
@@ -644,18 +715,19 @@
 !
 !--    Reorder input array for transposition
-!$OMP  PARALLEL PRIVATE ( i, j, k, l, m, zs )
+!$OMP  PARALLEL PRIVATE ( i, j, k, l, zs )
 !$OMP  DO
        DO  l = 0, pdims(1) - 1
-          m  = l * ( nyn_z - nys_z + 1 ) * ( nzt_y - nzb_y + 1 ) * &
-                   ( nxr_z - nxl_z + 1 )
           zs = 1 + l * ( nzt_y - nzb_y + 1 )
+          !$acc kernels present( f_in, work )
+          !$acc loop
           DO  j = nys_z, nyn_z
              DO  k = zs, zs + nzt_y - nzb_y
+                !$acc loop vector( 32 )
                 DO  i = nxl_z, nxr_z
-                   m = m + 1
-                   work(m) = f_in(i,j,k)
+                   work(i,k-zs+1,j,l) = f_in(i,j,k)
                 ENDDO
              ENDDO
           ENDDO
+          !$acc end kernels
        ENDDO
 !$OMP  END PARALLEL
@@ -665,7 +737,9 @@
        CALL cpu_log( log_point_s(32), 'mpi_alltoall', 'start' )
        IF ( collective_wait )  CALL MPI_BARRIER( comm2d, ierr )
-       CALL MPI_ALLTOALL( work(1),              sendrecvcount_yz, MPI_REAL, &
-                          f_inv(nxl_y,nzb_y,0), sendrecvcount_yz, MPI_REAL, &
+       !$acc update host( work )
+       CALL MPI_ALLTOALL( work(nxl_z,1,nys_z,0), sendrecvcount_yz, MPI_REAL, &
+                          f_inv(nxl_y,nzb_y,0),  sendrecvcount_yz, MPI_REAL, &
                           comm1dx, ierr )
+       !$acc update device( f_inv )
        CALL cpu_log( log_point_s(32), 'mpi_alltoall', 'stop' )
 #endif
@@ -676,6 +750,9 @@
 !$OMP  PARALLEL PRIVATE ( i, j, k )
 !$OMP  DO
+       !$acc kernels present( f_in )
+       !$acc loop
        DO  k = nzb_y, nzt_y
           DO  j = 0, ny
+             !$acc loop vector( 32 )
              DO  i = nxl_y, nxr_y
                 f_inv(i,k,j) = f_in(i,j,k)
@@ -683,4 +760,5 @@
           ENDDO
        ENDDO
+       !$acc end kernels
 !$OMP  END PARALLEL
 
@@ -691,6 +769,9 @@
 !$OMP  PARALLEL PRIVATE ( i, j, k )
 !$OMP  DO
+    !$acc kernels present( f_out )
+    !$acc loop
     DO  k = nzb_y, nzt_y
        DO  i = nxl_y, nxr_y
+          !$acc loop vector( 32 )
           DO  j = 0, ny
              f_out(j,i,k) = f_inv(i,k,j)
@@ -698,4 +779,5 @@
        ENDDO
     ENDDO
+    !$acc end kernels
 !$OMP  END PARALLEL
 
