Merge branch 'atmosphere/port_divergence_damping_3d' into develop (PR #1237)

This merge enables GPU execution of the atm_divergence_damping_3d subroutine
using OpenACC directives for the data movement and loops. A new timer,
'atm_divergence_damping_3d [ACC_data_xfer]', has been added for data transfers
in the atm_divergence_damping_3d subroutine.

Results from a regional simulation are bit-wise identical pre- and post-merge.

* atmosphere/port_divergence_damping_3d:
  Initial OpenACC port of atm_divergence_damping_3d subroutine

Author: mgduda

src/core_atmosphere/dynamics/mpas_atm_time_integration.F

@@ -212,6 +212,7 @@ subroutine mpas_atm_dynamics_init(domain)
       real (kind=RKIND), dimension(:), pointer :: invAreaCell
       integer, dimension(:), pointer :: bdyMaskCell
       integer, dimension(:), pointer :: bdyMaskEdge
+      real (kind=RKIND), dimension(:), pointer :: specZoneMaskEdge
       real (kind=RKIND), dimension(:), pointer :: invDvEdge
       real (kind=RKIND), dimension(:), pointer :: dcEdge
       real (kind=RKIND), dimension(:), pointer :: invDcEdge
@@ -295,6 +296,9 @@ subroutine mpas_atm_dynamics_init(domain)
       call mpas_pool_get_array(mesh, 'bdyMaskEdge', bdyMaskEdge)
       !$acc enter data copyin(bdyMaskEdge)
 
+      call mpas_pool_get_array(mesh, 'specZoneMaskEdge', specZoneMaskEdge)
+      !$acc enter data copyin(specZoneMaskEdge)
+
       call mpas_pool_get_array(mesh, 'invDvEdge', invDvEdge)
       !$acc enter data copyin(invDvEdge)
 
@@ -412,6 +416,7 @@ subroutine mpas_atm_dynamics_finalize(domain)
       real (kind=RKIND), dimension(:), pointer :: invAreaCell
       integer, dimension(:), pointer :: bdyMaskCell
       integer, dimension(:), pointer :: bdyMaskEdge
+      real (kind=RKIND), dimension(:), pointer :: specZoneMaskEdge
       real (kind=RKIND), dimension(:), pointer :: invDvEdge
       real (kind=RKIND), dimension(:), pointer :: dcEdge
       real (kind=RKIND), dimension(:), pointer :: invDcEdge
@@ -495,6 +500,9 @@ subroutine mpas_atm_dynamics_finalize(domain)
       call mpas_pool_get_array(mesh, 'bdyMaskEdge', bdyMaskEdge)
       !$acc exit data delete(bdyMaskEdge)
 
+      call mpas_pool_get_array(mesh, 'specZoneMaskEdge', specZoneMaskEdge)
+      !$acc exit data delete(specZoneMaskEdge)
+
       call mpas_pool_get_array(mesh, 'invDvEdge', invDvEdge)
       !$acc exit data delete(invDvEdge)
 
@@ -2696,8 +2704,10 @@ subroutine atm_divergence_damping_3d( state, diag, mesh, configs, dts, edgeStart
       real (kind=RKIND), dimension(:), pointer :: specZoneMaskEdge
 
       integer, dimension(:,:), pointer :: cellsOnEdge
-      integer, pointer :: nCellsSolve
-      integer, pointer :: nVertLevels
+      integer, pointer :: nCellsSolve_ptr
+      integer, pointer :: nVertLevels_ptr
+      integer :: nCellsSolve
+      integer :: nVertLevels
 
       real (kind=RKIND) :: divCell1, divCell2, rdts, coef_divdamp
       integer :: cell1, cell2, iEdge, k
@@ -2710,15 +2720,24 @@ subroutine atm_divergence_damping_3d( state, diag, mesh, configs, dts, edgeStart
       call mpas_pool_get_array(diag, 'rtheta_pp_old', rtheta_pp_old)
       call mpas_pool_get_array(diag, 'ru_p', ru_p)
 
-      call mpas_pool_get_dimension(mesh, 'nCellsSolve', nCellsSolve)
-      call mpas_pool_get_dimension(mesh, 'nVertLevels', nVertLevels)
+      call mpas_pool_get_dimension(mesh, 'nCellsSolve', nCellsSolve_ptr)
+      call mpas_pool_get_dimension(mesh, 'nVertLevels', nVertLevels_ptr)
 
       call mpas_pool_get_config(configs, 'config_smdiv', smdiv) 
       call mpas_pool_get_config(configs, 'config_len_disp', config_len_disp)
 
       rdts = 1.0_RKIND / dts
       coef_divdamp = 2.0_RKIND * smdiv * config_len_disp * rdts
 
+      nCellsSolve = nCellsSolve_ptr
+      nVertLevels = nVertLevels_ptr
+
+      MPAS_ACC_TIMER_START('atm_divergence_damping_3d [ACC_data_xfer]')
+      !$acc enter data copyin(ru_p, rtheta_pp, rtheta_pp_old, theta_m)
+      MPAS_ACC_TIMER_STOP('atm_divergence_damping_3d [ACC_data_xfer]')
+
+      !$acc parallel default(present)
+      !$acc loop gang worker
       do iEdge=edgeStart,edgeEnd ! MGD do we really just need edges touching owned cells?
 
          cell1 = cellsOnEdge(1,iEdge)
@@ -2728,6 +2747,7 @@ subroutine atm_divergence_damping_3d( state, diag, mesh, configs, dts, edgeStart
          if (cell1 <= nCellsSolve .or. cell2 <= nCellsSolve ) then
 
 !DIR$ IVDEP
+            !$acc loop vector
             do k=1,nVertLevels
 
 !!  unscaled 3d divergence damping
@@ -2745,6 +2765,13 @@ subroutine atm_divergence_damping_3d( state, diag, mesh, configs, dts, edgeStart
             end do
          end if ! edges for block-owned cells
       end do ! end loop over edges
+      !$acc end parallel
+
+      MPAS_ACC_TIMER_START('atm_divergence_damping_3d [ACC_data_xfer]')
+      !$acc exit data copyout(ru_p) &
+      !$acc           delete(rtheta_pp, rtheta_pp_old, theta_m)
+      MPAS_ACC_TIMER_STOP('atm_divergence_damping_3d [ACC_data_xfer]')
+      
 
    end subroutine atm_divergence_damping_3d