Initial OpenACC port of atm_divergence_damping_3d subroutine

This commit enables the GPU execution of the atm_divergence_damping_3d
subroutine using OpenACC directives for the data movements and loops. A new
timer, 'atm_divergence_damping_3d [ACC_data_xfer]', has been added for host-
GPU data transfers in this subroutine. This port follows these considerations:

- The data transfers for the invariant fields have been moved to
  mpas_atm_dynamics_init and finalize.
- Explicitly adding gang, worker and vector level parallelism to parallel
  constructs in atm_compute_vert_imp_coefs_work.
- The parallel constructs use default(present) clauses to avoid implicit
  data movements by the compiler. This change also requires dereferencing
  the pointers to nCellsSolve and nVertLevels, in order to avoid runtime
  error messages such as the following:

FATAL ERROR: data in PRESENT clause was not found on device 1:name=ncellssolve

Author: abishekg7

src/core_atmosphere/dynamics/mpas_atm_time_integration.F

@@ -212,6 +212,7 @@ subroutine mpas_atm_dynamics_init(domain)
       real (kind=RKIND), dimension(:), pointer :: invAreaCell
       integer, dimension(:), pointer :: bdyMaskCell
       integer, dimension(:), pointer :: bdyMaskEdge
+      real (kind=RKIND), dimension(:), pointer :: specZoneMaskEdge
       real (kind=RKIND), dimension(:), pointer :: invDvEdge
       real (kind=RKIND), dimension(:), pointer :: dcEdge
       real (kind=RKIND), dimension(:), pointer :: invDcEdge
@@ -295,6 +296,9 @@ subroutine mpas_atm_dynamics_init(domain)
       call mpas_pool_get_array(mesh, 'bdyMaskEdge', bdyMaskEdge)
       !$acc enter data copyin(bdyMaskEdge)
 
+      call mpas_pool_get_array(mesh, 'specZoneMaskEdge', specZoneMaskEdge)
+      !$acc enter data copyin(specZoneMaskEdge)
+
       call mpas_pool_get_array(mesh, 'invDvEdge', invDvEdge)
       !$acc enter data copyin(invDvEdge)
 
@@ -412,6 +416,7 @@ subroutine mpas_atm_dynamics_finalize(domain)
       real (kind=RKIND), dimension(:), pointer :: invAreaCell
       integer, dimension(:), pointer :: bdyMaskCell
       integer, dimension(:), pointer :: bdyMaskEdge
+      real (kind=RKIND), dimension(:), pointer :: specZoneMaskEdge
       real (kind=RKIND), dimension(:), pointer :: invDvEdge
       real (kind=RKIND), dimension(:), pointer :: dcEdge
       real (kind=RKIND), dimension(:), pointer :: invDcEdge
@@ -495,6 +500,9 @@ subroutine mpas_atm_dynamics_finalize(domain)
       call mpas_pool_get_array(mesh, 'bdyMaskEdge', bdyMaskEdge)
       !$acc exit data delete(bdyMaskEdge)
 
+      call mpas_pool_get_array(mesh, 'specZoneMaskEdge', specZoneMaskEdge)
+      !$acc exit data delete(specZoneMaskEdge)
+
       call mpas_pool_get_array(mesh, 'invDvEdge', invDvEdge)
       !$acc exit data delete(invDvEdge)
 
@@ -2696,8 +2704,10 @@ subroutine atm_divergence_damping_3d( state, diag, mesh, configs, dts, edgeStart
       real (kind=RKIND), dimension(:), pointer :: specZoneMaskEdge
 
       integer, dimension(:,:), pointer :: cellsOnEdge
-      integer, pointer :: nCellsSolve
-      integer, pointer :: nVertLevels
+      integer, pointer :: nCellsSolve_ptr
+      integer, pointer :: nVertLevels_ptr
+      integer :: nCellsSolve
+      integer :: nVertLevels
 
       real (kind=RKIND) :: divCell1, divCell2, rdts, coef_divdamp
       integer :: cell1, cell2, iEdge, k
@@ -2710,15 +2720,24 @@ subroutine atm_divergence_damping_3d( state, diag, mesh, configs, dts, edgeStart
       call mpas_pool_get_array(diag, 'rtheta_pp_old', rtheta_pp_old)
       call mpas_pool_get_array(diag, 'ru_p', ru_p)
 
-      call mpas_pool_get_dimension(mesh, 'nCellsSolve', nCellsSolve)
-      call mpas_pool_get_dimension(mesh, 'nVertLevels', nVertLevels)
+      call mpas_pool_get_dimension(mesh, 'nCellsSolve', nCellsSolve_ptr)
+      call mpas_pool_get_dimension(mesh, 'nVertLevels', nVertLevels_ptr)
 
       call mpas_pool_get_config(configs, 'config_smdiv', smdiv) 
       call mpas_pool_get_config(configs, 'config_len_disp', config_len_disp)
 
       rdts = 1.0_RKIND / dts
       coef_divdamp = 2.0_RKIND * smdiv * config_len_disp * rdts
 
+      nCellsSolve = nCellsSolve_ptr
+      nVertLevels = nVertLevels_ptr
+
+      MPAS_ACC_TIMER_START('atm_divergence_damping_3d [ACC_data_xfer]')
+      !$acc enter data copyin(ru_p, rtheta_pp, rtheta_pp_old, theta_m)
+      MPAS_ACC_TIMER_STOP('atm_divergence_damping_3d [ACC_data_xfer]')
+
+      !$acc parallel default(present)
+      !$acc loop gang worker
       do iEdge=edgeStart,edgeEnd ! MGD do we really just need edges touching owned cells?
 
          cell1 = cellsOnEdge(1,iEdge)
@@ -2728,6 +2747,7 @@ subroutine atm_divergence_damping_3d( state, diag, mesh, configs, dts, edgeStart
          if (cell1 <= nCellsSolve .or. cell2 <= nCellsSolve ) then
 
 !DIR$ IVDEP
+            !$acc loop vector
             do k=1,nVertLevels
 
 !!  unscaled 3d divergence damping
@@ -2745,6 +2765,13 @@ subroutine atm_divergence_damping_3d( state, diag, mesh, configs, dts, edgeStart
             end do
          end if ! edges for block-owned cells
       end do ! end loop over edges
+      !$acc end parallel
+
+      MPAS_ACC_TIMER_START('atm_divergence_damping_3d [ACC_data_xfer]')
+      !$acc exit data copyout(ru_p) &
+      !$acc           delete(rtheta_pp, rtheta_pp_old, theta_m)
+      MPAS_ACC_TIMER_STOP('atm_divergence_damping_3d [ACC_data_xfer]')
+      
 
    end subroutine atm_divergence_damping_3d