Merge branch 'atmosphere/acc_atm_zero_gradient_w_bdy_work' into develop (PR #1272)

This merge enables execution of the atm_zero_gradient_w_bdy_work subroutine on
GPUs through the addition of OpenACC directives.

* atmosphere/acc_atm_zero_gradient_w_bdy_work:
  Add OpenACC directives to enable executing atm_zero_gradient_w_bdy_work on GPUs.

Author: mgduda

src/core_atmosphere/dynamics/mpas_atm_time_integration.F

@@ -238,6 +238,7 @@ subroutine mpas_atm_dynamics_init(domain)
       real (kind=RKIND), dimension(:), pointer :: fzp
       real (kind=RKIND), dimension(:,:,:), pointer :: zb
       real (kind=RKIND), dimension(:,:,:), pointer :: zb3
+      integer, dimension(:), pointer :: nearestRelaxationCell
 #endif
 
 
@@ -376,6 +377,8 @@ subroutine mpas_atm_dynamics_init(domain)
       call mpas_pool_get_array(mesh, 'zb3', zb3)
       !$acc enter data copyin(zb3)
 
+      call mpas_pool_get_array(mesh, 'nearestRelaxationCell', nearestRelaxationCell)
+      !$acc enter data copyin(nearestRelaxationCell)
 #endif
 
    end subroutine mpas_atm_dynamics_init
@@ -450,6 +453,7 @@ subroutine mpas_atm_dynamics_finalize(domain)
       real (kind=RKIND), dimension(:), pointer :: fzp
       real (kind=RKIND), dimension(:,:,:), pointer :: zb
       real (kind=RKIND), dimension(:,:,:), pointer :: zb3
+      integer, dimension(:), pointer :: nearestRelaxationCell
 #endif
 
 
@@ -588,6 +592,8 @@ subroutine mpas_atm_dynamics_finalize(domain)
       call mpas_pool_get_array(mesh, 'zb3', zb3)
       !$acc exit data delete(zb3)
 
+      call mpas_pool_get_array(mesh, 'nearestRelaxationCell', nearestRelaxationCell)
+      !$acc exit data delete(nearestRelaxationCell)
 #endif
 
    end subroutine mpas_atm_dynamics_finalize
@@ -6555,14 +6561,27 @@ subroutine atm_zero_gradient_w_bdy_work( w, bdyMaskCell, nearestRelaxationCell,
 
       integer :: iCell, k
 
+      MPAS_ACC_TIMER_START('atm_zero_gradient_w_bdy_work [ACC_data_xfer]')
+      !$acc enter data copyin(w)
+      MPAS_ACC_TIMER_STOP('atm_zero_gradient_w_bdy_work [ACC_data_xfer]')
+
+      !$acc parallel default(present)
+      !$acc loop gang worker
       do iCell=cellSolveStart,cellSolveEnd
          if (bdyMaskCell(iCell) > nRelaxZone) then
 !DIR$ IVDEP
+            !$acc loop vector
             do k = 2, nVertLevels
               w(k,iCell) = w(k,nearestRelaxationCell(iCell))
             end do
          end if  
       end do
+      !$acc end parallel
+
+      MPAS_ACC_TIMER_START('atm_zero_gradient_w_bdy_work [ACC_data_xfer]')
+      !$acc exit data copyout(w)
+      MPAS_ACC_TIMER_STOP('atm_zero_gradient_w_bdy_work [ACC_data_xfer]')
+
 
    end subroutine atm_zero_gradient_w_bdy_work