Sixth reconciliation PR from production/RRFS.v1 + CCPP physics: wet()…

… bug fix and workaround for bad tsfco NOAA-EMC#916 (NOAA-EMC#896) * Parallel IO enhancements from GDIT * Add SW clear-sky downward flux at surface to available diagnostics * fix compilation error in GFS_diagnostics.F90 * update atmos_cubed_sphere to turn ENABLE_PARALLELRESTART OFF by default to avoid RT errors on some platforms (Hera, Hercules at least) * ccpp-physics - wet() bug fix and workaround for bad tsfco --------- Co-authored-by: Dustin Swales <[email protected]> Co-authored-by: Samuel Trahan <samuel,[email protected]>
LarissaReames-NOAA · Feb 12, 2025 · 7f55155 · 7f55155
1 parent c5c9420
commit 7f55155
Show file tree

Hide file tree

Showing 6 changed files with 83 additions and 2 deletions.
diff --git a/CMakeLists.txt b/CMakeLists.txt
@@ -64,6 +64,14 @@ if(NOT PARALLEL_NETCDF)
   list(APPEND _fv3atm_defs_private NO_PARALLEL_NETCDF)
 endif()
 
+if(ENABLE_PARALLELRESTART)
+  list(APPEND _fv3atm_defs_private ENABLE_PARALLELRESTART)
+endif()
+
+if(ENABLE_RRFS_WAR)
+  list(APPEND _fv3atm_defs_private ENABLE_RRFS_WAR)
+endif()
+
 if(MOVING_NEST)
   list(APPEND _fv3atm_defs_private MOVING_NEST)
   if(NOT HYDRO)

diff --git a/atmos_cubed_sphere b/atmos_cubed_sphere
diff --git a/ccpp/data/GFS_typedefs.F90 b/ccpp/data/GFS_typedefs.F90
@@ -523,6 +523,7 @@ module GFS_typedefs
     !--- In (physics only)
     real (kind=kind_phys), pointer :: sfcdsw(:)      => null()   !< total sky sfc downward sw flux ( w/m**2 )
                                                                  !< GFS_radtend_type%sfcfsw%dnfxc
+    real (kind=kind_phys), pointer :: sfcdswc(:)     => null()   !< total sky sfc downward sw flux assuming clear sky conditions( w/m**2 )
     real (kind=kind_phys), pointer :: sfcnsw(:)      => null()   !< total sky sfc netsw flx into ground(w/m**2)
                                                                  !< difference of dnfxc & upfxc from GFS_radtend_type%sfcfsw
     real (kind=kind_phys), pointer :: sfcdlw(:)      => null()   !< total sky sfc downward lw flux ( w/m**2 )
@@ -2032,6 +2033,7 @@ module GFS_typedefs
     real (kind=kind_phys), pointer :: dlwsfci(:)     => null()   !< instantaneous sfc dnwd lw flux ( w/m**2 )
     real (kind=kind_phys), pointer :: ulwsfci(:)     => null()   !< instantaneous sfc upwd lw flux ( w/m**2 )
     real (kind=kind_phys), pointer :: dswsfci(:)     => null()   !< instantaneous sfc dnwd sw flux ( w/m**2 )
+    real (kind=kind_phys), pointer :: dswsfcci(:)    => null()   !< instantaneous sfc dnwd sw flux ( w/m**2 ) (clear-sky)
     real (kind=kind_phys), pointer :: nswsfci(:)     => null()   !< instantaneous sfc net dnwd sw flux ( w/m**2 )
     real (kind=kind_phys), pointer :: uswsfci(:)     => null()   !< instantaneous sfc upwd sw flux ( w/m**2 )
     real (kind=kind_phys), pointer :: dusfci (:)     => null()   !< instantaneous u component of surface stress
@@ -2948,11 +2950,13 @@ subroutine coupling_create (Coupling, Model)
     Coupling%visbmui = clear_val
     Coupling%visdfui = clear_val
 
+    allocate (Coupling%sfcdswc (IM))
     allocate (Coupling%sfcdsw (IM))
     allocate (Coupling%sfcnsw (IM))
     allocate (Coupling%sfcdlw (IM))
     allocate (Coupling%sfculw (IM))
 
+    Coupling%sfcdswc = clear_val
     Coupling%sfcdsw = clear_val
     Coupling%sfcnsw = clear_val
     Coupling%sfcdlw = clear_val
@@ -7827,6 +7831,7 @@ subroutine diag_create (Diag, Model)
     allocate (Diag%dlwsfci (IM))
     allocate (Diag%ulwsfci (IM))
     allocate (Diag%dswsfci (IM))
+    allocate (Diag%dswsfcci(IM))
     allocate (Diag%nswsfci (IM))
     allocate (Diag%uswsfci (IM))
     allocate (Diag%dusfci  (IM))
@@ -8141,6 +8146,7 @@ subroutine diag_phys_zero (Diag, Model, linit, iauwindow_center)
     Diag%dlwsfci    = zero
     Diag%ulwsfci    = zero
     Diag%dswsfci    = zero
+    Diag%dswsfcci   = zero
     Diag%nswsfci    = zero
     Diag%uswsfci    = zero
     Diag%dusfci     = zero

diff --git a/ccpp/data/GFS_typedefs.meta b/ccpp/data/GFS_typedefs.meta
@@ -2480,6 +2480,13 @@
   dimensions = (horizontal_dimension)
   type = real
   kind = kind_phys
+[sfcdswc]
+  standard_name = surface_downwelling_shortwave_flux_on_radiation_timestep_assuming_clear_sky
+  long_name = total sky sfc downward sw flux assuming clear sky conditions
+  units = W m-2
+  dimensions = (horizontal_loop_extent)
+  type = real
+  kind = kind_phys
 [sfcnsw]
   standard_name = surface_net_downwelling_shortwave_flux_on_radiation_timestep
   long_name = total sky sfc netsw flx into ground
@@ -9185,6 +9192,13 @@
   dimensions = (horizontal_dimension)
   type = real
   kind = kind_phys
+[dswsfcci]
+  standard_name = surface_downwelling_shortwave_flux_assuming_clear_sky
+  long_name = surface downwelling shortwave flux at current time assuming clear sky
+  units = W m-2
+  dimensions = (horizontal_loop_extent)
+  type = real
+  kind = kind_phys
 [nswsfci]
   standard_name = surface_net_downwelling_shortwave_flux
   long_name = surface net downwelling shortwave flux at current time

diff --git a/ccpp/driver/GFS_diagnostics.F90 b/ccpp/driver/GFS_diagnostics.F90
@@ -364,6 +364,18 @@ subroutine GFS_externaldiag_populate (ExtDiag, Model, Statein, Stateout, Sfcprop
       ExtDiag(idx)%data(nb)%var2 => IntDiag%dswsfci(Model%chunk_begin(nb):Model%chunk_end(nb))
     enddo
 
+    idx = idx + 1
+    ExtDiag(idx)%axes = 2
+    ExtDiag(idx)%name = 'DSWRFCI'
+    ExtDiag(idx)%desc = 'instantaneous surface downward shortwave flux assuming clear sky'
+    ExtDiag(idx)%unit = 'w/m**2'
+    ExtDiag(idx)%mod_name = 'gfs_phys'
+    ExtDiag(idx)%intpl_method = 'bilinear'
+    allocate (ExtDiag(idx)%data(nblks))
+    do nb = 1,nblks
+      ExtDiag(idx)%data(nb)%var2 => IntDiag%dswsfcci(Model%chunk_begin(nb):Model%chunk_end(nb))
+    enddo    
+
     idx = idx + 1
     ExtDiag(idx)%axes = 2
     ExtDiag(idx)%name = 'USWRF'

diff --git a/io/fv3atm_restart_io.F90 b/io/fv3atm_restart_io.F90
@@ -19,7 +19,13 @@ module fv3atm_restart_io_mod
                                 read_restart, write_restart, write_data,     &
                                 get_global_io_domain_indices, get_dimension_size, &
                                 global_att_exists, get_global_attribute
-  use mpp_domains_mod,    only: domain2d
+#ifdef ENABLE_PARALLELRESTART
+  use mpp_domains_mod,    only: domain2d, mpp_get_domain_tile_commid, mpp_copy_domain, &
+                                  mpp_define_io_domain, mpp_get_layout
+#else
+  use mpp_domains_mod,    only: domain2d, mpp_copy_domain, &
+                                  mpp_define_io_domain, mpp_get_layout
+#endif
   use fv3atm_common_io,   only: create_2d_field_and_add_to_bundle, &
        create_3d_field_and_add_to_bundle, copy_from_gfs_data, axis_type
   use fv3atm_sfc_io
@@ -532,6 +538,7 @@ end subroutine fv3atm_checksum
   !!  Also calculates sncovr if it is not present in the restart file.
   subroutine sfc_prop_restart_read (Sfcprop, Atm_block, Model, fv_domain, warm_start, ignore_rst_cksum)
     use fv3atm_rrfs_sd_io
+    use atmosphere_mod,     only: Atm,mygrid
     implicit none
     !--- interface variable definitions
     type(GFS_sfcprop_type),    intent(inout) :: Sfcprop
@@ -558,12 +565,25 @@ subroutine sfc_prop_restart_read (Sfcprop, Atm_block, Model, fv_domain, warm_sta
 
     type(FmsNetcdfDomainFile_t) :: Oro_restart, Sfc_restart, dust12m_restart, emi_restart, rrfssd_restart
     type(FmsNetcdfDomainFile_t) :: Oro_ls_restart, Oro_ss_restart
+    type(domain2D) :: domain_for_read
+    integer :: read_layout(2)
 
     !--- OROGRAPHY FILE
 
     !--- open file
+#ifdef ENABLE_PARALLELRESTART
+    Oro_restart%use_collective = .true.
+    call mpp_get_layout(Atm(mygrid)%domain, read_layout)
+    call mpp_copy_domain(Atm(mygrid)%domain, domain_for_read)
+    call mpp_define_io_domain(domain_for_read, read_layout)
+    Oro_restart%tile_comm = mpp_get_domain_tile_commid(Atm(mygrid)%domain)
+
+    infile=trim(indir)//'/'//trim(fn_oro)
+    amiopen=open_file(Oro_restart, trim(infile), 'read', domain=domain_for_read, is_restart=.true., dont_add_res_to_filename=.true.)
+#else
     infile=trim(indir)//'/'//trim(fn_oro)
     amiopen=open_file(Oro_restart, trim(infile), 'read', domain=fv_domain, is_restart=.true., dont_add_res_to_filename=.true.)
+#endif
     if (.not.amiopen) call mpp_error( FATAL, 'Error with opening file '//trim(infile) )
 
     call oro%register(Model,Oro_restart,Atm_block)
@@ -669,8 +689,16 @@ subroutine sfc_prop_restart_read (Sfcprop, Atm_block, Model, fv_domain, warm_sta
     !--- SURFACE FILE
 
     !--- open file
+#ifdef ENABLE_PARALLELRESTART
+    Sfc_restart%use_collective = .true.
+    Sfc_restart%tile_comm = mpp_get_domain_tile_commid(Atm(mygrid)%domain)
+
+    infile=trim(indir)//'/'//trim(fn_srf)
+    amiopen=open_file(Sfc_restart, trim(infile), "read", domain=domain_for_read, is_restart=.true., dont_add_res_to_filename=.true.)
+#else
     infile=trim(indir)//'/'//trim(fn_srf)
     amiopen=open_file(Sfc_restart, trim(infile), "read", domain=fv_domain, is_restart=.true., dont_add_res_to_filename=.true.)
+#endif
     if( .not.amiopen ) call mpp_error(FATAL, 'Error opening file'//trim(infile))
 
     if (global_att_exists(Sfc_restart, "file_version")) then
@@ -840,6 +868,7 @@ end subroutine sfc_prop_restart_write
   !! restart variables with the GFDL FMS restart subsystem.
   !! Calls a GFDL FMS routine to restore the data from a restart file.
   subroutine phys_restart_read (GFS_Restart, Atm_block, Model, fv_domain, ignore_rst_cksum)
+    use atmosphere_mod,     only: Atm,mygrid
     implicit none
     !--- interface variable definitions
     type(GFS_restart_type),      intent(in) :: GFS_Restart
@@ -859,6 +888,8 @@ subroutine phys_restart_read (GFS_Restart, Atm_block, Model, fv_domain, ignore_r
 
     type(phy_data_type) :: phy
     type(FmsNetcdfDomainFile_t) :: Phy_restart
+    type(domain2D) :: domain_for_read
+    integer :: read_layout(2)
 
     isc = Atm_block%isc
     iec = Atm_block%iec
@@ -871,7 +902,17 @@ subroutine phys_restart_read (GFS_Restart, Atm_block, Model, fv_domain, ignore_r
 
     !--- open restart file and register axes
     fname = trim(indir)//'/'//trim(fn_phy)
+#ifdef ENABLE_PARALLELRESTART
+    Phy_restart%use_collective = .true.
+    call mpp_get_layout(Atm(mygrid)%domain, read_layout)
+    call mpp_copy_domain(Atm(mygrid)%domain, domain_for_read)
+    call mpp_define_io_domain(domain_for_read, read_layout)
+    Phy_restart%tile_comm = mpp_get_domain_tile_commid(Atm(mygrid)%domain)
+
+    amiopen=open_file(Phy_restart, trim(fname), 'read', domain=domain_for_read, is_restart=.true., dont_add_res_to_filename=.true.)
+#else
     amiopen=open_file(Phy_restart, trim(fname), 'read', domain=fv_domain, is_restart=.true., dont_add_res_to_filename=.true.)
+#endif
     if( amiopen ) then
       call register_axis(Phy_restart, 'xaxis_1', 'X')
       call register_axis(Phy_restart, 'yaxis_1', 'Y')
+10 −0		CMakeLists.txt
+23 −1		tools/external_ic.F90
+39 −2		tools/fv_io.F90
+4 −0		tools/fv_mp_mod.F90