ctsm5.3.069: Add SystemTests to run subset_data and then CTSM

.gitmodules

@@ -76,7 +76,7 @@ fxDONOTUSEurl = https://github.com/ESMCI/ccs_config_cesm.git
 [submodule "cime"]
 path = cime
 url = https://github.com/ESMCI/cime
-fxtag = cime6.1.111
+fxtag = cime6.1.112
 fxrequired = ToplevelRequired
 # Standard Fork to compare to with "git fleximod test" to ensure personal forks aren't committed
 fxDONOTUSEurl = https://github.com/ESMCI/cime

cime_config/SystemTests/subsetdata.py

@@ -0,0 +1,100 @@
+"""
+Parent class for CTSM-specific tests that first run the subset_data tool and then ensure
+that CTSM does not fail using the just-generated input files
+"""
+
+import os
+import sys
+import logging
+from CIME.SystemTests.system_tests_common import SystemTestsCommon
+from CIME.user_mod_support import apply_user_mods
+from CIME.XML.standard_module_setup import *
+
+# In case we need to import subset_data later
+_CTSM_PYTHON = os.path.join(
+    os.path.dirname(os.path.realpath(__file__)), os.pardir, os.pardir, "python"
+)
+sys.path.insert(1, _CTSM_PYTHON)
+
+logger = logging.getLogger(__name__)
+
+
+class SUBSETDATASHARED(SystemTestsCommon):
+    def __init__(self, case, subset_data_cmd):
+        """
+        initialize an object interface to the SMS system test
+        """
+        SystemTestsCommon.__init__(self, case)
+
+        # Check the test setup
+        if not self._case.get_value("LND_GRID") == "CLM_USRDAT":
+            raise RuntimeError("SUBSETDATA tests require resolution CLM_USRDAT")
+        if "serial" not in self._case.get_value("MPILIB"):
+            raise RuntimeError("SUBSETDATA tests require a serial MPILIB")
+        if "BGC-CROP" not in self._case.get_value("COMPSET"):
+            raise RuntimeError("SUBSETDATA tests require a BGC-CROP compset")
+
+        # Add standard subset_data arguments
+        out_dir = os.path.join(self._get_caseroot(), "subset_data_output")
+        self.usermods_dir = os.path.join(out_dir, "user_mods")
+        self.subset_data_cmd = subset_data_cmd + [
+            "--create-user-mods",
+            "--outdir",
+            out_dir,
+            "--user-mods-dir",
+            self.usermods_dir,
+            "--overwrite",
+        ]
+
+        # Run subset_data, if needed.
+        # It's needed during SETUP and/or NLCOMP phases if comparing/generating a baseline because
+        # the namelist comparison will require generating a namelist, and that will fail if we
+        # haven't specified our custom fsurdat and other stuff. By calling self._run_subset_data()
+        # only if the usermods directory doesn't yet exist, we avoid it being called every time the
+        # test class is initialized (which happens, e.g., in RUN phase).
+        # if not os.path.exists(self.usermods_dir):
+        #     self._run_subset_data()
+
+    def build_phase(self, sharedlib_only=False, model_only=False):
+        """
+        Run subset_data and then build the model
+        """
+
+        # Run subset_data.
+        # build_phase gets called twice:
+        # - once with sharedlib_only = True and
+        # - once with model_only = True
+        # Because we only need subset_data run once, we only do it for the sharedlib_only call.
+        # We could also check for the existence of the subset_data outputs, but that might lead to
+        # a situation where the user expects subset_data to be called but it's not. Better to run
+        # unnecessarily (e.g., if you fixed some FORTRAN code and just need to rebuild).
+        # In that same vein, yes we did run subset_data during the first time this test case was
+        # initialized (see __init__() above), but we're doing it again here just to be safe.
+        if sharedlib_only:
+            self._run_subset_data()
+
+        # Do the build
+        self.build_indv(sharedlib_only=sharedlib_only, model_only=model_only)
+
+    def _run_subset_data(self):
+        """
+        Run subset_data
+        """
+        # Import subset_data. Do it here rather than at top because otherwise the import will
+        # be attempted even during RUN phase.
+        # pylint: disable=wrong-import-position,import-outside-toplevel
+        from ctsm.subset_data import main as subset_data
+
+        # Run subset_data
+        sys.argv = self.subset_data_cmd
+        subset_data()
+
+        # Required so that CTSM doesn't fail
+        user_nl_clm_path = os.path.join(self.usermods_dir, "user_nl_clm")
+        with open(user_nl_clm_path, "a", encoding="utf-8") as user_nl_clm:
+            user_nl_clm.write("\ncheck_dynpft_consistency = .false.\n")
+
+        # Apply the user mods
+        self._case.flush(flushall=True)
+        apply_user_mods(self._get_caseroot(), self.usermods_dir)
+        self._case.read_xml()

cime_config/SystemTests/subsetdatapoint.py

@@ -0,0 +1,38 @@
+"""
+CTSM-specific test that first runs the subset_data point tool and then ensures
+that CTSM does not fail using the just-generated input files
+"""
+
+from subsetdata import SUBSETDATASHARED
+
+
+class SUBSETDATAPOINT(SUBSETDATASHARED):
+    def __init__(self, case):
+        """
+        initialize an object interface to the SMS system test
+        """
+
+        lat = 45.402252
+        lon = -92.798085
+
+        # Don't need to include things that are added during SUBSETDATASHARED.__init__()
+        subset_data_cmd = [
+            "tools/site_and_regional/subset_data",
+            "point",
+            "--lat",
+            str(lat),
+            "--lon",
+            str(lon),
+            "--create-surface",
+            "--crop",
+            "--create-landuse",
+            "--surf-year",
+            "1850",
+            "--create-datm",
+            "--datm-syr",
+            "1901",
+            "--datm-eyr",
+            "1901",
+        ]
+
+        super().__init__(case, subset_data_cmd)

cime_config/SystemTests/subsetdataregion.py

@@ -0,0 +1,41 @@
+"""
+CTSM-specific test that first runs the subset_data region tool and then ensures
+that CTSM does not fail using the just-generated input files
+"""
+
+from subsetdata import SUBSETDATASHARED
+
+
+class SUBSETDATAREGION(SUBSETDATASHARED):
+    def __init__(self, case):
+        """
+        initialize an object interface to the SMS system test
+        """
+
+        lat1 = -9
+        lat2 = -7
+        lon1 = 291
+        lon2 = 293
+
+        # Don't need to include things that are added during SUBSETDATASHARED.__init__()
+        subset_data_cmd = [
+            "tools/site_and_regional/subset_data",
+            "region",
+            "--lat1",
+            str(lat1),
+            "--lat2",
+            str(lat2),
+            "--lon1",
+            str(lon1),
+            "--lon2",
+            str(lon2),
+            "--create-mesh",
+            "--create-domain",
+            "--create-surface",
+            "--crop",
+            "--create-landuse",
+            "--surf-year",
+            "1850",
+        ]
+
+        super().__init__(case, subset_data_cmd)

cime_config/config_tests.xml

@@ -35,6 +35,26 @@ This defines various CTSM-specific system tests
     <HIST_N>$STOP_N</HIST_N>
   </test>
 
+  <test NAME="SUBSETDATAPOINT">
+    <DESC>Run CTSM with files generated by the subset_data point tool</DESC>
+    <INFO_DBUG>1</INFO_DBUG>
+    <DOUT_S>FALSE</DOUT_S>
+    <CONTINUE_RUN>FALSE</CONTINUE_RUN>
+    <REST_OPTION>never</REST_OPTION>
+    <HIST_OPTION>$STOP_OPTION</HIST_OPTION>
+    <HIST_N>$STOP_N</HIST_N>
+  </test>
+
+  <test NAME="SUBSETDATAREGION">
+    <DESC>Run CTSM with files generated by the subset_data region tool</DESC>
+    <INFO_DBUG>1</INFO_DBUG>
+    <DOUT_S>FALSE</DOUT_S>
+    <CONTINUE_RUN>FALSE</CONTINUE_RUN>
+    <REST_OPTION>never</REST_OPTION>
+    <HIST_OPTION>$STOP_OPTION</HIST_OPTION>
+    <HIST_N>$STOP_N</HIST_N>
+  </test>
+
   <test NAME="LGRAIN2">
     <DESC>CTSM Land model test to ensure that we can allocate and use a second grain pool</DESC>
     <INFO_DBUG>1</INFO_DBUG>

cime_config/testdefs/testlist_clm.xml

@@ -13,6 +13,7 @@
         rxcropmaturity: Short tests to be run during development related to prescribed crop calendars
         matrixcn: Tests exercising the matrix-CN capability
         aux_clm_mpi_serial: aux_clm tests using mpi-serial. Useful for redoing tests that failed due to https://github.com/ESCOMP/CTSM/issues/2916, after having replaced libraries/mpi-serial with a fresh copy.
+        subset_data: Tests exercising the subset_data tool and running CTSM with its output
         decomp_init: Initialization tests specifically for examining the PE layout decomposition initialization
         interim_restart: Tests having to do with interim restart capability.
 -->
@@ -4141,6 +4142,34 @@
     </options>
   </test>
 
+  <test name="SUBSETDATAPOINT_Ld5_D_Mmpi-serial" grid="CLM_USRDAT" compset="I2000Clm60BgcCropCrujra" testmods="clm-default">
+    <machines>
+      <machine name="derecho" compiler="intel" category="aux_clm"/>
+      <machine name="derecho" compiler="intel" category="aux_clm_mpi_serial"/>
+      <machine name="derecho" compiler="intel" category="clm_pymods"/>
+      <machine name="derecho" compiler="intel" category="subset_data"/>
+    </machines>
+    <options>
+      <option name="wallclock">00:20:00</option>
+      <option name="comment">Smoke test that first runs the subset_data point tool and then ensures that CTSM does not fail using the just-generated user mods.</option>
+      <option name="comment">This test invokes python code, so it should be run whenever changing python code (in addition to being run as part of aux_clm).</option>
+    </options>
+  </test>
+
+  <test name="SUBSETDATAREGION_Ld5_D_Mmpi-serial" grid="CLM_USRDAT" compset="I2000Clm60BgcCropCrujra" testmods="clm-default">
+    <machines>
+      <machine name="derecho" compiler="intel" category="aux_clm"/>
+      <machine name="derecho" compiler="intel" category="aux_clm_mpi_serial"/>
+      <machine name="derecho" compiler="intel" category="clm_pymods"/>
+      <machine name="derecho" compiler="intel" category="subset_data"/>
+    </machines>
+    <options>
+      <option name="wallclock">00:20:00</option>
+      <option name="comment">Smoke test that first runs the subset_data regiontool and then ensures that CTSM does not fail using the just-generated user mods.</option>
+      <option name="comment">This test invokes python code, so it should be run whenever changing python code (in addition to being run as part of aux_clm).</option>
+    </options>
+  </test>
+
   <test name="LILACSMOKE_D_Ld2" grid="f10_f10_mg37" compset="I2000Ctsm50NwpSpAsRs" testmods="clm-lilac">
     <machines>
       <machine name="derecho" compiler="intel" category="aux_clm"/>

doc/ChangeLog

@@ -1,4 +1,77 @@
 ===============================================================
+Tag name: ctsm5.3.069
+Originator(s): samrabin (Sam Rabin, UCAR/TSS)
+Date: Tue Aug 12 09:52:59 MDT 2025
+One-line Summary: Add SystemTests to run subset_data and then CTSM
+
+Purpose and description of changes
+----------------------------------
+
+Adds SUBSETDATAPOINT and SUBSETDATAREGION tests to aux_clm (and new subset_data suite). These run subset_data for either a point or a six-cell region, then run CTSM with the outputs.
+
+
+Significant changes to scientifically-supported configurations
+--------------------------------------------------------------
+
+Does this tag change answers significantly for any of the following physics configurations?
+(Details of any changes will be given in the "Answer changes" section below.)
+
+[ ] clm6_0
+
+[ ] clm5_0
+
+[ ] ctsm5_0-nwp
+
+[ ] clm4_5
+
+
+Bugs fixed
+----------
+
+List of CTSM issues fixed (include CTSM Issue # and description):
+- Resolves [Issue #1491: Add two new tests that run subset_data and then run a case from it](https://github.com/ESCOMP/CTSM/issues/1491)
+
+
+Notes of particular relevance for developers:
+---------------------------------------------
+NOTE: Be sure to review the steps in README.CHECKLIST.master_tags as well as the coding style in the Developers Guide
+[Remove any lines that don't apply. Remove entire section if nothing applies.]
+
+Caveats for developers (e.g., code that is duplicated that requires double maintenance):
+
+Changes to tests or testing:
+- Adds test SUBSETDATAPOINT_Ld5_D_Mmpi-serial.CLM_USRDAT.I2000Clm60BgcCropCrujra.derecho_intel.clm-default
+- Adds test SUBSETDATAREGION_Ld5_D_Mmpi-serial.CLM_USRDAT.I2000Clm60BgcCropCrujra.derecho_intel.clm-default
+- Adds new subset_data suite with those tests in it
+- Those tests are also in aux_clm
+
+Testing summary:
+----------------
+
+ [PASS means all tests PASS; OK means tests PASS other than expected fails.]
+
+  build-namelist tests (if CLMBuildNamelist.pm has changed):
+
+    derecho -
+
+  python testing (if python code has changed; see instructions in python/README.md; document testing done):
+
+    derecho - Unit, black, and pylint checks pass. Two system tests fail due to recent conda and mamba updates on Derecho; these have been fixed on b4b-dev with [Pull Request #3403: Fix py_env_create and tests by samsrabin](https://github.com/ESCOMP/CTSM/pull/3403)
+
+  regular tests (aux_clm: https://github.com/ESCOMP/CTSM/wiki/System-Testing-Guide#pre-merge-system-testing):
+
+    derecho ----- OK
+    izumi ------- OK
+
+
+Other details
+-------------
+
+Pull Requests that document the changes (include PR ids):
+- [Pull Request #3292: ctsm5.3.069: Add SystemTests to run subset_data and then CTSM by samsrabin](https://github.com/ESCOMP/CTSM/pull/3292)
+
+===============================================================
+===============================================================
 Tag name: ctsm5.3.068
 Originator(s): slevis (Samuel Levis,UCAR/TSS,303-665-1310)
 Date: Mon 11 Aug 2025 02:27:39 PM MDT

doc/ChangeSum

@@ -1,5 +1,6 @@
 Tag                   Who      Date  Summary
 ============================================================================================================================
+       ctsm5.3.069 samrabin 08/12/2025 Add SystemTests to run subset_data and then CTSM
        ctsm5.3.068   slevis 08/11/2025 Change megan_use_gamma_sm to default false
        ctsm5.3.067 samrabin 08/08/2025 (Mostly) fix interim restarts
        ctsm5.3.066   slevis 08/08/2025 Merge b4b-dev to master

python/ctsm/run_sys_tests.py

@@ -749,6 +749,14 @@ def _check_py_env(test_attributes):
         except ModuleNotFoundError as err:
             raise ModuleNotFoundError("modify_fsurdat" + err_msg) from err
 
+    # Check requirements for using subset_data Python module, if needed
+    subset_data_users = ["SUBSETDATAPOINT", "SUBSETDATAREGION"]
+    if any(any(u in t for u in subset_data_users) for t in test_attributes):
+        try:
+            import ctsm.subset_data
+        except ModuleNotFoundError as err:
+            raise ModuleNotFoundError("subset_data" + err_msg) from err
+
     # Check requirements for RXCROPMATURITY, if needed
     if any("RXCROPMATURITY" in t for t in test_attributes):
         try:

src/dyn_subgrid/dynFileMod.F90

@@ -7,6 +7,7 @@ module dynFileMod
   !
   ! !USES:
   use shr_log_mod    , only : errMsg => shr_log_errMsg
+  use clm_varctl     , only : fname_len
   use dynTimeInfoMod , only : time_info_type, year_position_type
   use ncdio_pio      , only : file_desc_t, ncd_pio_openfile, ncd_inqdid, ncd_inqdlen, ncd_io
   use abortutils     , only : endrun
@@ -56,7 +57,7 @@ type(dyn_file_type) function constructor(filename, year_position)
     type(year_position_type) , intent(in) :: year_position
     !
     ! !LOCAL VARIABLES:
-    character(len=256) :: locfn      ! local file name
+    character(len=fname_len) :: locfn  ! local file name
     integer :: ier                   ! error code
     integer :: ntimes                ! number of time samples
     integer :: varid                 ! netcdf variable ID