Changes to enable easily turning on CUPiD diagnostics from your CTSM case

.gitmodules

@@ -121,6 +121,18 @@ fxrequired = ToplevelRequired
 # Standard Fork to compare to with "git fleximod test" to ensure personal forks aren't committed
 fxDONOTUSEurl = https://github.com/ESMCI/mpi-serial
 
+[submodule "cupid"]
+# CUPiD does model post-processing
+path = tools/CUPiD
+#url = https://github.com/NCAR/CUPiD.git
+url = https://github.com/wwieder/CUPiD.git
+#fxtag = v0.6.0
+#fxtag = land_only
+fxtag = 30bd428b7139873382107b0a32624a7492e28a91
+fxrequired = ToplevelRequired
+# Standard Fork to compare to with "git fleximod test" to ensure personal forks aren't committed
+fxDONOTUSEurl = https://github.com/NCAR/CUPiD.git
+
 [submodule "doc-builder"]
 path = doc/doc-builder
 url = https://github.com/ESMCI/doc-builder

README

@@ -80,6 +80,7 @@ README ------------------- This file
 README.md ---------------- File that displays on github under https::/github.com/ESCOMP/CTSM.git
 README.rst --------------- File that displays under the project in github
 README_GITFLEXIMOD.rst --- Information on how to work with git-fleximod for CTSM
+README_DIAGNOSTICS.md ---- Quickstart on using CUPiD to plot diagnostics for your CTSM case
 WhatsNewInCTSM5.4.md ----- Overview document of the changes between ctsm5.3 and ctsm5.4
 Copyright ---------------- CESM Copyright file
 doc/UpdateChangeLog.pl --- Script to add documentation on a tag to the

README.DIAGNOSTICS

@@ -0,0 +1,51 @@
+# QuickStart to Running Diagnostics of your Case with CUPiD (CESM Unified Postprocessing and Diagnostics)
+
+[!NOTE]
+Land Diagnostics Cheat Sheet is here:
+
+This document has more details on more options and such
+
+https://docs.google.com/document/d/1ziZWGgaj9FxgR6WRHyCAZnzIc6-4ezqfUDDpQv4MzUI/edit?tab=t.0
+
+## Initial setup steps to do first time
+
+This step is something you need to do the first time you want to run CUPiD for a case.
+And you only need to do it again if the CUPiD environment changes.
+
+``` shell
+# Setup the conda environments
+# TODO: Make this use py_env_create?
+mamba env create --yes -f tools/cupid/environments/cupid-infrastructure.yml
+mamba env create --yes -f tools/cupid/environments/cupid-analysis.yml
+# Check that the environment is valid
+conda activate cupid-infrastructure
+which cupid-diagnostics
+# Should return something like:
+# $HOME/conda-envs/cupid-infrastructure/bin/cupid-diagnostics
+# If it returns "Command not found." something is wrong with the environment
+
+
+# Now make sure the conda environments can be used by Jupyter notebooks
+conda activate cupid-analysis
+python -m ipykernel install --user --name=cupid-analysis
+conda activate cupid-infrastructure
+python -m ipykernel install --user --name=cupid-infrastructure
+```
+
+## Create your case using the CUPiD user-mod
+
+This is similar to setting up any case, such as documented in the README and Quickstart guides.
+
+``` shell
+./create_newcase --case testIwCUPiD --res f09_t232 --compset I2000Clm60BgcCrop --user-mods-dirs clm-CUPiD
+```
+
+## Run CUPID in your case
+
+After st_archive has run do the following. With RUN_POSTPROCESSING set to TRUE this will happen
+with each case submission automatically. But, if you want to run it separately...
+
+
+``` shell
+./case.submit --only-job case.cupid
+```

cime_config/SystemTests/ctsm_test_status.py

@@ -0,0 +1,30 @@
+
+from CIME.test_status import *
+
+CUPID_PHASE = "CUPID"
+
+ALL_PHASES = [
+    CREATE_NEWCASE_PHASE,
+    XML_PHASE,
+    SETUP_PHASE,
+    NAMELIST_PHASE,
+    SHAREDLIB_BUILD_PHASE,
+    MODEL_BUILD_PHASE,
+    SUBMIT_PHASE,
+    RUN_PHASE,
+    COMPARE_PHASE,
+    BASELINE_PHASE,
+    THROUGHPUT_PHASE,
+    MEMCOMP_PHASE,
+    MEMLEAK_PHASE,
+    STARCHIVE_PHASE,
+    CUPID_PHASE,
+    GENERATE_PHASE,
+]
+
+# Extend the TestStatus class to include the CUPID phase and have CUPID phase within ALL_PHASES
+class CTSMTestStatus(TestStatus):
+
+    def __init__(self, case):
+        super().__init__(case, ALL_PHASES)
+        self._test_status = CTSMTestStatus(test_dir=case.get_value("CASEROOT"), test_name=self.case.get_value("CASEBASEID") )

cime_config/SystemTests/smsar.py

@@ -0,0 +1,40 @@
+from CIME.SystemTests.sms import SMS
+from ctsm_test_status import *
+
+
+class SMSAR(SMS):
+    def setup_indv(
+        self, clean=False, test_mode=False, reset=False, keep=False, disable_git=False
+    ):
+        """
+        Perform an individual setup
+        """
+        super().setup_indv(
+            clean=clean,
+            test_mode=test_mode,
+            reset=reset,
+            keep=keep,
+            disable_git=disable_git,
+        )
+        # Make sure the st_archiver is turned on, as CUPiD only runs after it runs
+        self._case.set_value("DOUT_S", True)
+
+    def run_indv(
+        self,
+        suffix="base",
+        st_archive=True,
+        cupid=True,
+        submit_resubmits=None,
+        keep_init_generated_files=False,
+    ):
+        """
+        Perform an individual run. Raises an EXCEPTION on fail.
+
+        Just add the CUPiD phase after the standard run.
+        """
+        super().run_indv(
+            suffix=suffix,
+            st_archive=st_archive,
+            submit_resubmits=submit_resubmits,
+            keep_init_generated_files=keep_init_generated_files,
+        )

cime_config/SystemTests/smscupid.py

@@ -0,0 +1,82 @@
+from CIME.SystemTests.sms import SMS
+from ctsm_test_status import *
+
+
+class SMSCUPID(SMS):
+    def __init__(self, case):
+        super().__init__(case)
+        self._test_status = CTSMTestStatus(
+            test_dir=self._case.get_value("CASEROOT"),
+            test_name=self._case.get_value("CASEBASEID"),
+        )
+
+    def setup_indv(
+        self, clean=False, test_mode=False, reset=False, keep=False, disable_git=False
+    ):
+        """
+        Perform an individual setup
+        """
+        super().setup_indv(
+            clean=clean,
+            test_mode=test_mode,
+            reset=reset,
+            keep=keep,
+            disable_git=disable_git,
+        )
+        # Make sure the st_archiver is turned on, as CUPiD only runs after it runs
+        self._case.set_value("DOUT_S", True)
+
+    def run_indv(
+        self,
+        suffix="base",
+        st_archive=True,
+        cupid=True,
+        submit_resubmits=None,
+        keep_init_generated_files=False,
+    ):
+        """
+        Perform an individual run. Raises an EXCEPTION on fail.
+
+        Just add the CUPiD phase after the standard run.
+        """
+        super().run_indv(
+            suffix=suffix,
+            st_archive=st_archive,
+            submit_resubmits=submit_resubmits,
+            keep_init_generated_files=keep_init_generated_files,
+        )
+        self._phase_modifying_call(CUPID_PHASE, self._cupid_case_test)
+
+    def case_test_cupid(self, testdir="cupid_test"):
+        # create the run directory testdir
+        if os.path.exists(testdir):
+            logger.info("Removing existing test directory {}".format(testdir))
+            shutil.rmtree(testdir)
+        # Check that the CUPid postprocessing directories and config file exist
+        cupid_dir = os.path.join(self._case.get_value("CASEROOT"), "cupid-postprocessing" )
+        notebooks_dir = os.path.join(cupid_dir, "compute_notebooks")
+        data_dir = os.path.join(cupid_dir, "temp_data")
+        for dir in [cupid_dir, notebooks_dir, data_dir]:
+           expect( os.isdir(dir),
+                   "CUPiD postprocessing directory {} does not exist".format(dir) )
+        cupid_config = os.path.join(cupid_dir, "config.yml")
+        expect( os.isfile(cupid_config),
+                "CUPiD config file {} does not exist".format(cupid_config) )
+        # TODO: Add more checks about more files that should exist
+
+
+        # TODO: Populate the testdir with data files, config files and notebooks from the cupid-postprocessing directory
+
+        # TODO: Save various files to the baseline directory to use for BASELINE comparison
+
+        return True
+
+    def _cupid_case_test(self):
+        # For the st_archiver this test is under the case object
+        # Here we create it in this object, but probably should be moved to under the case object in CIME
+        result = self.test_cupid()
+        with self._test_status:
+            if result:
+                self._test_status.set_status(CUPID_PHASE, TEST_PASS_STATUS)
+            else:
+                self._test_status.set_status(CUPID_PHASE, TEST_FAIL_STATUS)

cime_config/config_tests.xml

@@ -185,6 +185,32 @@ This defines various CTSM-specific system tests
     <HIST_N>$STOP_N</HIST_N>
   </test>
 
+  <test NAME="SMSAR">
+    <DESC>Run a smoke startup test with st_archive on</DESC>
+    <INFO_DBUG>1</INFO_DBUG>
+    <DOUT_S>TRUE</DOUT_S>
+    <CONTINUE_RUN>FALSE</CONTINUE_RUN>
+    <STOP_OPTION>nyears</STOP_OPTION>
+    <STOP_N>5</STOP_N>
+    <REST_OPTION>never</REST_OPTION>
+    <HIST_OPTION>$STOP_OPTION</HIST_OPTION>
+    <HIST_N>$STOP_N</HIST_N>
+  </test>
+
+  <test NAME="SMSCUPID">
+    <DESC>Run a smoke startup test with CUPiD on</DESC>
+    <INFO_DBUG>1</INFO_DBUG>
+    <DOUT_S>TRUE</DOUT_S>
+    <CONTINUE_RUN>FALSE</CONTINUE_RUN>
+    <RUN_POSTPROCESSING>TRUE</RUN_POSTPROCESSING>
+    <!-- TODO: Add CUPID Settings -->
+    <STOP_OPTION>nyears</STOP_OPTION>
+    <STOP_N>5</STOP_N>
+    <REST_OPTION>never</REST_OPTION>
+    <HIST_OPTION>$STOP_OPTION</HIST_OPTION>
+    <HIST_N>$STOP_N</HIST_N>
+  </test>
+
 <!--
 SSP    smoke CLM spinup test (only valid for CLM compsets with CLM45)
        do an initial spin test (setting CLM_ACCELERATED_SPINUP to on)

cime_config/testdefs/testlist_clm.xml

@@ -100,6 +100,33 @@
       <option name="comment">Repeat no anthropogenic compset with t232 and Crujra while the defaults have not changed</option>
     </options>
   </test>
+  <test name="SMSAR_PS" grid="f45_f45_mg37" compset="IHistClm60BgcCrujra" testmods="clm/CUPiD">
+    <machines>
+      <machine name="derecho" compiler="intel" category="aux_clm"/>
+    </machines>
+    <options>
+      <option name="wallclock">00:20:00</option>
+      <option name="comment">Simple smoke test running st_archive with CUPiD on for LND</option>
+    </options>
+  </test>
+  <test name="SMSAR_PS" grid="f45_f45_mg37" compset="IHistClm60BgcCrujra" testmods="clm/CUPiD_LDF">
+    <machines>
+      <machine name="derecho" compiler="intel" category="aux_clm"/>
+    </machines>
+    <options>
+      <option name="wallclock">00:20:00</option>
+      <option name="comment">Simple smoke test running st_archive with CUPiD on for the LDF</option>
+    </options>
+  </test>
+  <test name="SMSAR_PS" grid="f45_f45_mg37" compset="IHistClm60BgcCrujra" testmods="clm/CUPiD_ILAMB">
+    <machines>
+      <machine name="derecho" compiler="intel" category="aux_clm"/>
+    </machines>
+    <options>
+      <option name="wallclock">00:20:00</option>
+      <option name="comment">Simple smoke test running st_archive with CUPiD on for the ILAMB</option>
+    </options>
+  </test>
   <test name="SMS_D_Ld5_PS" grid="f09_f09_mt232" compset="I1850Clm60BgcCrop" testmods="clm/f09_ObscureStreamOpts">
     <machines>
       <machine name="derecho" compiler="gnu" category="aux_clm"/>

cime_config/testdefs/testmods_dirs/clm/CUPiD_ILAMB/include_user_mods

@@ -0,0 +1 @@
+../../../../usermods_dirs/clm/CUPiD

cime_config/testdefs/testmods_dirs/clm/CUPiD_ILAMB/shell_commands

@@ -0,0 +1,4 @@
+#!/bin/bash
+./xmlchange CUPID_ILAMB=TRUE
+./xmlchange CUPID_LND=FALSE
+

cime_config/testdefs/testmods_dirs/clm/CUPiD_LDF/include_user_mods

@@ -0,0 +1 @@
+../../../../usermods_dirs/clm/CUPiD

cime_config/testdefs/testmods_dirs/clm/CUPiD_LDF/shell_commands

@@ -0,0 +1,4 @@
+#!/bin/bash
+./xmlchange CUPID_LDF=TRUE
+./xmlchange CUPID_LND=FALSE
+

cime_config/testdefs/testmods_dirs/clm/CUPiD_ROF/include_user_mods

@@ -0,0 +1 @@
+../../../../usermods_dirs/clm/CUPiD

cime_config/testdefs/testmods_dirs/clm/CUPiD_ROF/shell_commands

@@ -0,0 +1,4 @@
+#!/bin/bash
+./xmlchange CUPID_ROF=TRUE
+./xmlchange CUPID_LND=FALSE
+

cime_config/usermods_dirs/clm/CUPiD/README

@@ -0,0 +1,2 @@
+This usermod turns on running CUPiD to get plots and diagnostics as part of a case. It sets the most common settings
+for a CTSM case.

cime_config/usermods_dirs/clm/CUPiD/include_user_mods

@@ -0,0 +1 @@
+../_includes/hist_2000-2005

cime_config/usermods_dirs/clm/CUPiD/shell_commands

@@ -0,0 +1,52 @@
+#!/bin/bash
+# Make sure DOUT_S is turned on, as CUPiD only runs after st_archive
+./xmlchange DOUT_S=TRUE
+
+# Enable postprocessing which is what turns on running CUPiD for a case
+./xmlchange RUN_POSTPROCESSING=TRUE
+
+
+LND_ROOT=`./xmlquery --value COMP_ROOT_DIR_LND`
+
+CUPID_ROOT=`./xmlquery --value CUPID_ROOT`
+# When I first tried this I thought I needed the following...
+if [ -z "$CUPID_ROOT" ]; then
+   echo "CUPID settings aren't done yet"
+   exit 5
+fi
+
+# For a test case, make sure DOUT_S is TRUE
+# TODO: Add a proper SystemTest so this is handled correctly in the test
+TEST=`./xmlquery --value TEST`
+if [ "$TEST" == "TRUE" ]; then
+   ./xmlchange DOUT_S=TRUE --file env_test.xml
+fi
+
+#
+# Set CUPiD parameters for what a CTSM case needs to do
+#
+
+# Set the standard CUPiD baseline case to compare to
+./xmlchange --force CUPID_BASELINE_CASE='$CASE'
+./xmlchange --force CUPID_BASE_STARTDATE='$RUN_STARTDATE'
+./xmlchange --force CUPID_BASE_STOP_N='$STOP_N'
+# TODO: Make sure STOP_OPTION is nyears
+./xmlchange --force CUPID_BASE_STOP_OPTION='$STOP_OPTION'
+# TODO: Have this set based on what CLM_BLDNML_OPTS has
+./xmlchange CUPID_RUN_TYPE=SP
+./xmlchange --force CUPID_STARTDATE='$RUN_STARTDATE'
+./xmlchange --force CUPID_STOP_N='$STOP_N'
+./xmlchange --force CUPID_STOP_OPTION='$STOP_OPTION'
+# TODO: This should maybe be turned on as a smoke test, and to check for the appropriate files in testing
+./xmlchange CUPID_GEN_HTML=TRUE
+# TODO: Test with running more tasks
+./xmlchange CUPID_NTASKS=1
+./xmlchange CUPID_RUN_ADF=FALSE
+./xmlchange CUPID_RUN_ALL=FALSE
+# Use the land_only example
+./xmlchange CUPID_EXAMPLE=land_only
+# TODO: Get CUPiD working with LDF under CESM
+./xmlchange CUPID_RUN_LDF=FALSE
+./xmlchange CUPID_RUN_LND=TRUE
+# TODO: Test if ROF can be run
+./xmlchange CUPID_RUN_ROF=TRUE

cime_config/usermods_dirs/clm/_includes/hist_2000-2005/shell_commands

@@ -0,0 +1,5 @@
+#!/bin/bash
+# Setup for a IHist case to start in 2000 and run for 5 years
+./xmlchange RUN_STARTDATE=2000-01-01
+./xmlchange STOP_OPTION=nyears
+./xmlchange STOP_N=5

tools/README.md

@@ -7,6 +7,7 @@ modification of CTSM input files.
 I.  General directory structure:
 
     `$CTSMROOT/tools`
+        CUPiD ------------ Postprocessing of cases (CESM Unified Postprocessing and Diagnostics)
         mksurfdata_esmf -- Create surface datasets.
 
         crop_calendars --- Regrid and process GGCMI sowing and harvest date files for use in CTSM.