Merge remote-tracking branch 'samsrabin/standardize-time-metadata' into standardize-time-metadata

Getting my local copy of this branch in sync with the remote.

Author: slevis-lmwg

cime_config/SystemTests/rxcropmaturity.py

@@ -106,7 +106,7 @@ def __init__(self, case):
         # Which conda environment should we use?
         self._get_conda_env()
 
-    def _run_phase(self, skip_gen=False):
+    def _run_phase(self, skip_gen=False, h1_inst=False):
         # Modeling this after the SSP test, we create a clone to be the case whose outputs we don't
         # want to be saved as baseline.
 
@@ -129,7 +129,7 @@ def _run_phase(self, skip_gen=False):
         self._set_active_case(case_gddgen)
 
         # Set up stuff that applies to both tests
-        self._setup_all()
+        self._setup_all(h1_inst)
 
         # Add stuff specific to GDD-Generating run
         logger.info("RXCROPMATURITY log:  modify user_nl files: generate GDDs")
@@ -196,7 +196,7 @@ def _run_phase(self, skip_gen=False):
         self._set_active_case(case_rxboth)
 
         # Set up stuff that applies to both tests
-        self._setup_all()
+        self._setup_all(h1_inst)
 
         # Add stuff specific to Prescribed Calendars run
         logger.info("RXCROPMATURITY log:  modify user_nl files: Prescribed Calendars")
@@ -264,7 +264,7 @@ def _get_rx_dates(self):
             logger.error(error_message)
             raise RuntimeError(error_message)
 
-    def _setup_all(self):
+    def _setup_all(self, h1_inst):
         logger.info("RXCROPMATURITY log:  _setup_all start")
 
         # Get some info
@@ -274,7 +274,7 @@ def _setup_all(self):
 
         # Set sowing dates file (and other crop calendar settings) for all runs
         logger.info("RXCROPMATURITY log:  modify user_nl files: all tests")
-        self._modify_user_nl_allruns()
+        self._modify_user_nl_allruns(h1_inst)
         logger.info("RXCROPMATURITY log:  _setup_all done")
 
     # Make a surface dataset that has every crop in every gridcell
@@ -399,7 +399,7 @@ def _run_check_rxboth_run(self, skip_gen):
             tool_path,
         )
 
-    def _modify_user_nl_allruns(self):
+    def _modify_user_nl_allruns(self, h1_inst):
         nl_additions = [
             "cropcals_rx = .true.",
             "cropcals_rx_adapt = .false.",
@@ -417,6 +417,8 @@ def _modify_user_nl_allruns(self):
             "hist_type1d_pertape(2) = 'PFTS'",
             "hist_dov2xy(2) = .false.",
         ]
+        if h1_inst:
+            nl_additions.append("hist_avgflag_pertape(2) = 'I'")
         self._append_to_user_nl_clm(nl_additions)
 
     def _run_generate_gdds(self, case_gddgen):

cime_config/SystemTests/rxcropmaturityinst.py

@@ -0,0 +1,6 @@
+from rxcropmaturity import RXCROPMATURITYSHARED
+
+
+class RXCROPMATURITYINST(RXCROPMATURITYSHARED):
+    def run_phase(self):
+        self._run_phase(h1_inst=True)

cime_config/SystemTests/rxcropmaturityskipgeninst.py

@@ -0,0 +1,6 @@
+from rxcropmaturity import RXCROPMATURITYSHARED
+
+
+class RXCROPMATURITYSKIPGENINST(RXCROPMATURITYSHARED):
+    def run_phase(self):
+        self._run_phase(skip_gen=True, h1_inst=True)

cime_config/config_tests.xml

@@ -145,6 +145,16 @@ This defines various CTSM-specific system tests
     <HIST_N>$STOP_N</HIST_N>
   </test>
 
+  <test NAME="RXCROPMATURITYINST">
+    <DESC>As RXCROPMATURITY but ensure instantaneous h1. Can be removed once instantaneous and other variables are on separate files.</DESC>
+    <INFO_DBUG>1</INFO_DBUG>
+    <DOUT_S>FALSE</DOUT_S>
+    <CONTINUE_RUN>FALSE</CONTINUE_RUN>
+    <REST_OPTION>never</REST_OPTION>
+    <HIST_OPTION>$STOP_OPTION</HIST_OPTION>
+    <HIST_N>$STOP_N</HIST_N>
+  </test>
+
   <test NAME="RXCROPMATURITYSKIPGEN">
     <DESC>As RXCROPMATURITY but don't actually generate GDDs. Allows short testing with existing GDD inputs.</DESC>
     <INFO_DBUG>1</INFO_DBUG>
@@ -155,6 +165,16 @@ This defines various CTSM-specific system tests
     <HIST_N>$STOP_N</HIST_N>
   </test>
 
+  <test NAME="RXCROPMATURITYSKIPGENINST">
+    <DESC>As RXCROPMATURITYSKIPGEN but ensure instantaneous h1. Can be removed once instantaneous and other variables are on separate files.</DESC>
+    <INFO_DBUG>1</INFO_DBUG>
+    <DOUT_S>FALSE</DOUT_S>
+    <CONTINUE_RUN>FALSE</CONTINUE_RUN>
+    <REST_OPTION>never</REST_OPTION>
+    <HIST_OPTION>$STOP_OPTION</HIST_OPTION>
+    <HIST_N>$STOP_N</HIST_N>
+  </test>
+
 <!--
 SSP    smoke CLM spinup test (only valid for CLM compsets with CLM45)
        do an initial spin test (setting CLM_ACCELERATED_SPINUP to on)

cime_config/testdefs/testlist_clm.xml

@@ -3603,6 +3603,17 @@
     </options>
   </test>
 
+  <test name="RXCROPMATURITYINST_Lm61" grid="f10_f10_mg37" compset="IHistClm60BgcCrop"  testmods="clm/cropMonthOutput">
+    <machines>
+      <machine name="derecho" compiler="intel" category="rxcropmaturity"/>
+      <machine name="derecho" compiler="intel" category="crop_calendars"/>
+    </machines>
+    <options>
+      <option name="wallclock">6:00:00</option>
+      <option name="comment">As RXCROPMATURITY, but ensure that h1 file is instantaneous. Can be removed once instantaneous and other variables are separated onto separate files.</option>
+    </options>
+  </test>
+
   <test name="RXCROPMATURITYSKIPGEN_Ld1097" grid="f10_f10_mg37" compset="IHistClm60BgcCrop"  testmods="clm/cropMonthOutput">
     <machines>
       <machine name="derecho" compiler="intel" category="aux_clm"/>
@@ -3615,6 +3626,17 @@
     </options>
   </test>
 
+  <test name="RXCROPMATURITYSKIPGENINST_Ld1097" grid="f10_f10_mg37" compset="IHistClm60BgcCrop"  testmods="clm/cropMonthOutput">
+    <machines>
+      <machine name="derecho" compiler="intel" category="rxcropmaturity"/>
+      <machine name="derecho" compiler="intel" category="crop_calendars"/>
+    </machines>
+    <options>
+      <option name="wallclock">00:45:00</option>
+      <option name="comment">As RXCROPMATURITYSKIPGEN, but ensure that h1 file is instantaneous. Can be removed once instantaneous and other variables are separated onto separate files.</option>
+    </options>
+  </test>
+
   <test name="ERP_D_P64x2_Ld10" grid="f10_f10_mg37" compset="I2000Clm60Bgc" testmods="clm/Hillslope">
     <machines>
       <machine name="derecho" compiler="intel" category="aux_clm"/>

python/ctsm/crop_calendars/convert_axis_time2gs.py

@@ -5,6 +5,7 @@
 import sys
 import numpy as np
 import xarray as xr
+from ctsm.crop_calendars.cropcal_utils import get_integer_years
 
 try:
     import pandas as pd
@@ -85,7 +86,7 @@ def set_up_ds_with_gs_axis(ds_in):
         if not any(x in ["mxsowings", "mxharvests"] for x in ds_in[var].dims):
             data_vars[var] = ds_in[var]
     # Set up the new dataset
-    gs_years = [t.year - 1 for t in ds_in.time.values[:-1]]
+    gs_years = get_integer_years(ds_in)[:-1]
     coords = ds_in.coords
     coords["gs"] = gs_years
     ds_out = xr.Dataset(data_vars=data_vars, coords=coords, attrs=ds_in.attrs)

python/ctsm/crop_calendars/cropcal_figs_module.py

@@ -13,6 +13,7 @@
 from matplotlib import cm
 import matplotlib.collections as mplcol
 
+# pylint: disable=abstract-class-instantiated
 
 # Colormaps (maps)
 cropcal_colors = {

python/ctsm/crop_calendars/cropcal_module.py

@@ -20,24 +20,16 @@ def check_and_trim_years(year_1, year_n, ds_in):
     """
     After importing a file, restrict it to years of interest.
     """
-    ### In annual outputs, file with name Y is actually results from year Y-1.
-    ### Note that time values refer to when it was SAVED. So 1981-01-01 is for year 1980.
-
-    def get_year_from_cftime(cftime_date):
-        # Subtract 1 because the date for annual files is when it was SAVED
-        return cftime_date.year - 1
 
     # Check that all desired years are included
-    if get_year_from_cftime(ds_in.time.values[0]) > year_1:
-        raise RuntimeError(
-            f"Requested year_1 is {year_1} but first year in outputs is "
-            + f"{get_year_from_cftime(ds_in.time.values[0])}"
-        )
-    if get_year_from_cftime(ds_in.time.values[-1]) < year_1:
+    year = utils.get_timestep_year(ds_in, ds_in.time.values[0])
+    if year > year_1:
         raise RuntimeError(
-            f"Requested year_n is {year_n} but last year in outputs is "
-            + f"{get_year_from_cftime(ds_in.time.values[-1])}"
+            f"Requested year_1 is {year_1} but first year in outputs is {year}"
         )
+    year = utils.get_timestep_year(ds_in, ds_in.time.values[-1])
+    if year < year_1:
+        raise RuntimeError(f"Requested year_n is {year_n} but last year in outputs is {year}")
 
     # Remove years outside range of interest
     ### Include an extra year at the end to finish out final seasons.

python/ctsm/crop_calendars/cropcal_utils.py

@@ -430,3 +430,45 @@ def make_lon_increasing(xr_obj):
         raise RuntimeError("Unable to rearrange longitude axis so it's monotonically increasing")
 
     return xr_obj.roll(lon=shift, roll_coords=True)
+
+
+def is_inst_file(dsa):
+    """
+    Check whether Dataset or DataArray has time data from an "instantaneous file"
+    """
+    return "at end of" in dsa["time"].attrs["long_name"]
+
+
+def get_beg_inst_timestep_year(timestep):
+    """
+    Get year associated with the BEGINNING of a timestep in an
+    instantaneous file
+    """
+    year = timestep.year
+
+    is_jan1 = timestep.dayofyr == 1
+    is_midnight = timestep.hour == timestep.minute == timestep.second == 0
+    if is_jan1 and is_midnight:
+        year -= 1
+
+    return year
+
+
+def get_timestep_year(dsa, timestep):
+    """
+    Get the year associated with a timestep, with different handling
+    depending on whether the file is instantaneous
+    """
+    if is_inst_file(dsa):
+        year = get_beg_inst_timestep_year(timestep)
+    else:
+        year = timestep.year
+    return year
+
+
+def get_integer_years(dsa):
+    """
+    Convert time axis to numpy array of integer years
+    """
+    out_array = [get_timestep_year(dsa, t) for t in dsa["time"].values]
+    return out_array

python/ctsm/crop_calendars/generate_gdds_functions.py

@@ -553,8 +553,8 @@ def import_and_process_1yr(
     clm_gdd_var = "GDDACCUM"
     my_vars = [clm_gdd_var, "GDDHARV"]
     patterns = [f"*h2.{this_year-1}-01*.nc", f"*h2.{this_year-1}-01*.nc.base"]
-    for p in patterns:
-        pattern = os.path.join(indir, p)
+    for pat in patterns:
+        pattern = os.path.join(indir, pat)
         h2_files = glob.glob(pattern)
         if h2_files:
             break