Merge pull request #1068 from MetOffice/1065_update_collapse_by_hour

Update collapse_by_hour for multiple cases

Author: daflack

src/CSET/operators/_utils.py

@@ -191,6 +191,38 @@ def fully_equalise_attributes(cubes: iris.cube.CubeList):
     return cubes
 
 
+def is_time_aggregatable(cube: iris.cube.Cube) -> bool:
+    """Determine whether a cube can be aggregated in time.
+
+    If a cube is aggregatable it will contain both a 'forecast_reference_time'
+    and 'forecast_period' coordinate as dimensional coordinates.
+
+    Arguments
+    ---------
+    cube: iris.cube.Cube
+        An iris cube which will be checked to see if it is aggregatable based
+        on a set of pre-defined dimensional time coordinates:
+        'forecast_period' and 'forecast_reference_time'.
+
+    Returns
+    -------
+    bool
+        If true, then the cube is aggregatable and contains dimensional
+        coordinates including both 'forecast_reference_time' and
+        'forecast_period'.
+    """
+    # Acceptable time coordinate names for aggregatable cube.
+    TEMPORAL_COORD_NAMES = ["forecast_period", "forecast_reference_time"]
+
+    # Coordinate names for the cube.
+    coord_names = [coord.name() for coord in cube.coords(dim_coords=True)]
+
+    # Check which temporal coordinates we have.
+    temporal_coords = [coord for coord in coord_names if coord in TEMPORAL_COORD_NAMES]
+    # Return whether both coordinates are in the temporal coordinates.
+    return len(temporal_coords) == 2
+
+
 def ensure_aggregatable_across_cases(
     cube: iris.cube.Cube | iris.cube.CubeList,
 ) -> iris.cube.Cube:
@@ -199,8 +231,8 @@ def ensure_aggregatable_across_cases(
     Arguments
     ---------
     cube: iris.cube.Cube | iris.cube.CubeList
-        If a Cube is provided a sub-operator is called to determine if the
-        cube has the necessary dimensional coordinates to be aggregateable.
+        If a Cube is provided it is checked to determine if it has the
+        the necessary dimensional coordinates to be aggregateable.
         These necessary coordinates are 'forecast_period' and
         'forecast_reference_time'.If a CubeList is provided a Cube is created
         by slicing over all time coordinates and the resulting list is merged
@@ -219,40 +251,6 @@ def ensure_aggregatable_across_cases(
         raised. The user should then provide a CubeList to be turned into an
         aggregatable cube to allow aggregation across multiple cases to occur.
     """
-
-    def is_time_aggregatable(cube: iris.cube.Cube) -> bool:
-        """Determine whether a cube can be aggregated in time.
-
-        If a cube is aggregatable it will contain both a 'forecast_reference_time'
-        and 'forecast_period' coordinate as dimensional coordinates.
-
-        Arguments
-        ---------
-        cube: iris.cube.Cube
-            An iris cube which will be checked to see if it is aggregatable based
-            on a set of pre-defined dimensional time coordinates:
-            'forecast_period' and 'forecast_reference_time'.
-
-        Returns
-        -------
-        bool
-            If true, then the cube is aggregatable and contains dimensional
-            coordinates including both 'forecast_reference_time' and
-            'forecast_period'.
-        """
-        # Acceptable time coordinate names for aggregatable cube.
-        TEMPORAL_COORD_NAMES = ["forecast_period", "forecast_reference_time"]
-
-        # Coordinate names for the cube.
-        coord_names = [coord.name() for coord in cube.coords(dim_coords=True)]
-
-        # Check which temporal coordinates we have.
-        temporal_coords = [
-            coord for coord in coord_names if coord in TEMPORAL_COORD_NAMES
-        ]
-        # Return whether both coordinates are in the temporal coordinates.
-        return len(temporal_coords) == 2
-
     # Check to see if a cube is input and if that cube is iterable.
     if isinstance(cube, iris.cube.Cube):
         if is_time_aggregatable(cube):

src/CSET/operators/collapse.py

@@ -20,8 +20,9 @@
 import iris.analysis
 import iris.coord_categorisation
 import iris.cube
+import iris.util
 
-from CSET.operators._utils import ensure_aggregatable_across_cases
+from CSET.operators._utils import ensure_aggregatable_across_cases, is_time_aggregatable
 
 
 def collapse(
@@ -88,6 +89,7 @@ def collapse_by_lead_time(
 
     First checks if the data can be aggregated by lead time easily. Then
     collapses by lead time for a specified method using the collapse function.
+    This operator provides the average of all T+1, T+2, etc.
 
     Arguments
     ---------
@@ -116,31 +118,50 @@ def collapse_by_lead_time(
     if method == "PERCENTILE":
         collapsed_cube = collapse(
             cube_to_collapse,
-            "forecast_period",
+            "forecast_reference_time",
             method,
             additional_percent=additional_percent,
         )
     else:
-        collapsed_cube = collapse(cube_to_collapse, "forecast_period", method)
+        collapsed_cube = collapse(cube_to_collapse, "forecast_reference_time", method)
     return collapsed_cube
 
 
 def collapse_by_hour_of_day(
-    cube: iris.cube.Cube,
+    cube: iris.cube.Cube | iris.cube.CubeList,
     method: str,
     additional_percent: float = None,
+    multi_case: bool = True,
     **kwargs,
 ) -> iris.cube.Cube:
     """Collapse a cube by hour of the day.
 
+    Collapses a cube by hour of the day in the time coordinates provided by the
+    model. It is useful for creating diurnal cycle plots. It aggregates all
+    00 UTC together regardless of lead time.
+
     Arguments
     ---------
-    cube: iris.cube.Cube
-        Cube to collapse and iterate over one dimension. It should contain only
-        one time dimension.
+    cube: iris.cube.Cube | iris.cube.CubeList
+        Cube to collapse and iterate over one dimension or CubeList to
+        convert to a cube and then collapse prior to aggregating by hour.
+        If a CubeList is provided multi_case must be set to True as the Cube List
+        should only contain cubes of multiple dates and not different variables
+        or models. A cube that only contains one time dimension must have
+        multi_case set to False as it contains only one forecast. A cube
+        containing two time dimensions, e.g., 'forecast_reference_time' and
+        'forecast_period' must have multi_case set to True as it will contain
+        multiple forecasts.
     method: str
         Type of collapse i.e. method: 'MEAN', 'MAX', 'MIN', 'MEDIAN',
         'PERCENTILE'. For 'PERCENTILE' the additional_percent must be specified.
+    multi_case: boolean, optional
+        Default is True. If True multiple cases will be aggregated by hour of
+        day; if False a single forecast will be aggregated by hour of day.
+        Information around the usage of multi_case is provided above under the
+        description for the cube argument. It is kept as an argument rather
+        than being automatically generated to maintain traceability for the
+        users actions.
 
     Returns
     -------
@@ -151,6 +172,10 @@ def collapse_by_hour_of_day(
     ------
     ValueError
         If additional_percent wasn't supplied while using PERCENTILE method.
+    TypeError
+        If a CubeList is given and multi_case is not True;
+        if a Cube is given and contains two time dimensions and multi_case is not True;
+        if a Cube is given and contains one time dimensions and multi_case is not False.
 
     Notes
     -----
@@ -163,10 +188,36 @@ def collapse_by_hour_of_day(
     To apply this operator successfully there must only be one time dimension.
     Should a MultiDim exception be raised the user first needs to apply the
     collapse operator to reduce the time dimensions before applying this
-    operator.
+    operator. If multi_case is true the collapse_by_lead_time operator is
+    applied and performs this step.
     """
     if method == "PERCENTILE" and additional_percent is None:
         raise ValueError("Must specify additional_percent")
+    elif (
+        isinstance(cube, iris.cube.Cube)
+        and is_time_aggregatable(cube)
+        and not multi_case
+    ):
+        raise TypeError(
+            "multi_case must be true for a cube containing two time dimensions"
+        )
+    elif (
+        isinstance(cube, iris.cube.Cube)
+        and not is_time_aggregatable(cube)
+        and multi_case
+    ):
+        raise TypeError(
+            "multi_case must be false for a cube containing one time dimension"
+        )
+    elif isinstance(cube, iris.cube.CubeList) and not multi_case:
+        raise TypeError("multi_case must be true for a CubeList")
+
+    if multi_case:
+        # Collapse by lead time to get a single time dimension.
+        cube = collapse_by_lead_time(
+            cube, method, additional_percent=additional_percent
+        )
+
     # Categorise the time coordinate by hour of the day.
     iris.coord_categorisation.add_hour(cube, "time", name="hour")
     # Aggregate by the new category coordinate.
@@ -176,10 +227,18 @@ def collapse_by_hour_of_day(
         )
     else:
         collapsed_cube = cube.aggregated_by("hour", getattr(iris.analysis, method))
+
     # Remove unnecessary time coordinates.
     collapsed_cube.remove_coord("time")
-    collapsed_cube.remove_coord("forecast_reference_time")
     collapsed_cube.remove_coord("forecast_period")
+    # Remove forecast_reference_time if a single case, as collapse_by_lead_time
+    # will have effectively done this if multi_case is True.
+    if not multi_case:
+        collapsed_cube.remove_coord("forecast_reference_time")
+
+    # Promote "hour" to dim_coord if monotonic.
+    if collapsed_cube.coord("hour").is_monotonic():
+        iris.util.promote_aux_coord_to_dim_coord(collapsed_cube, "hour")
     return collapsed_cube
 
 

tests/operators/test_collapse.py

@@ -46,6 +46,14 @@ def long_forecast_many_cubes() -> iris.cube.Cube:
     )
 
 
+@pytest.fixture()
+def medium_forecast() -> iris.cube.Cube:
+    """Get medium forecast with monotonic time coordinate."""
+    return iris.load_cube(
+        "tests/test_data/medium_forecast_air_temp_monotonic.nc", "air_temperature"
+    )
+
+
 def test_collapse(cube):
     """Reduces dimension of cube."""
     # Test collapsing a single coordinate.
@@ -80,28 +88,78 @@ def test_collapse_percentile(cube):
 
 def test_collapse_by_hour_of_day(long_forecast):
     """Convert and aggregates time dimension by hour of day."""
-    # Test collapsing a long forecast.
-    collapsed_cube = collapse.collapse_by_hour_of_day(long_forecast, "MEAN")
+    collapsed_cube = collapse.collapse_by_hour_of_day(
+        long_forecast, "MEAN", multi_case=False
+    )
     expected_cube = "<iris 'Cube' of air_temperature / (K) (-- : 24; grid_latitude: 3; grid_longitude: 3)>"
     assert repr(collapsed_cube) == expected_cube
 
 
+def test_collapse_by_hour_of_day_fail(long_forecast):
+    """Test failing due to multi_case set to True."""
+    with pytest.raises(TypeError):
+        collapse.collapse_by_hour_of_day(long_forecast, "MEAN")
+
+
 def test_collapse_by_hour_of_day_percentile(long_forecast):
     """Convert and aggregate time dimension by hour of day with percentiles."""
-    with pytest.raises(ValueError):
-        collapse.collapse_by_hour_of_day(long_forecast, "PERCENTILE")
     # Test collapsing long forecast.
     collapsed_cube = collapse.collapse_by_hour_of_day(
-        long_forecast, "PERCENTILE", additional_percent=[25, 75]
+        long_forecast, "PERCENTILE", additional_percent=[25, 75], multi_case=False
     )
     expected_cube = "<iris 'Cube' of air_temperature / (K) (percentile_over_hour: 2; -- : 24; grid_latitude: 3; grid_longitude: 3)>"
     assert repr(collapsed_cube) == expected_cube
 
 
+def test_collapse_by_hour_of_day_percentile_fail(long_forecast):
+    """Test failing due to non-specified additional_percent."""
+    with pytest.raises(ValueError):
+        collapse.collapse_by_hour_of_day(long_forecast, "PERCENTILE", multi_case=False)
+
+
+def test_collapse_by_hour_of_day_multi_forecast_cube(long_forecast_multi_day):
+    """Convert and aggregates time dimension by hour of day for a multi day cube."""
+    collapsed_cube = collapse.collapse_by_hour_of_day(long_forecast_multi_day, "MEAN")
+    expected_cube = "<iris 'Cube' of air_temperature / (K) (-- : 24; grid_latitude: 3; grid_longitude: 3)>"
+    assert repr(collapsed_cube) == expected_cube
+
+
+def test_collapse_by_hour_of_day_multi_forecast_cube_fail(long_forecast_multi_day):
+    """Test failing due to multi_case set to False."""
+    with pytest.raises(TypeError):
+        collapse.collapse_by_hour_of_day(
+            long_forecast_multi_day, "MEAN", multi_case=False
+        )
+
+
+def test_collapse_by_hour_of_day_multi_forecast_cubelist(long_forecast_many_cubes):
+    """Convert and aggregates time dimension by hour of day for a CubeList."""
+    collapsed_cube = collapse.collapse_by_hour_of_day(long_forecast_many_cubes, "MEAN")
+    expected_cube = "<iris 'Cube' of air_temperature / (K) (-- : 24; grid_latitude: 3; grid_longitude: 3)>"
+    assert repr(collapsed_cube) == expected_cube
+
+
+def test_collapse_by_hour_of_day_multi_forecast_cubelist_fail(long_forecast_many_cubes):
+    """Test failing due to multi_case set to False."""
+    with pytest.raises(TypeError):
+        collapse.collapse_by_hour_of_day(
+            long_forecast_many_cubes, "MEAN", multi_case=False
+        )
+
+
+def test_collapse_by_hour_of_day_monotonic_coords(medium_forecast):
+    """Convert and aggregates time dimension by hour of day with montonic coordinates."""
+    collapsed_cube = collapse.collapse_by_hour_of_day(
+        medium_forecast, "MEAN", multi_case=False
+    )
+    expected_cube = "<iris 'Cube' of air_temperature / (K) (hour: 24; grid_latitude: 3; grid_longitude: 3)>"
+    assert repr(collapsed_cube) == expected_cube
+
+
 def test_collapse_by_lead_time_single_cube(long_forecast_multi_day):
     """Check cube collapse by lead time."""
     calculated_cube = collapse.collapse(
-        long_forecast_multi_day, "forecast_period", "MEAN"
+        long_forecast_multi_day, "forecast_reference_time", "MEAN"
     )
     assert np.allclose(
         calculated_cube.data,
@@ -116,7 +174,7 @@ def test_collapse_by_lead_time_cube_list(
 ):
     """Check CubeList is made into an aggregatable cube and collapses by lead time."""
     calculated_cube = collapse.collapse(
-        long_forecast_multi_day, "forecast_period", "MEAN"
+        long_forecast_multi_day, "forecast_reference_time", "MEAN"
     )
     assert np.allclose(
         calculated_cube.data,
@@ -129,10 +187,11 @@ def test_collapse_by_lead_time_cube_list(
 def test_collapse_by_lead_time_single_cube_percentile(long_forecast_multi_day):
     """Check Cube collapse by lead time with percentiles."""
     calculated_cube = collapse.collapse(
-        long_forecast_multi_day, "forecast_period", "PERCENTILE", additional_percent=75
+        long_forecast_multi_day,
+        "forecast_reference_time",
+        "PERCENTILE",
+        additional_percent=75,
     )
-    with pytest.raises(ValueError):
-        collapse.collapse_by_lead_time(long_forecast_multi_day, "PERCENTILE")
     assert np.allclose(
         calculated_cube.data,
         collapse.collapse_by_lead_time(
@@ -143,15 +202,22 @@ def test_collapse_by_lead_time_single_cube_percentile(long_forecast_multi_day):
     )
 
 
+def test_collapse_by_lead_time_single_cube_percentile_fail(long_forecast_multi_day):
+    """Test fail by not setting additional percent."""
+    with pytest.raises(ValueError):
+        collapse.collapse_by_lead_time(long_forecast_multi_day, "PERCENTILE")
+
+
 def test_collapse_by_lead_time_cube_list_percentile(
     long_forecast_multi_day, long_forecast_many_cubes
 ):
     """Check CubeList is made into an aggregatable cube and collapses by lead time with percentiles."""
     calculated_cube = collapse.collapse(
-        long_forecast_multi_day, "forecast_period", "PERCENTILE", additional_percent=75
+        long_forecast_multi_day,
+        "forecast_reference_time",
+        "PERCENTILE",
+        additional_percent=75,
     )
-    with pytest.raises(ValueError):
-        collapse.collapse_by_lead_time(long_forecast_many_cubes, "PERCENTILE")
     assert np.allclose(
         calculated_cube.data,
         collapse.collapse_by_lead_time(