move delay_minutes to metamodel

Author: peterdudfield

src/nwp_consumer/internal/entities/modelmetadata.py

@@ -65,6 +65,12 @@ class ModelMetadata:
     to the repository.
     """
 
+    delay_minutes: int
+    """The model delay in minutes.
+
+    Note that this can be overwritten by using `with_delay_minutes`.
+    """
+
     chunk_count_overrides: dict[str, int] = dataclasses.field(default_factory=dict)
     """Mapping of dimension names to the desired number of chunks in that dimension.
 
@@ -201,6 +207,10 @@ def with_running_hours(self, hours: list[int]) -> "ModelMetadata":
         """Returns metadata for the given model with the given running hours."""
         return dataclasses.replace(self, running_hours=hours)
 
+    def with_delay_minutes(self, delay_minutes: int) -> "ModelMetadata":
+        """Returns metadata for the given model with the given delay minutes."""
+        return dataclasses.replace(self, delay_minutes=delay_minutes)
+
     def with_max_step(self, max_step: int) -> "ModelMetadata":
         """Returns metadata for the given model with the given max step."""
         return dataclasses.replace(
@@ -254,6 +264,7 @@ class Models:
             longitude=[float(f"{lon / 10:.2f}") for lon in range(-1800, 1800 + 1, 1)],
         ),
         running_hours=[0, 6, 12, 18],
+        delay_minutes=(60 * 26),  # 1 day, plus leeway
     )
     """ECMWF's High Resolution Integrated Forecast System."""
 
@@ -274,6 +285,7 @@ class Models:
             longitude=[v / 10 for v in range(-1800, 1800, 1)],
         ),
         running_hours=[0, 12],
+        delay_minutes=(60 * 26),  # 1 day, plus leeway
     )
     """Summary statistics from ECMWF's Ensemble Forecast System."""
 
@@ -303,6 +315,7 @@ class Models:
             longitude=[v / 10 for v in range(-1800, 1800, 1)],
         ),
         running_hours=[0, 6, 12, 18],
+        delay_minutes=(60 * 26),  # 1 day, plus leeway
     )
     """Full ensemble data from ECMWF's Ensemble Forecast System."""
 
@@ -335,6 +348,7 @@ class Models:
             longitude=[float(lon) for lon in range(-180, 180 + 1, 1)],
         ),
         running_hours=[0, 6, 12, 18],
+        delay_minutes=(60 * 5),  # 5 hours
     )
     """NCEP's Global Forecast System."""
 
@@ -372,6 +386,7 @@ class Models:
             # TODO: Change to -180 -> 180
         ),
         running_hours=[0, 6, 12, 18],
+        delay_minutes=(60 * 24 * 7) + (60 * 12),  # 7.5 days
     )
     """MetOffice's Unified Model, in the Global configuration, at a resolution of 17km."""
 
@@ -405,6 +420,7 @@ class Models:
             ],
         ),
         running_hours=[0, 12],
+        delay_minutes=300,
     )
     """MetOffice's Unified Model, in the Global configuration, at a resolution of 10km."""
 
@@ -438,6 +454,7 @@ class Models:
             x_osgb=[int(x) for x in np.arange(start=-239000, stop=857000, step=2000)],
         ),
         running_hours=list(range(0, 24, 6)),
+        delay_minutes=120,
     )
     """MetOffice's Unified Model in the UKV configuration, at a resolution of 2km"""
 
@@ -468,5 +485,6 @@ class Models:
             x_laea=[int(x) for x in np.arange(start=-576000, stop=332000 + 2000, step=2000)],
         ),
         running_hours=list(range(0, 24, 3)),  # Only first 12 steps available for hourly runs
+        delay_minutes=120,
     )
     """MetOffice's Unified Model in the UKV configuration, at a resolution of 2km"""

src/nwp_consumer/internal/entities/repometadata.py

@@ -40,12 +40,6 @@ class RawRepositoryMetadata:
     but rather are defined by pre-selected agreements with the provider.
     """
 
-    delay_minutes: int
-    """The approximate model delay in minutes.
-
-    This delay is the time between the running of the model and the time
-    at which the data is actually available."""
-
     required_env: list[str]
     """Environment variables required for usage."""
 
@@ -65,18 +59,23 @@ class RawRepositoryMetadata:
     available_models: dict[str, ModelMetadata]
     """A dictionary of available models and their metadata."""
 
-    def determine_latest_it_from(self, t: dt.datetime, running_hours: list[int]) -> dt.datetime:
+    def determine_latest_it_from(
+            self, t: dt.datetime,
+            running_hours: list[int],
+            delay_minutes:int) -> dt.datetime:
         """Determine the latest available initialization time from a given time.
 
         Args:
             t: The time from which to determine the latest initialization time.
             running_hours: A list of hours at which the model runs each day.
+            delay_minutes: The delay in minutes after the initialization time
+                before data is available.
 
         Returns:
             The latest available initialization time prior to the given time.
         """
         it = (
-            t.replace(minute=0, second=0, microsecond=0) - dt.timedelta(minutes=self.delay_minutes)
+            t.replace(minute=0, second=0, microsecond=0) - dt.timedelta(minutes=delay_minutes)
         ).replace(minute=0)
         while it.hour not in running_hours:
             it -= dt.timedelta(hours=1)

src/nwp_consumer/internal/entities/test_modelmetadata.py

@@ -27,6 +27,7 @@ def test_with_region(self) -> None:
                 longitude=[float(f"{lon / 10:.2f}") for lon in range(-1800, 1800 + 1, 1)],
             ),
             running_hours=[0, 6, 12, 18],
+            delay_minutes=60,
         )
 
         @dataclasses.dataclass

src/nwp_consumer/internal/entities/test_repometadata.py

@@ -13,7 +13,6 @@ class TestRawRepositoryMetadata(unittest.TestCase):
         name="test",
         is_archive=False,
         is_order_based=False,
-        delay_minutes=55,
         required_env=["TEST"],
         optional_env={"TEST": "test"},
         max_connections=1,
@@ -50,7 +49,9 @@ class TestCase:
 
         for test in tests:
             with self.subTest(name=test.name):
-                result = self.metadata.determine_latest_it_from(test.t, [0, 6, 12, 18])
+                result = self.metadata.determine_latest_it_from(test.t,
+                                                                [0, 6, 12, 18],
+                                                                delay_minutes=60)
                 self.assertEqual(result, test.expected)
 
 

src/nwp_consumer/internal/repositories/raw_repositories/ceda.py

@@ -117,7 +117,6 @@ def repository() -> entities.RawRepositoryMetadata:
             name="CEDA",
             is_archive=True,
             is_order_based=False,
-            delay_minutes=(60 * 24 * 7) + (60 * 12),  # 7.5 days
             max_connections=20,
             required_env=["CEDA_USER", "CEDA_PASS"],
             optional_env={},

src/nwp_consumer/internal/repositories/raw_repositories/ecmwf_mars.py

@@ -211,7 +211,6 @@ def repository() -> entities.RawRepositoryMetadata:
             name="ECMWF-MARS",
             is_archive=True,
             is_order_based=False,
-            delay_minutes=(60 * 26),  # 1 day, plus leeway
             max_connections=20,
             required_env=[
                 "ECMWF_API_KEY",

src/nwp_consumer/internal/repositories/raw_repositories/ecmwf_realtime.py

@@ -67,7 +67,6 @@ def repository() -> entities.RawRepositoryMetadata:
             name="ECMWF-Realtime-S3",
             is_archive=False,
             is_order_based=True,
-            delay_minutes=(60 * 7),  # 7 hours
             max_connections=100,
             required_env=[
                 "ECMWF_REALTIME_S3_ACCESS_KEY",
@@ -81,14 +80,16 @@ def repository() -> entities.RawRepositoryMetadata:
             },
             postprocess_options=entities.PostProcessOptions(),
             available_models={
-                "default": entities.Models.ECMWF_HRES_IFS_0P1DEGREE.with_region("uk-north60"),
-                "hres-ifs-uk": entities.Models.ECMWF_HRES_IFS_0P1DEGREE.with_region("uk-north60"),
+                "default": entities.Models.ECMWF_HRES_IFS_0P1DEGREE.with_region("uk-north60").
+                    with_delay_minutes(60 * 7),
+                "hres-ifs-uk": entities.Models.ECMWF_HRES_IFS_0P1DEGREE.with_region("uk-north60").
+                    with_delay_minutes(60 * 7),
                 "hres-ifs-india": entities.Models.ECMWF_HRES_IFS_0P1DEGREE.with_region(
                     "india",
-                ).with_chunk_count_overrides({"variable": 1}),
+                ).with_chunk_count_overrides({"variable": 1}).with_delay_minutes(60 * 7),
                 "hres-ifs-nl": entities.Models.ECMWF_HRES_IFS_0P1DEGREE.with_region(
                     "nl",
-                ).with_max_step(84),
+                ).with_max_step(84).with_delay_minutes(60 * 7),
             },
         )
 

src/nwp_consumer/internal/repositories/raw_repositories/mo_datahub.py

@@ -123,41 +123,6 @@
 
 log = logging.getLogger("nwp-consumer")
 
-class AvailableModel:
-    """Class to hold information about an available model."""
-    name: str
-    delay_minute: int
-    model: entities.ModelMetadata
-
-    def __init__(self, name: str, delay_minute: int, model: entities.ModelMetadata) -> None:
-        """Create a new instance."""
-        self.name = name
-        self.delay_minute = delay_minute
-        self.model = model
-
-# set up the available models
-available_models = []
-available_models.append(AvailableModel(name="default",
-                         delay_minute=300,
-                         model=entities.Models.MO_UM_GLOBAL_10KM.with_region("india")))
-available_models.append(AvailableModel(name="um-global-10km-india",
-                         delay_minute=300,
-                         model=entities.Models.MO_UM_GLOBAL_10KM.with_region("india")))
-available_models.append(AvailableModel(name="um-global-10km-uk",
-                         delay_minute=300,
-                         model=entities.Models.MO_UM_GLOBAL_10KM.with_region("uk")))
-available_models.append(AvailableModel(name="um-ukv-2km",
-                         delay_minute=120,
-                         model=entities.Models.MO_UM_UKV_2KM_LAEA))
-
-# get all model names
-available_models_names = [m.name for m in available_models]
-# get dict of delay minutes by model name
-delay_minutes_dict = {m.name: m.delay_minute for m in available_models}
-# get dict of model metadata by model name
-models_dict = {m.name: m.model for m in available_models}
-
-
 
 class MetOfficeDatahubRawRepository(ports.RawRepository):
     """Repository implementation for data from MetOffice's DataHub service."""
@@ -188,24 +153,34 @@ def __init__(self, order_id: str, api_key: str) -> None:
     @override
     def repository() -> entities.RawRepositoryMetadata:
 
-        requested_model: str = get_requested_model_name()
-
         return entities.RawRepositoryMetadata(
             name="MetOffice-Weather-Datahub",
             is_archive=False,
             is_order_based=True,
-            delay_minutes=delay_minutes_dict[requested_model],
             max_connections=10,
             required_env=["METOFFICE_API_KEY", "METOFFICE_ORDER_ID"],
             optional_env={"METOFFICE_DATASPEC": "1.1.0"},
             postprocess_options=entities.PostProcessOptions(),
-            available_models=models_dict,
+            available_models={
+                "default": entities.Models.MO_UM_GLOBAL_10KM.with_region("india"),
+                "um-global-10km-india": entities.Models.MO_UM_GLOBAL_10KM.with_region("india"),
+                "um-global-10km-uk": entities.Models.MO_UM_GLOBAL_10KM.with_region("uk"),
+                "um-ukv-2km": entities.Models.MO_UM_UKV_2KM_LAEA,
+            },
         )
 
     @staticmethod
     @override
     def model() -> entities.ModelMetadata:
-        requested_model: str = get_requested_model_name()
+        requested_model: str = os.getenv("MODEL", default="default")
+        if requested_model not in MetOfficeDatahubRawRepository.repository().available_models:
+            log.warn(
+                f"Unknown model '{requested_model}' requested, falling back to default. "
+                "MetOffice Datahub repository only supports "
+                f"'{list(MetOfficeDatahubRawRepository.repository().available_models.keys())}'. "
+                "Ensure MODEL environment variable is set to a valid model name.",
+            )
+            requested_model = "default"
         return MetOfficeDatahubRawRepository.repository().available_models[requested_model]
 
     @classmethod
@@ -542,16 +517,3 @@ def _convert_ukv(path: pathlib.Path) -> ResultE[list[xr.DataArray]]:
             )
 
         return Success([da])
-
-
-def get_requested_model_name() -> str:
-    """Get the requested model name from the environment variable."""
-    requested_model: str = os.getenv("MODEL", default="default")
-    if requested_model not in available_models_names:
-        log.warn(
-            f"Unknown model '{requested_model}' requested, falling back to default. "
-            f"MetOffice Datahub repository only supports '{available_models_names}'. "
-            "Ensure MODEL environment variable is set to a valid model name.",
-            )
-        requested_model = "default"
-    return requested_model

src/nwp_consumer/internal/repositories/raw_repositories/noaa_s3.py

@@ -55,7 +55,6 @@ def repository() -> entities.RawRepositoryMetadata:
             name="NOAA-GFS-S3",
             is_archive=False,
             is_order_based=False,
-            delay_minutes=(60 * 5),  # 5 hours
             max_connections=100,
             required_env=[],
             optional_env={},

src/nwp_consumer/internal/repositories/raw_repositories/test_mo_datahub.py

@@ -36,6 +36,7 @@ def test__download(self) -> None:
 
         self.assertIsInstance(dl_result, Success, msg=f"{dl_result!s}")
 
+    @patch.dict(os.environ, {"MODEL": "um-global-10km-india"}, clear=True)
     def test_convert(self) -> None:
         """Test the _convert method."""
 
@@ -92,6 +93,8 @@ class TestCase:
                 else:
                     self.assertIsInstance(region_result, Success, msg=f"{region_result}")
 
+
+
     @patch.dict(os.environ, {"MODEL": "um-ukv-2km"}, clear=True)
     def test_convert_ukv(self) -> None:
         @dataclasses.dataclass