From a6b5e1b3595fc98fa8270e325707137b9bbc909a Mon Sep 17 00:00:00 2001
From: Richa <41283476+14Richa@users.noreply.github.com>
Date: Wed, 31 Jul 2024 18:44:08 +0530
Subject: [PATCH 01/14] Add a national end point, that provides elexon BMRS
 solar foreacsts from elexonpy package (#347)

* added import statements of elexonpy

* [pre-commit.ci] auto fixes from pre-commit.com hooks

for more information, see https://pre-commit.ci

* added API for solar forecast

* [pre-commit.ci] auto fixes from pre-commit.com hooks

for more information, see https://pre-commit.ci

* minor fix

* [pre-commit.ci] auto fixes from pre-commit.com hooks

for more information, see https://pre-commit.ci

* minor fix

* [pre-commit.ci] auto fixes from pre-commit.com hooks

for more information, see https://pre-commit.ci

* changed function name

* added filter functionality

* [pre-commit.ci] auto fixes from pre-commit.com hooks

for more information, see https://pre-commit.ci

* added API link

* minor fix

* remove try except block

* changed bmrs to elexon

* minor fix

* resolving pre hook

* minor fix in docstring

* removed router

* minor fix in naming

* added elexonpy in requirement.txt file

* resolve hook error

* minor fix

* added response model

* fixed unit test

* remove plevel

---------

Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com>
---
 requirements.txt                  |   1 +
 src/main.py                       |   1 -
 src/national.py                   | 123 ++++++++++++++++++++++--------
 src/pydantic_models.py            |  24 +++++-
 src/tests/test_elexon_forecast.py |  63 +++++++++++++++
 5 files changed, 180 insertions(+), 32 deletions(-)
 create mode 100644 src/tests/test_elexon_forecast.py

diff --git a/requirements.txt b/requirements.txt
index b5c23b32..979d07dd 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -19,3 +19,4 @@ slowapi
 pathy==0.10.3
 fsspec
 s3fs
+elexonpy
diff --git a/src/main.py b/src/main.py
index d401d353..08ed289d 100644
--- a/src/main.py
+++ b/src/main.py
@@ -208,7 +208,6 @@ async def add_process_time_header(request: Request, call_next):
 # Dependency
 v0_route_solar = "/v0/solar/GB"
 v0_route_system = "/v0/system/GB"
-
 app.include_router(national_router, prefix=f"{v0_route_solar}/national")
 app.include_router(gsp_router, prefix=f"{v0_route_solar}/gsp")
 app.include_router(status_router, prefix=f"{v0_route_solar}")
diff --git a/src/national.py b/src/national.py
index b73da1fd..75156f50 100644
--- a/src/national.py
+++ b/src/national.py
@@ -1,10 +1,14 @@
 """National API routes"""
 
 import os
+from datetime import datetime, timedelta
 from typing import List, Optional, Union
 
+import pandas as pd
 import structlog
-from fastapi import APIRouter, Depends, HTTPException, Request, Security
+from elexonpy.api.generation_forecast_api import GenerationForecastApi
+from elexonpy.api_client import ApiClient
+from fastapi import APIRouter, Depends, HTTPException, Query, Request, Security
 from fastapi_auth0 import Auth0User
 from nowcasting_datamodel.read.read import get_latest_forecast_for_gsps
 from sqlalchemy.orm.session import Session
@@ -16,12 +20,17 @@
     get_session,
     get_truth_values_for_a_specific_gsp_from_database,
 )
-from pydantic_models import NationalForecast, NationalForecastValue, NationalYield
+from pydantic_models import (
+    NationalForecast,
+    NationalForecastValue,
+    NationalYield,
+    SolarForecastResponse,
+    SolarForecastValue,
+)
 from utils import N_CALLS_PER_HOUR, filter_forecast_values, format_datetime, format_plevels, limiter
 
 logger = structlog.stdlib.get_logger()
 
-
 adjust_limit = float(os.getenv("ADJUST_MW_LIMIT", 0.0))
 get_plevels = bool(os.getenv("GET_PLEVELS", True))
 
@@ -29,6 +38,80 @@
     tags=["National"],
 )
 
+# Initialize Elexon API client
+api_client = ApiClient()
+forecast_api = GenerationForecastApi(api_client)
+
+
+@router.get("/elexon", summary="Get elexon Solar Forecast")
+@limiter.limit(f"{N_CALLS_PER_HOUR}/hour")
+def get_elexon_forecast(
+    request: Request,
+    start_datetime_utc: datetime = Query(
+        default=datetime.utcnow() - timedelta(days=3), description="Start date and time in UTC"
+    ),
+    end_datetime_utc: datetime = Query(
+        default=datetime.utcnow() + timedelta(days=3), description="End date and time in UTC"
+    ),
+    process_type: str = Query("Day Ahead", description="Process type"),
+):
+    """
+    Fetch elexon Solar and wind(?) forecasts from the Elexon API.
+
+    Args:
+        request (Request): The request object containing metadata about the HTTP request.
+        start_datetime_utc (datetime): The start date and time in UTC.
+        end_datetime_utc (datetime): The end date and time in UTC.
+        process_type (str): The type of process (e.g., 'Day Ahead').
+
+    Returns:
+        SolarForecastResponse: The forecast data wrapped in a SolarForecastResponse model.
+    """
+    try:
+        response = forecast_api.forecast_generation_wind_and_solar_day_ahead_get(
+            _from=start_datetime_utc.isoformat(),
+            to=end_datetime_utc.isoformat(),
+            process_type=process_type,
+            format="json",
+        )
+
+        if not response.data:
+            return SolarForecastResponse(data=[])
+
+        df = pd.DataFrame([item.to_dict() for item in response.data])
+        logger.debug("DataFrame Columns: %s", df.columns)
+        logger.debug("DataFrame Sample: %s", df.head())
+
+        # Filter to include only solar forecasts
+        solar_df = df[df["business_type"] == "Solar generation"]
+        logger.debug("Filtered Solar DataFrame: %s", solar_df.head())
+
+        forecast_values = []
+        for _, row in solar_df.iterrows():
+            try:
+                forecast_values.append(
+                    SolarForecastValue(
+                        timestamp=pd.to_datetime(row["publish_time"]).to_pydatetime(),
+                        expected_power_generation_megawatts=row.get("quantity"),
+                        plevels=None,
+                    )
+                )
+            except KeyError as e:
+                logger.error("KeyError: %s. Data: %s", str(e), row)
+                raise HTTPException(status_code=500, detail="Internal Server Error")
+            except Exception as e:
+                logger.error(
+                    "Error during DataFrame to Model conversion: %s. Data: %s", str(e), row
+                )
+                raise HTTPException(status_code=500, detail="Internal Server Error")
+
+        result = SolarForecastResponse(data=forecast_values)
+        return result
+
+    except Exception as e:
+        logger.error("Unhandled exception: %s", str(e))
+        raise HTTPException(status_code=500, detail="Internal Server Error")
+
 
 @router.get(
     "/forecast",
@@ -47,7 +130,9 @@ def get_national_forecast(
     end_datetime_utc: Optional[str] = None,
     creation_limit_utc: Optional[str] = None,
 ) -> Union[NationalForecast, List[NationalForecastValue]]:
-    """Get the National Forecast
+    """
+
+    Fetch national forecasts.
 
     This route returns the most recent forecast for each _target_time_.
 
@@ -67,6 +152,9 @@ def get_national_forecast(
     - **creation_utc_limit**: optional, only return forecasts made before this datetime.
     Note you can only go 7 days back at the moment
 
+    Returns:
+        dict: The national forecast data.
+
     """
     logger.debug("Get national forecasts")
 
@@ -182,35 +270,10 @@ def get_national_pvlive(
 
     #### Parameters
     - **regime**: can choose __in-day__ or __day-after__
-    """
 
+    """
     logger.info(f"Get national PV Live estimates values " f"for regime {regime} for  {user}")
 
     return get_truth_values_for_a_specific_gsp_from_database(
         session=session, gsp_id=0, regime=regime
     )
-
-
-@router.get(
-    "/bmrs",
-    response_model=dict,
-    # dependencies=[Depends(get_auth_implicit_scheme())],
-    summary="Get BMRS Forecast",
-)
-# @cache_response
-@limiter.limit(f"{N_CALLS_PER_HOUR}/hour")
-def get_bmrs_forecast(
-    request: Request,
-    # session: Session = Depends(get_session),
-    # user: Auth0User = Security(get_user()),
-) -> dict:
-    """
-
-    This route returns the most recent BMRS forecast for each _target_time_.
-
-    #### Parameters
-
-    """
-    logger.debug("Get bmrs forecast")
-
-    return {"message": "This route is not yet implemented. Please check back later."}
diff --git a/src/pydantic_models.py b/src/pydantic_models.py
index ca01033c..10f9f64f 100644
--- a/src/pydantic_models.py
+++ b/src/pydantic_models.py
@@ -7,7 +7,7 @@
 
 from nowcasting_datamodel.models import Forecast, ForecastSQL, ForecastValue, Location, LocationSQL
 from nowcasting_datamodel.models.utils import EnhancedBaseModel
-from pydantic import Field, validator
+from pydantic import BaseModel, Field, validator
 
 logger = logging.getLogger(__name__)
 
@@ -214,3 +214,25 @@ class NationalForecast(Forecast):
     """One Forecast of generation at one timestamp"""
 
     forecast_values: List[NationalForecastValue] = Field(..., description="List of forecast values")
+
+
+class SolarForecastValue(BaseModel):
+    """Represents a single solar forecast entry"""
+
+    timestamp: datetime = Field(..., description="Timestamp of the forecast")
+    expected_power_generation_megawatts: Optional[float] = Field(
+        None, ge=0, description="Expected power generation in megawatts"
+    )
+
+    @validator("expected_power_generation_megawatts")
+    def result_check(cls, v):
+        """Round to 2 decimal places"""
+        if v is not None:
+            return round(v, 2)
+        return v
+
+
+class SolarForecastResponse(BaseModel):
+    """Wrapper for a list of solar forecast values"""
+
+    data: List[SolarForecastValue] = Field(..., description="List of solar forecast values")
diff --git a/src/tests/test_elexon_forecast.py b/src/tests/test_elexon_forecast.py
new file mode 100644
index 00000000..07f45017
--- /dev/null
+++ b/src/tests/test_elexon_forecast.py
@@ -0,0 +1,63 @@
+import pytest
+import requests
+import requests_mock
+
+API_URL = "/v0/solar/GB/national/elexon"
+
+
+@pytest.fixture
+def mock_data():
+    return {
+        "data": [
+            {
+                "timestamp": "2024-07-24T16:45:09+00:00",
+                "expected_power_generation_megawatts": 0,
+                "plevels": None,
+            },
+            {
+                "timestamp": "2024-07-24T16:45:09+00:00",
+                "expected_power_generation_megawatts": 0,
+                "plevels": None,
+            },
+        ]
+    }
+
+
+def test_get_elexon_forecast_with_data(mock_data):
+    with requests_mock.Mocker() as m:
+        url = (
+            f"{API_URL}?start_datetime_utc=2024-07-22T10:56:59.194610"
+            f"&end_datetime_utc=2024-07-28T10:56:59.194680"
+            f"&process_type=Day Ahead"
+        )
+        m.get(url, json=mock_data, headers={"Content-Type": "application/json"})
+
+        response = requests.get(url)
+        print("Response Headers:", response.headers)
+        # Assertions
+        assert response.status_code == 200
+        assert response.headers.get("Content-Type") == "application/json"
+        assert response.json() == mock_data
+
+
+@pytest.fixture
+def empty_mock_data():
+    return {"data": []}
+
+
+def test_get_elexon_forecast_no_data(empty_mock_data):
+    with requests_mock.Mocker() as m:
+        url = (
+            f"{API_URL}?start_datetime_utc=2024-07-22T10:56:59.194610"
+            f"&end_datetime_utc=2024-07-28T10:56:59.194680"
+            f"&process_type=Day Ahead"
+        )
+
+        m.get(url, json=empty_mock_data, headers={"Content-Type": "application/json"})
+
+        response = requests.get(url)
+        print("Response Headers:", response.headers)
+        # Assertions
+        assert response.status_code == 200
+        assert response.headers.get("Content-Type") == "application/json"
+        assert response.json() == empty_mock_data

From 3f4a6ba964d4c88a7c99efd7ba34ce407884a1e1 Mon Sep 17 00:00:00 2001
From: peterdudfield <peter.dudfield@hotmail.com>
Date: Wed, 31 Jul 2024 14:27:18 +0100
Subject: [PATCH 02/14] first try at mocking elexon api

---
 src/tests/test_elexon_forecast.py | 8 +++-----
 1 file changed, 3 insertions(+), 5 deletions(-)

diff --git a/src/tests/test_elexon_forecast.py b/src/tests/test_elexon_forecast.py
index 07f45017..666b34ce 100644
--- a/src/tests/test_elexon_forecast.py
+++ b/src/tests/test_elexon_forecast.py
@@ -23,16 +23,14 @@ def mock_data():
     }
 
 
-def test_get_elexon_forecast_with_data(mock_data):
+def test_get_elexon_forecast_with_data(mock_data, api_client):
     with requests_mock.Mocker() as m:
         url = (
-            f"{API_URL}?start_datetime_utc=2024-07-22T10:56:59.194610"
-            f"&end_datetime_utc=2024-07-28T10:56:59.194680"
-            f"&process_type=Day Ahead"
+            f"https://data.elexon.co.uk/bmrs/api/v1/forecast/generation/wind-and-solar/day-ahead"
         )
         m.get(url, json=mock_data, headers={"Content-Type": "application/json"})
 
-        response = requests.get(url)
+        response = api_client.get(url)
         print("Response Headers:", response.headers)
         # Assertions
         assert response.status_code == 200

From 45431fab8999d52b799684ed8d2be6aaef66856c Mon Sep 17 00:00:00 2001
From: peterdudfield <peter.dudfield@hotmail.com>
Date: Wed, 31 Jul 2024 14:33:29 +0100
Subject: [PATCH 03/14] upadte url for api

---
 src/tests/test_elexon_forecast.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/tests/test_elexon_forecast.py b/src/tests/test_elexon_forecast.py
index 666b34ce..52422d44 100644
--- a/src/tests/test_elexon_forecast.py
+++ b/src/tests/test_elexon_forecast.py
@@ -30,7 +30,7 @@ def test_get_elexon_forecast_with_data(mock_data, api_client):
         )
         m.get(url, json=mock_data, headers={"Content-Type": "application/json"})
 
-        response = api_client.get(url)
+        response = api_client.get('/v0/solar/GB/national/elexon')
         print("Response Headers:", response.headers)
         # Assertions
         assert response.status_code == 200

From edf4107408d42e9cb50c831c59d73604860c7d93 Mon Sep 17 00:00:00 2001
From: peterdudfield <peter.dudfield@hotmail.com>
Date: Wed, 31 Jul 2024 15:31:45 +0100
Subject: [PATCH 04/14] update test, remove try and except

---
 src/national.py                   |  78 ++++++++++-----------
 src/tests/test_elexon_forecast.py | 112 +++++++++++++++---------------
 2 files changed, 92 insertions(+), 98 deletions(-)

diff --git a/src/national.py b/src/national.py
index 75156f50..9c53956a 100644
--- a/src/national.py
+++ b/src/national.py
@@ -41,6 +41,7 @@
 # Initialize Elexon API client
 api_client = ApiClient()
 forecast_api = GenerationForecastApi(api_client)
+forecast_generation_wind_and_solar_day_ahead_get = forecast_api.forecast_generation_wind_and_solar_day_ahead_get
 
 
 @router.get("/elexon", summary="Get elexon Solar Forecast")
@@ -67,50 +68,45 @@ def get_elexon_forecast(
     Returns:
         SolarForecastResponse: The forecast data wrapped in a SolarForecastResponse model.
     """
-    try:
-        response = forecast_api.forecast_generation_wind_and_solar_day_ahead_get(
-            _from=start_datetime_utc.isoformat(),
-            to=end_datetime_utc.isoformat(),
-            process_type=process_type,
-            format="json",
-        )
 
-        if not response.data:
-            return SolarForecastResponse(data=[])
-
-        df = pd.DataFrame([item.to_dict() for item in response.data])
-        logger.debug("DataFrame Columns: %s", df.columns)
-        logger.debug("DataFrame Sample: %s", df.head())
-
-        # Filter to include only solar forecasts
-        solar_df = df[df["business_type"] == "Solar generation"]
-        logger.debug("Filtered Solar DataFrame: %s", solar_df.head())
-
-        forecast_values = []
-        for _, row in solar_df.iterrows():
-            try:
-                forecast_values.append(
-                    SolarForecastValue(
-                        timestamp=pd.to_datetime(row["publish_time"]).to_pydatetime(),
-                        expected_power_generation_megawatts=row.get("quantity"),
-                        plevels=None,
-                    )
-                )
-            except KeyError as e:
-                logger.error("KeyError: %s. Data: %s", str(e), row)
-                raise HTTPException(status_code=500, detail="Internal Server Error")
-            except Exception as e:
-                logger.error(
-                    "Error during DataFrame to Model conversion: %s. Data: %s", str(e), row
-                )
-                raise HTTPException(status_code=500, detail="Internal Server Error")
+    response = forecast_generation_wind_and_solar_day_ahead_get(
+        _from=start_datetime_utc.isoformat(),
+        to=end_datetime_utc.isoformat(),
+        process_type=process_type,
+        format="json",
+    )
+
+    if not response.data:
+        return SolarForecastResponse(data=[])
 
-        result = SolarForecastResponse(data=forecast_values)
-        return result
+    df = pd.DataFrame([item.to_dict() for item in response.data])
+    logger.debug("DataFrame Columns: %s", df.columns)
+    logger.debug("DataFrame Sample: %s", df.head())
+
+    # Filter to include only solar forecasts
+    solar_df = df[df["business_type"] == "Solar generation"]
+    logger.debug("Filtered Solar DataFrame: %s", solar_df.head())
+
+    forecast_values = []
+    for _, row in solar_df.iterrows():
+        try:
+            forecast_values.append(
+                SolarForecastValue(
+                    timestamp=pd.to_datetime(row["publish_time"]).to_pydatetime(),
+                    expected_power_generation_megawatts=row.get("quantity"),
+                )
+            )
+        except KeyError as e:
+            logger.error("KeyError: %s. Data: %s", str(e), row)
+            raise HTTPException(status_code=500, detail="Internal Server Error")
+        except Exception as e:
+            logger.error(
+                "Error during DataFrame to Model conversion: %s. Data: %s", str(e), row
+            )
+            raise HTTPException(status_code=500, detail="Internal Server Error")
 
-    except Exception as e:
-        logger.error("Unhandled exception: %s", str(e))
-        raise HTTPException(status_code=500, detail="Internal Server Error")
+    result = SolarForecastResponse(data=forecast_values)
+    return result
 
 
 @router.get(
diff --git a/src/tests/test_elexon_forecast.py b/src/tests/test_elexon_forecast.py
index 52422d44..96d6ba24 100644
--- a/src/tests/test_elexon_forecast.py
+++ b/src/tests/test_elexon_forecast.py
@@ -1,61 +1,59 @@
-import pytest
-import requests
-import requests_mock
+from unittest.mock import patch
+
+import pandas as pd
+
+from pydantic_models import BaseModel
+from typing import Optional
+
+# InsightsApiModelsResponsesResponseWithMetadata1InsightsApiModelsResponsesTransparencyDayAheadGenerationForWindAndSolar
 
 API_URL = "/v0/solar/GB/national/elexon"
 
 
-@pytest.fixture
-def mock_data():
-    return {
-        "data": [
-            {
-                "timestamp": "2024-07-24T16:45:09+00:00",
-                "expected_power_generation_megawatts": 0,
-                "plevels": None,
-            },
-            {
-                "timestamp": "2024-07-24T16:45:09+00:00",
-                "expected_power_generation_megawatts": 0,
-                "plevels": None,
-            },
-        ]
-    }
-
-
-def test_get_elexon_forecast_with_data(mock_data, api_client):
-    with requests_mock.Mocker() as m:
-        url = (
-            f"https://data.elexon.co.uk/bmrs/api/v1/forecast/generation/wind-and-solar/day-ahead"
-        )
-        m.get(url, json=mock_data, headers={"Content-Type": "application/json"})
-
-        response = api_client.get('/v0/solar/GB/national/elexon')
-        print("Response Headers:", response.headers)
-        # Assertions
-        assert response.status_code == 200
-        assert response.headers.get("Content-Type") == "application/json"
-        assert response.json() == mock_data
-
-
-@pytest.fixture
-def empty_mock_data():
-    return {"data": []}
-
-
-def test_get_elexon_forecast_no_data(empty_mock_data):
-    with requests_mock.Mocker() as m:
-        url = (
-            f"{API_URL}?start_datetime_utc=2024-07-22T10:56:59.194610"
-            f"&end_datetime_utc=2024-07-28T10:56:59.194680"
-            f"&process_type=Day Ahead"
-        )
-
-        m.get(url, json=empty_mock_data, headers={"Content-Type": "application/json"})
-
-        response = requests.get(url)
-        print("Response Headers:", response.headers)
-        # Assertions
-        assert response.status_code == 200
-        assert response.headers.get("Content-Type") == "application/json"
-        assert response.json() == empty_mock_data
+class MockClass(BaseModel):
+
+    publish_time: str
+    quantity: float
+    business_type: Optional[str] = "Solar generation"
+
+    def to_dict(self):
+        return self.__dict__
+
+
+mock_data = [
+    MockClass(
+        **{
+            "publish_time": "2024-07-24T16:45:09+00:00",
+            "quantity": 0,
+        }
+    ),
+    MockClass(
+        **{
+            "publish_time": "2024-07-24T16:45:09+00:00",
+            "quantity": 0,
+        }
+    ),
+]
+
+
+class MockResponse:
+    def __init__(self):
+        self.data = mock_data
+
+
+@patch("national.forecast_generation_wind_and_solar_day_ahead_get")
+def test_get_elexon_forecast_with_data(mock_function, api_client):
+    mock_function.return_value = MockResponse()
+
+    response = api_client.get("/v0/solar/GB/national/elexon")
+    print("Response Headers:", response.headers)
+    # Assertions
+    assert response.status_code == 200
+    assert response.headers.get("Content-Type") == "application/json"
+
+    api_data = response.json()["data"]
+    assert len(api_data) == len(mock_data)
+    for i in range(len(api_data)):
+        assert api_data[i]["expected_power_generation_megawatts"] == mock_data[i].quantity
+        assert pd.Timestamp(api_data[i]["timestamp"]) == pd.Timestamp(mock_data[i].publish_time)
+

From 5af3f6e31578aa7331381b0456809d56d97fa238 Mon Sep 17 00:00:00 2001
From: "pre-commit-ci[bot]"
 <66853113+pre-commit-ci[bot]@users.noreply.github.com>
Date: Wed, 31 Jul 2024 14:35:49 +0000
Subject: [PATCH 05/14] [pre-commit.ci] auto fixes from pre-commit.com hooks

for more information, see https://pre-commit.ci
---
 src/national.py                   | 8 ++++----
 src/tests/test_elexon_forecast.py | 1 -
 2 files changed, 4 insertions(+), 5 deletions(-)

diff --git a/src/national.py b/src/national.py
index 9c53956a..45347d82 100644
--- a/src/national.py
+++ b/src/national.py
@@ -41,7 +41,9 @@
 # Initialize Elexon API client
 api_client = ApiClient()
 forecast_api = GenerationForecastApi(api_client)
-forecast_generation_wind_and_solar_day_ahead_get = forecast_api.forecast_generation_wind_and_solar_day_ahead_get
+forecast_generation_wind_and_solar_day_ahead_get = (
+    forecast_api.forecast_generation_wind_and_solar_day_ahead_get
+)
 
 
 @router.get("/elexon", summary="Get elexon Solar Forecast")
@@ -100,9 +102,7 @@ def get_elexon_forecast(
             logger.error("KeyError: %s. Data: %s", str(e), row)
             raise HTTPException(status_code=500, detail="Internal Server Error")
         except Exception as e:
-            logger.error(
-                "Error during DataFrame to Model conversion: %s. Data: %s", str(e), row
-            )
+            logger.error("Error during DataFrame to Model conversion: %s. Data: %s", str(e), row)
             raise HTTPException(status_code=500, detail="Internal Server Error")
 
     result = SolarForecastResponse(data=forecast_values)
diff --git a/src/tests/test_elexon_forecast.py b/src/tests/test_elexon_forecast.py
index 96d6ba24..cf3697b4 100644
--- a/src/tests/test_elexon_forecast.py
+++ b/src/tests/test_elexon_forecast.py
@@ -56,4 +56,3 @@ def test_get_elexon_forecast_with_data(mock_function, api_client):
     for i in range(len(api_data)):
         assert api_data[i]["expected_power_generation_megawatts"] == mock_data[i].quantity
         assert pd.Timestamp(api_data[i]["timestamp"]) == pd.Timestamp(mock_data[i].publish_time)
-

From fea937453953f07cb9fcacdbe23ade4f357a61bb Mon Sep 17 00:00:00 2001
From: peterdudfield <peter.dudfield@hotmail.com>
Date: Wed, 31 Jul 2024 15:43:42 +0100
Subject: [PATCH 06/14] isort

---
 src/tests/test_elexon_forecast.py | 4 +---
 1 file changed, 1 insertion(+), 3 deletions(-)

diff --git a/src/tests/test_elexon_forecast.py b/src/tests/test_elexon_forecast.py
index cf3697b4..e00f1c07 100644
--- a/src/tests/test_elexon_forecast.py
+++ b/src/tests/test_elexon_forecast.py
@@ -1,11 +1,9 @@
+from typing import Optional
 from unittest.mock import patch
 
 import pandas as pd
 
 from pydantic_models import BaseModel
-from typing import Optional
-
-# InsightsApiModelsResponsesResponseWithMetadata1InsightsApiModelsResponsesTransparencyDayAheadGenerationForWindAndSolar
 
 API_URL = "/v0/solar/GB/national/elexon"
 

From faa79400d8b557ffb5e4c88c1630d08d6b1086f6 Mon Sep 17 00:00:00 2001
From: peterdudfield <peter.dudfield@hotmail.com>
Date: Wed, 31 Jul 2024 16:07:13 +0100
Subject: [PATCH 07/14] and try and except

---
 src/national.py | 18 ++++++++++++------
 1 file changed, 12 insertions(+), 6 deletions(-)

diff --git a/src/national.py b/src/national.py
index 45347d82..6ee28f01 100644
--- a/src/national.py
+++ b/src/national.py
@@ -71,12 +71,18 @@ def get_elexon_forecast(
         SolarForecastResponse: The forecast data wrapped in a SolarForecastResponse model.
     """
 
-    response = forecast_generation_wind_and_solar_day_ahead_get(
-        _from=start_datetime_utc.isoformat(),
-        to=end_datetime_utc.isoformat(),
-        process_type=process_type,
-        format="json",
-    )
+    try:
+        response = forecast_generation_wind_and_solar_day_ahead_get(
+            _from=start_datetime_utc.isoformat(),
+            to=end_datetime_utc.isoformat(),
+            process_type=process_type,
+            format="json",
+        )
+    except Exception as e:
+        logger.error("Unhandled exception when collecting ELexon Data: %s", str(e))
+        raise HTTPException(
+            status_code=500, detail="Internal Server Error when collecting Elexon Data"
+        )
 
     if not response.data:
         return SolarForecastResponse(data=[])

From deb714d3413f7d65c0c03c091829ee0c0ee13422 Mon Sep 17 00:00:00 2001
From: peterdudfield <peter.dudfield@hotmail.com>
Date: Thu, 1 Aug 2024 10:29:26 +0100
Subject: [PATCH 08/14] add intergation test

---
 src/tests/test_elexon_forecast.py | 22 ++++++++++++++++++++--
 1 file changed, 20 insertions(+), 2 deletions(-)

diff --git a/src/tests/test_elexon_forecast.py b/src/tests/test_elexon_forecast.py
index e00f1c07..5b0eec5a 100644
--- a/src/tests/test_elexon_forecast.py
+++ b/src/tests/test_elexon_forecast.py
@@ -2,8 +2,10 @@
 from unittest.mock import patch
 
 import pandas as pd
+import pytest
+
+from pydantic_models import BaseModel, SolarForecastResponse
 
-from pydantic_models import BaseModel
 
 API_URL = "/v0/solar/GB/national/elexon"
 
@@ -40,7 +42,7 @@ def __init__(self):
 
 
 @patch("national.forecast_generation_wind_and_solar_day_ahead_get")
-def test_get_elexon_forecast_with_data(mock_function, api_client):
+def test_get_elexon_forecast_mock(mock_function, api_client):
     mock_function.return_value = MockResponse()
 
     response = api_client.get("/v0/solar/GB/national/elexon")
@@ -54,3 +56,19 @@ def test_get_elexon_forecast_with_data(mock_function, api_client):
     for i in range(len(api_data)):
         assert api_data[i]["expected_power_generation_megawatts"] == mock_data[i].quantity
         assert pd.Timestamp(api_data[i]["timestamp"]) == pd.Timestamp(mock_data[i].publish_time)
+
+
+@pytest.mark.integration
+def test_get_elexon_forecast(api_client):
+
+    response = api_client.get("/v0/solar/GB/national/elexon")
+
+    # Assertions
+    assert response.status_code == 200
+    assert response.headers.get("Content-Type") == "application/json"
+
+    api_data = response.json()["data"]
+
+    solar_forecast = SolarForecastResponse(**response.json())
+
+    assert len(solar_forecast.data) > 0

From 13a916d4d607e249d8612f3a039558aad5465243 Mon Sep 17 00:00:00 2001
From: peterdudfield <peter.dudfield@hotmail.com>
Date: Thu, 1 Aug 2024 10:36:18 +0100
Subject: [PATCH 09/14] lint

---
 src/tests/test_elexon_forecast.py | 3 ---
 1 file changed, 3 deletions(-)

diff --git a/src/tests/test_elexon_forecast.py b/src/tests/test_elexon_forecast.py
index 5b0eec5a..c456d1d2 100644
--- a/src/tests/test_elexon_forecast.py
+++ b/src/tests/test_elexon_forecast.py
@@ -6,7 +6,6 @@
 
 from pydantic_models import BaseModel, SolarForecastResponse
 
-
 API_URL = "/v0/solar/GB/national/elexon"
 
 
@@ -67,8 +66,6 @@ def test_get_elexon_forecast(api_client):
     assert response.status_code == 200
     assert response.headers.get("Content-Type") == "application/json"
 
-    api_data = response.json()["data"]
-
     solar_forecast = SolarForecastResponse(**response.json())
 
     assert len(solar_forecast.data) > 0

From 0fb7505cbfd0520ea127181f7c6468032e634a1b Mon Sep 17 00:00:00 2001
From: peterdudfield <peter.dudfield@hotmail.com>
Date: Mon, 12 Aug 2024 16:26:35 +0100
Subject: [PATCH 10/14] PR comments, move elexon to below forecast and pvlive

---
 src/national.py                   | 145 +++++++++++++++---------------
 src/tests/test_elexon_forecast.py |   8 +-
 2 files changed, 76 insertions(+), 77 deletions(-)

diff --git a/src/national.py b/src/national.py
index 6ee28f01..996e3baf 100644
--- a/src/national.py
+++ b/src/national.py
@@ -40,81 +40,12 @@
 
 # Initialize Elexon API client
 api_client = ApiClient()
-forecast_api = GenerationForecastApi(api_client)
-forecast_generation_wind_and_solar_day_ahead_get = (
-    forecast_api.forecast_generation_wind_and_solar_day_ahead_get
+elexon_forecast_api = GenerationForecastApi(api_client)
+elexon_forecast_generation_wind_and_solar_day_ahead_get = (
+    elexon_forecast_api.forecast_generation_wind_and_solar_day_ahead_get
 )
 
 
-@router.get("/elexon", summary="Get elexon Solar Forecast")
-@limiter.limit(f"{N_CALLS_PER_HOUR}/hour")
-def get_elexon_forecast(
-    request: Request,
-    start_datetime_utc: datetime = Query(
-        default=datetime.utcnow() - timedelta(days=3), description="Start date and time in UTC"
-    ),
-    end_datetime_utc: datetime = Query(
-        default=datetime.utcnow() + timedelta(days=3), description="End date and time in UTC"
-    ),
-    process_type: str = Query("Day Ahead", description="Process type"),
-):
-    """
-    Fetch elexon Solar and wind(?) forecasts from the Elexon API.
-
-    Args:
-        request (Request): The request object containing metadata about the HTTP request.
-        start_datetime_utc (datetime): The start date and time in UTC.
-        end_datetime_utc (datetime): The end date and time in UTC.
-        process_type (str): The type of process (e.g., 'Day Ahead').
-
-    Returns:
-        SolarForecastResponse: The forecast data wrapped in a SolarForecastResponse model.
-    """
-
-    try:
-        response = forecast_generation_wind_and_solar_day_ahead_get(
-            _from=start_datetime_utc.isoformat(),
-            to=end_datetime_utc.isoformat(),
-            process_type=process_type,
-            format="json",
-        )
-    except Exception as e:
-        logger.error("Unhandled exception when collecting ELexon Data: %s", str(e))
-        raise HTTPException(
-            status_code=500, detail="Internal Server Error when collecting Elexon Data"
-        )
-
-    if not response.data:
-        return SolarForecastResponse(data=[])
-
-    df = pd.DataFrame([item.to_dict() for item in response.data])
-    logger.debug("DataFrame Columns: %s", df.columns)
-    logger.debug("DataFrame Sample: %s", df.head())
-
-    # Filter to include only solar forecasts
-    solar_df = df[df["business_type"] == "Solar generation"]
-    logger.debug("Filtered Solar DataFrame: %s", solar_df.head())
-
-    forecast_values = []
-    for _, row in solar_df.iterrows():
-        try:
-            forecast_values.append(
-                SolarForecastValue(
-                    timestamp=pd.to_datetime(row["publish_time"]).to_pydatetime(),
-                    expected_power_generation_megawatts=row.get("quantity"),
-                )
-            )
-        except KeyError as e:
-            logger.error("KeyError: %s. Data: %s", str(e), row)
-            raise HTTPException(status_code=500, detail="Internal Server Error")
-        except Exception as e:
-            logger.error("Error during DataFrame to Model conversion: %s. Data: %s", str(e), row)
-            raise HTTPException(status_code=500, detail="Internal Server Error")
-
-    result = SolarForecastResponse(data=forecast_values)
-    return result
-
-
 @router.get(
     "/forecast",
     response_model=Union[NationalForecast, List[NationalForecastValue]],
@@ -279,3 +210,73 @@ def get_national_pvlive(
     return get_truth_values_for_a_specific_gsp_from_database(
         session=session, gsp_id=0, regime=regime
     )
+
+
+
+@router.get("/elexon", summary="Get elexon Solar Forecast")
+@limiter.limit(f"{N_CALLS_PER_HOUR}/hour")
+def get_elexon_forecast(
+    request: Request,
+    start_datetime_utc: datetime = Query(
+        default=datetime.utcnow() - timedelta(days=3), description="Start date and time in UTC"
+    ),
+    end_datetime_utc: datetime = Query(
+        default=datetime.utcnow() + timedelta(days=3), description="End date and time in UTC"
+    ),
+    process_type: str = Query("Day Ahead", description="Process type"),
+):
+    """
+    Fetch elexon Solar forecasts from the Elexon API.
+
+    #### Parameters:
+    - **start_datetime_utc** (datetime): The start date and time in UTC.
+    - **end_datetime_utc** (datetime): The end date and time in UTC.
+    - **process_type** (str): The type of process
+            (e.g., 'Day Ahead', 'Intraday Process' or  'Intraday Total').
+
+    Returns:
+        SolarForecastResponse: The forecast data wrapped in a SolarForecastResponse model.
+    """
+
+    try:
+        response = elexon_forecast_generation_wind_and_solar_day_ahead_get(
+            _from=start_datetime_utc.isoformat(),
+            to=end_datetime_utc.isoformat(),
+            process_type=process_type,
+            format="json",
+        )
+    except Exception as e:
+        logger.error("Unhandled exception when collecting ELexon Data: %s", str(e))
+        raise HTTPException(
+            status_code=500, detail="Internal Server Error when collecting Elexon Data"
+        )
+
+    if not response.data:
+        return SolarForecastResponse(data=[])
+
+    df = pd.DataFrame([item.to_dict() for item in response.data])
+    logger.debug("DataFrame Columns: %s", df.columns)
+    logger.debug("DataFrame Sample: %s", df.head())
+
+    # Filter to include only solar forecasts
+    solar_df = df[df["business_type"] == "Solar generation"]
+    logger.debug("Filtered Solar DataFrame: %s", solar_df.head())
+
+    forecast_values = []
+    for _, row in solar_df.iterrows():
+        try:
+            forecast_values.append(
+                SolarForecastValue(
+                    timestamp=pd.to_datetime(row["start_time"]).to_pydatetime(),
+                    expected_power_generation_megawatts=row.get("quantity"),
+                )
+            )
+        except KeyError as e:
+            logger.error("KeyError: %s. Data: %s", str(e), row)
+            raise HTTPException(status_code=500, detail="Internal Server Error")
+        except Exception as e:
+            logger.error("Error during DataFrame to Model conversion: %s. Data: %s", str(e), row)
+            raise HTTPException(status_code=500, detail="Internal Server Error")
+
+    result = SolarForecastResponse(data=forecast_values)
+    return result
diff --git a/src/tests/test_elexon_forecast.py b/src/tests/test_elexon_forecast.py
index c456d1d2..46aa61ab 100644
--- a/src/tests/test_elexon_forecast.py
+++ b/src/tests/test_elexon_forecast.py
@@ -6,8 +6,6 @@
 
 from pydantic_models import BaseModel, SolarForecastResponse
 
-API_URL = "/v0/solar/GB/national/elexon"
-
 
 class MockClass(BaseModel):
 
@@ -22,13 +20,13 @@ def to_dict(self):
 mock_data = [
     MockClass(
         **{
-            "publish_time": "2024-07-24T16:45:09+00:00",
+            "start_time": "2024-07-24T16:00:00+00:00",
             "quantity": 0,
         }
     ),
     MockClass(
         **{
-            "publish_time": "2024-07-24T16:45:09+00:00",
+            "start_time": "2024-07-24T16:30:00+00:00",
             "quantity": 0,
         }
     ),
@@ -40,7 +38,7 @@ def __init__(self):
         self.data = mock_data
 
 
-@patch("national.forecast_generation_wind_and_solar_day_ahead_get")
+@patch("national.elexon_forecast_generation_wind_and_solar_day_ahead_get")
 def test_get_elexon_forecast_mock(mock_function, api_client):
     mock_function.return_value = MockResponse()
 

From d19dee10e13d4fb42232fdc60f2757685dc78a66 Mon Sep 17 00:00:00 2001
From: "pre-commit-ci[bot]"
 <66853113+pre-commit-ci[bot]@users.noreply.github.com>
Date: Mon, 12 Aug 2024 15:26:57 +0000
Subject: [PATCH 11/14] [pre-commit.ci] auto fixes from pre-commit.com hooks

for more information, see https://pre-commit.ci
---
 src/national.py | 1 -
 1 file changed, 1 deletion(-)

diff --git a/src/national.py b/src/national.py
index 996e3baf..acabbe39 100644
--- a/src/national.py
+++ b/src/national.py
@@ -212,7 +212,6 @@ def get_national_pvlive(
     )
 
 
-
 @router.get("/elexon", summary="Get elexon Solar Forecast")
 @limiter.limit(f"{N_CALLS_PER_HOUR}/hour")
 def get_elexon_forecast(

From 8401d817248c739ad5fc879129a635cb55ad0da6 Mon Sep 17 00:00:00 2001
From: peterdudfield <peter.dudfield@hotmail.com>
Date: Mon, 12 Aug 2024 16:29:01 +0100
Subject: [PATCH 12/14] change docker-compose to docker compose

---
 .github/workflows/test-docker.yaml | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/.github/workflows/test-docker.yaml b/.github/workflows/test-docker.yaml
index cb07a9ab..54ecf201 100644
--- a/.github/workflows/test-docker.yaml
+++ b/.github/workflows/test-docker.yaml
@@ -15,10 +15,10 @@ jobs:
         uses: actions/checkout@v2
 
       - name: Build Docker image
-        run: docker-compose -f test-docker-compose.yml build
+        run: docker compose -f test-docker-compose.yml build
 
       - name: Run tests inside the container
-        run: docker-compose -f test-docker-compose.yml run api
+        run: docker compose -f test-docker-compose.yml run api
 
       - name: Copy coverage
         run: |

From ceadff6ca7ab6211f7d1b31f51418e10f58fc7ac Mon Sep 17 00:00:00 2001
From: peterdudfield <peter.dudfield@hotmail.com>
Date: Mon, 12 Aug 2024 16:31:50 +0100
Subject: [PATCH 13/14] fix test

---
 src/tests/test_elexon_forecast.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/src/tests/test_elexon_forecast.py b/src/tests/test_elexon_forecast.py
index 46aa61ab..77214208 100644
--- a/src/tests/test_elexon_forecast.py
+++ b/src/tests/test_elexon_forecast.py
@@ -9,7 +9,7 @@
 
 class MockClass(BaseModel):
 
-    publish_time: str
+    start_time: str
     quantity: float
     business_type: Optional[str] = "Solar generation"
 
@@ -52,7 +52,7 @@ def test_get_elexon_forecast_mock(mock_function, api_client):
     assert len(api_data) == len(mock_data)
     for i in range(len(api_data)):
         assert api_data[i]["expected_power_generation_megawatts"] == mock_data[i].quantity
-        assert pd.Timestamp(api_data[i]["timestamp"]) == pd.Timestamp(mock_data[i].publish_time)
+        assert pd.Timestamp(api_data[i]["timestamp"]) == pd.Timestamp(mock_data[i].start_time)
 
 
 @pytest.mark.integration

From 9afb2f1b3d624d2e3b361b723a8ed041cf0a6589 Mon Sep 17 00:00:00 2001
From: braddf <bradbdf@gmail.com>
Date: Tue, 13 Aug 2024 12:23:10 +0100
Subject: [PATCH 14/14] Minor change to mock method on real class

---
 src/national.py                   |  5 +----
 src/tests/test_elexon_forecast.py | 24 ++++++++++++------------
 2 files changed, 13 insertions(+), 16 deletions(-)

diff --git a/src/national.py b/src/national.py
index acabbe39..b9c0d5fa 100644
--- a/src/national.py
+++ b/src/national.py
@@ -41,9 +41,6 @@
 # Initialize Elexon API client
 api_client = ApiClient()
 elexon_forecast_api = GenerationForecastApi(api_client)
-elexon_forecast_generation_wind_and_solar_day_ahead_get = (
-    elexon_forecast_api.forecast_generation_wind_and_solar_day_ahead_get
-)
 
 
 @router.get(
@@ -238,7 +235,7 @@ def get_elexon_forecast(
     """
 
     try:
-        response = elexon_forecast_generation_wind_and_solar_day_ahead_get(
+        response = elexon_forecast_api.forecast_generation_wind_and_solar_day_ahead_get(
             _from=start_datetime_utc.isoformat(),
             to=end_datetime_utc.isoformat(),
             process_type=process_type,
diff --git a/src/tests/test_elexon_forecast.py b/src/tests/test_elexon_forecast.py
index 77214208..c0e4f25f 100644
--- a/src/tests/test_elexon_forecast.py
+++ b/src/tests/test_elexon_forecast.py
@@ -1,14 +1,16 @@
 from typing import Optional
-from unittest.mock import patch
+from unittest.mock import MagicMock, patch
 
 import pandas as pd
 import pytest
+from elexonpy.api.generation_forecast_api import GenerationForecastApi
 
 from pydantic_models import BaseModel, SolarForecastResponse
 
+API_URL = "/v0/solar/GB/national/elexon"
 
-class MockClass(BaseModel):
 
+class MockClass(BaseModel):
     start_time: str
     quantity: float
     business_type: Optional[str] = "Solar generation"
@@ -31,18 +33,17 @@ def to_dict(self):
         }
     ),
 ]
+mock_response = MagicMock()
+mock_response.data = mock_data
 
 
-class MockResponse:
-    def __init__(self):
-        self.data = mock_data
-
-
-@patch("national.elexon_forecast_generation_wind_and_solar_day_ahead_get")
+@patch.object(GenerationForecastApi, "forecast_generation_wind_and_solar_day_ahead_get")
 def test_get_elexon_forecast_mock(mock_function, api_client):
-    mock_function.return_value = MockResponse()
+    # Set mock_response
+    mock_function.return_value = mock_response
 
-    response = api_client.get("/v0/solar/GB/national/elexon")
+    # Call the API endpoint
+    response = api_client.get(API_URL)
     print("Response Headers:", response.headers)
     # Assertions
     assert response.status_code == 200
@@ -57,8 +58,7 @@ def test_get_elexon_forecast_mock(mock_function, api_client):
 
 @pytest.mark.integration
 def test_get_elexon_forecast(api_client):
-
-    response = api_client.get("/v0/solar/GB/national/elexon")
+    response = api_client.get(API_URL)
 
     # Assertions
     assert response.status_code == 200