From 5ab9cdc83550f369bd828f8e8844deb85aa4a13d Mon Sep 17 00:00:00 2001 From: braddf Date: Tue, 28 Jan 2025 11:56:07 +0000 Subject: [PATCH 01/14] Amend docker setup for local fake data --- README.md | 38 +++++++++++++++++++++++--------------- docker-compose.yml | 24 +++++++++++++----------- script/fake_data.py | 16 +++++++++++++++- src/database.py | 7 +++++-- 4 files changed, 56 insertions(+), 29 deletions(-) diff --git a/README.md b/README.md index 4d11444a..f11c13ba 100644 --- a/README.md +++ b/README.md @@ -62,25 +62,33 @@ python3 -m venv ./venv source venv/bin/activate ``` -Install Requirements and Run +### Running the API -```bash -pip install -r requirements.txt -cd src && uvicorn main:app --reload -``` - -Warning: -If you don't have a local database set up, you can leave the `DB_URL` string empty (default not set) -and set `FAKE=True` and the API will return fake data. This is a work in progress, -so some routes might be need to be updated - -### Docker - 🛑 Currently non-functional, needs updating to migrate database to match datamodel +#### Option 1: Docker + 🟢 __Preferred method__ 1. Make sure docker is installed on your system. 2. Use `docker-compose up` - in the main directory to start up the application. -3. You will now be able to access it on `http://localhost:80` + in the main directory with the optional `--build` flag to build the image the first time + to start up the application. This builds the image, sets up the database, seeds some fake data + and starts the API. +3. You will now be able to access it on `http://localhost:8000` +4. The API should restart automatically when you make changes to the code, but the fake + data currently is static. To seed new fake data, just manually restart the API. + +#### Option 2: Running the API with a local database + +To set up the API with a local database, you will need to: + - start your own local postgres instance on your machine + - set `FAKE=1` in the `.env` file + - set `DB_URL` to your local postgres instance in the `.env` file + - run the following commands to install required packages, create the tables in your local postgres instance, and populate them with fake data: + +```bash +pip install -r requirements.txt +cd src +uvicorn main:app --reload +``` ### Running the test suite diff --git a/docker-compose.yml b/docker-compose.yml index 6e1db58a..7ba396db 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -2,6 +2,7 @@ version: "3" services: postgres: + platform: linux/amd64 image: postgres:14.5 restart: always environment: @@ -10,12 +11,12 @@ services: ports: - "5432:5432" - datamodel: - image: openclimatefix/nowcasting_datamodel - environment: - - DB_URL=postgresql://postgres:postgres@postgres:5432/postgres - depends_on: - - "postgres" + # datamodel: + # image: openclimatefix/nowcasting_datamodel + # environment: + # - DB_URL=postgresql://postgres:postgres@postgres:5432/postgres + # depends_on: + # - "postgres" api: build: @@ -23,18 +24,19 @@ services: dockerfile: Dockerfile # image: openclimatefix/nowcasting_api:0.1.7 container_name: nowcasting_api - command: bash -c "sleep 5 + command: bash -c "sleep 2 && python script/fake_data.py - && uvicorn src.main:app --host 0.0.0.0 --port 8000" + && uvicorn src.main:app --reload --host 0.0.0.0 --port 8000" ports: - 8000:8000 environment: - DB_URL=postgresql://postgres:postgres@postgres:5432/postgres - - AUTH0_DOMAIN=nowcasting-dev.eu.auth0.com - - AUTH0_API_AUDIENCE=https://nowcasting-api-eu-auth0.com/ + # - AUTH0_DOMAIN=nowcasting-dev.eu.auth0.com + # - AUTH0_API_AUDIENCE=https://nowcasting-api-eu-auth0.com/ + # - FAKE=1 volumes: - ./src/:/app/src - ./script/:/app/script depends_on: - "postgres" - - "datamodel" +# - "datamodel" diff --git a/script/fake_data.py b/script/fake_data.py index 2da14cf4..9f93019a 100644 --- a/script/fake_data.py +++ b/script/fake_data.py @@ -20,13 +20,27 @@ ForecastValueSQL, ) from nowcasting_datamodel.models.models import StatusSQL +from sqlalchemy import inspect from src.utils import floor_30_minutes_dt now = floor_30_minutes_dt(datetime.now(tz=timezone.utc)) +if os.getenv("DB_URL") is None: + raise ValueError("Please set the DB_URL environment variable") + +if not os.getenv("DB_URL") == "postgresql://postgres:postgres@postgres:5432/postgres": + raise ValueError("This script should only be run in the local docker container") + connection = DatabaseConnection(url=os.getenv("DB_URL", "not_set")) +print(f"Creating fake data at {now}") + +# Check database has been created, if not run the migrations +if not inspect(connection.engine).has_table("status"): + connection.create_all() + + with connection.get_session() as session: session.query(StatusSQL).delete() session.query(ForecastValueLatestSQL).delete() @@ -48,7 +62,7 @@ make_fake_gsp_yields(gsp_ids=range(0, N_GSPS), session=session, t0_datetime_utc=now) # 3. make status - status = StatusSQL(status="warning", message="this is all fake data") + status = StatusSQL(status="warning", message="Local Quartz API serving fake data") session.add(status) session.commit() diff --git a/src/database.py b/src/database.py index bb778216..f330b07b 100644 --- a/src/database.py +++ b/src/database.py @@ -45,6 +45,8 @@ ) from utils import filter_forecast_values, floor_30_minutes_dt, get_start_datetime +logger = structlog.stdlib.get_logger() + class BaseDBConnection(abc.ABC): """This is a base class for database connections with one static method get_connection(). @@ -289,9 +291,10 @@ def get_latest_forecast_values_for_a_specific_gsp_from_database( session=session, gsp_id=gsp_id, start_datetime=start_datetime, - model_name="blend", + model_name="fake_model", end_datetime=end_datetime_utc, ) + logger.info(f"forecast_values: {forecast_values}") else: if creation_utc_limit is not None and creation_utc_limit < datetime.now( @@ -311,7 +314,7 @@ def get_latest_forecast_values_for_a_specific_gsp_from_database( start_datetime=start_datetime, end_datetime=end_datetime_utc, forecast_horizon_minutes=forecast_horizon_minutes, - model_name="blend", + model_name="fake_model", model=model, only_return_latest=True, created_utc_limit=creation_utc_limit, From 8fa200c897e0c594cebc0f0f62cdb7ec4a22f7f0 Mon Sep 17 00:00:00 2001 From: braddf Date: Tue, 28 Jan 2025 12:07:05 +0000 Subject: [PATCH 02/14] revert model name change --- src/database.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/database.py b/src/database.py index f330b07b..21a8a594 100644 --- a/src/database.py +++ b/src/database.py @@ -291,7 +291,7 @@ def get_latest_forecast_values_for_a_specific_gsp_from_database( session=session, gsp_id=gsp_id, start_datetime=start_datetime, - model_name="fake_model", + model_name="blend", end_datetime=end_datetime_utc, ) logger.info(f"forecast_values: {forecast_values}") From 4ae42c0f5fbd1de33aee3ddc40d91a485c795efb Mon Sep 17 00:00:00 2001 From: braddf Date: Tue, 28 Jan 2025 12:39:23 +0000 Subject: [PATCH 03/14] revert model name change #2 --- src/database.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/database.py b/src/database.py index 21a8a594..98510ff0 100644 --- a/src/database.py +++ b/src/database.py @@ -314,7 +314,7 @@ def get_latest_forecast_values_for_a_specific_gsp_from_database( start_datetime=start_datetime, end_datetime=end_datetime_utc, forecast_horizon_minutes=forecast_horizon_minutes, - model_name="fake_model", + model_name="blend", model=model, only_return_latest=True, created_utc_limit=creation_utc_limit, From a7c59f4290cdc47adb62ae0e96e418584b87e0c7 Mon Sep 17 00:00:00 2001 From: braddf Date: Tue, 28 Jan 2025 12:48:30 +0000 Subject: [PATCH 04/14] set fake data model name to be queried --- script/fake_data.py | 1 + 1 file changed, 1 insertion(+) diff --git a/script/fake_data.py b/script/fake_data.py index 9f93019a..79556aad 100644 --- a/script/fake_data.py +++ b/script/fake_data.py @@ -53,6 +53,7 @@ make_fake_forecasts( gsp_ids=range(0, N_GSPS), session=session, + model_name="blend", t0_datetime_utc=now, add_latest=True, historic=True, From cbff1a6be211b71bb9fbd4658420c30160a9bb07 Mon Sep 17 00:00:00 2001 From: braddf Date: Wed, 29 Jan 2025 18:12:05 +0000 Subject: [PATCH 05/14] correct fake data datetime issues --- requirements.txt | 2 +- src/gsp.py | 19 +++++++++++++------ src/national.py | 25 ++++++++++++++++++++++--- src/utils.py | 7 +++++++ 4 files changed, 43 insertions(+), 10 deletions(-) diff --git a/requirements.txt b/requirements.txt index b340e768..87d16b64 100644 --- a/requirements.txt +++ b/requirements.txt @@ -4,7 +4,7 @@ uvicorn[standard] pydantic numpy requests -nowcasting_datamodel==1.5.62 +nowcasting_datamodel==1.5.59 sqlalchemy psycopg2-binary geopandas diff --git a/src/gsp.py b/src/gsp.py index f7e7074f..c25bf368 100644 --- a/src/gsp.py +++ b/src/gsp.py @@ -1,7 +1,6 @@ """Get GSP boundary data from eso """ import os -from datetime import datetime, timezone from typing import List, Optional, Union import structlog @@ -31,8 +30,8 @@ from utils import ( N_CALLS_PER_HOUR, N_SLOW_CALLS_PER_HOUR, - floor_30_minutes_dt, format_datetime, + get_rounded_30_min_before_now, limiter, ) @@ -104,7 +103,9 @@ def get_all_available_forecasts( if gsp_ids is None: gsp_ids = [int(gsp_id) for gsp_id in range(1, GSP_TOTAL)] - make_fake_forecasts(gsp_ids=gsp_ids, session=session) + make_fake_forecasts( + gsp_ids=gsp_ids, session=session, t0_datetime_utc=get_rounded_30_min_before_now() + ) logger.info(f"Get forecasts for all gsps. The option is {historic=} for user {user}") @@ -217,7 +218,9 @@ def get_forecasts_for_a_specific_gsp( returns the latest forecast made 60 minutes before the target time) """ if is_fake(): - make_fake_forecast(gsp_id=gsp_id, session=session) + make_fake_forecast( + gsp_id=gsp_id, session=session, t0_datetime_utc=get_rounded_30_min_before_now() + ) logger.info(f"Get forecasts for gsp id {gsp_id} forecast of forecast with only values.") logger.info(f"This is for user {user}") @@ -292,7 +295,9 @@ def get_truths_for_all_gsps( if gsp_ids is None: gsp_ids = [int(gsp_id) for gsp_id in range(1, GSP_TOTAL)] - make_fake_gsp_yields(gsp_ids=gsp_ids, session=session) + make_fake_gsp_yields( + gsp_ids=gsp_ids, session=session, t0_datetime_utc=get_rounded_30_min_before_now() + ) logger.info(f"Get PV Live estimates values for all gsp id and regime {regime} for user {user}") @@ -376,7 +381,9 @@ def get_truths_for_a_specific_gsp( """ if is_fake(): - make_fake_forecast(gsp_id=gsp_id, session=session) + make_fake_forecast( + gsp_id=gsp_id, session=session, t0_datetime_utc=get_rounded_30_min_before_now() + ) logger.info( f"Get PV Live estimates values for gsp id {gsp_id} " f"and regime {regime} for user {user}" diff --git a/src/national.py b/src/national.py index ef03d28e..a689a45f 100644 --- a/src/national.py +++ b/src/national.py @@ -28,7 +28,14 @@ SolarForecastResponse, SolarForecastValue, ) -from utils import N_CALLS_PER_HOUR, filter_forecast_values, format_datetime, format_plevels, limiter +from utils import ( + N_CALLS_PER_HOUR, + filter_forecast_values, + format_datetime, + format_plevels, + get_rounded_30_min_before_now, + limiter, +) logger = structlog.stdlib.get_logger() @@ -95,7 +102,15 @@ def get_national_forecast( logger.debug("Get national forecasts") if is_fake: - make_fake_forecast(gsp_id=0, session=session) + fake_forecast = make_fake_forecast( + gsp_id=0, + model_name="blend", + session=session, + t0_datetime_utc=get_rounded_30_min_before_now(), + add_latest=True, + ) + # add the forecast to the session, as this single fake function doesn't by default + session.add(fake_forecast) start_datetime_utc = format_datetime(start_datetime_utc) end_datetime_utc = format_datetime(end_datetime_utc) @@ -214,7 +229,11 @@ def get_national_pvlive( logger.info(f"Get national PV Live estimates values " f"for regime {regime} for {user}") if is_fake(): - make_fake_gsp_yields(gsp_ids=[0], session=session) + make_fake_gsp_yields( + gsp_ids=[0], + session=session, + t0_datetime_utc=get_rounded_30_min_before_now(), + ) return get_truth_values_for_a_specific_gsp_from_database( session=session, gsp_id=0, regime=regime diff --git a/src/utils.py b/src/utils.py index 248abc6c..2aebac34 100644 --- a/src/utils.py +++ b/src/utils.py @@ -43,6 +43,13 @@ def floor_30_minutes_dt(dt): return dt +def get_rounded_30_min_before_now(): + """ + Get the rounded 30 minutes before now, e.g. 14:00, 14:30, 15:00, etc. + """ + return floor_30_minutes_dt(datetime.now(tz=utc)) + + def floor_6_hours_dt(dt: datetime): """ Floor a datetime by 6 hours. From ce759769a732c0b12585c5c46eced744484dbedd Mon Sep 17 00:00:00 2001 From: braddf Date: Thu, 30 Jan 2025 09:26:41 +0000 Subject: [PATCH 06/14] remove unneeded log --- src/database.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/src/database.py b/src/database.py index 98510ff0..bb778216 100644 --- a/src/database.py +++ b/src/database.py @@ -45,8 +45,6 @@ ) from utils import filter_forecast_values, floor_30_minutes_dt, get_start_datetime -logger = structlog.stdlib.get_logger() - class BaseDBConnection(abc.ABC): """This is a base class for database connections with one static method get_connection(). @@ -294,7 +292,6 @@ def get_latest_forecast_values_for_a_specific_gsp_from_database( model_name="blend", end_datetime=end_datetime_utc, ) - logger.info(f"forecast_values: {forecast_values}") else: if creation_utc_limit is not None and creation_utc_limit < datetime.now( From 4efc4aef0b83f4b0540b8436fcb37b8fc27a3b0e Mon Sep 17 00:00:00 2001 From: braddf Date: Thu, 30 Jan 2025 09:30:59 +0000 Subject: [PATCH 07/14] add missing parentheses to is_fake call --- src/national.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/national.py b/src/national.py index a689a45f..2538de0c 100644 --- a/src/national.py +++ b/src/national.py @@ -101,7 +101,7 @@ def get_national_forecast( """ logger.debug("Get national forecasts") - if is_fake: + if is_fake(): fake_forecast = make_fake_forecast( gsp_id=0, model_name="blend", From 6e6260ef3fe7fadf24a98341fd4395ef84e22541 Mon Sep 17 00:00:00 2001 From: braddf Date: Mon, 3 Feb 2025 14:47:27 +0000 Subject: [PATCH 08/14] remove unneeded non-pure util function --- src/gsp.py | 19 ++++++++++++++----- src/national.py | 8 ++++---- 2 files changed, 18 insertions(+), 9 deletions(-) diff --git a/src/gsp.py b/src/gsp.py index c25bf368..0b4c503f 100644 --- a/src/gsp.py +++ b/src/gsp.py @@ -1,6 +1,7 @@ """Get GSP boundary data from eso """ import os +from datetime import UTC, datetime from typing import List, Optional, Union import structlog @@ -30,8 +31,8 @@ from utils import ( N_CALLS_PER_HOUR, N_SLOW_CALLS_PER_HOUR, + floor_30_minutes_dt, format_datetime, - get_rounded_30_min_before_now, limiter, ) @@ -104,7 +105,9 @@ def get_all_available_forecasts( gsp_ids = [int(gsp_id) for gsp_id in range(1, GSP_TOTAL)] make_fake_forecasts( - gsp_ids=gsp_ids, session=session, t0_datetime_utc=get_rounded_30_min_before_now() + gsp_ids=gsp_ids, + session=session, + t0_datetime_utc=floor_30_minutes_dt(datetime.now(tz=UTC)), ) logger.info(f"Get forecasts for all gsps. The option is {historic=} for user {user}") @@ -219,7 +222,9 @@ def get_forecasts_for_a_specific_gsp( """ if is_fake(): make_fake_forecast( - gsp_id=gsp_id, session=session, t0_datetime_utc=get_rounded_30_min_before_now() + gsp_id=gsp_id, + session=session, + t0_datetime_utc=floor_30_minutes_dt(datetime.now(tz=UTC)), ) logger.info(f"Get forecasts for gsp id {gsp_id} forecast of forecast with only values.") @@ -296,7 +301,9 @@ def get_truths_for_all_gsps( gsp_ids = [int(gsp_id) for gsp_id in range(1, GSP_TOTAL)] make_fake_gsp_yields( - gsp_ids=gsp_ids, session=session, t0_datetime_utc=get_rounded_30_min_before_now() + gsp_ids=gsp_ids, + session=session, + t0_datetime_utc=floor_30_minutes_dt(datetime.now(tz=UTC)), ) logger.info(f"Get PV Live estimates values for all gsp id and regime {regime} for user {user}") @@ -382,7 +389,9 @@ def get_truths_for_a_specific_gsp( if is_fake(): make_fake_forecast( - gsp_id=gsp_id, session=session, t0_datetime_utc=get_rounded_30_min_before_now() + gsp_id=gsp_id, + session=session, + t0_datetime_utc=floor_30_minutes_dt(datetime.now(tz=UTC)), ) logger.info( diff --git a/src/national.py b/src/national.py index 2538de0c..ef2c160d 100644 --- a/src/national.py +++ b/src/national.py @@ -1,7 +1,7 @@ """National API routes""" import os -from datetime import datetime, timedelta +from datetime import UTC, datetime, timedelta from typing import List, Optional, Union import pandas as pd @@ -31,9 +31,9 @@ from utils import ( N_CALLS_PER_HOUR, filter_forecast_values, + floor_30_minutes_dt, format_datetime, format_plevels, - get_rounded_30_min_before_now, limiter, ) @@ -106,7 +106,7 @@ def get_national_forecast( gsp_id=0, model_name="blend", session=session, - t0_datetime_utc=get_rounded_30_min_before_now(), + t0_datetime_utc=floor_30_minutes_dt(datetime.now(tz=UTC)), add_latest=True, ) # add the forecast to the session, as this single fake function doesn't by default @@ -232,7 +232,7 @@ def get_national_pvlive( make_fake_gsp_yields( gsp_ids=[0], session=session, - t0_datetime_utc=get_rounded_30_min_before_now(), + t0_datetime_utc=floor_30_minutes_dt(datetime.now(tz=UTC)), ) return get_truth_values_for_a_specific_gsp_from_database( From 2de0bfd4b843b4f95472c48bb26c0e205c610767 Mon Sep 17 00:00:00 2001 From: braddf Date: Thu, 6 Feb 2025 10:17:39 +0000 Subject: [PATCH 09/14] remove unneeded non-pure util function --- src/utils.py | 7 ------- 1 file changed, 7 deletions(-) diff --git a/src/utils.py b/src/utils.py index 2aebac34..248abc6c 100644 --- a/src/utils.py +++ b/src/utils.py @@ -43,13 +43,6 @@ def floor_30_minutes_dt(dt): return dt -def get_rounded_30_min_before_now(): - """ - Get the rounded 30 minutes before now, e.g. 14:00, 14:30, 15:00, etc. - """ - return floor_30_minutes_dt(datetime.now(tz=utc)) - - def floor_6_hours_dt(dt: datetime): """ Floor a datetime by 6 hours. From cadfdb8d5d763243c1c10039b95a4ce72c24d89f Mon Sep 17 00:00:00 2001 From: braddf Date: Thu, 6 Mar 2025 13:55:36 +0000 Subject: [PATCH 10/14] refactor to run with fake data cron in docker --- Dockerfile | 6 +- docker-compose-local-datamodel.yml | 56 +++++++++++++ docker-compose.yml | 24 ++++-- .../QUARTZSOLAR_LOGO_SECONDARY_BLACK_1.png | Bin nowcasting_api/__init__.py | 0 {src => nowcasting_api}/auth_utils.py | 0 {src => nowcasting_api}/cache.py | 0 {src => nowcasting_api}/favicon.ico | Bin {src => nowcasting_api}/gsp.py | 76 +++++++++--------- {src => nowcasting_api}/main.py | 2 +- {src => nowcasting_api}/national.py | 32 ++++---- {src => nowcasting_api}/pydantic_models.py | 0 {src => nowcasting_api}/redoc_theme.py | 0 {src => nowcasting_api}/status.py | 0 {src => nowcasting_api}/system.py | 0 {src => nowcasting_api}/test.db-journal | Bin {src => nowcasting_api}/tests/conftest.py | 0 .../tests/fake/test_gsp_fake.py | 0 .../tests/fake/test_national_fake.py | 0 .../tests/test_database.py | 0 .../tests/test_elexon_forecast.py | 0 {src => nowcasting_api}/tests/test_gsp.py | 0 {src => nowcasting_api}/tests/test_main.py | 0 .../tests/test_merged_routes.py | 0 .../tests/test_national.py | 0 {src => nowcasting_api}/tests/test_status.py | 0 {src => nowcasting_api}/tests/test_system.py | 0 {src => nowcasting_api}/tests/test_utils.py | 0 {src => nowcasting_api}/utils.py | 2 +- requirements.txt | 2 +- script/fake_data.py | 40 ++++++--- 31 files changed, 167 insertions(+), 73 deletions(-) create mode 100644 docker-compose-local-datamodel.yml rename {src => nowcasting_api}/QUARTZSOLAR_LOGO_SECONDARY_BLACK_1.png (100%) create mode 100644 nowcasting_api/__init__.py rename {src => nowcasting_api}/auth_utils.py (100%) rename {src => nowcasting_api}/cache.py (100%) rename {src => nowcasting_api}/favicon.ico (100%) rename {src => nowcasting_api}/gsp.py (89%) rename {src => nowcasting_api}/main.py (99%) rename {src => nowcasting_api}/national.py (94%) rename {src => nowcasting_api}/pydantic_models.py (100%) rename {src => nowcasting_api}/redoc_theme.py (100%) rename {src => nowcasting_api}/status.py (100%) rename {src => nowcasting_api}/system.py (100%) rename {src => nowcasting_api}/test.db-journal (100%) rename {src => nowcasting_api}/tests/conftest.py (100%) rename {src => nowcasting_api}/tests/fake/test_gsp_fake.py (100%) rename {src => nowcasting_api}/tests/fake/test_national_fake.py (100%) rename {src => nowcasting_api}/tests/test_database.py (100%) rename {src => nowcasting_api}/tests/test_elexon_forecast.py (100%) rename {src => nowcasting_api}/tests/test_gsp.py (100%) rename {src => nowcasting_api}/tests/test_main.py (100%) rename {src => nowcasting_api}/tests/test_merged_routes.py (100%) rename {src => nowcasting_api}/tests/test_national.py (100%) rename {src => nowcasting_api}/tests/test_status.py (100%) rename {src => nowcasting_api}/tests/test_system.py (100%) rename {src => nowcasting_api}/tests/test_utils.py (100%) rename {src => nowcasting_api}/utils.py (98%) diff --git a/Dockerfile b/Dockerfile index c8c99fe0..3aacad7f 100644 --- a/Dockerfile +++ b/Dockerfile @@ -13,11 +13,11 @@ RUN pip install -r /app/requirements.txt WORKDIR /app # copy files over -COPY ./src /app/src +COPY nowcasting_api /app/nowcasting_api COPY ./script /app/script # pin coverage RUN pip install -U coverage -# make sure 'src' is in python path - this is so imports work -ENV PYTHONPATH=${PYTHONPATH}:/app/src +# make sure 'nowcasting_api' is in python path - this is so imports work +ENV PYTHONPATH=${PYTHONPATH}:/app/nowcasting_api diff --git a/docker-compose-local-datamodel.yml b/docker-compose-local-datamodel.yml new file mode 100644 index 00000000..8b2dd3b4 --- /dev/null +++ b/docker-compose-local-datamodel.yml @@ -0,0 +1,56 @@ +services: + postgres: + platform: linux/amd64 + image: postgres:14.5 + restart: always + environment: + - POSTGRES_USER=postgres + - POSTGRES_PASSWORD=postgres + ports: + - "5432:5432" + + # datamodel: + # image: openclimatefix/nowcasting_datamodel + # environment: + # - DB_URL=postgresql://postgres:postgres@postgres:5432/postgres + # depends_on: + # - "postgres" + + api: + build: + context: . + dockerfile: Dockerfile + # image: openclimatefix/nowcasting_api:0.1.7 + container_name: nowcasting_api + command: bash -c "sleep 2 + && apt-get update + && apt-get install -y cron + && echo 'starting cron' + && cron + && pip install file:/app/nowcasting_datamodel + && sleep 2 + && python script/fake_data.py + && uvicorn nowcasting_api.main:app --reload --host 0.0.0.0 --port 8000" + ports: + - "8000:8000" + environment: + - DB_URL=postgresql://postgres:postgres@postgres:5432/postgres + # - AUTH0_DOMAIN=nowcasting-dev.eu.auth0.com + # - AUTH0_API_AUDIENCE=https://nowcasting-api-eu-auth0.com/ + volumes: + - ./nowcasting_api/:/app/nowcasting_api + - ./script/:/app/script + - ../nowcasting_datamodel/:/app/nowcasting_datamodel + working_dir: /app + configs: + - source: crontab + target: /etc/cron.d/crontab + mode: 0644 + depends_on: + - "postgres" +# - "datamodel" + +configs: + crontab: + content: | + */15 * * * * root PYTHONPATH=/app DB_URL=postgresql://postgres:postgres@postgres:5432/postgres /usr/local/bin/python -m script.fake_data > /proc/1/fd/1 2>/proc/1/fd/2 diff --git a/docker-compose.yml b/docker-compose.yml index 7ba396db..2d0e9175 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -1,5 +1,3 @@ -version: "3" - services: postgres: platform: linux/amd64 @@ -25,18 +23,32 @@ services: # image: openclimatefix/nowcasting_api:0.1.7 container_name: nowcasting_api command: bash -c "sleep 2 + && apt-get update + && apt-get install -y cron + && cron + && sleep 2 && python script/fake_data.py - && uvicorn src.main:app --reload --host 0.0.0.0 --port 8000" + && uvicorn nowcasting_api.main:app --reload --host 0.0.0.0 --port 8000" ports: - - 8000:8000 + - "8000:8000" environment: - DB_URL=postgresql://postgres:postgres@postgres:5432/postgres # - AUTH0_DOMAIN=nowcasting-dev.eu.auth0.com # - AUTH0_API_AUDIENCE=https://nowcasting-api-eu-auth0.com/ - # - FAKE=1 volumes: - - ./src/:/app/src + - ./nowcasting_api/:/app/nowcasting_api - ./script/:/app/script + # - ../nowcasting_datamodel/:/app/nowcasting_datamodel + working_dir: /app + configs: + - source: crontab + target: /etc/cron.d/crontab + mode: 0644 depends_on: - "postgres" # - "datamodel" + +configs: + crontab: + content: | + */15 * * * * root PYTHONPATH=/app DB_URL=postgresql://postgres:postgres@postgres:5432/postgres /usr/local/bin/python -m script.fake_data > /proc/1/fd/1 2>/proc/1/fd/2 diff --git a/src/QUARTZSOLAR_LOGO_SECONDARY_BLACK_1.png b/nowcasting_api/QUARTZSOLAR_LOGO_SECONDARY_BLACK_1.png similarity index 100% rename from src/QUARTZSOLAR_LOGO_SECONDARY_BLACK_1.png rename to nowcasting_api/QUARTZSOLAR_LOGO_SECONDARY_BLACK_1.png diff --git a/nowcasting_api/__init__.py b/nowcasting_api/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/src/auth_utils.py b/nowcasting_api/auth_utils.py similarity index 100% rename from src/auth_utils.py rename to nowcasting_api/auth_utils.py diff --git a/src/cache.py b/nowcasting_api/cache.py similarity index 100% rename from src/cache.py rename to nowcasting_api/cache.py diff --git a/src/favicon.ico b/nowcasting_api/favicon.ico similarity index 100% rename from src/favicon.ico rename to nowcasting_api/favicon.ico diff --git a/src/gsp.py b/nowcasting_api/gsp.py similarity index 89% rename from src/gsp.py rename to nowcasting_api/gsp.py index 0b4c503f..c1f82cc6 100644 --- a/src/gsp.py +++ b/nowcasting_api/gsp.py @@ -61,8 +61,8 @@ def is_fake(): response_model=Union[ManyForecasts, List[OneDatetimeManyForecastValues]], dependencies=[Depends(get_auth_implicit_scheme())], ) -@cache_response -@limiter.limit(f"{N_SLOW_CALLS_PER_HOUR}/hour") +# @cache_response +# @limiter.limit(f"{N_SLOW_CALLS_PER_HOUR}/hour") def get_all_available_forecasts( request: Request, historic: Optional[bool] = True, @@ -100,15 +100,15 @@ def get_all_available_forecasts( if gsp_ids == "": gsp_ids = None - if is_fake(): - if gsp_ids is None: - gsp_ids = [int(gsp_id) for gsp_id in range(1, GSP_TOTAL)] - - make_fake_forecasts( - gsp_ids=gsp_ids, - session=session, - t0_datetime_utc=floor_30_minutes_dt(datetime.now(tz=UTC)), - ) + # if is_fake(): + # if gsp_ids is None: + # gsp_ids = [int(gsp_id) for gsp_id in range(1, 10)] + # + # fake_forecasts = make_fake_forecasts( + # gsp_ids=gsp_ids, + # session=session, + # t0_datetime_utc=floor_30_minutes_dt(datetime.now(tz=UTC)), + # ) logger.info(f"Get forecasts for all gsps. The option is {historic=} for user {user}") @@ -130,8 +130,12 @@ def get_all_available_forecasts( creation_utc_limit=creation_limit_utc, ) + logger.info(f"Got forecasts for all gsps. The option is {historic=} for user {user}") + if not compact: + logger.info(f"Normalizing forecasts") forecasts.normalize() + logger.info(f"Normalized forecasts") logger.info( f"Got {len(forecasts.forecasts)} forecasts for all gsps. " @@ -169,8 +173,8 @@ def get_forecasts_for_a_specific_gsp_old_route( ) -> Union[Forecast, List[ForecastValue]]: """Redirects old API route to new route /v0/solar/GB/gsp/{gsp_id}/forecast""" - if is_fake(): - make_fake_forecast(gsp_id=gsp_id, session=session) + # if is_fake(): + # make_fake_forecast(gsp_id=gsp_id, session=session) return get_forecasts_for_a_specific_gsp( request=request, @@ -220,12 +224,12 @@ def get_forecasts_for_a_specific_gsp( - **creation_utc_limit**: optional, only return forecasts made before this datetime. returns the latest forecast made 60 minutes before the target time) """ - if is_fake(): - make_fake_forecast( - gsp_id=gsp_id, - session=session, - t0_datetime_utc=floor_30_minutes_dt(datetime.now(tz=UTC)), - ) + # if is_fake(): + # make_fake_forecast( + # gsp_id=gsp_id, + # session=session, + # t0_datetime_utc=floor_30_minutes_dt(datetime.now(tz=UTC)), + # ) logger.info(f"Get forecasts for gsp id {gsp_id} forecast of forecast with only values.") logger.info(f"This is for user {user}") @@ -296,15 +300,15 @@ def get_truths_for_all_gsps( if isinstance(gsp_ids, str): gsp_ids = [int(gsp_id) for gsp_id in gsp_ids.split(",")] - if is_fake(): - if gsp_ids is None: - gsp_ids = [int(gsp_id) for gsp_id in range(1, GSP_TOTAL)] - - make_fake_gsp_yields( - gsp_ids=gsp_ids, - session=session, - t0_datetime_utc=floor_30_minutes_dt(datetime.now(tz=UTC)), - ) + # if is_fake(): + # if gsp_ids is None: + # gsp_ids = [int(gsp_id) for gsp_id in range(1, GSP_TOTAL)] + # + # make_fake_gsp_yields( + # gsp_ids=gsp_ids, + # session=session, + # t0_datetime_utc=floor_30_minutes_dt(datetime.now(tz=UTC)), + # ) logger.info(f"Get PV Live estimates values for all gsp id and regime {regime} for user {user}") @@ -339,8 +343,8 @@ def get_truths_for_a_specific_gsp_old_route( ) -> List[GSPYield]: """Redirects old API route to new route /v0/solar/GB/gsp/{gsp_id}/pvlive""" - if is_fake(): - make_fake_gsp_yields(gsp_ids=[gsp_id], session=session) + # if is_fake(): + # make_fake_gsp_yields(gsp_ids=[gsp_id], session=session) return get_truths_for_a_specific_gsp( request=request, @@ -387,12 +391,12 @@ def get_truths_for_a_specific_gsp( If not set, defaults to N_HISTORY_DAYS env var, which if not set defaults to yesterday. """ - if is_fake(): - make_fake_forecast( - gsp_id=gsp_id, - session=session, - t0_datetime_utc=floor_30_minutes_dt(datetime.now(tz=UTC)), - ) + # if is_fake(): + # make_fake_forecast( + # gsp_id=gsp_id, + # session=session, + # t0_datetime_utc=floor_30_minutes_dt(datetime.now(tz=UTC)), + # ) logger.info( f"Get PV Live estimates values for gsp id {gsp_id} " f"and regime {regime} for user {user}" diff --git a/src/main.py b/nowcasting_api/main.py similarity index 99% rename from src/main.py rename to nowcasting_api/main.py index a29f9a08..059a04d6 100644 --- a/src/main.py +++ b/nowcasting_api/main.py @@ -25,7 +25,7 @@ structlog.configure( wrapper_class=structlog.make_filtering_bound_logger( - getattr(logging, os.getenv("LOGLEVEL", "INFO")) + getattr(logging, os.getenv("LOGLEVEL", "DEBUG")) ), processors=[ structlog.processors.EventRenamer("message", replace_by="_event"), diff --git a/src/national.py b/nowcasting_api/national.py similarity index 94% rename from src/national.py rename to nowcasting_api/national.py index ef2c160d..7dbcd029 100644 --- a/src/national.py +++ b/nowcasting_api/national.py @@ -101,16 +101,16 @@ def get_national_forecast( """ logger.debug("Get national forecasts") - if is_fake(): - fake_forecast = make_fake_forecast( - gsp_id=0, - model_name="blend", - session=session, - t0_datetime_utc=floor_30_minutes_dt(datetime.now(tz=UTC)), - add_latest=True, - ) - # add the forecast to the session, as this single fake function doesn't by default - session.add(fake_forecast) + # if is_fake(): + # fake_forecast = make_fake_forecast( + # gsp_id=0, + # model_name="blend", + # session=session, + # t0_datetime_utc=floor_30_minutes_dt(datetime.now(tz=UTC)), + # add_latest=True, + # ) + # # add the forecast to the session, as this single fake function doesn't by default + # session.add(fake_forecast) start_datetime_utc = format_datetime(start_datetime_utc) end_datetime_utc = format_datetime(end_datetime_utc) @@ -228,12 +228,12 @@ def get_national_pvlive( """ logger.info(f"Get national PV Live estimates values " f"for regime {regime} for {user}") - if is_fake(): - make_fake_gsp_yields( - gsp_ids=[0], - session=session, - t0_datetime_utc=floor_30_minutes_dt(datetime.now(tz=UTC)), - ) + # if is_fake(): + # make_fake_gsp_yields( + # gsp_ids=[0], + # session=session, + # t0_datetime_utc=floor_30_minutes_dt(datetime.now(tz=UTC)), + # ) return get_truth_values_for_a_specific_gsp_from_database( session=session, gsp_id=0, regime=regime diff --git a/src/pydantic_models.py b/nowcasting_api/pydantic_models.py similarity index 100% rename from src/pydantic_models.py rename to nowcasting_api/pydantic_models.py diff --git a/src/redoc_theme.py b/nowcasting_api/redoc_theme.py similarity index 100% rename from src/redoc_theme.py rename to nowcasting_api/redoc_theme.py diff --git a/src/status.py b/nowcasting_api/status.py similarity index 100% rename from src/status.py rename to nowcasting_api/status.py diff --git a/src/system.py b/nowcasting_api/system.py similarity index 100% rename from src/system.py rename to nowcasting_api/system.py diff --git a/src/test.db-journal b/nowcasting_api/test.db-journal similarity index 100% rename from src/test.db-journal rename to nowcasting_api/test.db-journal diff --git a/src/tests/conftest.py b/nowcasting_api/tests/conftest.py similarity index 100% rename from src/tests/conftest.py rename to nowcasting_api/tests/conftest.py diff --git a/src/tests/fake/test_gsp_fake.py b/nowcasting_api/tests/fake/test_gsp_fake.py similarity index 100% rename from src/tests/fake/test_gsp_fake.py rename to nowcasting_api/tests/fake/test_gsp_fake.py diff --git a/src/tests/fake/test_national_fake.py b/nowcasting_api/tests/fake/test_national_fake.py similarity index 100% rename from src/tests/fake/test_national_fake.py rename to nowcasting_api/tests/fake/test_national_fake.py diff --git a/src/tests/test_database.py b/nowcasting_api/tests/test_database.py similarity index 100% rename from src/tests/test_database.py rename to nowcasting_api/tests/test_database.py diff --git a/src/tests/test_elexon_forecast.py b/nowcasting_api/tests/test_elexon_forecast.py similarity index 100% rename from src/tests/test_elexon_forecast.py rename to nowcasting_api/tests/test_elexon_forecast.py diff --git a/src/tests/test_gsp.py b/nowcasting_api/tests/test_gsp.py similarity index 100% rename from src/tests/test_gsp.py rename to nowcasting_api/tests/test_gsp.py diff --git a/src/tests/test_main.py b/nowcasting_api/tests/test_main.py similarity index 100% rename from src/tests/test_main.py rename to nowcasting_api/tests/test_main.py diff --git a/src/tests/test_merged_routes.py b/nowcasting_api/tests/test_merged_routes.py similarity index 100% rename from src/tests/test_merged_routes.py rename to nowcasting_api/tests/test_merged_routes.py diff --git a/src/tests/test_national.py b/nowcasting_api/tests/test_national.py similarity index 100% rename from src/tests/test_national.py rename to nowcasting_api/tests/test_national.py diff --git a/src/tests/test_status.py b/nowcasting_api/tests/test_status.py similarity index 100% rename from src/tests/test_status.py rename to nowcasting_api/tests/test_status.py diff --git a/src/tests/test_system.py b/nowcasting_api/tests/test_system.py similarity index 100% rename from src/tests/test_system.py rename to nowcasting_api/tests/test_system.py diff --git a/src/tests/test_utils.py b/nowcasting_api/tests/test_utils.py similarity index 100% rename from src/tests/test_utils.py rename to nowcasting_api/tests/test_utils.py diff --git a/src/utils.py b/nowcasting_api/utils.py similarity index 98% rename from src/utils.py rename to nowcasting_api/utils.py index 248abc6c..51f7afc6 100644 --- a/src/utils.py +++ b/nowcasting_api/utils.py @@ -20,7 +20,7 @@ limiter = Limiter(key_func=get_remote_address) N_CALLS_PER_HOUR = os.getenv("N_CALLS_PER_HOUR", 3600) # 1 call per second -N_SLOW_CALLS_PER_HOUR = os.getenv("N_SLOW_CALLS_PER_HOUR", 60) # 1 call per minute +N_SLOW_CALLS_PER_HOUR = os.getenv("N_SLOW_CALLS_PER_HOUR", 6000) # 1 call per minute def floor_30_minutes_dt(dt): diff --git a/requirements.txt b/requirements.txt index 87d16b64..b4f01307 100644 --- a/requirements.txt +++ b/requirements.txt @@ -4,7 +4,7 @@ uvicorn[standard] pydantic numpy requests -nowcasting_datamodel==1.5.59 +nowcasting_datamodel==1.5.63 sqlalchemy psycopg2-binary geopandas diff --git a/script/fake_data.py b/script/fake_data.py index 79556aad..ceccba5a 100644 --- a/script/fake_data.py +++ b/script/fake_data.py @@ -9,20 +9,30 @@ 3. a warning status of this is fake data """ +import sys + +sys.path.append("/app/nowcasting_api") + import os -from datetime import datetime, timezone +from datetime import datetime, timezone, UTC from nowcasting_datamodel.connection import DatabaseConnection -from nowcasting_datamodel.fake import make_fake_forecasts, make_fake_gsp_yields +from nowcasting_datamodel.fake import ( + generate_fake_forecasts, + make_fake_gsp_yields, + N_FAKE_FORECASTS, +) from nowcasting_datamodel.models.forecast import ( ForecastSQL, ForecastValueLatestSQL, + ForecastValueSevenDaysSQL, ForecastValueSQL, ) from nowcasting_datamodel.models.models import StatusSQL +from nowcasting_datamodel.save.save import save as save_forecasts from sqlalchemy import inspect -from src.utils import floor_30_minutes_dt +from nowcasting_api.utils import floor_30_minutes_dt now = floor_30_minutes_dt(datetime.now(tz=timezone.utc)) @@ -43,21 +53,33 @@ with connection.get_session() as session: session.query(StatusSQL).delete() + # TODO: maybe only delete data older than 4hrs to keep N-hr forecasts but keep DB small + session.query(ForecastValueSevenDaysSQL).delete() session.query(ForecastValueLatestSQL).delete() session.query(ForecastValueSQL).delete() session.query(ForecastSQL).delete() - N_GSPS = 10 + N_GSPS = 317 # 1. make fake forecasts - make_fake_forecasts( + forecasts = generate_fake_forecasts( gsp_ids=range(0, N_GSPS), session=session, model_name="blend", - t0_datetime_utc=now, + t0_datetime_utc=floor_30_minutes_dt(datetime.now(tz=UTC)), add_latest=True, historic=True, ) + save_forecasts(forecasts, session) + + non_historic_forecasts = generate_fake_forecasts( + gsp_ids=range(0, N_GSPS), + session=session, + model_name="blend", + t0_datetime_utc=floor_30_minutes_dt(datetime.now(tz=UTC)), + historic=False, + ) + save_forecasts(non_historic_forecasts, session) # 2. make gsp yields make_fake_gsp_yields(gsp_ids=range(0, N_GSPS), session=session, t0_datetime_utc=now) @@ -69,6 +91,6 @@ session.commit() assert len(session.query(StatusSQL).all()) == 1 - assert len(session.query(ForecastValueLatestSQL).all()) == 112 * N_GSPS - assert len(session.query(ForecastValueSQL).all()) == 112 * N_GSPS - assert len(session.query(ForecastSQL).all()) == N_GSPS + assert len(session.query(ForecastSQL).all()) == N_GSPS * 2 + assert len(session.query(ForecastValueLatestSQL).all()) == N_GSPS * N_FAKE_FORECASTS + assert len(session.query(ForecastValueSQL).all()) == N_GSPS * N_FAKE_FORECASTS * 2 From 8111b776c671fdbb3458a28b27ffc1cb8cd75856 Mon Sep 17 00:00:00 2001 From: braddf Date: Thu, 6 Mar 2025 14:02:14 +0000 Subject: [PATCH 11/14] update tests directory imports --- nowcasting_api/tests/conftest.py | 6 +++--- nowcasting_api/tests/fake/test_gsp_fake.py | 2 +- nowcasting_api/tests/fake/test_national_fake.py | 4 ++-- nowcasting_api/tests/test_database.py | 6 +++++- nowcasting_api/tests/test_elexon_forecast.py | 2 +- nowcasting_api/tests/test_gsp.py | 6 +++--- nowcasting_api/tests/test_main.py | 2 +- nowcasting_api/tests/test_merged_routes.py | 4 ++-- nowcasting_api/tests/test_national.py | 6 +++--- nowcasting_api/tests/test_status.py | 4 ++-- nowcasting_api/tests/test_system.py | 4 ++-- nowcasting_api/tests/test_utils.py | 9 +++++++-- 12 files changed, 32 insertions(+), 23 deletions(-) diff --git a/nowcasting_api/tests/conftest.py b/nowcasting_api/tests/conftest.py index 5d3f4dc0..8531ccfc 100644 --- a/nowcasting_api/tests/conftest.py +++ b/nowcasting_api/tests/conftest.py @@ -8,9 +8,9 @@ from nowcasting_datamodel.fake import make_fake_forecasts from nowcasting_datamodel.models.base import Base_PV -from auth_utils import get_auth_implicit_scheme, get_user -from database import get_session -from main import app +from nowcasting_api.auth_utils import get_auth_implicit_scheme, get_user +from nowcasting_api.database import get_session +from nowcasting_api.main import app @pytest.fixture diff --git a/nowcasting_api/tests/fake/test_gsp_fake.py b/nowcasting_api/tests/fake/test_gsp_fake.py index 79780bd6..fc1e9ad4 100644 --- a/nowcasting_api/tests/fake/test_gsp_fake.py +++ b/nowcasting_api/tests/fake/test_gsp_fake.py @@ -1,6 +1,6 @@ from nowcasting_datamodel.models import ForecastValue, LocationWithGSPYields, ManyForecasts -from gsp import GSP_TOTAL, is_fake +from nowcasting_api.gsp import GSP_TOTAL, is_fake def test_is_fake_specific_gsp(monkeypatch, api_client, gsp_id=1): diff --git a/nowcasting_api/tests/fake/test_national_fake.py b/nowcasting_api/tests/fake/test_national_fake.py index bd8ba8bb..5c4e20c4 100644 --- a/nowcasting_api/tests/fake/test_national_fake.py +++ b/nowcasting_api/tests/fake/test_national_fake.py @@ -2,8 +2,8 @@ from freezegun import freeze_time -from national import is_fake -from pydantic_models import NationalForecastValue, NationalYield +from nowcasting_api.national import is_fake +from nowcasting_api.pydantic_models import NationalForecastValue, NationalYield def test_is_fake_national_all_available_forecasts(monkeypatch, api_client): diff --git a/nowcasting_api/tests/test_database.py b/nowcasting_api/tests/test_database.py index 898cb3e4..959af45c 100644 --- a/nowcasting_api/tests/test_database.py +++ b/nowcasting_api/tests/test_database.py @@ -3,7 +3,11 @@ from freezegun import freeze_time from nowcasting_datamodel.read.read import national_gb_label -from database import get_forecasts_for_a_specific_gsp_from_database, get_gsp_system, get_session +from nowcasting_api.database import ( + get_forecasts_for_a_specific_gsp_from_database, + get_gsp_system, + get_session, +) def test_get_session(): diff --git a/nowcasting_api/tests/test_elexon_forecast.py b/nowcasting_api/tests/test_elexon_forecast.py index c0e4f25f..cdc1e731 100644 --- a/nowcasting_api/tests/test_elexon_forecast.py +++ b/nowcasting_api/tests/test_elexon_forecast.py @@ -5,7 +5,7 @@ import pytest from elexonpy.api.generation_forecast_api import GenerationForecastApi -from pydantic_models import BaseModel, SolarForecastResponse +from nowcasting_api.pydantic_models import BaseModel, SolarForecastResponse API_URL = "/v0/solar/GB/national/elexon" diff --git a/nowcasting_api/tests/test_gsp.py b/nowcasting_api/tests/test_gsp.py index 9d7b6811..26d45b5a 100644 --- a/nowcasting_api/tests/test_gsp.py +++ b/nowcasting_api/tests/test_gsp.py @@ -17,9 +17,9 @@ from nowcasting_datamodel.save.save import save_all_forecast_values_seven_days from nowcasting_datamodel.save.update import update_all_forecast_latest -from database import get_session -from main import app -from pydantic_models import GSPYieldGroupByDatetime, OneDatetimeManyForecastValues +from nowcasting_api.database import get_session +from nowcasting_api.main import app +from nowcasting_api.pydantic_models import GSPYieldGroupByDatetime, OneDatetimeManyForecastValues @freeze_time("2022-01-01") diff --git a/nowcasting_api/tests/test_main.py b/nowcasting_api/tests/test_main.py index 1e10bc69..cbe5309c 100644 --- a/nowcasting_api/tests/test_main.py +++ b/nowcasting_api/tests/test_main.py @@ -1,6 +1,6 @@ """ Test for main app """ -from main import version +from nowcasting_api.main import version def test_read_main(api_client): diff --git a/nowcasting_api/tests/test_merged_routes.py b/nowcasting_api/tests/test_merged_routes.py index dad18c5e..cb3f652c 100644 --- a/nowcasting_api/tests/test_merged_routes.py +++ b/nowcasting_api/tests/test_merged_routes.py @@ -7,8 +7,8 @@ from nowcasting_datamodel.models import ForecastValue, ForecastValueLatestSQL from nowcasting_datamodel.read.read_models import get_model -from database import get_session -from main import app +from nowcasting_api.database import get_session +from nowcasting_api.main import app @freeze_time("2022-06-01") diff --git a/nowcasting_api/tests/test_national.py b/nowcasting_api/tests/test_national.py index fe462df5..86bbf840 100644 --- a/nowcasting_api/tests/test_national.py +++ b/nowcasting_api/tests/test_national.py @@ -10,9 +10,9 @@ from nowcasting_datamodel.save.save import save_all_forecast_values_seven_days from nowcasting_datamodel.save.update import update_all_forecast_latest -from database import get_session -from main import app -from pydantic_models import NationalForecast, NationalForecastValue +from nowcasting_api.database import get_session +from nowcasting_api.main import app +from nowcasting_api.pydantic_models import NationalForecast, NationalForecastValue def test_read_latest_national_values(db_session, api_client): diff --git a/nowcasting_api/tests/test_status.py b/nowcasting_api/tests/test_status.py index fd26e35a..99d2e95a 100644 --- a/nowcasting_api/tests/test_status.py +++ b/nowcasting_api/tests/test_status.py @@ -18,8 +18,8 @@ UserSQL, ) -from database import get_session -from main import app +from nowcasting_api.database import get_session +from nowcasting_api.main import app client = TestClient(app) diff --git a/nowcasting_api/tests/test_system.py b/nowcasting_api/tests/test_system.py index ca3ec269..b32f91a9 100644 --- a/nowcasting_api/tests/test_system.py +++ b/nowcasting_api/tests/test_system.py @@ -4,8 +4,8 @@ from nowcasting_datamodel.models import Location from nowcasting_datamodel.read.read import get_location -from database import get_session -from main import app +from nowcasting_api.database import get_session +from nowcasting_api.main import app def test_get_gsp_systems(db_session, api_client): diff --git a/nowcasting_api/tests/test_utils.py b/nowcasting_api/tests/test_utils.py index d4e01c27..2828bbfb 100644 --- a/nowcasting_api/tests/test_utils.py +++ b/nowcasting_api/tests/test_utils.py @@ -5,8 +5,13 @@ from freezegun import freeze_time -from pydantic_models import NationalForecastValue -from utils import floor_30_minutes_dt, format_plevels, get_start_datetime, traces_sampler +from nowcasting_api.pydantic_models import NationalForecastValue +from nowcasting_api.utils import ( + floor_30_minutes_dt, + format_plevels, + get_start_datetime, + traces_sampler, +) LOWER_LIMIT_MINUTE = 0 UPPER_LIMIT_MINUTE = 60 From d4d85a0cce8f8162c659066e1cfe96b9e41c22a3 Mon Sep 17 00:00:00 2001 From: braddf Date: Thu, 6 Mar 2025 14:11:34 +0000 Subject: [PATCH 12/14] update cache.py import --- nowcasting_api/cache.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nowcasting_api/cache.py b/nowcasting_api/cache.py index f15cbc83..f16bdad3 100644 --- a/nowcasting_api/cache.py +++ b/nowcasting_api/cache.py @@ -8,7 +8,7 @@ import structlog -from database import save_api_call_to_db +from nowcasting_api.database import save_api_call_to_db logger = structlog.stdlib.get_logger() From bf869a61edc995fbb96409bb86613f16e7435ebb Mon Sep 17 00:00:00 2001 From: braddf Date: Thu, 6 Mar 2025 14:20:57 +0000 Subject: [PATCH 13/14] revert cache.py import and commit database.py rename --- nowcasting_api/cache.py | 2 +- {src => nowcasting_api}/database.py | 3 +-- 2 files changed, 2 insertions(+), 3 deletions(-) rename {src => nowcasting_api}/database.py (99%) diff --git a/nowcasting_api/cache.py b/nowcasting_api/cache.py index f16bdad3..f15cbc83 100644 --- a/nowcasting_api/cache.py +++ b/nowcasting_api/cache.py @@ -8,7 +8,7 @@ import structlog -from nowcasting_api.database import save_api_call_to_db +from database import save_api_call_to_db logger = structlog.stdlib.get_logger() diff --git a/src/database.py b/nowcasting_api/database.py similarity index 99% rename from src/database.py rename to nowcasting_api/database.py index bb778216..e908f0e7 100644 --- a/src/database.py +++ b/nowcasting_api/database.py @@ -33,8 +33,6 @@ from nowcasting_datamodel.read.read_gsp import get_gsp_yield, get_gsp_yield_by_location from nowcasting_datamodel.read.read_user import get_user as get_user_from_db from nowcasting_datamodel.save.update import N_GSP -from sqlalchemy.orm.session import Session - from pydantic_models import ( GSPYield, GSPYieldGroupByDatetime, @@ -43,6 +41,7 @@ convert_forecasts_to_many_datetime_many_generation, convert_location_sql_to_many_datetime_many_generation, ) +from sqlalchemy.orm.session import Session from utils import filter_forecast_values, floor_30_minutes_dt, get_start_datetime From fe3b05fe13fcc2e90ef4e478a7358602c4ca02dd Mon Sep 17 00:00:00 2001 From: braddf Date: Fri, 7 Mar 2025 16:29:22 +0000 Subject: [PATCH 14/14] remove is_fake functions/usage and tests --- nowcasting_api/gsp.py | 71 ++------------- nowcasting_api/national.py | 48 ++-------- nowcasting_api/tests/fake/test_gsp_fake.py | 88 ------------------- .../tests/fake/test_national_fake.py | 47 ---------- 4 files changed, 17 insertions(+), 237 deletions(-) delete mode 100644 nowcasting_api/tests/fake/test_gsp_fake.py delete mode 100644 nowcasting_api/tests/fake/test_national_fake.py diff --git a/nowcasting_api/gsp.py b/nowcasting_api/gsp.py index c1f82cc6..0cdcc13a 100644 --- a/nowcasting_api/gsp.py +++ b/nowcasting_api/gsp.py @@ -1,18 +1,9 @@ """Get GSP boundary data from eso """ import os -from datetime import UTC, datetime from typing import List, Optional, Union import structlog -from dotenv import load_dotenv -from fastapi import APIRouter, Depends, Request, Security, status -from fastapi.responses import Response -from fastapi_auth0 import Auth0User -from nowcasting_datamodel.fake import make_fake_forecast, make_fake_forecasts, make_fake_gsp_yields -from nowcasting_datamodel.models import Forecast, ForecastValue, ManyForecasts -from sqlalchemy.orm.session import Session - from auth_utils import get_auth_implicit_scheme, get_user from cache import cache_response from database import ( @@ -22,19 +13,19 @@ get_truth_values_for_a_specific_gsp_from_database, get_truth_values_for_all_gsps_from_database, ) +from dotenv import load_dotenv +from fastapi import APIRouter, Depends, Request, Security, status +from fastapi.responses import Response +from fastapi_auth0 import Auth0User +from nowcasting_datamodel.models import Forecast, ForecastValue, ManyForecasts from pydantic_models import ( GSPYield, GSPYieldGroupByDatetime, LocationWithGSPYields, OneDatetimeManyForecastValues, ) -from utils import ( - N_CALLS_PER_HOUR, - N_SLOW_CALLS_PER_HOUR, - floor_30_minutes_dt, - format_datetime, - limiter, -) +from sqlalchemy.orm.session import Session +from utils import N_CALLS_PER_HOUR, format_datetime, limiter GSP_TOTAL = 317 @@ -50,11 +41,6 @@ NationalYield = GSPYield -def is_fake(): - """Start FAKE environment""" - return int(os.environ.get("FAKE", 0)) - - # corresponds to route /v0/solar/GB/gsp/forecast/all/ @router.get( "/forecast/all/", @@ -100,16 +86,6 @@ def get_all_available_forecasts( if gsp_ids == "": gsp_ids = None - # if is_fake(): - # if gsp_ids is None: - # gsp_ids = [int(gsp_id) for gsp_id in range(1, 10)] - # - # fake_forecasts = make_fake_forecasts( - # gsp_ids=gsp_ids, - # session=session, - # t0_datetime_utc=floor_30_minutes_dt(datetime.now(tz=UTC)), - # ) - logger.info(f"Get forecasts for all gsps. The option is {historic=} for user {user}") start_datetime_utc = format_datetime(start_datetime_utc) @@ -133,9 +109,9 @@ def get_all_available_forecasts( logger.info(f"Got forecasts for all gsps. The option is {historic=} for user {user}") if not compact: - logger.info(f"Normalizing forecasts") + logger.info("Normalizing forecasts") forecasts.normalize() - logger.info(f"Normalized forecasts") + logger.info("Normalized forecasts") logger.info( f"Got {len(forecasts.forecasts)} forecasts for all gsps. " @@ -173,9 +149,6 @@ def get_forecasts_for_a_specific_gsp_old_route( ) -> Union[Forecast, List[ForecastValue]]: """Redirects old API route to new route /v0/solar/GB/gsp/{gsp_id}/forecast""" - # if is_fake(): - # make_fake_forecast(gsp_id=gsp_id, session=session) - return get_forecasts_for_a_specific_gsp( request=request, gsp_id=gsp_id, @@ -224,12 +197,6 @@ def get_forecasts_for_a_specific_gsp( - **creation_utc_limit**: optional, only return forecasts made before this datetime. returns the latest forecast made 60 minutes before the target time) """ - # if is_fake(): - # make_fake_forecast( - # gsp_id=gsp_id, - # session=session, - # t0_datetime_utc=floor_30_minutes_dt(datetime.now(tz=UTC)), - # ) logger.info(f"Get forecasts for gsp id {gsp_id} forecast of forecast with only values.") logger.info(f"This is for user {user}") @@ -300,16 +267,6 @@ def get_truths_for_all_gsps( if isinstance(gsp_ids, str): gsp_ids = [int(gsp_id) for gsp_id in gsp_ids.split(",")] - # if is_fake(): - # if gsp_ids is None: - # gsp_ids = [int(gsp_id) for gsp_id in range(1, GSP_TOTAL)] - # - # make_fake_gsp_yields( - # gsp_ids=gsp_ids, - # session=session, - # t0_datetime_utc=floor_30_minutes_dt(datetime.now(tz=UTC)), - # ) - logger.info(f"Get PV Live estimates values for all gsp id and regime {regime} for user {user}") start_datetime_utc = format_datetime(start_datetime_utc) @@ -343,9 +300,6 @@ def get_truths_for_a_specific_gsp_old_route( ) -> List[GSPYield]: """Redirects old API route to new route /v0/solar/GB/gsp/{gsp_id}/pvlive""" - # if is_fake(): - # make_fake_gsp_yields(gsp_ids=[gsp_id], session=session) - return get_truths_for_a_specific_gsp( request=request, gsp_id=gsp_id, @@ -391,13 +345,6 @@ def get_truths_for_a_specific_gsp( If not set, defaults to N_HISTORY_DAYS env var, which if not set defaults to yesterday. """ - # if is_fake(): - # make_fake_forecast( - # gsp_id=gsp_id, - # session=session, - # t0_datetime_utc=floor_30_minutes_dt(datetime.now(tz=UTC)), - # ) - logger.info( f"Get PV Live estimates values for gsp id {gsp_id} " f"and regime {regime} for user {user}" ) diff --git a/nowcasting_api/national.py b/nowcasting_api/national.py index 7dbcd029..48abe17e 100644 --- a/nowcasting_api/national.py +++ b/nowcasting_api/national.py @@ -1,19 +1,11 @@ """National API routes""" import os -from datetime import UTC, datetime, timedelta +from datetime import datetime, timedelta from typing import List, Optional, Union import pandas as pd import structlog -from elexonpy.api.generation_forecast_api import GenerationForecastApi -from elexonpy.api_client import ApiClient -from fastapi import APIRouter, Depends, HTTPException, Query, Request, Security -from fastapi_auth0 import Auth0User -from nowcasting_datamodel.fake import make_fake_forecast, make_fake_gsp_yields -from nowcasting_datamodel.read.read import get_latest_forecast_for_gsps -from sqlalchemy.orm.session import Session - from auth_utils import get_auth_implicit_scheme, get_user from cache import cache_response from database import ( @@ -21,6 +13,11 @@ get_session, get_truth_values_for_a_specific_gsp_from_database, ) +from elexonpy.api.generation_forecast_api import GenerationForecastApi +from elexonpy.api_client import ApiClient +from fastapi import APIRouter, Depends, HTTPException, Query, Request, Security +from fastapi_auth0 import Auth0User +from nowcasting_datamodel.read.read import get_latest_forecast_for_gsps from pydantic_models import ( NationalForecast, NationalForecastValue, @@ -28,14 +25,8 @@ SolarForecastResponse, SolarForecastValue, ) -from utils import ( - N_CALLS_PER_HOUR, - filter_forecast_values, - floor_30_minutes_dt, - format_datetime, - format_plevels, - limiter, -) +from sqlalchemy.orm.session import Session +from utils import N_CALLS_PER_HOUR, filter_forecast_values, format_datetime, format_plevels, limiter logger = structlog.stdlib.get_logger() @@ -51,11 +42,6 @@ elexon_forecast_api = GenerationForecastApi(api_client) -def is_fake(): - """Start FAKE environment""" - return int(os.environ.get("FAKE", 0)) - - @router.get( "/forecast", response_model=Union[NationalForecast, List[NationalForecastValue]], @@ -101,17 +87,6 @@ def get_national_forecast( """ logger.debug("Get national forecasts") - # if is_fake(): - # fake_forecast = make_fake_forecast( - # gsp_id=0, - # model_name="blend", - # session=session, - # t0_datetime_utc=floor_30_minutes_dt(datetime.now(tz=UTC)), - # add_latest=True, - # ) - # # add the forecast to the session, as this single fake function doesn't by default - # session.add(fake_forecast) - start_datetime_utc = format_datetime(start_datetime_utc) end_datetime_utc = format_datetime(end_datetime_utc) creation_limit_utc = format_datetime(creation_limit_utc) @@ -228,13 +203,6 @@ def get_national_pvlive( """ logger.info(f"Get national PV Live estimates values " f"for regime {regime} for {user}") - # if is_fake(): - # make_fake_gsp_yields( - # gsp_ids=[0], - # session=session, - # t0_datetime_utc=floor_30_minutes_dt(datetime.now(tz=UTC)), - # ) - return get_truth_values_for_a_specific_gsp_from_database( session=session, gsp_id=0, regime=regime ) diff --git a/nowcasting_api/tests/fake/test_gsp_fake.py b/nowcasting_api/tests/fake/test_gsp_fake.py deleted file mode 100644 index fc1e9ad4..00000000 --- a/nowcasting_api/tests/fake/test_gsp_fake.py +++ /dev/null @@ -1,88 +0,0 @@ -from nowcasting_datamodel.models import ForecastValue, LocationWithGSPYields, ManyForecasts - -from nowcasting_api.gsp import GSP_TOTAL, is_fake - - -def test_is_fake_specific_gsp(monkeypatch, api_client, gsp_id=1): - """### Test FAKE environment specific _gsp_id_ routes are populating - with fake data. - - #### Parameters - - **gsp_id**: Please set to any non-zero integer that is <= GSP_TOTAL - """ - - monkeypatch.setenv("FAKE", "1") - assert is_fake() == 1 - - # Specific _gsp_id_ route/endpoint for successful connection - response = api_client.get(f"/v0/solar/GB/gsp/{gsp_id}/forecast") - assert response.status_code == 200 - - forecast_value = [ForecastValue(**f) for f in response.json()] - assert forecast_value is not None - - # Disable is_fake environment - monkeypatch.setenv("FAKE", "0") - - -def test_is_fake_get_truths_for_a_specific_gsp(monkeypatch, api_client, gsp_id=1): - """### Test FAKE environment specific _gsp_id_ routes are populating - with fake data. - - #### Parameters - - **gsp_id**: Please set to any non-zero integer that is <= GSP_TOTAL - """ - - monkeypatch.setenv("FAKE", "1") - assert is_fake() == 1 - - # Specific _gsp_id_ route/endpoint for successful connection - response = api_client.get(f"/v0/solar/GB/gsp/{gsp_id}/pvlive") - assert response.status_code == 200 - - forecast_value = [ForecastValue(**f) for f in response.json()] - assert forecast_value is not None - - # Disable is_fake environment - monkeypatch.setenv("FAKE", "0") - - -def test_is_fake_all_available_forecasts(monkeypatch, api_client): - """Test FAKE environment for all GSPs are populating - with fake data. - """ - - monkeypatch.setenv("FAKE", "1") - assert is_fake() == 1 - - # Connect to DB endpoint - response = api_client.get("/v0/solar/GB/gsp/forecast/all/") - assert response.status_code == 200 - - all_forecasts = ManyForecasts(**response.json()) - assert all_forecasts is not None - - # Disable is_fake environment - monkeypatch.setenv("FAKE", "0") - - -def test_is_fake_get_truths_for_all_gsps( - monkeypatch, api_client, gsp_ids=list(range(1, GSP_TOTAL)) -): - """Test FAKE environment for all GSPs for yesterday and today - are populating with fake data. - """ - - monkeypatch.setenv("FAKE", "1") - assert is_fake() == 1 - - # Connect to DB endpoint - gsp_ids_str = ", ".join(map(str, gsp_ids)) - response = api_client.get(f"/v0/solar/GB/gsp/pvlive/all?gsp_ids={gsp_ids_str}") - assert response.status_code == 200 - - all_forecasts = [LocationWithGSPYields(**f) for f in response.json()] - assert all_forecasts is not None - - # Disable is_fake environment - monkeypatch.setenv("FAKE", "0") diff --git a/nowcasting_api/tests/fake/test_national_fake.py b/nowcasting_api/tests/fake/test_national_fake.py deleted file mode 100644 index 5c4e20c4..00000000 --- a/nowcasting_api/tests/fake/test_national_fake.py +++ /dev/null @@ -1,47 +0,0 @@ -""" Test for main app """ - -from freezegun import freeze_time - -from nowcasting_api.national import is_fake -from nowcasting_api.pydantic_models import NationalForecastValue, NationalYield - - -def test_is_fake_national_all_available_forecasts(monkeypatch, api_client): - """Test FAKE environment for all GSPs are populating - with fake data. - """ - - monkeypatch.setenv("FAKE", "1") - assert is_fake() == 1 - # Connect to DB endpoint - response = api_client.get("/v0/solar/GB/national/forecast") - assert response.status_code == 200 - - national_forecast_values = [NationalForecastValue(**f) for f in response.json()] - assert national_forecast_values is not None - - # Disable is_fake environment - monkeypatch.setenv("FAKE", "0") - - -# The freeze time is needed so the cahce doesnt interact with the test in test_national.py -# Ideally we would not have this -@freeze_time("2021-12-01") -def test_is_fake_national_get_truths_for_all_gsps(monkeypatch, api_client): - """Test FAKE environment for all GSPs for yesterday and today - are populating with fake data. - """ - - monkeypatch.setenv("FAKE", "1") - assert is_fake() == 1 - # Connect to DB endpoint - response = api_client.get("/v0/solar/GB/national/pvlive/") - assert response.status_code == 200 - - print(response.json()) - - national_forecast_values = [NationalYield(**f) for f in response.json()] - assert national_forecast_values is not None - - # Disable is_fake environment - monkeypatch.setenv("FAKE", "0")