From 3bf311225f21bca9e1f253c7eaba07266bb70da7 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Thu, 12 Dec 2024 16:53:36 +0300 Subject: [PATCH 01/10] Update AirQo exceedance production image tag to prod-ecb85570-1734011546 --- k8s/exceedance/values-prod-airqo.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/k8s/exceedance/values-prod-airqo.yaml b/k8s/exceedance/values-prod-airqo.yaml index 9a874bf284..6bdadeb1ee 100644 --- a/k8s/exceedance/values-prod-airqo.yaml +++ b/k8s/exceedance/values-prod-airqo.yaml @@ -4,6 +4,6 @@ app: configmap: env-exceedance-production image: repository: eu.gcr.io/airqo-250220/airqo-exceedance-job - tag: prod-7ca041eb-1734008736 + tag: prod-ecb85570-1734011546 nameOverride: '' fullnameOverride: '' From 90c59d32f06cc8f43b1c354f1c121a0250e2b985 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Thu, 12 Dec 2024 16:53:46 +0300 Subject: [PATCH 02/10] Update KCCA exceedance production image tag to prod-ecb85570-1734011546 --- k8s/exceedance/values-prod-kcca.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/k8s/exceedance/values-prod-kcca.yaml b/k8s/exceedance/values-prod-kcca.yaml index 2d0c18e90d..464f44ebae 100644 --- a/k8s/exceedance/values-prod-kcca.yaml +++ b/k8s/exceedance/values-prod-kcca.yaml @@ -4,6 +4,6 @@ app: configmap: env-exceedance-production image: repository: eu.gcr.io/airqo-250220/kcca-exceedance-job - tag: prod-7ca041eb-1734008736 + tag: prod-ecb85570-1734011546 nameOverride: '' fullnameOverride: '' From 45ad5cb2ce4e2eda91488fdd35da4cfad0da8a44 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Thu, 12 Dec 2024 16:54:09 +0300 Subject: [PATCH 03/10] Update workflows staging image tag to stage-76f24f1c-1734011507 --- k8s/workflows/values-stage.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/k8s/workflows/values-stage.yaml b/k8s/workflows/values-stage.yaml index 0c841cf60c..5552069735 100644 --- a/k8s/workflows/values-stage.yaml +++ b/k8s/workflows/values-stage.yaml @@ -10,7 +10,7 @@ images: initContainer: eu.gcr.io/airqo-250220/airqo-stage-workflows-xcom redisContainer: eu.gcr.io/airqo-250220/airqo-stage-redis containers: eu.gcr.io/airqo-250220/airqo-stage-workflows - tag: stage-c43059fe-1734008689 + tag: stage-76f24f1c-1734011507 nameOverride: '' fullnameOverride: '' podAnnotations: {} From 7db3b89fd41369000a5e52039773b6373a704fe8 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Thu, 12 Dec 2024 16:54:25 +0300 Subject: [PATCH 04/10] Update device registry production image tag to prod-ecb85570-1734011546 --- k8s/device-registry/values-prod.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/k8s/device-registry/values-prod.yaml b/k8s/device-registry/values-prod.yaml index 8f5c278184..55606be458 100644 --- a/k8s/device-registry/values-prod.yaml +++ b/k8s/device-registry/values-prod.yaml @@ -6,7 +6,7 @@ app: replicaCount: 3 image: repository: eu.gcr.io/airqo-250220/airqo-device-registry-api - tag: prod-7ca041eb-1734008736 + tag: prod-ecb85570-1734011546 nameOverride: '' fullnameOverride: '' podAnnotations: {} From 09f7e67dc5cc74fcb2648bec98da7755d2fe433e Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Thu, 12 Dec 2024 16:54:32 +0300 Subject: [PATCH 05/10] Update website production image tag to prod-ecb85570-1734011546 --- k8s/website/values-prod.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/k8s/website/values-prod.yaml b/k8s/website/values-prod.yaml index 8a160bb136..db797b9f98 100644 --- a/k8s/website/values-prod.yaml +++ b/k8s/website/values-prod.yaml @@ -6,7 +6,7 @@ app: replicaCount: 3 image: repository: eu.gcr.io/airqo-250220/airqo-website-api - tag: prod-7ca041eb-1734008736 + tag: prod-ecb85570-1734011546 nameOverride: '' fullnameOverride: '' podAnnotations: {} From 3e5aba49bf190318cfe7aa715dbd9144d7cf5c23 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Thu, 12 Dec 2024 16:55:03 +0300 Subject: [PATCH 06/10] Update workflows prod image tag to prod-ecb85570-1734011546 --- k8s/workflows/values-prod.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/k8s/workflows/values-prod.yaml b/k8s/workflows/values-prod.yaml index 2fa29dd7b0..d597a7dc9c 100644 --- a/k8s/workflows/values-prod.yaml +++ b/k8s/workflows/values-prod.yaml @@ -10,7 +10,7 @@ images: initContainer: eu.gcr.io/airqo-250220/airqo-workflows-xcom redisContainer: eu.gcr.io/airqo-250220/airqo-redis containers: eu.gcr.io/airqo-250220/airqo-workflows - tag: prod-7ca041eb-1734008736 + tag: prod-ecb85570-1734011546 nameOverride: '' fullnameOverride: '' podAnnotations: {} From 3db5b6fdcf8e92715e1519dbc7c0a38d58e8e0ce Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Thu, 12 Dec 2024 16:55:17 +0300 Subject: [PATCH 07/10] Update analytics production image tag to prod-ecb85570-1734011546 --- k8s/analytics/values-prod.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/k8s/analytics/values-prod.yaml b/k8s/analytics/values-prod.yaml index 72b6247050..da52c50e1c 100644 --- a/k8s/analytics/values-prod.yaml +++ b/k8s/analytics/values-prod.yaml @@ -8,7 +8,7 @@ images: celeryWorker: eu.gcr.io/airqo-250220/airqo-analytics-celery-worker reportJob: eu.gcr.io/airqo-250220/airqo-analytics-report-job devicesSummaryJob: eu.gcr.io/airqo-250220/airqo-analytics-devices-summary-job - tag: prod-7ca041eb-1734008736 + tag: prod-ecb85570-1734011546 api: name: airqo-analytics-api label: analytics-api From d8d6e18f97e08c79123a187e18ceb61562b92339 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Thu, 12 Dec 2024 16:56:07 +0300 Subject: [PATCH 08/10] Update predict production image tag to prod-ecb85570-1734011546 --- k8s/predict/values-prod.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/k8s/predict/values-prod.yaml b/k8s/predict/values-prod.yaml index 6cec31b67d..f59a3cc823 100644 --- a/k8s/predict/values-prod.yaml +++ b/k8s/predict/values-prod.yaml @@ -7,7 +7,7 @@ images: predictJob: eu.gcr.io/airqo-250220/airqo-predict-job trainJob: eu.gcr.io/airqo-250220/airqo-train-job predictPlaces: eu.gcr.io/airqo-250220/airqo-predict-places-air-quality - tag: prod-7ca041eb-1734008736 + tag: prod-ecb85570-1734011546 api: name: airqo-prediction-api label: prediction-api From 2df1eac1b1c5ddae67c45b873344c1a493cdc988 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Thu, 12 Dec 2024 17:01:07 +0300 Subject: [PATCH 09/10] Update spatial production image tag to prod-ecb85570-1734011546 --- k8s/spatial/values-prod.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/k8s/spatial/values-prod.yaml b/k8s/spatial/values-prod.yaml index 057f1feff3..e85fb0d1a0 100644 --- a/k8s/spatial/values-prod.yaml +++ b/k8s/spatial/values-prod.yaml @@ -6,7 +6,7 @@ app: replicaCount: 3 image: repository: eu.gcr.io/airqo-250220/airqo-spatial-api - tag: prod-7ca041eb-1734008736 + tag: prod-ecb85570-1734011546 nameOverride: '' fullnameOverride: '' podAnnotations: {} From 5f317bdad825a120c4ea5b82d9c9f377b2044193 Mon Sep 17 00:00:00 2001 From: NicholasTurner23 Date: Thu, 12 Dec 2024 17:21:59 +0300 Subject: [PATCH 10/10] Add last updated time to track updates --- src/workflows/airqo_etl_utils/meta_data_utils.py | 7 ++++--- src/workflows/airqo_etl_utils/schema/devices.json | 15 --------------- 2 files changed, 4 insertions(+), 18 deletions(-) diff --git a/src/workflows/airqo_etl_utils/meta_data_utils.py b/src/workflows/airqo_etl_utils/meta_data_utils.py index 51cf1d3712..b54ea6a485 100644 --- a/src/workflows/airqo_etl_utils/meta_data_utils.py +++ b/src/workflows/airqo_etl_utils/meta_data_utils.py @@ -5,11 +5,12 @@ from .constants import Tenant from .data_validator import DataValidationUtils from .weather_data_utils import WeatherDataUtils +from datetime import datetime, timezone class MetaDataUtils: @staticmethod - def extract_devices_from_api(tenant: Tenant = Tenant.ALL) -> pd.DataFrame: + def extract_devices_from_api() -> pd.DataFrame: devices = AirQoApi().get_devices() dataframe = pd.json_normalize(devices) dataframe = dataframe[ @@ -20,13 +21,13 @@ def extract_devices_from_api(tenant: Tenant = Tenant.ALL) -> pd.DataFrame: "site_id", "device_id", "device_number", - "name", "description", "device_manufacturer", "device_category", ] ] - + dataframe["name"] = dataframe["device_id"] + dataframe["last_updated"] = datetime.now(timezone.utc) dataframe = DataValidationUtils.remove_outliers(dataframe) return dataframe diff --git a/src/workflows/airqo_etl_utils/schema/devices.json b/src/workflows/airqo_etl_utils/schema/devices.json index 44a7a76fc8..04f7312d0a 100644 --- a/src/workflows/airqo_etl_utils/schema/devices.json +++ b/src/workflows/airqo_etl_utils/schema/devices.json @@ -1,9 +1,4 @@ [ - { - "name": "tenant", - "type": "STRING", - "mode": "REQUIRED" - }, { "name": "network", "type": "STRING", @@ -19,16 +14,6 @@ "type": "FLOAT", "mode": "NULLABLE" }, - { - "name": "approximate_latitude", - "type": "FLOAT", - "mode": "NULLABLE" - }, - { - "name": "approximate_longitude", - "type": "FLOAT", - "mode": "NULLABLE" - }, { "name": "site_id", "type": "STRING",