diff --git a/k8s/analytics/values-prod.yaml b/k8s/analytics/values-prod.yaml index 72b6247050..da52c50e1c 100644 --- a/k8s/analytics/values-prod.yaml +++ b/k8s/analytics/values-prod.yaml @@ -8,7 +8,7 @@ images: celeryWorker: eu.gcr.io/airqo-250220/airqo-analytics-celery-worker reportJob: eu.gcr.io/airqo-250220/airqo-analytics-report-job devicesSummaryJob: eu.gcr.io/airqo-250220/airqo-analytics-devices-summary-job - tag: prod-7ca041eb-1734008736 + tag: prod-ecb85570-1734011546 api: name: airqo-analytics-api label: analytics-api diff --git a/k8s/device-registry/values-prod.yaml b/k8s/device-registry/values-prod.yaml index 8f5c278184..55606be458 100644 --- a/k8s/device-registry/values-prod.yaml +++ b/k8s/device-registry/values-prod.yaml @@ -6,7 +6,7 @@ app: replicaCount: 3 image: repository: eu.gcr.io/airqo-250220/airqo-device-registry-api - tag: prod-7ca041eb-1734008736 + tag: prod-ecb85570-1734011546 nameOverride: '' fullnameOverride: '' podAnnotations: {} diff --git a/k8s/exceedance/values-prod-airqo.yaml b/k8s/exceedance/values-prod-airqo.yaml index 9a874bf284..6bdadeb1ee 100644 --- a/k8s/exceedance/values-prod-airqo.yaml +++ b/k8s/exceedance/values-prod-airqo.yaml @@ -4,6 +4,6 @@ app: configmap: env-exceedance-production image: repository: eu.gcr.io/airqo-250220/airqo-exceedance-job - tag: prod-7ca041eb-1734008736 + tag: prod-ecb85570-1734011546 nameOverride: '' fullnameOverride: '' diff --git a/k8s/exceedance/values-prod-kcca.yaml b/k8s/exceedance/values-prod-kcca.yaml index 2d0c18e90d..464f44ebae 100644 --- a/k8s/exceedance/values-prod-kcca.yaml +++ b/k8s/exceedance/values-prod-kcca.yaml @@ -4,6 +4,6 @@ app: configmap: env-exceedance-production image: repository: eu.gcr.io/airqo-250220/kcca-exceedance-job - tag: prod-7ca041eb-1734008736 + tag: prod-ecb85570-1734011546 nameOverride: '' fullnameOverride: '' diff --git a/k8s/predict/values-prod.yaml b/k8s/predict/values-prod.yaml index 6cec31b67d..f59a3cc823 100644 --- a/k8s/predict/values-prod.yaml +++ b/k8s/predict/values-prod.yaml @@ -7,7 +7,7 @@ images: predictJob: eu.gcr.io/airqo-250220/airqo-predict-job trainJob: eu.gcr.io/airqo-250220/airqo-train-job predictPlaces: eu.gcr.io/airqo-250220/airqo-predict-places-air-quality - tag: prod-7ca041eb-1734008736 + tag: prod-ecb85570-1734011546 api: name: airqo-prediction-api label: prediction-api diff --git a/k8s/spatial/values-prod.yaml b/k8s/spatial/values-prod.yaml index 057f1feff3..e85fb0d1a0 100644 --- a/k8s/spatial/values-prod.yaml +++ b/k8s/spatial/values-prod.yaml @@ -6,7 +6,7 @@ app: replicaCount: 3 image: repository: eu.gcr.io/airqo-250220/airqo-spatial-api - tag: prod-7ca041eb-1734008736 + tag: prod-ecb85570-1734011546 nameOverride: '' fullnameOverride: '' podAnnotations: {} diff --git a/k8s/website/values-prod.yaml b/k8s/website/values-prod.yaml index 8a160bb136..db797b9f98 100644 --- a/k8s/website/values-prod.yaml +++ b/k8s/website/values-prod.yaml @@ -6,7 +6,7 @@ app: replicaCount: 3 image: repository: eu.gcr.io/airqo-250220/airqo-website-api - tag: prod-7ca041eb-1734008736 + tag: prod-ecb85570-1734011546 nameOverride: '' fullnameOverride: '' podAnnotations: {} diff --git a/k8s/workflows/values-prod.yaml b/k8s/workflows/values-prod.yaml index 2fa29dd7b0..d597a7dc9c 100644 --- a/k8s/workflows/values-prod.yaml +++ b/k8s/workflows/values-prod.yaml @@ -10,7 +10,7 @@ images: initContainer: eu.gcr.io/airqo-250220/airqo-workflows-xcom redisContainer: eu.gcr.io/airqo-250220/airqo-redis containers: eu.gcr.io/airqo-250220/airqo-workflows - tag: prod-7ca041eb-1734008736 + tag: prod-ecb85570-1734011546 nameOverride: '' fullnameOverride: '' podAnnotations: {} diff --git a/k8s/workflows/values-stage.yaml b/k8s/workflows/values-stage.yaml index 0c841cf60c..5552069735 100644 --- a/k8s/workflows/values-stage.yaml +++ b/k8s/workflows/values-stage.yaml @@ -10,7 +10,7 @@ images: initContainer: eu.gcr.io/airqo-250220/airqo-stage-workflows-xcom redisContainer: eu.gcr.io/airqo-250220/airqo-stage-redis containers: eu.gcr.io/airqo-250220/airqo-stage-workflows - tag: stage-c43059fe-1734008689 + tag: stage-76f24f1c-1734011507 nameOverride: '' fullnameOverride: '' podAnnotations: {} diff --git a/src/workflows/airqo_etl_utils/meta_data_utils.py b/src/workflows/airqo_etl_utils/meta_data_utils.py index c5cdd8691b..b54ea6a485 100644 --- a/src/workflows/airqo_etl_utils/meta_data_utils.py +++ b/src/workflows/airqo_etl_utils/meta_data_utils.py @@ -5,11 +5,12 @@ from .constants import Tenant from .data_validator import DataValidationUtils from .weather_data_utils import WeatherDataUtils +from datetime import datetime, timezone class MetaDataUtils: @staticmethod - def extract_devices_from_api(network: str = "all") -> pd.DataFrame: + def extract_devices_from_api() -> pd.DataFrame: devices = AirQoApi().get_devices() dataframe = pd.json_normalize(devices) dataframe = dataframe[ @@ -26,6 +27,7 @@ def extract_devices_from_api(network: str = "all") -> pd.DataFrame: ] ] dataframe["name"] = dataframe["device_id"] + dataframe["last_updated"] = datetime.now(timezone.utc) dataframe = DataValidationUtils.remove_outliers(dataframe) return dataframe diff --git a/src/workflows/airqo_etl_utils/schema/devices.json b/src/workflows/airqo_etl_utils/schema/devices.json index 44a7a76fc8..04f7312d0a 100644 --- a/src/workflows/airqo_etl_utils/schema/devices.json +++ b/src/workflows/airqo_etl_utils/schema/devices.json @@ -1,9 +1,4 @@ [ - { - "name": "tenant", - "type": "STRING", - "mode": "REQUIRED" - }, { "name": "network", "type": "STRING", @@ -19,16 +14,6 @@ "type": "FLOAT", "mode": "NULLABLE" }, - { - "name": "approximate_latitude", - "type": "FLOAT", - "mode": "NULLABLE" - }, - { - "name": "approximate_longitude", - "type": "FLOAT", - "mode": "NULLABLE" - }, { "name": "site_id", "type": "STRING",