From 30f5f2784570503c39ab60a1c112e99f610ae60f Mon Sep 17 00:00:00 2001 From: NicholasTurner23 Date: Wed, 5 Feb 2025 12:09:29 +0300 Subject: [PATCH 1/2] Default values only for na --- src/workflows/airqo_etl_utils/airqo_utils.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/src/workflows/airqo_etl_utils/airqo_utils.py b/src/workflows/airqo_etl_utils/airqo_utils.py index 8beb5d6217..9b0b572c4d 100644 --- a/src/workflows/airqo_etl_utils/airqo_utils.py +++ b/src/workflows/airqo_etl_utils/airqo_utils.py @@ -606,9 +606,7 @@ def calibrate_data(data: pd.DataFrame) -> pd.DataFrame: ] # TODO: Need to opt for a different approach eg forward fill, can't do here as df only has data of last 1 hour. Perhaps use raw data only? - default_values = {col: 0 for col in columns_to_fill} - data = data.assign(**default_values) - # data[columns_to_fill] = data[columns_to_fill].fillna(0) + data[columns_to_fill] = data[columns_to_fill].fillna(0) # additional input columns for calibration data["avg_pm2_5"] = data[["s1_pm2_5", "s2_pm2_5"]].mean(axis=1).round(2) From 2c146729fd985790d19cf20aca1aa121f1a75b2f Mon Sep 17 00:00:00 2001 From: NicholasTurner23 Date: Wed, 5 Feb 2025 12:11:01 +0300 Subject: [PATCH 2/2] Cleanup --- src/workflows/airqo_etl_utils/airqo_utils.py | 1 + 1 file changed, 1 insertion(+) diff --git a/src/workflows/airqo_etl_utils/airqo_utils.py b/src/workflows/airqo_etl_utils/airqo_utils.py index 9b0b572c4d..9ff268a1d1 100644 --- a/src/workflows/airqo_etl_utils/airqo_utils.py +++ b/src/workflows/airqo_etl_utils/airqo_utils.py @@ -606,6 +606,7 @@ def calibrate_data(data: pd.DataFrame) -> pd.DataFrame: ] # TODO: Need to opt for a different approach eg forward fill, can't do here as df only has data of last 1 hour. Perhaps use raw data only? + # Fill nas for the specified fields. data[columns_to_fill] = data[columns_to_fill].fillna(0) # additional input columns for calibration