Nans nchs:

* add missing column handling
* update export function and tests
* gitignore cache csvs

Author: dshemetov

nchs_mortality/.gitignore

@@ -5,6 +5,9 @@ params.json
 
 # Do not commit output files
 receiving/*.csv
+daily_receiving/*.csv
+cache/*.csv
+daily_cache/*.csv
 
 # Do not commit test files
 tests/receiving/*.csv

nchs_mortality/delphi_nchs_mortality/export.py

@@ -28,7 +28,11 @@ def export_csv(df, geo_name, sensor, export_dir, start_date):
         t = Week.fromdate(pd.to_datetime(str(date)))
         date_short = "weekly_" + str(t.year) + str(t.week).zfill(2)
         export_fn = f"{date_short}_{geo_name}_{sensor}.csv"
-        result_df = df[df["timestamp"] == date][["geo_id", "val", "se", "sample_size"]]
+        expected_columns = [
+            "geo_id", "val", "se", "sample_size",
+            "missing_val", "missing_se", "missing_sample_size"
+        ]
+        result_df = df[df["timestamp"] == date][expected_columns]
         result_df.to_csv(f"{export_dir}/{export_fn}",
                          index=False,
                          float_format="%.8f")

nchs_mortality/delphi_nchs_mortality/run.py

@@ -9,7 +9,7 @@
 from typing import Dict, Any
 
 import numpy as np
-from delphi_utils import S3ArchiveDiffer, get_structured_logger
+from delphi_utils import S3ArchiveDiffer, get_structured_logger, Nans
 
 from .archive_diffs import arch_diffs
 from .constants import (METRICS, SENSOR_NAME_MAP,
@@ -18,6 +18,18 @@
 from .pull import pull_nchs_mortality_data
 
 
+def add_nancodes(df):
+    """Add nancodes to the dataframe."""
+    # Default missingness codes
+    df["missing_val"] = Nans.NOT_MISSING
+    df["missing_se"] = Nans.NOT_APPLICABLE
+    df["missing_sample_size"] = Nans.NOT_APPLICABLE
+
+    # Mark any remaining nans with unknown
+    remaining_nans_mask = df["val"].isnull()
+    df.loc[remaining_nans_mask, "missing_val"] = Nans.UNKNOWN
+    return df
+
 def run_module(params: Dict[str, Any]):
     """Run module for processing NCHS mortality data.
 
@@ -67,7 +79,8 @@ def run_module(params: Dict[str, Any]):
             df["val"] = df[metric]
             df["se"] = np.nan
             df["sample_size"] = np.nan
-            df = df[~df["val"].isnull()]
+            df = add_nancodes(df)
+            # df = df[~df["val"].isnull()]
             sensor_name = "_".join([SENSOR_NAME_MAP[metric]])
             export_csv(
                 df,
@@ -86,7 +99,8 @@ def run_module(params: Dict[str, Any]):
                     df["val"] = df[metric] / df["population"] * INCIDENCE_BASE
                 df["se"] = np.nan
                 df["sample_size"] = np.nan
-                df = df[~df["val"].isnull()]
+                df = add_nancodes(df)
+                # df = df[~df["val"].isnull()]
                 sensor_name = "_".join([SENSOR_NAME_MAP[metric], sensor])
                 export_csv(
                     df,

nchs_mortality/tests/test_export.py

@@ -4,6 +4,7 @@
 import pandas as pd
 
 from delphi_nchs_mortality.export import export_csv
+from delphi_utils import Nans
 
 
 class TestExport:
@@ -16,7 +17,10 @@ def test_export(self):
                 "val": [0, 2, 3, 5, 10, 12],
                 "timestamp": [datetime(2020, 6, 2), datetime(2020, 6, 9)] * 3,
                 "se": [0.01, 0.02, 0.01, 0.01, 0.005, 0.01],
-                "sample_size": [100, 200, 500, 50, 80, 10]
+                "sample_size": [100, 200, 500, 50, 80, 10],
+                "missing_val": [Nans.NOT_MISSING] * 6,
+                "missing_se": [Nans.NOT_MISSING] * 6,
+                "missing_sample_size": [Nans.NOT_MISSING] * 6,
             }
         )
 
@@ -34,7 +38,11 @@ def test_export(self):
 
         output_data = pd.read_csv(join("./receiving", expected_name))
 
-        assert (output_data.columns == ["geo_id", "val", "se", "sample_size"]).all()
+        expected_columns = [
+            "geo_id", "val", "se", "sample_size",
+            "missing_val", "missing_se", "missing_sample_size"
+        ]
+        assert (output_data.columns == expected_columns).all()
         assert (output_data.geo_id == ["a", "b", "c"]).all()
         assert (output_data.se.values == [0.01, 0.01, 0.005]).all()
         assert (output_data.sample_size.values == [100, 500, 80]).all()
@@ -45,7 +53,7 @@ def test_export(self):
 
         output_data = pd.read_csv(join("./receiving", expected_name))
 
-        assert (output_data.columns == ["geo_id", "val", "se", "sample_size"]).all()
+        assert (output_data.columns == expected_columns).all()
         assert (output_data.geo_id == ["a", "b", "c"]).all()
         assert (output_data.se.values == [0.02, 0.01, 0.01]).all()
         assert (output_data.sample_size.values == [200, 50, 10]).all()

nchs_mortality/tests/test_run.py

@@ -60,4 +60,8 @@ def test_output_file_format(self, run_as_module, date):
             df = pd.read_csv(
                 join(output_folder, "weekly_202026_state_deaths_covid_incidence_prop.csv")
             )
-            assert (df.columns.values == ["geo_id", "val", "se", "sample_size"]).all()
+            expected_columns = [
+                "geo_id", "val", "se", "sample_size",
+                "missing_val", "missing_se", "missing_sample_size"
+            ]
+            assert (df.columns.values == expected_columns).all()