Nans NCHS: add nancodes, missing columns

Author: dshemetov

Jenkinsfile

@@ -19,7 +19,10 @@ pipeline {
     stages {
         stage('Build and Package') {
             when {
+                anyOf {
                 branch "main";
+                branch "main-nan-testing";
+                }
             }
             steps {
                 script {
@@ -34,7 +37,26 @@ pipeline {
         }
         stage('Deploy staging') {
             when {
+                anyOf {
                 branch "main";
+                branch "main-nan-testing";
+                }
+            }
+            steps {
+                script {
+                    indicator_list.each { indicator ->
+                        deploy_staging[indicator] = {
+                            sh "jenkins/deploy-staging.sh ${indicator}"
+                        }
+                    }
+                    parallel deploy_staging
+                }
+            }
+        }
+        stage('Deploy staging [main-nan-testing]') {
+            when {
+                branch "main-nan-testing";
+                }
             }
             steps {
                 script {
@@ -45,6 +67,7 @@ pipeline {
                     }
                     parallel deploy_staging
                 }
+                sh "jenkins/deploy-staging-api-match-list.sh"
             }
         }
         stage('Deploy production') {

nchs_mortality/.gitignore

@@ -5,6 +5,9 @@ params.json
 
 # Do not commit output files
 receiving/*.csv
+daily_receiving/*.csv
+cache/*.csv
+daily_cache/*.csv
 
 # Do not commit test files
 tests/receiving/*.csv

nchs_mortality/delphi_nchs_mortality/run.py

@@ -9,14 +9,26 @@
 from typing import Dict, Any
 
 import numpy as np
-from delphi_utils import S3ArchiveDiffer, get_structured_logger, create_export_csv
+from delphi_utils import S3ArchiveDiffer, get_structured_logger, create_export_csv, Nans
 
 from .archive_diffs import arch_diffs
 from .constants import (METRICS, SENSOR_NAME_MAP,
                         SENSORS, INCIDENCE_BASE, GEO_RES)
 from .pull import pull_nchs_mortality_data
 
 
+def add_nancodes(df):
+    """Add nancodes to the dataframe."""
+    # Default missingness codes
+    df["missing_val"] = Nans.NOT_MISSING
+    df["missing_se"] = Nans.NOT_APPLICABLE
+    df["missing_sample_size"] = Nans.NOT_APPLICABLE
+
+    # Mark any remaining nans with unknown
+    remaining_nans_mask = df["val"].isnull()
+    df.loc[remaining_nans_mask, "missing_val"] = Nans.OTHER
+    return df
+
 def run_module(params: Dict[str, Any]):
     """Run module for processing NCHS mortality data.
 
@@ -67,7 +79,8 @@ def run_module(params: Dict[str, Any]):
             df["val"] = df[metric]
             df["se"] = np.nan
             df["sample_size"] = np.nan
-            df = df[~df["val"].isnull()]
+            df = add_nancodes(df)
+            # df = df[~df["val"].isnull()]
             sensor_name = "_".join([SENSOR_NAME_MAP[metric]])
             dates = create_export_csv(
                 df,
@@ -91,7 +104,8 @@ def run_module(params: Dict[str, Any]):
                     df["val"] = df[metric] / df["population"] * INCIDENCE_BASE
                 df["se"] = np.nan
                 df["sample_size"] = np.nan
-                df = df[~df["val"].isnull()]
+                df = add_nancodes(df)
+                # df = df[~df["val"].isnull()]
                 sensor_name = "_".join([SENSOR_NAME_MAP[metric], sensor])
                 dates = create_export_csv(
                     df,

nchs_mortality/tests/test_run.py

@@ -60,4 +60,8 @@ def test_output_file_format(self, run_as_module, date):
             df = pd.read_csv(
                 join(output_folder, "weekly_202026_state_deaths_covid_incidence_prop.csv")
             )
-            assert (df.columns.values == ["geo_id", "val", "se", "sample_size"]).all()
+            expected_columns = [
+                "geo_id", "val", "se", "sample_size",
+                "missing_val", "missing_se", "missing_sample_size"
+            ]
+            assert (df.columns.values == expected_columns).all()