RVDSS fix flu subtype behaviour and inf's (#1752)

* When there are 0 flu tests, subtype percent positivity should be null

* calculate total positive flu types from positive subtype tests

* add current season to list of seasons to be patched

* change infs to None

* replace() single values to avoid stray NaNs remaining

---------

Co-authored-by: nmdefries <42820733+nmdefries@users.noreply.github.com>

Author: cchuong

src/acquisition/rvdss/database.py

@@ -87,8 +87,13 @@ def update(data, logger):
                                   database="epidata")
     cur = cnx.cursor()
 
+    # Convert infinities to NaN first (comparison-based replace handles inf on
+    # numeric columns), then map every NaN to None. Passing multiple keys to a
+    # single replace() call routes through pandas' multi-value code path, which
+    # does not reliably convert NaN -> None, leaving stray NaNs that break the
+    # SQL insert ("Unknown column 'nan'").
     data = data.reset_index(
-      ).replace({np.nan: None})
+      ).replace([np.inf, -np.inf], np.nan).replace({np.nan: None})
     rvdss_cols_subset = [col for col in data.columns if col in rvdss_cols]
     data_dict = data.to_dict(orient = "records")
 

src/acquisition/rvdss/pull_historic.py

@@ -232,9 +232,13 @@ def create_detections_table(table,modified_date,week_number,week_end_date,start_
                     'geo_type':geo_types})
 
     table.columns =[re.sub(" ","_",col) for col in table.columns]
+    
+    # calculate total flu positive tests from a and b positive tests
+    if "flua_positive_tests" in table.columns and "flub_positive_tests" in table.columns:
+        table["flu_positive_tests"] = table["flua_positive_tests"] + table["flub_positive_tests"]
+    
     return(table)
 
-
 def create_percent_positive_detection_table(table,modified_date,start_year, flu=False,overwrite_weeks=False):
     table = deduplicate_rows(table)
     table.columns = [re.sub(' +', ' ',col) for col in table.columns]
@@ -300,6 +304,10 @@ def create_percent_positive_detection_table(table,modified_date,start_year, flu=
 
         table["flu_positive_tests"] =  table["flua_positive_tests"] +  table["flub_positive_tests"]
         table["flu_pct_positive"] =   (table["flu_positive_tests"]/table["flu_tests"])*100
+        
+        table.loc[table["flu_tests"] == 0,"flua_pct_positive"] = np.nan  
+        table.loc[table["flu_tests"] == 0,"flub_pct_positive"] = np.nan  
+        
     else:
         table[virus+"_positive_tests"] = (table[virus+"_pct_positive"]/100) *table[virus+"_tests"]
 

src/acquisition/rvdss/run.py

@@ -137,7 +137,6 @@ def main():
         "-pat",
         action="store_true",
         help="patch in missings seasons, which are unarchived and must be obtained through csvs"
-        # TODO: Look into passing a file name
     )
     # fmt: on
     args = parser.parse_args()
@@ -167,7 +166,7 @@ def main():
     if historical_flag:
         update_historical_data(logger)
     if patch_flag:
-        patch_seasons(season_start_years=[2013,2014,2024], logger=logger)
+        patch_seasons(season_start_years=[2013,2014,2024,2025], logger=logger)
 
 
 if __name__ == "__main__":

testdata/acquisition/rvdss/formatted_detections.csv

@@ -1,42 +1,42 @@
-epiweek,time_value,issue,geo_type,geo_value,flu_tests,fluah1n1pdm09_positive_tests,fluah3_positive_tests,fluauns_positive_tests,flua_positive_tests,flub_positive_tests,rsv_tests,rsv_positive_tests,hpiv_tests,hpiv1_positive_tests,hpiv2_positive_tests,hpiv3_positive_tests,hpiv4_positive_tests,hpivother_positive_tests,adv_tests,adv_positive_tests,hmpv_tests,hmpv_positive_tests,evrv_tests,evrv_positive_tests,hcov_tests,hcov_positive_tests
-201703,2017-01-21,2017-01-25,province,nl,87,0,4,4,8,0,87,7,87,0,1,1,0,0,87,0,87,3,87,1,NA,NA
-201703,2017-01-21,2017-01-25,province,pe,58,0,17,0,17,0,58,6,2,0,0,0,0,0,2,0,2,0,2,0,2,1
-201703,2017-01-21,2017-01-25,province,ns,119,0,0,12,12,0,109,18,20,0,0,1,0,0,20,0,20,0,20,1,20,7
-201703,2017-01-21,2017-01-25,province,nb,247,0,0,43,43,0,249,33,57,0,0,1,0,0,57,2,57,1,57,5,57,7
-201703,2017-01-21,2017-01-25,region,atlantic,511,0,21,59,80,0,503,64,166,0,1,3,0,0,166,2,166,4,166,7,79,15
-201703,2017-01-21,2017-01-25,lab,région nord est,330,0,0,18,18,0,293,62,0,0,0,0,0,0,0,0,0,0,NA,NA,0,0
-201703,2017-01-21,2017-01-25,lab,québec chaudière appalaches,608,0,0,93,93,2,380,83,139,0,0,5,0,0,140,2,108,0,NA,NA,108,0
-201703,2017-01-21,2017-01-25,lab,centre du québec,659,0,49,102,151,4,490,118,0,0,0,0,0,0,0,0,0,0,NA,NA,0,0
-201703,2017-01-21,2017-01-25,lab,montréal laval,1653,0,0,300,300,6,1101,167,632,1,2,3,3,0,644,24,613,28,NA,NA,549,27
-201703,2017-01-21,2017-01-25,lab,ouest du québec,403,0,0,75,75,2,147,33,0,0,0,0,0,0,0,0,0,0,NA,NA,0,0
-201703,2017-01-21,2017-01-25,lab,montérégie,342,0,0,69,69,1,285,49,0,0,0,0,0,0,0,0,0,0,NA,NA,0,0
-201703,2017-01-21,2017-01-25,province,qc,3995,0,49,657,706,15,2696,512,771,1,2,8,3,0,784,26,721,28,NA,NA,657,27
-201703,2017-01-21,2017-01-25,lab,ottawa phl,64,0,14,0,14,0,64,7,64,0,0,1,0,0,64,0,64,9,64,1,64,16
-201703,2017-01-21,2017-01-25,lab,cheo ottawa,304,0,0,42,42,0,304,60,29,0,0,0,0,0,29,0,51,3,29,1,29,2
-201703,2017-01-21,2017-01-25,lab,kingston phl,76,0,17,0,17,0,76,11,76,1,4,2,0,0,76,1,74,4,76,3,74,11
-201703,2017-01-21,2017-01-25,lab,uhn mount sinai hospital,433,0,0,71,71,0,433,24,18,0,0,0,0,0,0,0,18,0,0,0,0,0
-201703,2017-01-21,2017-01-25,lab,cphl toronto,1559,2,613,1,616,3,1174,102,1174,0,2,7,2,0,1174,9,1169,36,1174,28,1169,98
-201703,2017-01-21,2017-01-25,lab,sick kids hospital toronto,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0
-201703,2017-01-21,2017-01-25,lab,sunnybrook womens college hsc,127,0,12,10,22,0,127,10,127,0,1,1,0,0,127,0,127,4,127,1,127,12
-201703,2017-01-21,2017-01-25,lab,sault ste marie phl,24,0,4,0,4,0,24,5,24,0,0,2,0,0,24,0,24,1,24,0,24,3
-201703,2017-01-21,2017-01-25,lab,timmins phl,14,0,1,0,1,0,14,3,14,0,0,1,0,0,14,0,14,1,14,1,14,2
-201703,2017-01-21,2017-01-25,lab,st josephs london,100,0,0,16,16,0,100,6,18,0,1,0,0,0,0,0,18,2,0,0,0,0
-201703,2017-01-21,2017-01-25,lab,london phl,399,2,100,2,104,1,399,65,399,0,2,8,4,0,399,6,397,17,399,13,397,57
-201703,2017-01-21,2017-01-25,lab,orillia phl,228,0,38,0,38,0,227,42,227,0,1,3,1,0,227,5,226,8,227,15,226,22
-201703,2017-01-21,2017-01-25,lab,thunder bay phl,36,0,7,0,7,0,32,6,32,0,0,0,0,0,32,0,32,1,32,2,32,6
-201703,2017-01-21,2017-01-25,lab,sudbury phl,62,0,7,0,7,0,62,6,62,0,0,1,0,0,62,0,60,0,62,4,60,5
-201703,2017-01-21,2017-01-25,lab,hamilton phl,201,1,54,2,57,3,190,32,190,0,0,2,0,0,190,3,186,6,190,13,186,17
-201703,2017-01-21,2017-01-25,lab,peterborough phl,139,0,31,1,32,0,139,21,139,0,0,4,0,0,139,0,139,3,139,9,139,9
-201703,2017-01-21,2017-01-25,province,on,3766,5,898,145,1048,7,3365,400,2593,1,11,32,7,0,2557,24,2599,95,2557,91,2541,260
-201703,2017-01-21,2017-01-25,province,mb,333,0,7,11,18,0,331,92,55,0,0,1,0,0,55,0,44,2,55,1,44,11
-201703,2017-01-21,2017-01-25,lab,regina,536,0,101,10,111,0,536,113,536,0,0,13,3,0,536,9,536,7,536,28,536,50
-201703,2017-01-21,2017-01-25,lab,saskatoon,217,0,0,32,32,1,217,40,37,0,0,1,0,0,37,1,37,0,37,2,37,0
-201703,2017-01-21,2017-01-25,province,sk,753,0,101,42,143,1,753,153,573,0,0,14,3,0,573,10,573,7,573,30,573,50
-201703,2017-01-21,2017-01-25,province,ab,1336,0,202,48,250,4,1336,185,1336,33,0,0,0,0,1336,9,1336,10,1336,30,1336,30
-201703,2017-01-21,2017-01-25,region,prairies,2422,0,310,101,411,5,2420,430,1964,33,0,15,3,0,1964,19,1953,19,1964,61,1953,91
-201703,2017-01-21,2017-01-25,province,bc,1360,0,55,362,417,20,1360,149,277,2,3,11,6,0,277,2,277,22,277,30,277,27
-201703,2017-01-21,2017-01-25,province,yt,27,0,2,4,6,0,27,7,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA
-201703,2017-01-21,2017-01-25,province,nt,32,0,7,1,8,0,32,6,32,1,0,0,0,0,32,1,32,0,32,1,32,3
-201703,2017-01-21,2017-01-25,province,nu,44,0,8,6,14,0,44,2,44,0,3,0,0,0,44,2,44,0,44,16,44,4
-201703,2017-01-21,2017-01-25,region,territories,103,0,17,11,28,0,103,15,76,1,3,0,0,0,76,3,76,0,76,17,76,7
-201703,2017-01-21,2017-01-25,nation,ca,12157,5,1350,1335,2690,47,10447,1570,5847,38,20,69,19,0,5824,76,5792,168,5040,206,5583,427
+epiweek,time_value,issue,geo_type,geo_value,flu_tests,fluah1n1pdm09_positive_tests,fluah3_positive_tests,fluauns_positive_tests,flua_positive_tests,flub_positive_tests,rsv_tests,rsv_positive_tests,hpiv_tests,hpiv1_positive_tests,hpiv2_positive_tests,hpiv3_positive_tests,hpiv4_positive_tests,hpivother_positive_tests,adv_tests,adv_positive_tests,hmpv_tests,hmpv_positive_tests,evrv_tests,evrv_positive_tests,hcov_tests,hcov_positive_tests,flu_positive_tests
+201703,2017-01-21,2017-01-25,province,nl,87,0,4,4,8,0,87,7,87,0,1,1,0,0,87,0,87,3,87,1,NA,NA,8
+201703,2017-01-21,2017-01-25,province,pe,58,0,17,0,17,0,58,6,2,0,0,0,0,0,2,0,2,0,2,0,2,1,17
+201703,2017-01-21,2017-01-25,province,ns,119,0,0,12,12,0,109,18,20,0,0,1,0,0,20,0,20,0,20,1,20,7,12
+201703,2017-01-21,2017-01-25,province,nb,247,0,0,43,43,0,249,33,57,0,0,1,0,0,57,2,57,1,57,5,57,7,43
+201703,2017-01-21,2017-01-25,region,atlantic,511,0,21,59,80,0,503,64,166,0,1,3,0,0,166,2,166,4,166,7,79,15,80
+201703,2017-01-21,2017-01-25,lab,région nord est,330,0,0,18,18,0,293,62,0,0,0,0,0,0,0,0,0,0,NA,NA,0,0,18
+201703,2017-01-21,2017-01-25,lab,québec chaudière appalaches,608,0,0,93,93,2,380,83,139,0,0,5,0,0,140,2,108,0,NA,NA,108,0,95
+201703,2017-01-21,2017-01-25,lab,centre du québec,659,0,49,102,151,4,490,118,0,0,0,0,0,0,0,0,0,0,NA,NA,0,0,155
+201703,2017-01-21,2017-01-25,lab,montréal laval,1653,0,0,300,300,6,1101,167,632,1,2,3,3,0,644,24,613,28,NA,NA,549,27,306
+201703,2017-01-21,2017-01-25,lab,ouest du québec,403,0,0,75,75,2,147,33,0,0,0,0,0,0,0,0,0,0,NA,NA,0,0,77
+201703,2017-01-21,2017-01-25,lab,montérégie,342,0,0,69,69,1,285,49,0,0,0,0,0,0,0,0,0,0,NA,NA,0,0,70
+201703,2017-01-21,2017-01-25,province,qc,3995,0,49,657,706,15,2696,512,771,1,2,8,3,0,784,26,721,28,NA,NA,657,27,721
+201703,2017-01-21,2017-01-25,lab,ottawa phl,64,0,14,0,14,0,64,7,64,0,0,1,0,0,64,0,64,9,64,1,64,16,14
+201703,2017-01-21,2017-01-25,lab,cheo ottawa,304,0,0,42,42,0,304,60,29,0,0,0,0,0,29,0,51,3,29,1,29,2,42
+201703,2017-01-21,2017-01-25,lab,kingston phl,76,0,17,0,17,0,76,11,76,1,4,2,0,0,76,1,74,4,76,3,74,11,17
+201703,2017-01-21,2017-01-25,lab,uhn mount sinai hospital,433,0,0,71,71,0,433,24,18,0,0,0,0,0,0,0,18,0,0,0,0,0,71
+201703,2017-01-21,2017-01-25,lab,cphl toronto,1559,2,613,1,616,3,1174,102,1174,0,2,7,2,0,1174,9,1169,36,1174,28,1169,98,619
+201703,2017-01-21,2017-01-25,lab,sick kids hospital toronto,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0
+201703,2017-01-21,2017-01-25,lab,sunnybrook womens college hsc,127,0,12,10,22,0,127,10,127,0,1,1,0,0,127,0,127,4,127,1,127,12,22
+201703,2017-01-21,2017-01-25,lab,sault ste marie phl,24,0,4,0,4,0,24,5,24,0,0,2,0,0,24,0,24,1,24,0,24,3,4
+201703,2017-01-21,2017-01-25,lab,timmins phl,14,0,1,0,1,0,14,3,14,0,0,1,0,0,14,0,14,1,14,1,14,2,1
+201703,2017-01-21,2017-01-25,lab,st josephs london,100,0,0,16,16,0,100,6,18,0,1,0,0,0,0,0,18,2,0,0,0,0,16
+201703,2017-01-21,2017-01-25,lab,london phl,399,2,100,2,104,1,399,65,399,0,2,8,4,0,399,6,397,17,399,13,397,57,105
+201703,2017-01-21,2017-01-25,lab,orillia phl,228,0,38,0,38,0,227,42,227,0,1,3,1,0,227,5,226,8,227,15,226,22,38
+201703,2017-01-21,2017-01-25,lab,thunder bay phl,36,0,7,0,7,0,32,6,32,0,0,0,0,0,32,0,32,1,32,2,32,6,7
+201703,2017-01-21,2017-01-25,lab,sudbury phl,62,0,7,0,7,0,62,6,62,0,0,1,0,0,62,0,60,0,62,4,60,5,7
+201703,2017-01-21,2017-01-25,lab,hamilton phl,201,1,54,2,57,3,190,32,190,0,0,2,0,0,190,3,186,6,190,13,186,17,60
+201703,2017-01-21,2017-01-25,lab,peterborough phl,139,0,31,1,32,0,139,21,139,0,0,4,0,0,139,0,139,3,139,9,139,9,32
+201703,2017-01-21,2017-01-25,province,on,3766,5,898,145,1048,7,3365,400,2593,1,11,32,7,0,2557,24,2599,95,2557,91,2541,260,1055
+201703,2017-01-21,2017-01-25,province,mb,333,0,7,11,18,0,331,92,55,0,0,1,0,0,55,0,44,2,55,1,44,11,18
+201703,2017-01-21,2017-01-25,lab,regina,536,0,101,10,111,0,536,113,536,0,0,13,3,0,536,9,536,7,536,28,536,50,111
+201703,2017-01-21,2017-01-25,lab,saskatoon,217,0,0,32,32,1,217,40,37,0,0,1,0,0,37,1,37,0,37,2,37,0,33
+201703,2017-01-21,2017-01-25,province,sk,753,0,101,42,143,1,753,153,573,0,0,14,3,0,573,10,573,7,573,30,573,50,144
+201703,2017-01-21,2017-01-25,province,ab,1336,0,202,48,250,4,1336,185,1336,33,0,0,0,0,1336,9,1336,10,1336,30,1336,30,254
+201703,2017-01-21,2017-01-25,region,prairies,2422,0,310,101,411,5,2420,430,1964,33,0,15,3,0,1964,19,1953,19,1964,61,1953,91,416
+201703,2017-01-21,2017-01-25,province,bc,1360,0,55,362,417,20,1360,149,277,2,3,11,6,0,277,2,277,22,277,30,277,27,437
+201703,2017-01-21,2017-01-25,province,yt,27,0,2,4,6,0,27,7,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,6
+201703,2017-01-21,2017-01-25,province,nt,32,0,7,1,8,0,32,6,32,1,0,0,0,0,32,1,32,0,32,1,32,3,8
+201703,2017-01-21,2017-01-25,province,nu,44,0,8,6,14,0,44,2,44,0,3,0,0,0,44,2,44,0,44,16,44,4,14
+201703,2017-01-21,2017-01-25,region,territories,103,0,17,11,28,0,103,15,76,1,3,0,0,0,76,3,76,0,76,17,76,7,28
+201703,2017-01-21,2017-01-25,nation,ca,12157,5,1350,1335,2690,47,10447,1570,5847,38,20,69,19,0,5824,76,5792,168,5040,206,5583,427,2737

testdata/acquisition/rvdss/formatted_flu_positive_tests.csv

@@ -2,12 +2,12 @@ epiweek,time_value,issue,geo_type,geo_value,flua_pct_positive,flub_pct_positive,
 201635,2016-09-03,2017-01-25,nation,ca,0.11,0,1862,2.0482,0,2.0482,0.11
 201635,2016-09-03,2017-01-25,region,atlantic,0,0,79,0,0,0,0
 201635,2016-09-03,2017-01-25,province,qc,0,0,431,0,0,0,0
-201635,2016-09-03,2017-01-25,province,on,0.18,0,568,1.0224,0,1.0224,0.18
+201635,2016-09-03,2017-01-25,province,on,NA,NA,0,0,0,0,NA
 201635,2016-09-03,2017-01-25,region,prairies,0.16,0,616,0.9856,0,0.9856,0.16
 201635,2016-09-03,2017-01-25,province,bc,0,0,151,0,0,0,0
 201635,2016-09-03,2017-01-25,region,territories,0,0,17,0,0,0,0
 201636,2016-09-10,2017-01-25,nation,ca,0.49,0.55,1630,7.986999999999999,8.965000000000002,16.952,1.04
-201636,2016-09-10,2017-01-25,region,atlantic,0,0,55,0,0,0,0
+201636,2016-09-10,2017-01-25,region,atlantic,NA,NA,0,0,0,0,NA
 201636,2016-09-10,2017-01-25,province,qc,0.25,0,400,1,0,1,0.25
 201636,2016-09-10,2017-01-25,province,on,0.21,0.62,481,1.0101,2.9821999999999997,3.9923,0.83
 201636,2016-09-10,2017-01-25,region,prairies,0.18,0.7,571,1.0278,3.997,5.0248,0.88

testdata/acquisition/rvdss/week3_20162017.html

@@ -14875,7 +14875,7 @@
         <p class="p1"><span class="s1"></span><br></p>
       </td>
       <td valign="middle" class="td2">
-        <p class="p2"><span class="s1">&lt;td&gt;568&lt;/td&gt;</span></p>
+        <p class="p2"><span class="s1">&lt;td&gt;0&lt;/td&gt;</span></p>
       </td>
     </tr>
     <tr>
@@ -15027,7 +15027,7 @@
         <p class="p1"><span class="s1"></span><br></p>
       </td>
       <td valign="middle" class="td2">
-        <p class="p2"><span class="s1">&lt;td&gt;55&lt;/td&gt;</span></p>
+        <p class="p2"><span class="s1">&lt;td&gt;0&lt;/td&gt;</span></p>
       </td>
     </tr>
     <tr>