diff --git a/docs/CHANGELOG.md b/docs/CHANGELOG.md index 205eedf0..1473faaf 100755 --- a/docs/CHANGELOG.md +++ b/docs/CHANGELOG.md @@ -1,6 +1,22 @@ All notable changes to this project will be documented in this file. We follow the [Semantic Versioning 2.0.0](http://semver.org/) format. +## v4.5.14.3 - 2025-01-31 - [PR#1413](https://github.com/NOAA-OWP/inundation-mapping/pull/1413) + +Implements a denylist for flow-based CatFIM (that uses the same conventions as the existing denylist functionality used in stage-based CatFIM. Adds CMUG1 to the denylist for flow-based CatFIM. + +### Additions +- `tools/catfim/ahps_restricted_sites.csv`: Renamed from `stage_based_ahps_restricted_sites.csv`. Added an additional column, `catfim_type`, that specifies whether a site should be restricted for flow-based CatFIM (`flow`), stage-based CatFIM (`stage`), or both (`both`). + +### Changes +- `tools/catfim/generate_categorical_fim.py`: Update the `load_restricted_sites()` function to handle restricted sites for both flow- and stage-based CatFIM. +- `tools/catfim/generate_categorical_fim_flows.py`: Add restricted sites filtration to flow-based CatFIM processing. + +### Removals +- `tools/catfim/stage_based_ahps_restricted_sites.csv`: Renamed to `ahps_restricted_sites.csv` + +

+ ## v4.5.14.2 - 2025-01-24 - [PR#1178](https://github.com/NOAA-OWP/inundation-mapping/pull/1178) ### Summary diff --git a/tools/catfim/ahps_restricted_sites.csv b/tools/catfim/ahps_restricted_sites.csv new file mode 100644 index 00000000..7f3d6c63 --- /dev/null +++ b/tools/catfim/ahps_restricted_sites.csv @@ -0,0 +1,70 @@ +nws_lid,restricted_reason,catfim_type +# RULES: Ensure that the top line has no comments above the header line above: +# -- Careful not to add commas other then the one after the lid ID. +# -- Don't allow for any duplicate LID values in this list please. +# +# Comment lines can be added as any independent line in this csv. +# +AABDB,Test Site,stage +ADLG1,Stage thresholds seem to be based on sea level and not channel thalweg,stage +AUGG1,Stage thresholds seem to be based on sea level and not channel thalweg,stage +BAXG1,Stage thresholds seem to be based on sea level and not channel thalweg,stage +BRKTS,Test Site,stage +CNNN6,Pool elevation thresholds. Disabled at request of MARFC,stage +DMBT2,Test Site,stage +DMSF1,Tidal Gauge,stage +DVNIA,Test Site,both +GVDA1,bad data in API,stage +HFMW4,gage relocated. check again later for review,stage +HLZU1,bad data in API,stage +HNXCA,Test Site,stage +HRAG1,Stage thresholds seem to be based on sea level and not channel thalweg,stage +HUSKR,Test Site,stage +HZT00,Test Site,stage +HZT02,Test Site,stage +ILMNA,Test Site,stage +JTEST,Test Site,stage +JYNT2,Test Site,stage +KLNQ9,Outside U.S.,both +LAMF1,Stage thresholds seem to be based on sea level and not channel thalweg.,stage +MCVA3,historical gauge,both +NVRN6,Pool elevation thresholds. Disabled at request of MARFC.,stage +PEPN6,Pool elevation thresholds. Disabled at request of MARFC.,stage +PRXQ9,Outside U.S.,both +ONDN6,Inundation issues,stage +QUTG1,Stage thresholds seem to be based on sea level and not channel thalweg,stage +RCYF1,Out of Service,both +RWBW3, historical gauge,stage +SMSV2,point not in AHPS,both +STNG1,Stage thresholds seem to be based on sea level and not channel thalweg,stage +TESM8,Test Site,stage +TEST,Test Site,stage +TEST1,Test Site,stage +TEST11,Test Site,stage +TEST2,Test Site,stage +TEST3,Test Site,stage +TEST4,Test Site,stage +TEST9,Test Site,stage +TESTA,Test Site,stage +TESTB,Test Site,stage +TESTC,Test Site,stage +TESTPT1,Test Site,stage +TETM8,Test Site,stage +TS2W3,Test Site,stage +TS3W3,Test Site,stage +TS4W3,Test Site,stage +TSTMO,Test Site,stage +TSTN6,Test Site,stage +TSTSP,Test Site,stage +TSTUP,Test Site,stage +TTARX,Test Site,both +VEFNV,Test Site,both +WBYA1,Out of Service,both +WLSA1, bad data in API,stage +WUNTS, not a real gauge,both +XXXN3,Test Site,stage +YDAQ9,Outside U.S.,both +YLKA2,Discontinued Gage,stage +YWRQ9,Outside U.S.,both +ZZZT2,Test Site,stage +CMUG1,Out of Service,both diff --git a/tools/catfim/generate_categorical_fim.py b/tools/catfim/generate_categorical_fim.py index 2261c37a..1cd75bdc 100755 --- a/tools/catfim/generate_categorical_fim.py +++ b/tools/catfim/generate_categorical_fim.py @@ -247,13 +247,13 @@ def process_generate_categorical_fim( # STAGE-BASED if is_stage_based: # Generate Stage-Based CatFIM mapping - # does flows and inundation (mapping) + # does flows and inundation (mapping) catfim_sites_file_path = os.path.join(output_mapping_dir, 'stage_based_catfim_sites.gpkg') if step_num <= 1: - df_restricted_sites = load_restricted_sites() + df_restricted_sites = load_restricted_sites(is_stage_based) generate_stage_based_categorical_fim( output_catfim_dir, @@ -300,6 +300,9 @@ def process_generate_categorical_fim( job_flows = job_number_huc * job_number_inundate if step_num <= 1: + + df_restricted_sites = load_restricted_sites(is_stage_based) + generate_flows( output_catfim_dir, nwm_us_search, @@ -310,6 +313,7 @@ def process_generate_categorical_fim( valid_ahps_hucs, nwm_metafile, FLOG.LOG_FILE_PATH, + df_restricted_sites, ) end = time.time() elapsed_time = (end - start) / 60 @@ -1167,16 +1171,18 @@ def __calc_stage_intervals(non_rec_stage_values_df, past_major_interval_cap, huc return interval_recs -def load_restricted_sites(): +def load_restricted_sites(is_stage_based): """ - At this point, only stage based uses this. But a arg of "catfim_type (stage or flow) or something - can be added later. + Previously, only stage based used this. It is now being used by stage-based and flow-based (1/24/25) + + The 'catfim_type' column can have three different values: 'stage', 'flow', and 'both'. This determines + whether the site should be filtered out for stage-based CatFIM, flow-based CatFIM, or both of them. Returns: a dataframe for the restricted lid and the reason why: - "nws_lid", "restricted_reason" + 'nws_lid', 'restricted_reason', 'catfim_type' """ - file_name = "stage_based_ahps_restricted_sites.csv" + file_name = "ahps_restricted_sites.csv" current_script_folder = os.path.dirname(__file__) file_path = os.path.join(current_script_folder, file_name) @@ -1184,6 +1190,12 @@ def load_restricted_sites(): df_restricted_sites['nws_lid'].fillna("", inplace=True) df_restricted_sites['restricted_reason'].fillna("", inplace=True) + df_restricted_sites['catfim_type'].fillna("", inplace=True) + + # remove extra empty spaces on either side of all cellls + df_restricted_sites['nws_lid'] = df_restricted_sites['nws_lid'].str.strip() + df_restricted_sites['restricted_reason'] = df_restricted_sites['restricted_reason'].str.strip() + df_restricted_sites['catfim_type'] = df_restricted_sites['catfim_type'].str.strip() # Need to drop the comment lines before doing any more processing df_restricted_sites.drop( @@ -1195,11 +1207,13 @@ def load_restricted_sites(): # There are enough conditions and a low number of rows that it is easier to # test / change them via a for loop indexs_for_recs_to_be_removed_from_list = [] + + # Clean up dataframe for ind, row in df_restricted_sites.iterrows(): nws_lid = row['nws_lid'] restricted_reason = row['restricted_reason'] - if len(nws_lid) != 5: # could be just a blank row in the + if len(nws_lid) != 5: # Invalid row, could be just a blank row in the file FLOG.warning( f"From the ahps_restricted_sites list, an invalid nws_lid value of '{nws_lid}'" " and has dropped from processing" @@ -1213,14 +1227,22 @@ def load_restricted_sites(): df_restricted_sites.at[ind, 'restricted_reason'] = restricted_reason FLOG.warning(f"{restricted_reason}. Lid is '{nws_lid}'") continue - # end for + # end loop - # Invalid records (not dropping, just completely invalid recs from the csv) + # Invalid records in CSV (not dropping, just completely invalid recs from the csv) # Could be just blank rows from the csv if len(indexs_for_recs_to_be_removed_from_list) > 0: df_restricted_sites = df_restricted_sites.drop(indexs_for_recs_to_be_removed_from_list).reset_index() - # print(df_restricted_sites.head(10)) + # Filter df_restricted_sites by CatFIM type + if is_stage_based == True: # Keep rows where 'catfim_type' is either 'stage' or 'both' + df_restricted_sites = df_restricted_sites[df_restricted_sites['catfim_type'].isin(['stage', 'both'])] + + else: # Keep rows where 'catfim_type' is either 'flow' or 'both' + df_restricted_sites = df_restricted_sites[df_restricted_sites['catfim_type'].isin(['flow', 'both'])] + + # Remove catfim_type column + df_restricted_sites.drop('catfim_type', axis=1, inplace=True) return df_restricted_sites @@ -1526,6 +1548,7 @@ def generate_stage_based_categorical_fim( lst_hucs, nwm_metafile, str(FLOG.LOG_FILE_PATH), + df_restricted_sites, ) ) diff --git a/tools/catfim/generate_categorical_fim_flows.py b/tools/catfim/generate_categorical_fim_flows.py index 005cc0aa..eac86610 100755 --- a/tools/catfim/generate_categorical_fim_flows.py +++ b/tools/catfim/generate_categorical_fim_flows.py @@ -66,6 +66,7 @@ def generate_flows_for_huc( nwm_flows_df, parent_log_output_file, child_log_file_prefix, + df_restricted_sites, ): try: @@ -127,6 +128,17 @@ def generate_flows_for_huc( # Convert lid to lower case lid = lid.lower() + # Check whether LID is in the restricted sites list + found_restrict_lid = df_restricted_sites.loc[df_restricted_sites['nws_lid'] == lid.upper()] + + # Assume only one rec for now, fix later + if len(found_restrict_lid) > 0: + reason = found_restrict_lid.iloc[0, found_restrict_lid.columns.get_loc("restricted_reason")] + msg = ':' + reason + all_messages.append(lid + msg) + MP_LOG.warning(huc_lid_id + msg) + continue + # TODO: Jun 17, 2024 - This gets recalled for every huc but only uses the nws_list. # Move this somewhere outside the huc list so it doesn't need to be called over and over again @@ -363,6 +375,7 @@ def generate_flows( lst_hucs, nwm_metafile, log_output_file, + df_restricted_sites, ): # TODO; Most docstrings like this are now very outdated and need updating @@ -509,6 +522,7 @@ def generate_flows( nwm_flows_region_df, log_output_file, child_log_file_prefix, + df_restricted_sites, ) # end ProcessPoolExecutor diff --git a/tools/catfim/stage_based_ahps_restricted_sites.csv b/tools/catfim/stage_based_ahps_restricted_sites.csv deleted file mode 100644 index db5cb88f..00000000 --- a/tools/catfim/stage_based_ahps_restricted_sites.csv +++ /dev/null @@ -1,69 +0,0 @@ -nws_lid,restricted_reason -# RULES: Ensure that the top line has no comments above the header line above: -# -- Careful not to add commas other then the one after the lid ID. -# -- Don't allow for any duplicate LID values in this list please. -# -# Comment lines can be added as any independent line in this csv. -# -AABDB,Test Site -ADLG1,Stage thresholds seem to be based on sea level and not channel thalweg -AUGG1,Stage thresholds seem to be based on sea level and not channel thalweg -BAXG1,Stage thresholds seem to be based on sea level and not channel thalweg -BRKTS,Test Site -CNNN6,Pool elevation thresholds. Disabled at request of MARFC -DMBT2,Test Site -DMSF1,Tidal Gauge -DVNIA,Test Site -GVDA1,bad data in API -HFMW4,gage relocated. check again later for review -HLZU1,bad data in API -HNXCA,Test Site -HRAG1,Stage thresholds seem to be based on sea level and not channel thalweg -HUSKR,Test Site -HZT00,Test Site -HZT02,Test Site -ILMNA,Test Site -JTEST,Test Site -JYNT2,Test Site -KLNQ9,Outside U.S. -LAMF1,Stage thresholds seem to be based on sea level and not channel thalweg. -MCVA3,historical gauge -NVRN6,Pool elevation thresholds. Disabled at request of MARFC. -PEPN6,Pool elevation thresholds. Disabled at request of MARFC. -PRXQ9,Outside U.S. -ONDN6,Inundation issues -QUTG1,Stage thresholds seem to be based on sea level and not channel thalweg -RCYF1,Out of Service -RWBW3, historical gauge -SMSV2,point not in AHPS -STNG1,Stage thresholds seem to be based on sea level and not channel thalweg -TESM8,Test Site -TEST,Test Site -TEST1,Test Site -TEST11,Test Site -TEST2,Test Site -TEST3,Test Site -TEST4,Test Site -TEST9,Test Site -TESTA,Test Site -TESTB,Test Site -TESTC,Test Site -TESTPT1,Test Site -TETM8,Test Site -TS2W3,Test Site -TS3W3,Test Site -TS4W3,Test Site -TSTMO,Test Site -TSTN6,Test Site -TSTSP,Test Site -TSTUP,Test Site -TTARX,Test Site -VEFNV,Test Site -WBYA1,Out of Service -WLSA1, bad data in API -WUNTS, not a real gauge -XXXN3,Test Site -YDAQ9,Outside U.S. -YLKA2,Discontinued Gage -YWRQ9,Outside U.S. -ZZZT2,Test Site