From c7981ae52acfe9bea767310c1ae2537f0a9da1d8 Mon Sep 17 00:00:00 2001 From: Matt Luck Date: Fri, 27 Dec 2024 07:08:59 -0700 Subject: [PATCH 1/5] Fix test_cases and add env variables for AK --- src/bash_variables.env | 6 ++++- tools/run_test_case.py | 40 +++++++++++++++++++++++----------- tools/synthesize_test_cases.py | 3 +++ 3 files changed, 35 insertions(+), 14 deletions(-) diff --git a/src/bash_variables.env b/src/bash_variables.env index fff5d785..bc4eb027 100644 --- a/src/bash_variables.env +++ b/src/bash_variables.env @@ -13,6 +13,8 @@ export input_DEM=${inputsDir}/dems/3dep_dems/10m_50 export input_DEM_Alaska=${inputsDir}/dems/3dep_dems/10m_South_Alaska/20240912/FIM_3dep_dem_South_Alaska_10m.vrt export input_DEM_domain=${inputsDir}/dems/3dep_dems/10m_5070/20240916/DEM_Domain.gpkg export input_DEM_domain_Alaska=${inputsDir}/dems/3dep_dems/10m_South_Alaska/20240912/DEM_Domain.gpkg +export input_landsea=${inputsDir}/landsea/water_polygons_us.gpkg +export input_landsea_Alaska=${inputsDir}/landsea/water_polygons_alaska.gpkg export input_GL_boundaries=${inputsDir}/landsea/gl_water_polygons.gpkg export input_NLD=${inputsDir}/nld_vectors/System_Routes_NLDFS_5070_230314.gpkg export input_NLD_Alaska=${inputsDir}/nld_vectors/System_Routes_NLDFS_3338_230314.gpkg @@ -27,13 +29,15 @@ export input_nwm_flows_Alaska=${inputsDir}/nwm_hydrofabric/nwm_f export input_nwm_headwaters=${inputsDir}/nwm_hydrofabric/nwm_headwaters.gpkg export input_nwm_headwaters_Alaska=${inputsDir}/nwm_hydrofabric/nwm_headwaters_alaska.gpkg export input_nwm_lakes=${inputsDir}/nwm_hydrofabric/nwm_lakes.gpkg -export input_nwm_lakes_Alaska=${inputsDir}/nwm_hydrofabric/nwm_waterbodies_alaska.gpkg +export input_nwm_lakes_Alaska=${inputsDir}/nwm_hydrofabric/nwm_waterbodies_alaska.gpkg export input_WBD_gdb=${inputsDir}/wbd/WBD_National_EPSG_5070_WBDHU8_clip_dem_domain.gpkg export input_WBD_gdb_Alaska=${inputsDir}/wbd/WBD_National_South_Alaska.gpkg export input_calib_points_dir=${inputsDir}/rating_curve/water_edge_database/calibration_points/ export bathymetry_file=${inputsDir}/bathymetry/bathymetric_adjustment_data.gpkg export osm_bridges=${inputsDir}/osm/bridges/241002/osm_all_bridges.gpkg +export nws_lid=${inputsDir}/ahps_sites/nws_lid.gpkg + # input file location with nwm feature_id and recurrence flow values export bankfull_flows_file=${inputsDir}/rating_curve/bankfull_flows/nwm3_high_water_threshold_cms.csv diff --git a/tools/run_test_case.py b/tools/run_test_case.py index efb2f22e..4180b89d 100755 --- a/tools/run_test_case.py +++ b/tools/run_test_case.py @@ -54,6 +54,9 @@ def magnitudes(self): def huc_data(self): '''Returns a dict of HUC8, magnitudes, and sites.''' huc_mags = {} + if not os.path.exists(self.validation_data): + return huc_mags + for huc in os.listdir(self.validation_data): if not re.match(r'\d{8}', huc): continue @@ -126,19 +129,30 @@ def __init__(self, test_id, version, archive=True): # Benchmark data path self.benchmark_dir = os.path.join(self.validation_data, self.huc) - # Create list of shapefile paths to use as exclusion areas. - self.mask_dict = { - 'levees': { - 'path': '/data/inputs/nld_vectors/Levee_protected_areas.gpkg', - 'buffer': None, - 'operation': 'exclude', - }, - 'waterbodies': { - 'path': '/data/inputs/nwm_hydrofabric/nwm_lakes.gpkg', - 'buffer': None, - 'operation': 'exclude', - }, - } + if self.huc[:2] == '19': + self.mask_dict = { + 'levees': { + 'path': os.getenv('input_nld_levee_protected_areas_Alaska'), + 'buffer': None, + 'operation': 'exclude', + }, + 'waterbodies': { + # 'path': '/data/inputs/nwm_hydrofabric/nwm_lakes.gpkg', + 'path': os.getenv('input_nwm_lakes_Alaska'), + 'buffer': None, + 'operation': 'exclude', + }, + } + + else: + self.mask_dict = { + 'levees': { + 'path': os.getenv('input_nld_levee_protected_areas'), + 'buffer': None, + 'operation': 'exclude', + }, + 'waterbodies': {'path': os.getenv('input_nwm_lakes'), 'buffer': None, 'operation': 'exclude'}, + } @classmethod def list_all_test_cases(cls, version, archive, benchmark_categories=[]): diff --git a/tools/synthesize_test_cases.py b/tools/synthesize_test_cases.py index ab734bf7..9118f56f 100755 --- a/tools/synthesize_test_cases.py +++ b/tools/synthesize_test_cases.py @@ -115,6 +115,9 @@ def create_master_metrics_csv( # Iterate through 5 benchmark sources for benchmark_source in ['ble', 'nws', 'usgs', 'ifc', 'ras2fim']: benchmark_test_case_dir = os.path.join(TEST_CASES_DIR, benchmark_source + '_test_cases') + if not os.path.exists(benchmark_test_case_dir): + continue + test_cases_list = [d for d in os.listdir(benchmark_test_case_dir) if re.match(r'\d{8}_\w{3,7}', d)] if benchmark_source in ['ble', 'ifc', 'ras2fim']: From f6c263174a7fcd19547d96b2713856b142999ad5 Mon Sep 17 00:00:00 2001 From: Matt Luck Date: Fri, 27 Dec 2024 08:22:31 -0700 Subject: [PATCH 2/5] Update Changelog --- docs/CHANGELOG.md | 15 +++++++++++++++ src/bash_variables.env | 22 +++++++++++----------- 2 files changed, 26 insertions(+), 11 deletions(-) diff --git a/docs/CHANGELOG.md b/docs/CHANGELOG.md index 4e2da186..74a66cbd 100755 --- a/docs/CHANGELOG.md +++ b/docs/CHANGELOG.md @@ -1,6 +1,21 @@ All notable changes to this project will be documented in this file. We follow the [Semantic Versioning 2.0.0](http://semver.org/) format. +## v4.x.x.x - 2024-12-27 - [PR#1387](https://github.com/NOAA-OWP/inundation-mapping/pull/1387) + +Fixes two issues in test_cases: +1. An error in `synthesize_test_cases` and `run_test_case` if any directories of the 5 benchmark sources (BLE, NWS, IFC, USGS, or ras2fim) do not exist. This issue was originally discovered and fixed in #1178, but is being elevated to its own PR here. Fixes #1386. +2. Updated `run_test_cases` to accommodate levee and waterbody masking in Alaska. As part of these changes, hardcoded paths were replaced by environment variables. + +### Changes + +- `src/bash_variables.env`: Added environment variables for Alaska waterbody and levee masks and two variables to replace fixed paths in an upcoming PR (#1178). +- `tools/` + - `run_test_case.py`: Fixed error if missing validation data. Updated masking data to include Alaska. + - `synthesize_test_cases.py`: Fixed error if missing validation data. + +

+ ## v4.5.13.1 - 2024-12-13 - [PR#1361](https://github.com/NOAA-OWP/inundation-mapping/pull/1361) This PR was triggered by two dep-bot PR's. One for Tornado, one for aiohttp. Upon further research, these two exist only as dependencies for Jupyter and Jupyterlab which were very out of date. Upgrading Jupyter/JupyterLab took care of the other two. diff --git a/src/bash_variables.env b/src/bash_variables.env index bc4eb027..5aaa4676 100644 --- a/src/bash_variables.env +++ b/src/bash_variables.env @@ -2,8 +2,8 @@ export res=10 # Default raster cell size in meters -export DEFAULT_FIM_PROJECTION_CRS=EPSG:5070 -export ALASKA_CRS=EPSG:3338 +export DEFAULT_FIM_PROJECTION_CRS=EPSG:5070 +export ALASKA_CRS=EPSG:3338 # NOTE: $inputsDir is defined in Dockerfile @@ -39,26 +39,26 @@ export osm_bridges=${inputsDir}/osm/bridges/241002/os export nws_lid=${inputsDir}/ahps_sites/nws_lid.gpkg # input file location with nwm feature_id and recurrence flow values -export bankfull_flows_file=${inputsDir}/rating_curve/bankfull_flows/nwm3_high_water_threshold_cms.csv +export bankfull_flows_file=${inputsDir}/rating_curve/bankfull_flows/nwm3_high_water_threshold_cms.csv # input file location with nwm feature_id and channel roughness and overbank roughness attributes -export vmann_input_file=${inputsDir}/rating_curve/variable_roughness/mannings_global_nwm3.csv +export vmann_input_file=${inputsDir}/rating_curve/variable_roughness/mannings_global_nwm3.csv # input file location with nwm feature_id and recurrence flow values -export nwm_recur_file=${inputsDir}/rating_curve/nwm_recur_flows/nwm3_17C_recurrence_flows_cfs.csv +export nwm_recur_file=${inputsDir}/rating_curve/nwm_recur_flows/nwm3_17C_recurrence_flows_cfs.csv # input file location with usgs rating curve database -export usgs_rating_curve_csv=${inputsDir}/usgs_gages/usgs_rating_curves.csv +export usgs_rating_curve_csv=${inputsDir}/usgs_gages/usgs_rating_curves.csv # input file locations for ras2fim locations and rating curve data -export ras2fim_input_dir=${inputsDir}/rating_curve/ras2fim_exports/v2_0 -export ras_rating_curve_csv_filename=reformat_ras_rating_curve_table.csv -export ras_rating_curve_gpkg_filename=reformat_ras_rating_curve_points.gpkg +export ras2fim_input_dir=${inputsDir}/rating_curve/ras2fim_exports/v2_0 +export ras_rating_curve_csv_filename=reformat_ras_rating_curve_table.csv +export ras_rating_curve_gpkg_filename=reformat_ras_rating_curve_points.gpkg -export fim_obs_pnt_data=${inputsDir}/rating_curve/water_edge_database/usgs_nws_benchmark_points_cleaned.gpkg +export fim_obs_pnt_data=${inputsDir}/rating_curve/water_edge_database/usgs_nws_benchmark_points_cleaned.gpkg # Input file location with HUC, nwm feature_id and manual calibration coefficients -export man_calb_file=${inputsDir}/rating_curve/manual_calibration_coefficients.csv +export man_calb_file=${inputsDir}/rating_curve/manual_calibration_coefficients.csv # Use pyarrow export PYOGRIO_USE_ARROW=1 From 4f21a3e86cef5b24d0e705161b7b50ca9b023459 Mon Sep 17 00:00:00 2001 From: Matt Luck Date: Thu, 2 Jan 2025 09:15:21 -0700 Subject: [PATCH 3/5] Edit environment variables --- docs/CHANGELOG.md | 2 +- src/bash_variables.env | 2 -- 2 files changed, 1 insertion(+), 3 deletions(-) diff --git a/docs/CHANGELOG.md b/docs/CHANGELOG.md index 74a66cbd..226a1e80 100755 --- a/docs/CHANGELOG.md +++ b/docs/CHANGELOG.md @@ -9,7 +9,7 @@ Fixes two issues in test_cases: ### Changes -- `src/bash_variables.env`: Added environment variables for Alaska waterbody and levee masks and two variables to replace fixed paths in an upcoming PR (#1178). +- `src/bash_variables.env`: Added environment variables for Alaska waterbody and levee masks. - `tools/` - `run_test_case.py`: Fixed error if missing validation data. Updated masking data to include Alaska. - `synthesize_test_cases.py`: Fixed error if missing validation data. diff --git a/src/bash_variables.env b/src/bash_variables.env index 5aaa4676..740f268e 100644 --- a/src/bash_variables.env +++ b/src/bash_variables.env @@ -13,8 +13,6 @@ export input_DEM=${inputsDir}/dems/3dep_dems/10m_50 export input_DEM_Alaska=${inputsDir}/dems/3dep_dems/10m_South_Alaska/20240912/FIM_3dep_dem_South_Alaska_10m.vrt export input_DEM_domain=${inputsDir}/dems/3dep_dems/10m_5070/20240916/DEM_Domain.gpkg export input_DEM_domain_Alaska=${inputsDir}/dems/3dep_dems/10m_South_Alaska/20240912/DEM_Domain.gpkg -export input_landsea=${inputsDir}/landsea/water_polygons_us.gpkg -export input_landsea_Alaska=${inputsDir}/landsea/water_polygons_alaska.gpkg export input_GL_boundaries=${inputsDir}/landsea/gl_water_polygons.gpkg export input_NLD=${inputsDir}/nld_vectors/System_Routes_NLDFS_5070_230314.gpkg export input_NLD_Alaska=${inputsDir}/nld_vectors/System_Routes_NLDFS_3338_230314.gpkg From c3555c6168b4fddbd5fdf9aee7199e73e5714e34 Mon Sep 17 00:00:00 2001 From: Matt Luck Date: Thu, 2 Jan 2025 09:23:32 -0700 Subject: [PATCH 4/5] Edit environment variables --- src/bash_variables.env | 2 -- 1 file changed, 2 deletions(-) diff --git a/src/bash_variables.env b/src/bash_variables.env index 740f268e..de0851f8 100644 --- a/src/bash_variables.env +++ b/src/bash_variables.env @@ -34,8 +34,6 @@ export input_calib_points_dir=${inputsDir}/rating_curve/water_ed export bathymetry_file=${inputsDir}/bathymetry/bathymetric_adjustment_data.gpkg export osm_bridges=${inputsDir}/osm/bridges/241002/osm_all_bridges.gpkg -export nws_lid=${inputsDir}/ahps_sites/nws_lid.gpkg - # input file location with nwm feature_id and recurrence flow values export bankfull_flows_file=${inputsDir}/rating_curve/bankfull_flows/nwm3_high_water_threshold_cms.csv From 519d3b36985512e2b1c6327d23e5a24369f891bb Mon Sep 17 00:00:00 2001 From: Matt Luck Date: Thu, 2 Jan 2025 10:06:22 -0700 Subject: [PATCH 5/5] Revert bash_variables --- docs/CHANGELOG.md | 1 - src/bash_variables.env | 24 ++++++++++++------------ 2 files changed, 12 insertions(+), 13 deletions(-) diff --git a/docs/CHANGELOG.md b/docs/CHANGELOG.md index 226a1e80..34025e9f 100755 --- a/docs/CHANGELOG.md +++ b/docs/CHANGELOG.md @@ -9,7 +9,6 @@ Fixes two issues in test_cases: ### Changes -- `src/bash_variables.env`: Added environment variables for Alaska waterbody and levee masks. - `tools/` - `run_test_case.py`: Fixed error if missing validation data. Updated masking data to include Alaska. - `synthesize_test_cases.py`: Fixed error if missing validation data. diff --git a/src/bash_variables.env b/src/bash_variables.env index de0851f8..fff5d785 100644 --- a/src/bash_variables.env +++ b/src/bash_variables.env @@ -2,8 +2,8 @@ export res=10 # Default raster cell size in meters -export DEFAULT_FIM_PROJECTION_CRS=EPSG:5070 -export ALASKA_CRS=EPSG:3338 +export DEFAULT_FIM_PROJECTION_CRS=EPSG:5070 +export ALASKA_CRS=EPSG:3338 # NOTE: $inputsDir is defined in Dockerfile @@ -27,7 +27,7 @@ export input_nwm_flows_Alaska=${inputsDir}/nwm_hydrofabric/nwm_f export input_nwm_headwaters=${inputsDir}/nwm_hydrofabric/nwm_headwaters.gpkg export input_nwm_headwaters_Alaska=${inputsDir}/nwm_hydrofabric/nwm_headwaters_alaska.gpkg export input_nwm_lakes=${inputsDir}/nwm_hydrofabric/nwm_lakes.gpkg -export input_nwm_lakes_Alaska=${inputsDir}/nwm_hydrofabric/nwm_waterbodies_alaska.gpkg +export input_nwm_lakes_Alaska=${inputsDir}/nwm_hydrofabric/nwm_waterbodies_alaska.gpkg export input_WBD_gdb=${inputsDir}/wbd/WBD_National_EPSG_5070_WBDHU8_clip_dem_domain.gpkg export input_WBD_gdb_Alaska=${inputsDir}/wbd/WBD_National_South_Alaska.gpkg export input_calib_points_dir=${inputsDir}/rating_curve/water_edge_database/calibration_points/ @@ -35,26 +35,26 @@ export bathymetry_file=${inputsDir}/bathymetry/bathymetri export osm_bridges=${inputsDir}/osm/bridges/241002/osm_all_bridges.gpkg # input file location with nwm feature_id and recurrence flow values -export bankfull_flows_file=${inputsDir}/rating_curve/bankfull_flows/nwm3_high_water_threshold_cms.csv +export bankfull_flows_file=${inputsDir}/rating_curve/bankfull_flows/nwm3_high_water_threshold_cms.csv # input file location with nwm feature_id and channel roughness and overbank roughness attributes -export vmann_input_file=${inputsDir}/rating_curve/variable_roughness/mannings_global_nwm3.csv +export vmann_input_file=${inputsDir}/rating_curve/variable_roughness/mannings_global_nwm3.csv # input file location with nwm feature_id and recurrence flow values -export nwm_recur_file=${inputsDir}/rating_curve/nwm_recur_flows/nwm3_17C_recurrence_flows_cfs.csv +export nwm_recur_file=${inputsDir}/rating_curve/nwm_recur_flows/nwm3_17C_recurrence_flows_cfs.csv # input file location with usgs rating curve database -export usgs_rating_curve_csv=${inputsDir}/usgs_gages/usgs_rating_curves.csv +export usgs_rating_curve_csv=${inputsDir}/usgs_gages/usgs_rating_curves.csv # input file locations for ras2fim locations and rating curve data -export ras2fim_input_dir=${inputsDir}/rating_curve/ras2fim_exports/v2_0 -export ras_rating_curve_csv_filename=reformat_ras_rating_curve_table.csv -export ras_rating_curve_gpkg_filename=reformat_ras_rating_curve_points.gpkg +export ras2fim_input_dir=${inputsDir}/rating_curve/ras2fim_exports/v2_0 +export ras_rating_curve_csv_filename=reformat_ras_rating_curve_table.csv +export ras_rating_curve_gpkg_filename=reformat_ras_rating_curve_points.gpkg -export fim_obs_pnt_data=${inputsDir}/rating_curve/water_edge_database/usgs_nws_benchmark_points_cleaned.gpkg +export fim_obs_pnt_data=${inputsDir}/rating_curve/water_edge_database/usgs_nws_benchmark_points_cleaned.gpkg # Input file location with HUC, nwm feature_id and manual calibration coefficients -export man_calb_file=${inputsDir}/rating_curve/manual_calibration_coefficients.csv +export man_calb_file=${inputsDir}/rating_curve/manual_calibration_coefficients.csv # Use pyarrow export PYOGRIO_USE_ARROW=1