From 638543acbcb18ab7761816e24d6f09fe26c3da7c Mon Sep 17 00:00:00 2001 From: Seth Lawler Date: Mon, 27 Jan 2025 10:02:47 -0500 Subject: [PATCH 01/71] update nomenclature during review --- docs/source/user_guide.rst | 4 ++-- hecstac/events/ffrd.py | 8 ++++---- hecstac/hms/item.py | 3 ++- new_ffrd_event_item.py | 4 ++-- 4 files changed, 10 insertions(+), 9 deletions(-) diff --git a/docs/source/user_guide.rst b/docs/source/user_guide.rst index a8191e6..a3e921e 100644 --- a/docs/source/user_guide.rst +++ b/docs/source/user_guide.rst @@ -63,7 +63,7 @@ The following snippet provides an example of how to create stac items for an eve from pystac import Item from hecstac.common.logger import initialize_logger - from hecstac.events.ffrd import FFRDEventItem + from hecstac.events.ffrd import EventItem if __name__ == "__main__": initialize_logger() @@ -98,7 +98,7 @@ The following snippet provides an example of how to create stac items for an eve ffrd_event_item_id = f"{realization}-{block_group}-{event_id}" dest_href = f"//{ffrd_event_item_id}.json" - ffrd_event_item = FFRDEventItem( + ffrd_event_item = EventItem( realization=realization, block_group=block_group, event_id=event_id, diff --git a/hecstac/events/ffrd.py b/hecstac/events/ffrd.py index c67efee..fcbbd23 100644 --- a/hecstac/events/ffrd.py +++ b/hecstac/events/ffrd.py @@ -16,10 +16,10 @@ from hecstac.ras.assets import RAS_EXTENSION_MAPPING -class FFRDEventItem(Item): - FFRD_REALIZATION = "FFRD:realization" - FFRD_BLOCK_GROUP = "FFRD:block_group" - FFRD_EVENT = "FFRD:event" +class EventItem(Item): + REALIZATION = "montecarlo:realization" + BLOCK_GROUP = "montecarlo:block_group" + MC_EVENT = "montecarlo:event" def __init__( self, diff --git a/hecstac/hms/item.py b/hecstac/hms/item.py index 54b4b0d..50a2021 100644 --- a/hecstac/hms/item.py +++ b/hecstac/hms/item.py @@ -29,6 +29,7 @@ class HMSModelItem(Item): PROJECT_VERSION = "hms:version" PROJECT_DESCRIPTION = "hms:description" PROJECT_UNITS = "hms:unit_system" + SUMMARY = "hms:summary" def __init__(self, hms_project_file, item_id: str, simplify_geometry: bool = True): @@ -85,7 +86,7 @@ def _properties(self): if self.pf.basins[0].epsg: logging.warning("No EPSG code found in basin file.") properties["proj:wkt"] = self.pf.basins[0].wkt - properties["hms:summary"] = self.pf.file_counts + properties[SUMMARY] = self.pf.file_counts return properties @property diff --git a/new_ffrd_event_item.py b/new_ffrd_event_item.py index 94e6967..8a01258 100644 --- a/new_ffrd_event_item.py +++ b/new_ffrd_event_item.py @@ -1,7 +1,7 @@ from pystac import Item from hecstac.common.logger import initialize_logger -from hecstac.events.ffrd import FFRDEventItem +from hecstac.events.ffrd import EventItem if __name__ == "__main__": initialize_logger() @@ -38,7 +38,7 @@ ffrd_event_item_id = f"{realization}-{block_group}-{event_id}" dest_href = f"/Users/slawler/Downloads/duwamish/{ffrd_event_item_id}.json" - ffrd_event_item = FFRDEventItem( + ffrd_event_item = EventItem( realization=realization, block_group=block_group, event_id=event_id, From 332a88463fc3b699957e5dc3ab5fc10924930ca2 Mon Sep 17 00:00:00 2001 From: Stevenray Janke Date: Mon, 27 Jan 2025 20:09:55 -0500 Subject: [PATCH 02/71] Fix thumbnail --- hecstac/ras/assets.py | 64 ++++++++++++++----------------------------- 1 file changed, 20 insertions(+), 44 deletions(-) diff --git a/hecstac/ras/assets.py b/hecstac/ras/assets.py index dca5732..0fc3bbc 100644 --- a/hecstac/ras/assets.py +++ b/hecstac/ras/assets.py @@ -4,6 +4,7 @@ import contextily as ctx import geopandas as gpd +import io import matplotlib.pyplot as plt from matplotlib.lines import Line2D from pystac import MediaType @@ -352,7 +353,7 @@ def __init__(self, href: str, **kwargs): for key, value in { VERSION: self.hdf_object.file_version, UNITS: self.hdf_object.units_system, - # REFERENCE_LINES: self.hdf_object.reference_lines,#TODO: fix this + REFERENCE_LINES: list(self.hdf_object.reference_lines["refln_name"]), }.items() if value } @@ -453,33 +454,22 @@ def _add_thumbnail_asset(self, filepath: str) -> None: def thumbnail( self, - add_asset: bool, - write: bool, layers: list, title: str = "Model_Thumbnail", - add_usgs_properties: bool = False, crs="EPSG:4326", thumbnail_dest: str = None, ): """Create a thumbnail figure for each geometry hdf file, including various geospatial layers such as USGS gages, mesh areas, - breaklines, and boundary condition (BC) lines. If `add_asset` or `write` - is `True`, the function saves the thumbnail to a file and optionally - adds it as an asset. + breaklines, and boundary condition (BC) lines. Parameters ---------- - add_asset : bool - Whether to add the thumbnail as an asset in the asset dictionary. If true then it also writes the thumbnail to a file. - write : bool - Whether to save the thumbnail image to a file. layers : list A list of model layers to include in the thumbnail plot. Options include "usgs_gages", "mesh_areas", "breaklines", and "bc_lines". title : str, optional Title of the figure, by default "Model Thumbnail". - add_usgs_properties : bool, optional - If usgs_gages is included in layers, adds USGS metadata to the STAC item properties. Defaults to false. """ fig, ax = plt.subplots(figsize=(12, 12)) @@ -494,28 +484,17 @@ def thumbnail( # gages_gdf = self.get_usgs_data(False, geom_asset=geom_asset) # gages_gdf_geo = gages_gdf.to_crs(self.crs) # legend_handles += self._plot_usgs_gages(ax, gages_gdf_geo) - # else: - # if not hasattr(geom_asset, layer): - # raise AttributeError(f"Layer {layer} not found in {geom_asset.hdf_file}") - - # if layer == "mesh_areas": - # layer_data = geom_asset.mesh_areas(self.crs, return_gdf=True) - # else: - # layer_data = getattr(geom_asset, layer) - - # if layer_data.crs is None: - # layer_data.set_crs(self.crs, inplace=True) - # layer_data_geo = layer_data.to_crs(self.crs) if layer == "mesh_areas": - mesh_areas_data = self.mesh_areas(crs, return_gdf=True) - legend_handles += self._plot_mesh_areas(ax, mesh_areas_data) + mesh_areas_data = self.hdf_object.mesh_areas(crs, return_gdf=True) + mesh_areas_geo = mesh_areas_data.to_crs(crs) + legend_handles += self._plot_mesh_areas(ax, mesh_areas_geo) elif layer == "breaklines": - breaklines_data = self.breaklines + breaklines_data = self.hdf_object.breaklines breaklines_data_geo = breaklines_data.to_crs(crs) legend_handles += self._plot_breaklines(ax, breaklines_data_geo) elif layer == "bc_lines": - bc_lines_data = self.bc_lines + bc_lines_data = self.hdf_object.bc_lines bc_lines_data_geo = bc_lines_data.to_crs(crs) legend_handles += self._plot_bc_lines(ax, bc_lines_data_geo) except Exception as e: @@ -533,23 +512,20 @@ def thumbnail( ax.set_ylabel("Latitude") ax.legend(handles=legend_handles, loc="center left", bbox_to_anchor=(1, 0.5)) - if add_asset or write: - hdf_ext = os.path.basename(self.href).split(".")[-2] - filename = f"thumbnail_{hdf_ext}.png" - base_dir = os.path.dirname(thumbnail_dest) - filepath = os.path.join(base_dir, filename) - - # if filepath.startswith("s3://"): - # img_data = io.BytesIO() - # fig.savefig(img_data, format="png", bbox_inches="tight") - # img_data.seek(0) - # save_bytes_s3(img_data, filepath) - # else: + hdf_ext = os.path.basename(self.href).split(".")[-2] + filename = f"thumbnail_{hdf_ext}.png" + base_dir = os.path.dirname(thumbnail_dest) + filepath = os.path.join(base_dir, filename) + + if filepath.startswith("s3://"): + img_data = io.BytesIO() + fig.savefig(img_data, format="png", bbox_inches="tight") + img_data.seek(0) + save_bytes_s3(img_data, filepath) + else: os.makedirs(base_dir, exist_ok=True) fig.savefig(filepath, dpi=80, bbox_inches="tight") - - if add_asset: - return self._add_thumbnail_asset(filepath) + return self._add_thumbnail_asset(filepath) class RunFileAsset(GenericAsset): From e9c68fe55c1fa23ff4abbda76a6515f925c40af6 Mon Sep 17 00:00:00 2001 From: Stevenray Janke Date: Mon, 27 Jan 2025 20:10:10 -0500 Subject: [PATCH 03/71] Update for item thumbnail, crs --- new_ras_item.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/new_ras_item.py b/new_ras_item.py index 234cff7..b332a57 100644 --- a/new_ras_item.py +++ b/new_ras_item.py @@ -17,11 +17,12 @@ def sanitize_catalog_assets(item: RASModelItem) -> RASModelItem: if __name__ == "__main__": initialize_logger() - ras_project_file = "/Users/slawler/Downloads/model-library-2/ffrd-duwamish/checkpoint-validation/hydraulics/duwamish-20250106/Duwamish_17110013.prj" + ras_project_file = "ElkMiddle/ElkMiddle.prj" item_id = Path(ras_project_file).stem - ras_item = RASModelItem(ras_project_file, item_id) + ras_item = RASModelItem(ras_project_file, item_id, crs="EPSG:4326") ras_item = sanitize_catalog_assets(ras_item) + ras_item.add_model_thumbnail(["mesh_areas", "breaklines", "bc_lines"]) fs = ras_item.scan_model_dir() ras_item.add_ras_asset() From a2e7dbe62ed6e59d423b86dec2fd5dae76d7d1b8 Mon Sep 17 00:00:00 2001 From: Stevenray Janke Date: Mon, 27 Jan 2025 20:10:58 -0500 Subject: [PATCH 04/71] Fix geometries --- hecstac/ras/item.py | 90 +++++++++++++++++++++++++-------------------- 1 file changed, 51 insertions(+), 39 deletions(-) diff --git a/hecstac/ras/item.py b/hecstac/ras/item.py index 8bc7669..59406e9 100644 --- a/hecstac/ras/item.py +++ b/hecstac/ras/item.py @@ -8,7 +8,9 @@ from pystac.extensions.projection import ProjectionExtension from pystac.extensions.storage import StorageExtension from shapely import Polygon, box, simplify, to_geojson, union_all - +from shapely import Geometry +from pyproj import CRS, Transformer +from shapely.ops import transform from hecstac.common.path_manager import LocalPathManager from hecstac.ras.parser import ProjectFile @@ -44,7 +46,7 @@ class RASModelItem(Item): RAS_HAS_2D = "ras:has_2d" RAS_DATETIME_SOURCE = "ras:datetime_source" - def __init__(self, ras_project_file, item_id: str, simplify_geometry: bool = True): + def __init__(self, ras_project_file, item_id: str, crs: str, simplify_geometry: bool = True): self._project = None self.assets = {} @@ -52,14 +54,19 @@ def __init__(self, ras_project_file, item_id: str, simplify_geometry: bool = Tru self.thumbnail_paths = [] self.geojson_paths = [] self.extra_fields = {} + self._geom_files = [] self.stac_extensions = None self.pm = LocalPathManager(Path(ras_project_file).parent) self._href = self.pm.item_path(item_id) + self.crs = crs self.ras_project_file = ras_project_file self._simplify_geometry = simplify_geometry self.pf = ProjectFile(self.ras_project_file) + self.factory = AssetFactory(RAS_EXTENSION_MAPPING) + self.has_1d = None + self.has_2d = None super().__init__( Path(self.ras_project_file).stem, @@ -69,7 +76,6 @@ def __init__(self, ras_project_file, item_id: str, simplify_geometry: bool = Tru self._properties, href=self._href, ) - # derived_assets = self.add_model_thumbnail() TODO: implement this method ras_asset_files = self.scan_model_dir() @@ -77,6 +83,8 @@ def __init__(self, ras_project_file, item_id: str, simplify_geometry: bool = Tru if fpath and fpath != self._href: self.add_ras_asset(fpath) + self._geometry + def _register_extensions(self) -> None: ProjectionExtension.add_to(self) StorageExtension.add_to(self) @@ -84,19 +92,18 @@ def _register_extensions(self) -> None: @property def _properties(self) -> None: """Properties for the RAS STAC item.""" - properties = {} properties = {} + properties[self.RAS_HAS_1D] = self.has_1d + properties[self.RAS_HAS_2D] = self.has_2d properties[self.PROJECT_TITLE] = self.pf.project_title properties[self.PROJECT_VERSION] = self.pf.ras_version - properties[self.PROJECT_DESCRIPTION] = self.pf.project_description - properties[self.PROJECT_STATUS] = self.pf.project_status - properties[self.MODEL_UNITS] = self.pf.project_units + # properties[self.PROJECT_DESCRIPTION] = self.pf.project_description + # properties[self.PROJECT_STATUS] = self.pf.project_status + # properties[self.MODEL_UNITS] = self.pf.project_units # self.properties[RAS_DATETIME_SOURCE] = self.datetime_source - # self.properties[RAS_HAS_1D] = self.has_1d - # self.properties[RAS_HAS_2D] = self.has_2d - # once all assets are created, populate associations between assets + # TODO: once all assets are created, populate associations between assets return properties @property @@ -115,13 +122,12 @@ def _geometry(self) -> dict | None: """ # if geometry is equal to null placeholder, continue, else return current value geometries = [] - if 2 == 3: - # if self.has_2d: + + if self.has_2d: geometries.append(self.parse_2d_geom()) # if hdf file is not present, get concave hull of cross sections and use as geometry - # if self.has_1d: - if 1 == 2: + if self.has_1d: geometries.append(self.parse_1d_geom()) if len(geometries) == 0: @@ -130,9 +136,10 @@ def _geometry(self) -> dict | None: unioned_geometry = union_all(geometries) if self._simplify_geometry: - unioned_geometry = simplify(unioned_geometry, self.simplify_tolerance) + unioned_geometry = simplify(unioned_geometry, 0.001) - return json.loads(to_geojson(unioned_geometry)) + self.geometry = json.loads(to_geojson(unioned_geometry)) + self.bbox = unioned_geometry.bounds @property def _datetime(self) -> datetime: @@ -152,14 +159,11 @@ def datetime_source(self) -> str: self._datetime_source = "model_geometry" return self._datetime_source - def add_model_thumbnail(self, add_asset: bool, write: bool, layers: list, title: str = "Model_Thumbnail"): + def add_model_thumbnail(self, layers: list, title: str = "Model_Thumbnail"): for geom in self._geom_files: if isinstance(geom, GeometryHdfAsset): - if add_asset: - self.assets["thumbnail"] = geom.thumbnail( - add_asset=add_asset, write=write, layers=layers, title=title, thumbnail_dest=self.href - ) + self.assets["thumbnail"] = geom.thumbnail(layers=layers, title=title, thumbnail_dest=self._href) def add_ras_asset(self, fpath: str = "") -> None: """Add an asset to the HMS STAC item.""" @@ -177,35 +181,43 @@ def add_ras_asset(self, fpath: str = "") -> None: f"Only one project asset is allowed. Found {str(asset)} when {str(self._project)} was already set." ) self._project = asset + elif isinstance(asset, GeometryHdfAsset): + self.has_2d = True + self._geom_files.append(asset) + elif isinstance(asset, GeometryAsset): + self.has_1d = True + self._geom_files.append(asset) + + def _geometry_to_wgs84(self, geom: Geometry) -> Geometry: + pyproj_crs = CRS.from_user_input(self.crs) + wgs_crs = CRS.from_authority("EPSG", "4326") + if pyproj_crs != wgs_crs: + transformer = Transformer.from_crs(pyproj_crs, wgs_crs, True) + return transform(transformer.transform, geom) + return geom def parse_1d_geom(self): logging.info("Creating geometry using 1d text file cross sections") concave_hull_polygons: list[Polygon] = [] - for geom_asset in self.geometry_files: + for geom_asset in self._geom_files: if isinstance(geom_asset, GeometryAsset): - if self.simplify_tolerance: - concave_hull = simplify( - self._geometry_to_wgs84(geom_asset.concave_hull), - self.simplify_tolerance, - ) - else: - concave_hull = self._geometry_to_wgs84(geom_asset.concave_hull) - concave_hull_polygons.append(concave_hull) - return self._geometry + try: + concave_hull = self._geometry_to_wgs84(geom_asset.geomf.concave_hull) + concave_hull_polygons.append(concave_hull) + except ValueError: + logging.warning(f"Could not extract geometry from {geom_asset.href}") + + return union_all(concave_hull_polygons) def parse_2d_geom(self): logging.info("Creating 2D geometry elements using hdf file mesh areas") mesh_area_polygons: list[Polygon] = [] - for geom_asset in self.geometry_files: + for geom_asset in self._geom_files: if isinstance(geom_asset, GeometryHdfAsset): - if self.simplify_tolerance: - mesh_areas = simplify( - self._geometry_to_wgs84(geom_asset.mesh_areas(self.crs)), - self.simplify_tolerance, - ) - else: - mesh_areas = self._geometry_to_wgs84(geom_asset.mesh_areas(self.crs)) + + mesh_areas = self._geometry_to_wgs84(geom_asset.hdf_object.mesh_areas(self.crs)) mesh_area_polygons.append(mesh_areas) + return union_all(mesh_area_polygons) def ensure_projection_schema(self) -> None: From bd761b2937fce0e6c5bae909e52781c0531642ca Mon Sep 17 00:00:00 2001 From: Stevenray Janke Date: Mon, 27 Jan 2025 20:11:45 -0500 Subject: [PATCH 05/71] Fix parse classes --- hecstac/ras/parser.py | 67 +++++++++++++++++-------------------------- 1 file changed, 27 insertions(+), 40 deletions(-) diff --git a/hecstac/ras/parser.py b/hecstac/ras/parser.py index 67773a8..d605a1a 100644 --- a/hecstac/ras/parser.py +++ b/hecstac/ras/parser.py @@ -10,7 +10,7 @@ import numpy as np import pandas as pd from pystac import Asset -from rashdf import RasHdf, RasPlanHdf +from rashdf import RasHdf, RasPlanHdf, RasGeomHdf from shapely import LineString, MultiPolygon, Point, Polygon, make_valid, union_all from shapely.ops import unary_union @@ -861,23 +861,27 @@ def has_1d(self) -> bool: def concave_hull(self) -> Polygon: """Compute and return the concave hull (polygon) for cross sections.""" polygons = [] - xs_gdf = pd.concat([xs.gdf for xs in self.cross_sections.values()], ignore_index=True) - for river_reach in xs_gdf["river_reach"].unique(): - xs_subset: gpd.GeoSeries = xs_gdf[xs_gdf["river_reach"] == river_reach] - points = xs_subset.boundary.explode(index_parts=True).unstack() - points_last_xs = [Point(coord) for coord in xs_subset["geometry"].iloc[-1].coords] - points_first_xs = [Point(coord) for coord in xs_subset["geometry"].iloc[0].coords[::-1]] - polygon = Polygon(points_first_xs + list(points[0]) + points_last_xs + list(points[1])[::-1]) - if isinstance(polygon, MultiPolygon): - polygons += list(polygon.geoms) - else: - polygons.append(polygon) - if len(self.junctions) > 0: - for junction in self.junctions.values(): - for _, j in junction.gdf.iterrows(): - polygons.append(self.junction_hull(xs_gdf, j)) - out_hull = union_all([make_valid(p) for p in polygons]) - return out_hull + if self.cross_sections: + xs_gdf = pd.concat([xs.gdf for xs in self.cross_sections.values()], ignore_index=True) + + for river_reach in xs_gdf["river_reach"].unique(): + xs_subset: gpd.GeoSeries = xs_gdf[xs_gdf["river_reach"] == river_reach] + points = xs_subset.boundary.explode(index_parts=True).unstack() + points_last_xs = [Point(coord) for coord in xs_subset["geometry"].iloc[-1].coords] + points_first_xs = [Point(coord) for coord in xs_subset["geometry"].iloc[0].coords[::-1]] + polygon = Polygon(points_first_xs + list(points[0]) + points_last_xs + list(points[1])[::-1]) + if isinstance(polygon, MultiPolygon): + polygons += list(polygon.geoms) + else: + polygons.append(polygon) + if len(self.junctions) > 0: + for junction in self.junctions.values(): + for _, j in junction.gdf.iterrows(): + polygons.append(self.junction_hull(xs_gdf, j)) + out_hull = union_all([make_valid(p) for p in polygons]) + return out_hull + else: + raise ValueError(f"No cross sections found for {self.fpath}. Cannot calculate geometry") def junction_hull(self, xs_gdf: gpd.GeoDataFrame, junction: gpd.GeoSeries) -> Polygon: """Compute and return the concave hull (polygon) for a juction.""" @@ -1007,32 +1011,15 @@ class QuasiUnsteadyFlowFile: class RASHDFFile: """Base class for HDF assets (Plan and Geometry HDF files).""" - def __init__(self, fpath): + def __init__(self, fpath, hdf_constructor): self.fpath = fpath - self.hdf_object = RasHdf(fpath) + self.hdf_object = hdf_constructor(fpath) self._root_attrs: dict | None = None self._geom_attrs: dict | None = None self._structures_attrs: dict | None = None self._2d_flow_attrs: dict | None = None - # def populate( - # self, - # optional_property_dict: dict[str, str], - # required_property_dict: dict[str, str], - # ) -> dict: - # extra_fields = {} - # # go through dictionary of stac property names and class property names, only adding property to extra fields if the value is not None - # for stac_property_name, class_property_name in optional_property_dict.items(): - # property_value = getattr(self, class_property_name) - # if property_value != None: - # extra_fields[stac_property_name] = property_value - # # go through dictionary of stac property names and class property names, adding all properties to extra fields regardless of value - # for stac_property_name, class_property_name in required_property_dict.items(): - # property_value = getattr(self, class_property_name) - # extra_fields[stac_property_name] = property_value - # return extra_fields - @property def file_version(self) -> str | None: if self._root_attrs == None: @@ -1202,7 +1189,7 @@ def associate_related_assets(self, asset_dict: dict[str, Asset]) -> None: class PlanHDFFile(RASHDFFile): def __init__(self, fpath: str, **kwargs): - super().__init__(fpath, **kwargs) + super().__init__(fpath, RasPlanHdf, **kwargs) self.hdf_object = RasPlanHdf(fpath) self._plan_info_attrs = None @@ -1443,9 +1430,9 @@ def meteorology_units(self): class GeometryHDFFile(RASHDFFile): def __init__(self, fpath: str, **kwargs): - super().__init__(fpath, **kwargs) + super().__init__(fpath, RasGeomHdf, **kwargs) - self.hdf_object = RasPlanHdf(fpath) + self.hdf_object = RasGeomHdf(fpath) self._plan_info_attrs = None self._plan_parameters_attrs = None self._meteorology_attrs = None From 450d2c810f6b97c9387b3cc902219d28236ad4a0 Mon Sep 17 00:00:00 2001 From: Seth Lawler Date: Tue, 28 Jan 2025 09:39:46 -0500 Subject: [PATCH 06/71] add error handling --- docs/source/requirements.txt | Bin 3714 -> 3744 bytes hecstac/common/asset_factory.py | 1 + hecstac/ras/assets.py | 4 +- hecstac/ras/item.py | 74 +++++++++++++++++--------------- hecstac/ras/parser.py | 31 +++++++------ hecstac/ras/utils.py | 10 +++-- 6 files changed, 67 insertions(+), 53 deletions(-) diff --git a/docs/source/requirements.txt b/docs/source/requirements.txt index 71051e803a7d6e3e52303e27b0213945fd4b90cf..608044e662bb32557034a981e9b71bfd5d843559 100644 GIT binary patch delta 32 kcmZpYT_C$*0;g~WLkdG0gDnsmFz7Lu1F_-eR?fdH0D`**n*aa+ delta 12 TcmZ1=+a$YT0_Wy+oS#?#AjAa* diff --git a/hecstac/common/asset_factory.py b/hecstac/common/asset_factory.py index 1cb9128..0aaad59 100644 --- a/hecstac/common/asset_factory.py +++ b/hecstac/common/asset_factory.py @@ -66,4 +66,5 @@ def create_ras_asset(self, fpath: str): logging.debug(f"Matched {pattern} for {Path(fpath).name}: {asset_class}") return asset_class(href=fpath, title=Path(fpath).name) + logging.warning(f"Unable to pattern match asset for file {fpath}") return GenericAsset(href=fpath, title=Path(fpath).name) diff --git a/hecstac/ras/assets.py b/hecstac/ras/assets.py index 0fc3bbc..9416b77 100644 --- a/hecstac/ras/assets.py +++ b/hecstac/ras/assets.py @@ -1,10 +1,10 @@ +import io import logging import os import re import contextily as ctx import geopandas as gpd -import io import matplotlib.pyplot as plt from matplotlib.lines import Line2D from pystac import MediaType @@ -218,7 +218,7 @@ def __init__(self, href: str, **kwargs): self.extra_fields = { key: value for key, value in { - TITLE: self.flowf.geom_title, + TITLE: self.flowf.flow_title, N_PROFILES: self.flowf.n_profiles, }.items() if value diff --git a/hecstac/ras/item.py b/hecstac/ras/item.py index 59406e9..8308034 100644 --- a/hecstac/ras/item.py +++ b/hecstac/ras/item.py @@ -4,13 +4,13 @@ import os from pathlib import Path +from pyproj import CRS, Transformer from pystac import Item from pystac.extensions.projection import ProjectionExtension from pystac.extensions.storage import StorageExtension -from shapely import Polygon, box, simplify, to_geojson, union_all -from shapely import Geometry -from pyproj import CRS, Transformer +from shapely import Geometry, Polygon, box, simplify, to_geojson, union_all from shapely.ops import transform + from hecstac.common.path_manager import LocalPathManager from hecstac.ras.parser import ProjectFile @@ -98,9 +98,9 @@ def _properties(self) -> None: properties[self.RAS_HAS_2D] = self.has_2d properties[self.PROJECT_TITLE] = self.pf.project_title properties[self.PROJECT_VERSION] = self.pf.ras_version - # properties[self.PROJECT_DESCRIPTION] = self.pf.project_description - # properties[self.PROJECT_STATUS] = self.pf.project_status - # properties[self.MODEL_UNITS] = self.pf.project_units + properties[self.PROJECT_DESCRIPTION] = self.pf.project_description + properties[self.PROJECT_STATUS] = self.pf.project_status + properties[self.MODEL_UNITS] = self.pf.project_units # self.properties[RAS_DATETIME_SOURCE] = self.datetime_source # TODO: once all assets are created, populate associations between assets @@ -167,34 +167,40 @@ def add_model_thumbnail(self, layers: list, title: str = "Model_Thumbnail"): def add_ras_asset(self, fpath: str = "") -> None: """Add an asset to the HMS STAC item.""" - if os.path.exists(fpath): - try: - asset = self.factory.create_ras_asset(fpath) - logging.debug(f"Adding asset {str(asset)}") - except TypeError as e: - logging.error(f"Error creating asset for {fpath}: {e}") - if asset is not None: - self.add_asset(asset.title, asset) - if isinstance(asset, ProjectAsset): - if self._project is not None: - logging.error( - f"Only one project asset is allowed. Found {str(asset)} when {str(self._project)} was already set." - ) - self._project = asset - elif isinstance(asset, GeometryHdfAsset): - self.has_2d = True - self._geom_files.append(asset) - elif isinstance(asset, GeometryAsset): - self.has_1d = True - self._geom_files.append(asset) - - def _geometry_to_wgs84(self, geom: Geometry) -> Geometry: - pyproj_crs = CRS.from_user_input(self.crs) - wgs_crs = CRS.from_authority("EPSG", "4326") - if pyproj_crs != wgs_crs: - transformer = Transformer.from_crs(pyproj_crs, wgs_crs, True) - return transform(transformer.transform, geom) - return geom + if not os.path.exists(fpath): + logging.warning(f"File not found: {fpath}") + return + try: + asset = self.factory.create_ras_asset(fpath) + logging.debug(f"Adding asset {str(asset)}") + except TypeError as e: + logging.error(f"Error creating asset for {fpath}: {e}") + return + + if asset: + self.add_asset(asset.title, asset) + if isinstance(asset, ProjectAsset): + if self._project is not None: + logging.error( + f"Only one project asset is allowed. Found {str(asset)} when {str(self._project)} was already set." + ) + self._project = asset + elif isinstance(asset, GeometryHdfAsset): + # TODO: if mesh areas exist there are 2d...these can be in text or hdf files. + pass + # self.has_2d = True + # self._geom_files.append(asset) + elif isinstance(asset, GeometryAsset): + self.has_1d = True + self._geom_files.append(asset) + + # def _geometry_to_wgs84(self, geom: Geometry) -> Geometry: + # pyproj_crs = CRS.from_user_input(self.crs) + # wgs_crs = CRS.from_authority("EPSG", "4326") + # if pyproj_crs != wgs_crs: + # transformer = Transformer.from_crs(pyproj_crs, wgs_crs, True) + # return transform(transformer.transform, geom) + # return geom def parse_1d_geom(self): logging.info("Creating geometry using 1d text file cross sections") diff --git a/hecstac/ras/parser.py b/hecstac/ras/parser.py index d605a1a..d03460d 100644 --- a/hecstac/ras/parser.py +++ b/hecstac/ras/parser.py @@ -10,7 +10,7 @@ import numpy as np import pandas as pd from pystac import Asset -from rashdf import RasHdf, RasPlanHdf, RasGeomHdf +from rashdf import RasGeomHdf, RasHdf, RasPlanHdf from shapely import LineString, MultiPolygon, Point, Polygon, make_valid, union_all from shapely.ops import unary_union @@ -655,11 +655,11 @@ def project_title(self) -> str: @property def project_description(self) -> str: - return search_contents(self.file_lines, "Model Description", token=":") + return search_contents(self.file_lines, "Model Description", token=":", require_one=False) @property def project_status(self) -> str: - return search_contents(self.file_lines, "Status of Model", token=":") + return search_contents(self.file_lines, "Status of Model", token=":", require_one=False) @property def project_units(self) -> str | None: @@ -670,7 +670,7 @@ def project_units(self) -> str | None: @property def plan_current(self) -> str | None: try: - suffix = search_contents(self.file_lines, "Current Plan", expect_one=True) + suffix = search_contents(self.file_lines, "Current Plan", expect_one=True, require_one=False) return self.name_from_suffix(suffix) except Exception: logging.warning("Ras model has no current plan") @@ -678,9 +678,11 @@ def plan_current(self) -> str | None: @property def ras_version(self) -> str | None: - version = search_contents(self.file_lines, "Program Version", token="=", expect_one=False) + version = search_contents(self.file_lines, "Program Version", token="=", expect_one=False, require_one=False) if version == []: - version = search_contents(self.file_lines, "Program and Version", token=":", expect_one=False) + version = search_contents( + self.file_lines, "Program and Version", token=":", expect_one=False, require_one=False + ) if version == []: logging.warning("Unable to parse project version") return "N/A" @@ -699,17 +701,17 @@ def geometry_files(self) -> list[str]: @property def steady_flow_files(self) -> list[str]: - suffixes = search_contents(self.file_lines, "Flow File", expect_one=False) + suffixes = search_contents(self.file_lines, "Flow File", expect_one=False, require_one=False) return [name_from_suffix(self.fpath, i) for i in suffixes] @property def quasi_unsteady_flow_files(self) -> list[str]: - suffixes = search_contents(self.file_lines, "QuasiSteady File", expect_one=False) + suffixes = search_contents(self.file_lines, "QuasiSteady File", expect_one=False, require_one=False) return [name_from_suffix(self.fpath, i) for i in suffixes] @property def unsteady_flow_files(self) -> list[str]: - suffixes = search_contents(self.file_lines, "Unsteady File", expect_one=False) + suffixes = search_contents(self.file_lines, "Unsteady File", expect_one=False, require_one=False) return [name_from_suffix(self.fpath, i) for i in suffixes] @@ -751,7 +753,7 @@ def breach_locations(self) -> dict: Breach Loc= , , ,True,HH_DamEmbankment """ breach_dict = {} - matches = search_contents(self.file_lines, "Breach Loc", expect_one=False) + matches = search_contents(self.file_lines, "Breach Loc", expect_one=False, require_one=False) for line in matches: parts = line.split(",") if len(parts) >= 4: @@ -764,9 +766,10 @@ def breach_locations(self) -> dict: class GeometryFile: """HEC-RAS Geometry file asset.""" - def __init__(self, fpath): + def __init__(self, fpath, crs): # TODO: Compare with HMS implementation self.fpath = fpath + self.crs = crs with open(fpath, "r") as f: self.file_lines = f.readlines() @@ -993,7 +996,9 @@ def boundary_locations(self) -> list: @property def reference_lines(self): - return search_contents(self.file_lines, "Observed Rating Curve=Name=Ref Line", token=":", expect_one=False) + return search_contents( + self.file_lines, "Observed Rating Curve=Name=Ref Line", token=":", expect_one=False, require_one=False + ) class QuasiUnsteadyFlowFile: @@ -1447,6 +1452,6 @@ def reference_lines(self) -> gpd.GeoDataFrame | None: ref_lines = self.hdf_object.reference_lines() if ref_lines is None or ref_lines.empty: - raise ValueError("No reference lines found.") + logging.warning("No reference lines found.") else: return ref_lines diff --git a/hecstac/ras/utils.py b/hecstac/ras/utils.py index 03273ac..24627c6 100644 --- a/hecstac/ras/utils.py +++ b/hecstac/ras/utils.py @@ -19,7 +19,9 @@ def is_ras_prj(url: str) -> bool: return False -def search_contents(lines: list[str], search_string: str, token: str = "=", expect_one: bool = True) -> list[str] | str: +def search_contents( + lines: list[str], search_string: str, token: str = "=", expect_one: bool = True, require_one: bool = True +) -> list[str] | str: """Split a line by a token and returns the second half of the line if the search_string is found in the first half.""" results = [] for line in lines: @@ -27,9 +29,9 @@ def search_contents(lines: list[str], search_string: str, token: str = "=", expe results.append(line.split(token)[1]) if expect_one and len(results) > 1: - raise ValueError(f"expected 1 result, got {len(results)}") - elif expect_one and len(results) == 0: - raise ValueError("expected 1 result, no results found") + raise ValueError(f"expected 1 result for {search_string}, got {len(results)} results") + elif require_one and len(results) == 0: + raise ValueError(f"1 result for {search_string} is required, no results found") elif expect_one and len(results) == 1: return results[0] else: From 0f11e09cbb967f3177e15e3a0fb5407c88356df9 Mon Sep 17 00:00:00 2001 From: Seth Lawler Date: Tue, 28 Jan 2025 10:01:49 -0500 Subject: [PATCH 07/71] pair coding tests --- hecstac/ras/assets.py | 2 ++ hecstac/ras/item.py | 15 +++++++-------- hecstac/ras/parser.py | 6 ++++++ 3 files changed, 15 insertions(+), 8 deletions(-) diff --git a/hecstac/ras/assets.py b/hecstac/ras/assets.py index 9416b77..acda01e 100644 --- a/hecstac/ras/assets.py +++ b/hecstac/ras/assets.py @@ -346,6 +346,7 @@ def __init__(self, href: str, **kwargs): description = kwargs.get("description", "The HEC-RAS geometry HDF file.") super().__init__(href, roles=roles, description=description, **kwargs) + import json self.hdf_object = GeometryHDFFile(self.href) self.extra_fields = { @@ -353,6 +354,7 @@ def __init__(self, href: str, **kwargs): for key, value in { VERSION: self.hdf_object.file_version, UNITS: self.hdf_object.units_system, + "proj:wkt": json.dumps(self.hdf_object.crs), REFERENCE_LINES: list(self.hdf_object.reference_lines["refln_name"]), }.items() if value diff --git a/hecstac/ras/item.py b/hecstac/ras/item.py index 8308034..344204c 100644 --- a/hecstac/ras/item.py +++ b/hecstac/ras/item.py @@ -70,7 +70,7 @@ def __init__(self, ras_project_file, item_id: str, crs: str, simplify_geometry: super().__init__( Path(self.ras_project_file).stem, - self._geometry, + NULL_STAC_GEOMETRY, self._bbox, self._datetime, self._properties, @@ -187,9 +187,8 @@ def add_ras_asset(self, fpath: str = "") -> None: self._project = asset elif isinstance(asset, GeometryHdfAsset): # TODO: if mesh areas exist there are 2d...these can be in text or hdf files. - pass - # self.has_2d = True - # self._geom_files.append(asset) + self.has_2d = True + self._geom_files.append(asset) elif isinstance(asset, GeometryAsset): self.has_1d = True self._geom_files.append(asset) @@ -208,8 +207,8 @@ def parse_1d_geom(self): for geom_asset in self._geom_files: if isinstance(geom_asset, GeometryAsset): try: - concave_hull = self._geometry_to_wgs84(geom_asset.geomf.concave_hull) - concave_hull_polygons.append(concave_hull) + # concave_hull = self._geometry_to_wgs84(geom_asset.geomf.concave_hull) + concave_hull_polygons.append(geom_asset.geomf.concave_hull) except ValueError: logging.warning(f"Could not extract geometry from {geom_asset.href}") @@ -221,8 +220,8 @@ def parse_2d_geom(self): for geom_asset in self._geom_files: if isinstance(geom_asset, GeometryHdfAsset): - mesh_areas = self._geometry_to_wgs84(geom_asset.hdf_object.mesh_areas(self.crs)) - mesh_area_polygons.append(mesh_areas) + # mesh_areas = self._geometry_to_wgs84(geom_asset.hdf_object.mesh_areas(self.crs)) + mesh_area_polygons.append(geom_asset.hdf_object.mesh_areas(self.crs)) return union_all(mesh_area_polygons) diff --git a/hecstac/ras/parser.py b/hecstac/ras/parser.py index d03460d..96d829b 100644 --- a/hecstac/ras/parser.py +++ b/hecstac/ras/parser.py @@ -9,6 +9,7 @@ import geopandas as gpd import numpy as np import pandas as pd +from pyproj import CRS from pystac import Asset from rashdf import RasGeomHdf, RasHdf, RasPlanHdf from shapely import LineString, MultiPolygon, Point, Polygon, make_valid, union_all @@ -1438,6 +1439,11 @@ def __init__(self, fpath: str, **kwargs): super().__init__(fpath, RasGeomHdf, **kwargs) self.hdf_object = RasGeomHdf(fpath) + self.crs = CRS.from_user_input(self.hdf_object.projection()) + if not self.crs: + logging.info(f"No projection found in file {fpath}") + else: + logging.debug(f"Projection found in file {fpath}: {self.crs}") self._plan_info_attrs = None self._plan_parameters_attrs = None self._meteorology_attrs = None From f9577f74f9528e97ad381d1a8f8ebc59019bd28a Mon Sep 17 00:00:00 2001 From: Stevenray Janke Date: Tue, 28 Jan 2025 14:54:07 -0500 Subject: [PATCH 08/71] Start crs fix --- hecstac/ras/assets.py | 33 ++++++++++++++------- hecstac/ras/item.py | 67 ++++++++++++++++++++++++------------------- hecstac/ras/parser.py | 31 +++++++++++++------- new_ras_item.py | 10 ++++--- 4 files changed, 87 insertions(+), 54 deletions(-) diff --git a/hecstac/ras/assets.py b/hecstac/ras/assets.py index acda01e..b568f13 100644 --- a/hecstac/ras/assets.py +++ b/hecstac/ras/assets.py @@ -8,6 +8,7 @@ import matplotlib.pyplot as plt from matplotlib.lines import Line2D from pystac import MediaType +from pyproj import CRS from hecstac.common.asset_factory import GenericAsset from hecstac.ras.parser import ( @@ -169,6 +170,7 @@ class GeometryAsset(GenericAsset): def __init__(self, href: str, crs: str = None, **kwargs): # self.pyproj_crs = self.validate_crs(crs) + self.crs = crs roles = kwargs.get("roles", []) + ["geometry-file", "ras-file"] description = kwargs.get( "description", @@ -178,7 +180,7 @@ def __init__(self, href: str, crs: str = None, **kwargs): super().__init__(href, roles=roles, description=description, **kwargs) self.href = href - self.geomf = GeometryFile(self.href) + self.geomf = GeometryFile(self.href, self.crs) self.extra_fields = { key: value for key, value in { @@ -186,11 +188,11 @@ def __init__(self, href: str, crs: str = None, **kwargs): VERSION: self.geomf.geom_version, HAS_1D: self.geomf.has_1d, HAS_2D: self.geomf.has_2d, - RIVERS: self.geomf.rivers, - REACHES: self.geomf.reaches, - JUNCTIONS: self.geomf.junctions, - CROSS_SECTIONS: self.geomf.cross_sections, - STRUCTURES: self.geomf.structures, + # RIVERS: self.geomf.rivers, + # REACHES: self.geomf.reaches, + # JUNCTIONS: self.geomf.junctions, + # CROSS_SECTIONS: self.geomf.cross_sections, + # STRUCTURES: self.geomf.structures, # STORAGE_AREAS: self.geomf.storage_areas, #TODO: fix this # CONNECTIONS: self.geomf.connections,#TODO: fix this # BREACH_LOCATIONS: self.planf.breach_locations, @@ -341,20 +343,31 @@ class GeometryHdfAsset(GenericAsset): regex_parse_str = r".+\.g\d{2}\.hdf$" - def __init__(self, href: str, **kwargs): + def __init__(self, href: str, crs: str = None, **kwargs): + logging.info("Initializing hdf class") roles = kwargs.get("roles", []) + ["geometry-hdf-file"] description = kwargs.get("description", "The HEC-RAS geometry HDF file.") super().__init__(href, roles=roles, description=description, **kwargs) - import json + logging.info("initializing hdf file") + self.crs = crs + + logging.warning(f"crs has been set to {self.crs}") + self.hdf_object = GeometryHDFFile(self.href, self.crs) + logging.info("reading mesh areas...") + try: + if self.hdf_object.mesh_areas(): + self.has_2d = True + except ValueError: + logging.info(f"Could not read mesh areas for {self.href}") + self.has_2d = False - self.hdf_object = GeometryHDFFile(self.href) self.extra_fields = { key: value for key, value in { VERSION: self.hdf_object.file_version, UNITS: self.hdf_object.units_system, - "proj:wkt": json.dumps(self.hdf_object.crs), + # "proj:wkt": json.load(self.hdf_object.crs), REFERENCE_LINES: list(self.hdf_object.reference_lines["refln_name"]), }.items() if value diff --git a/hecstac/ras/item.py b/hecstac/ras/item.py index 344204c..383890b 100644 --- a/hecstac/ras/item.py +++ b/hecstac/ras/item.py @@ -46,7 +46,7 @@ class RASModelItem(Item): RAS_HAS_2D = "ras:has_2d" RAS_DATETIME_SOURCE = "ras:datetime_source" - def __init__(self, ras_project_file, item_id: str, crs: str, simplify_geometry: bool = True): + def __init__(self, ras_project_file, item_id: str, crs: str = None, simplify_geometry: bool = True): self._project = None self.assets = {} @@ -65,13 +65,13 @@ def __init__(self, ras_project_file, item_id: str, crs: str, simplify_geometry: self.pf = ProjectFile(self.ras_project_file) self.factory = AssetFactory(RAS_EXTENSION_MAPPING) - self.has_1d = None - self.has_2d = None + self.has_1d = False + self.has_2d = False super().__init__( Path(self.ras_project_file).stem, NULL_STAC_GEOMETRY, - self._bbox, + NULL_STAC_BBOX, self._datetime, self._properties, href=self._href, @@ -81,6 +81,7 @@ def __init__(self, ras_project_file, item_id: str, crs: str, simplify_geometry: for fpath in ras_asset_files: if fpath and fpath != self._href: + logging.info(f"processing {fpath}") self.add_ras_asset(fpath) self._geometry @@ -106,12 +107,6 @@ def _properties(self) -> None: # TODO: once all assets are created, populate associations between assets return properties - @property - def _bbox(self) -> tuple[float, float, float, float]: - if self._geometry == NULL_STAC_GEOMETRY: - return NULL_STAC_BBOX - return self._geometry.bounds - @property def _geometry(self) -> dict | None: """ @@ -144,8 +139,8 @@ def _geometry(self) -> dict | None: @property def _datetime(self) -> datetime: """The datetime for the HMS STAC item.""" - # date = datetime.strptime(self.pf.basins[0].header.attrs["Last Modified Date"], "%d %B %Y") - # time = datetime.strptime(self.pf.basins[0].header.attrs["Last Modified Time"], "%H:%M:%S").time() + # date = datetime.datetime.strptime(self.pf.basins[0].header.attrs["Last Modified Date"], "%d %B %Y") + # time = datetime.datetime.strptime(self.pf.basins[0].header.attrs["Last Modified Time"], "%H:%M:%S").time() return datetime.datetime.now() @property @@ -186,20 +181,29 @@ def add_ras_asset(self, fpath: str = "") -> None: ) self._project = asset elif isinstance(asset, GeometryHdfAsset): - # TODO: if mesh areas exist there are 2d...these can be in text or hdf files. - self.has_2d = True - self._geom_files.append(asset) + # if crs is None, use the crs from the 2d geom hdf file. + if self.crs is None: + self.crs = asset.hdf_object.crs + else: + logging.warning("settomg crs") + asset.crs = self.crs + if asset.has_2d: + self.has_2d = True + self.properties[self.RAS_HAS_2D] = True + self._geom_files.append(asset) elif isinstance(asset, GeometryAsset): - self.has_1d = True - self._geom_files.append(asset) - - # def _geometry_to_wgs84(self, geom: Geometry) -> Geometry: - # pyproj_crs = CRS.from_user_input(self.crs) - # wgs_crs = CRS.from_authority("EPSG", "4326") - # if pyproj_crs != wgs_crs: - # transformer = Transformer.from_crs(pyproj_crs, wgs_crs, True) - # return transform(transformer.transform, geom) - # return geom + if asset.geomf.has_1d: + self.has_1d = True + self.properties[self.RAS_HAS_1D] = True + self._geom_files.append(asset) + + def _geometry_to_wgs84(self, geom: Geometry) -> Geometry: + pyproj_crs = CRS.from_user_input(self.crs) + wgs_crs = CRS.from_authority("EPSG", "4326") + if pyproj_crs != wgs_crs: + transformer = Transformer.from_crs(pyproj_crs, wgs_crs, True) + return transform(transformer.transform, geom) + return geom def parse_1d_geom(self): logging.info("Creating geometry using 1d text file cross sections") @@ -207,8 +211,13 @@ def parse_1d_geom(self): for geom_asset in self._geom_files: if isinstance(geom_asset, GeometryAsset): try: - # concave_hull = self._geometry_to_wgs84(geom_asset.geomf.concave_hull) - concave_hull_polygons.append(geom_asset.geomf.concave_hull) + logging.info("Getting concave hull") + geom_asset.crs = self.crs + logging.info(geom_asset.crs) + concave_hull = geom_asset.geomf.concave_hull + logging.info("Concave hull retrieved") + concave_hull = self._geometry_to_wgs84(concave_hull) + concave_hull_polygons.append(concave_hull) except ValueError: logging.warning(f"Could not extract geometry from {geom_asset.href}") @@ -220,8 +229,8 @@ def parse_2d_geom(self): for geom_asset in self._geom_files: if isinstance(geom_asset, GeometryHdfAsset): - # mesh_areas = self._geometry_to_wgs84(geom_asset.hdf_object.mesh_areas(self.crs)) - mesh_area_polygons.append(geom_asset.hdf_object.mesh_areas(self.crs)) + mesh_areas = self._geometry_to_wgs84(geom_asset.hdf_object.mesh_areas(self.crs)) + mesh_area_polygons.append(mesh_areas) return union_all(mesh_area_polygons) diff --git a/hecstac/ras/parser.py b/hecstac/ras/parser.py index 96d829b..0efa6da 100644 --- a/hecstac/ras/parser.py +++ b/hecstac/ras/parser.py @@ -10,6 +10,7 @@ import numpy as np import pandas as pd from pyproj import CRS +from pyproj.exceptions import CRSError from pystac import Asset from rashdf import RasGeomHdf, RasHdf, RasPlanHdf from shapely import LineString, MultiPolygon, Point, Polygon, make_valid, union_all @@ -798,7 +799,7 @@ def rivers(self) -> dict[str, "River"]: @property def reaches(self) -> dict[str, "Reach"]: """A dictionary of the reaches contained in the HEC-RAS geometry file.""" - river_reaches = search_contents(self.file_lines, "River Reach", expect_one=False) + river_reaches = search_contents(self.file_lines, "River Reach", expect_one=False, require_one=False) return {river_reach: Reach(self.file_lines, river_reach, self.crs) for river_reach in river_reaches} @property @@ -867,13 +868,16 @@ def concave_hull(self) -> Polygon: polygons = [] if self.cross_sections: xs_gdf = pd.concat([xs.gdf for xs in self.cross_sections.values()], ignore_index=True) - + # logging.info(xs_gdf) for river_reach in xs_gdf["river_reach"].unique(): + logging.info(f"river reach: {river_reach}") xs_subset: gpd.GeoSeries = xs_gdf[xs_gdf["river_reach"] == river_reach] + logging.info(xs_subset) points = xs_subset.boundary.explode(index_parts=True).unstack() points_last_xs = [Point(coord) for coord in xs_subset["geometry"].iloc[-1].coords] points_first_xs = [Point(coord) for coord in xs_subset["geometry"].iloc[0].coords[::-1]] polygon = Polygon(points_first_xs + list(points[0]) + points_last_xs + list(points[1])[::-1]) + logging.info("got polygon") if isinstance(polygon, MultiPolygon): polygons += list(polygon.geoms) else: @@ -1146,14 +1150,14 @@ def two_d_flow_cell_minimum_size(self) -> int | None: self._2d_flow_attrs = self.hdf_object.get_geom_2d_flow_area_attrs() return int(np.sqrt(self._2d_flow_attrs.get("Cell Minimum Size"))) - def mesh_areas(self, crs, return_gdf=False) -> gpd.GeoDataFrame | Polygon | MultiPolygon: + def mesh_areas(self, crs=None, return_gdf=False) -> gpd.GeoDataFrame | Polygon | MultiPolygon: mesh_areas = self.hdf_object.mesh_cell_polygons() if mesh_areas is None or mesh_areas.empty: raise ValueError("No mesh areas found.") - if mesh_areas.crs and mesh_areas.crs != crs: - mesh_areas = mesh_areas.to_crs(crs) + if mesh_areas.crs is None and crs is not None: + mesh_areas = mesh_areas.set_crs(crs) if return_gdf: return mesh_areas @@ -1435,15 +1439,20 @@ def meteorology_units(self): class GeometryHDFFile(RASHDFFile): - def __init__(self, fpath: str, **kwargs): + def __init__(self, fpath: str, crs=None, **kwargs): super().__init__(fpath, RasGeomHdf, **kwargs) self.hdf_object = RasGeomHdf(fpath) - self.crs = CRS.from_user_input(self.hdf_object.projection()) - if not self.crs: - logging.info(f"No projection found in file {fpath}") - else: - logging.debug(f"Projection found in file {fpath}: {self.crs}") + self.crs = crs + logging.info("creating pyproj crs") + if self.crs is None: + try: + self.crs = CRS.from_user_input(self.hdf_object.projection()) + logging.debug(f"Projection found in file {fpath}: {self.crs}") + except CRSError: + raise ValueError(f"CRS was not given and could not be extracted from from {fpath}") + + logging.info("done getting crs") self._plan_info_attrs = None self._plan_parameters_attrs = None self._meteorology_attrs = None diff --git a/new_ras_item.py b/new_ras_item.py index b332a57..e81204a 100644 --- a/new_ras_item.py +++ b/new_ras_item.py @@ -15,14 +15,16 @@ def sanitize_catalog_assets(item: RASModelItem) -> RASModelItem: return item +crs = 'PROJCS["NAD_1983_StatePlane_California_III_FIPS_0403_Feet",GEOGCS["GCS_North_American_1983",DATUM["D_North_American_1983",SPHEROID["GRS_1980",6378137,298.257222101]],PRIMEM["Greenwich",0],UNIT["Degree",0.0174532925199432955]],PROJECTION["Lambert_Conformal_Conic"],PARAMETER["False_Easting",6561666.666666666],PARAMETER["False_Northing",1640416.666666667],PARAMETER["Central_Meridian",-120.5],PARAMETER["Standard_Parallel_1",37.06666666666667],PARAMETER["Standard_Parallel_2",38.43333333333333],PARAMETER["Latitude_Of_Origin",36.5],UNIT["Foot_US",0.304800609601219241]]' + if __name__ == "__main__": - initialize_logger() - ras_project_file = "ElkMiddle/ElkMiddle.prj" + initialize_logger(logging.DEBUG) + ras_project_file = "Baxter/Baxter.prj" item_id = Path(ras_project_file).stem - ras_item = RASModelItem(ras_project_file, item_id, crs="EPSG:4326") + ras_item = RASModelItem(ras_project_file, item_id, crs) ras_item = sanitize_catalog_assets(ras_item) - ras_item.add_model_thumbnail(["mesh_areas", "breaklines", "bc_lines"]) + # ras_item.add_model_thumbnail(["mesh_areas", "breaklines", "bc_lines"]) fs = ras_item.scan_model_dir() ras_item.add_ras_asset() From bf325ecd60ee0a77282a3ad2da2159624c20e3c8 Mon Sep 17 00:00:00 2001 From: Stevenray Janke Date: Tue, 28 Jan 2025 19:06:24 -0500 Subject: [PATCH 09/71] fix 2d crs --- hecstac/ras/assets.py | 44 ++++++++++++++++++++++++++++--------------- hecstac/ras/item.py | 19 +++++++++++-------- hecstac/ras/parser.py | 25 +++++++++++------------- 3 files changed, 51 insertions(+), 37 deletions(-) diff --git a/hecstac/ras/assets.py b/hecstac/ras/assets.py index b568f13..f73c44f 100644 --- a/hecstac/ras/assets.py +++ b/hecstac/ras/assets.py @@ -9,7 +9,8 @@ from matplotlib.lines import Line2D from pystac import MediaType from pyproj import CRS - +from pyproj.exceptions import CRSError +import json from hecstac.common.asset_factory import GenericAsset from hecstac.ras.parser import ( GeometryFile, @@ -27,6 +28,7 @@ TITLE = "ras:title" UNITS = "ras:units" VERSION = "ras:version" +PROJECTION = "proj:wkt" PLAN_FILE = "ras:plan_file" GEOMETRY_FILE = "ras:geometry_file" @@ -344,35 +346,47 @@ class GeometryHdfAsset(GenericAsset): regex_parse_str = r".+\.g\d{2}\.hdf$" def __init__(self, href: str, crs: str = None, **kwargs): - logging.info("Initializing hdf class") roles = kwargs.get("roles", []) + ["geometry-hdf-file"] description = kwargs.get("description", "The HEC-RAS geometry HDF file.") super().__init__(href, roles=roles, description=description, **kwargs) - logging.info("initializing hdf file") + self.hdf_object = GeometryHDFFile(self.href) self.crs = crs - - logging.warning(f"crs has been set to {self.crs}") - self.hdf_object = GeometryHDFFile(self.href, self.crs) - logging.info("reading mesh areas...") - try: - if self.hdf_object.mesh_areas(): - self.has_2d = True - except ValueError: - logging.info(f"Could not read mesh areas for {self.href}") - self.has_2d = False + self.has_2d = None + if self.crs is None: + try: + self.crs = CRS.from_user_input(self.hdf_object.projection) + logging.info(f"crs has been set to {self.crs}") + except CRSError: + logging.warning(f"Could not extract crs from {self.href}") self.extra_fields = { key: value for key, value in { VERSION: self.hdf_object.file_version, UNITS: self.hdf_object.units_system, - # "proj:wkt": json.load(self.hdf_object.crs), - REFERENCE_LINES: list(self.hdf_object.reference_lines["refln_name"]), + PROJECTION: self.crs.to_wkt() if self.crs is not None else None, + REFERENCE_LINES: ( + list(self.hdf_object.reference_lines["refln_name"]) + if not self.hdf_object.reference_lines.empty + else None + ), }.items() if value } + @property + def check_2d(self): + try: + logging.info("reading mesh areas...") + logging.info("crs is {self.crs}") + + if self.hdf_object.mesh_areas(self.crs): + return True + except ValueError: + logging.info(f"No mesh areas found for {self.href}") + return False + def _plot_mesh_areas(self, ax, mesh_polygons: gpd.GeoDataFrame) -> list[Line2D]: """ Plots mesh areas on the given axes. diff --git a/hecstac/ras/item.py b/hecstac/ras/item.py index 383890b..70c1448 100644 --- a/hecstac/ras/item.py +++ b/hecstac/ras/item.py @@ -181,19 +181,22 @@ def add_ras_asset(self, fpath: str = "") -> None: ) self._project = asset elif isinstance(asset, GeometryHdfAsset): - # if crs is None, use the crs from the 2d geom hdf file. - if self.crs is None: - self.crs = asset.hdf_object.crs - else: - logging.warning("settomg crs") + # if crs is None, use the crs from the 2d geom hdf file if it exists. + if self.crs is None and asset.crs is None: + pass + elif self.crs is None and asset.crs is not None: + self.crs = asset.crs + self._geom_files.append(asset) + elif self.crs: asset.crs = self.crs - if asset.has_2d: + self._geom_files.append(asset) + + if asset.check_2d: self.has_2d = True self.properties[self.RAS_HAS_2D] = True - self._geom_files.append(asset) elif isinstance(asset, GeometryAsset): if asset.geomf.has_1d: - self.has_1d = True + self.has_1d = False self.properties[self.RAS_HAS_1D] = True self._geom_files.append(asset) diff --git a/hecstac/ras/parser.py b/hecstac/ras/parser.py index 0efa6da..e5bc635 100644 --- a/hecstac/ras/parser.py +++ b/hecstac/ras/parser.py @@ -693,12 +693,12 @@ def ras_version(self) -> str | None: @property def plan_files(self) -> list[str]: - suffixes = search_contents(self.file_lines, "Plan File", expect_one=False) + suffixes = search_contents(self.file_lines, "Plan File", expect_one=False, require_one=False) return [name_from_suffix(self.fpath, i) for i in suffixes] @property def geometry_files(self) -> list[str]: - suffixes = search_contents(self.file_lines, "Geom File", expect_one=False) + suffixes = search_contents(self.file_lines, "Geom File", expect_one=False, require_one=False) return [name_from_suffix(self.fpath, i) for i in suffixes] @property @@ -1152,13 +1152,16 @@ def two_d_flow_cell_minimum_size(self) -> int | None: def mesh_areas(self, crs=None, return_gdf=False) -> gpd.GeoDataFrame | Polygon | MultiPolygon: - mesh_areas = self.hdf_object.mesh_cell_polygons() + mesh_areas = self.hdf_object.mesh_areas() if mesh_areas is None or mesh_areas.empty: raise ValueError("No mesh areas found.") if mesh_areas.crs is None and crs is not None: mesh_areas = mesh_areas.set_crs(crs) + elif mesh_areas.crs is None and crs is None: + raise CRSError("Mesh areas have no CRS and have none to be set to") + if return_gdf: return mesh_areas else: @@ -1439,24 +1442,18 @@ def meteorology_units(self): class GeometryHDFFile(RASHDFFile): - def __init__(self, fpath: str, crs=None, **kwargs): + def __init__(self, fpath: str, **kwargs): super().__init__(fpath, RasGeomHdf, **kwargs) self.hdf_object = RasGeomHdf(fpath) - self.crs = crs - logging.info("creating pyproj crs") - if self.crs is None: - try: - self.crs = CRS.from_user_input(self.hdf_object.projection()) - logging.debug(f"Projection found in file {fpath}: {self.crs}") - except CRSError: - raise ValueError(f"CRS was not given and could not be extracted from from {fpath}") - - logging.info("done getting crs") self._plan_info_attrs = None self._plan_parameters_attrs = None self._meteorology_attrs = None + @property + def projection(self): + return self.hdf_object.projection() + @property def cross_sections(self) -> int | None: pass From f8adc79cc35e753033b2ebf50789e94a598a303e Mon Sep 17 00:00:00 2001 From: Stevenray Janke Date: Wed, 29 Jan 2025 16:36:29 -0500 Subject: [PATCH 10/71] Fix asset hrefs --- new_ras_item.py | 27 ++++++++++++++++----------- 1 file changed, 16 insertions(+), 11 deletions(-) diff --git a/new_ras_item.py b/new_ras_item.py index e81204a..b03cc7b 100644 --- a/new_ras_item.py +++ b/new_ras_item.py @@ -1,30 +1,35 @@ import logging from pathlib import Path - from hecstac.common.logger import initialize_logger from hecstac.ras.item import RASModelItem def sanitize_catalog_assets(item: RASModelItem) -> RASModelItem: """ - Forces the asset paths in the catalog relative to item root. + Forces the asset paths in the catalog to be relative to the item root. """ - for asset in item.assets.values(): - if item.pm.model_root_dir in asset.href: - asset.href = asset.href.replace(item.pm.item_dir, ".") - return item + item_dir = Path(item.pm.item_dir).resolve() + for _, asset in item.assets.items(): + + asset_path = Path(asset.href).resolve() + if asset_path.is_relative_to(item_dir): + asset.href = str(asset_path.relative_to(item_dir)) + + elif asset.href.startswith(f"{item_dir.name}/"): + asset.href = asset.href.replace(f"{item_dir.name}/", "", 1) + + return item -crs = 'PROJCS["NAD_1983_StatePlane_California_III_FIPS_0403_Feet",GEOGCS["GCS_North_American_1983",DATUM["D_North_American_1983",SPHEROID["GRS_1980",6378137,298.257222101]],PRIMEM["Greenwich",0],UNIT["Degree",0.0174532925199432955]],PROJECTION["Lambert_Conformal_Conic"],PARAMETER["False_Easting",6561666.666666666],PARAMETER["False_Northing",1640416.666666667],PARAMETER["Central_Meridian",-120.5],PARAMETER["Standard_Parallel_1",37.06666666666667],PARAMETER["Standard_Parallel_2",38.43333333333333],PARAMETER["Latitude_Of_Origin",36.5],UNIT["Foot_US",0.304800609601219241]]' if __name__ == "__main__": - initialize_logger(logging.DEBUG) - ras_project_file = "Baxter/Baxter.prj" + initialize_logger() + ras_project_file = "ElkMiddle/ElkMiddle.prj" item_id = Path(ras_project_file).stem - ras_item = RASModelItem(ras_project_file, item_id, crs) + ras_item = RASModelItem(ras_project_file, item_id, crs=None) ras_item = sanitize_catalog_assets(ras_item) - # ras_item.add_model_thumbnail(["mesh_areas", "breaklines", "bc_lines"]) + # ras_item.add_model_thumbnails(["mesh_areas", "breaklines", "bc_lines"]) fs = ras_item.scan_model_dir() ras_item.add_ras_asset() From f63d71eefe43bc04aa725c6148477353833bd86f Mon Sep 17 00:00:00 2001 From: Stevenray Janke Date: Wed, 29 Jan 2025 16:37:46 -0500 Subject: [PATCH 11/71] Clean logging, update thumbnail creation to work with new crs handling --- hecstac/ras/assets.py | 46 ++++++++++++++++++------------------------- 1 file changed, 19 insertions(+), 27 deletions(-) diff --git a/hecstac/ras/assets.py b/hecstac/ras/assets.py index f73c44f..89f48b2 100644 --- a/hecstac/ras/assets.py +++ b/hecstac/ras/assets.py @@ -1,4 +1,3 @@ -import io import logging import os import re @@ -10,7 +9,6 @@ from pystac import MediaType from pyproj import CRS from pyproj.exceptions import CRSError -import json from hecstac.common.asset_factory import GenericAsset from hecstac.ras.parser import ( GeometryFile, @@ -368,7 +366,7 @@ def __init__(self, href: str, crs: str = None, **kwargs): PROJECTION: self.crs.to_wkt() if self.crs is not None else None, REFERENCE_LINES: ( list(self.hdf_object.reference_lines["refln_name"]) - if not self.hdf_object.reference_lines.empty + if self.hdf_object.reference_lines is not None and not self.hdf_object.reference_lines.empty else None ), }.items() @@ -377,14 +375,16 @@ def __init__(self, href: str, crs: str = None, **kwargs): @property def check_2d(self): + """Check if the geometry asset has 2d geometry, if yes then return True and set has_2d to True.""" try: - logging.info("reading mesh areas...") - logging.info("crs is {self.crs}") + logging.debug(f"reading mesh areas using crs {self.crs}...") if self.hdf_object.mesh_areas(self.crs): + self.has_2d = True return True except ValueError: - logging.info(f"No mesh areas found for {self.href}") + logging.warning(f"No mesh areas found for {self.href}") + self.has_2d = False return False def _plot_mesh_areas(self, ax, mesh_polygons: gpd.GeoDataFrame) -> list[Line2D]: @@ -474,7 +474,7 @@ def _add_thumbnail_asset(self, filepath: str) -> None: return GenericAsset( href=filename, - title="Model Thumbnail", + title=filename.split(".")[0], description="Thumbnail image for the model", media_type=media_type, roles=["thumbnail"], @@ -488,7 +488,7 @@ def thumbnail( crs="EPSG:4326", thumbnail_dest: str = None, ): - """Create a thumbnail figure for each geometry hdf file, including + """Create a thumbnail figure for a geometry hdf file, including various geospatial layers such as USGS gages, mesh areas, breaklines, and boundary condition (BC) lines. @@ -506,25 +506,17 @@ def thumbnail( for layer in layers: try: - # if layer == "usgs_gages": - # if add_usgs_properties: - # gages_gdf = self.get_usgs_data(True, geom_asset=geom_asset) - # else: - # gages_gdf = self.get_usgs_data(False, geom_asset=geom_asset) - # gages_gdf_geo = gages_gdf.to_crs(self.crs) - # legend_handles += self._plot_usgs_gages(ax, gages_gdf_geo) - if layer == "mesh_areas": - mesh_areas_data = self.hdf_object.mesh_areas(crs, return_gdf=True) - mesh_areas_geo = mesh_areas_data.to_crs(crs) + mesh_areas_data = self.hdf_object.mesh_cells + mesh_areas_geo = mesh_areas_data.set_crs(self.crs) legend_handles += self._plot_mesh_areas(ax, mesh_areas_geo) elif layer == "breaklines": breaklines_data = self.hdf_object.breaklines - breaklines_data_geo = breaklines_data.to_crs(crs) + breaklines_data_geo = breaklines_data.set_crs(self.crs) legend_handles += self._plot_breaklines(ax, breaklines_data_geo) elif layer == "bc_lines": bc_lines_data = self.hdf_object.bc_lines - bc_lines_data_geo = bc_lines_data.to_crs(crs) + bc_lines_data_geo = bc_lines_data.set_crs(self.crs) legend_handles += self._plot_bc_lines(ax, bc_lines_data_geo) except Exception as e: logging.warning(f"Warning: Failed to process layer '{layer}' for {self.href}: {e}") @@ -532,7 +524,7 @@ def thumbnail( # Add OpenStreetMap basemap ctx.add_basemap( ax, - crs=crs, + crs=self.crs, source=ctx.providers.OpenStreetMap.Mapnik, alpha=0.4, ) @@ -547,10 +539,12 @@ def thumbnail( filepath = os.path.join(base_dir, filename) if filepath.startswith("s3://"): - img_data = io.BytesIO() - fig.savefig(img_data, format="png", bbox_inches="tight") - img_data.seek(0) - save_bytes_s3(img_data, filepath) + pass + # TODO add thumbnail s3 functionality + # img_data = io.BytesIO() + # fig.savefig(img_data, format="png", bbox_inches="tight") + # img_data.seek(0) + # save_bytes_s3(img_data, filepath) else: os.makedirs(base_dir, exist_ok=True) fig.savefig(filepath, dpi=80, bbox_inches="tight") @@ -851,8 +845,6 @@ class RasMapperFileAsset(GenericAsset): def __init__(self, href: str, *args, **kwargs): roles = ["ras-mapper-file", "ras-file", MediaType.TEXT] description = "RAS Mapper file." - media_type = MediaType.TEXT - extra_fields = kwargs.get("extra_fields", {}) super().__init__(href, roles=roles, description=description, *args, **kwargs) From 9dbc9cc36d1fe144b7bf3db6fe10f6af6ebbb318 Mon Sep 17 00:00:00 2001 From: Stevenray Janke Date: Wed, 29 Jan 2025 16:38:09 -0500 Subject: [PATCH 12/71] Add mesh cells property, clean --- hecstac/ras/parser.py | 38 ++++++++++++++++++++++++-------------- 1 file changed, 24 insertions(+), 14 deletions(-) diff --git a/hecstac/ras/parser.py b/hecstac/ras/parser.py index e5bc635..afb8e9b 100644 --- a/hecstac/ras/parser.py +++ b/hecstac/ras/parser.py @@ -9,10 +9,9 @@ import geopandas as gpd import numpy as np import pandas as pd -from pyproj import CRS from pyproj.exceptions import CRSError from pystac import Asset -from rashdf import RasGeomHdf, RasHdf, RasPlanHdf +from rashdf import RasGeomHdf, RasPlanHdf from shapely import LineString, MultiPolygon, Point, Polygon, make_valid, union_all from shapely.ops import unary_union @@ -868,16 +867,12 @@ def concave_hull(self) -> Polygon: polygons = [] if self.cross_sections: xs_gdf = pd.concat([xs.gdf for xs in self.cross_sections.values()], ignore_index=True) - # logging.info(xs_gdf) for river_reach in xs_gdf["river_reach"].unique(): - logging.info(f"river reach: {river_reach}") xs_subset: gpd.GeoSeries = xs_gdf[xs_gdf["river_reach"] == river_reach] - logging.info(xs_subset) points = xs_subset.boundary.explode(index_parts=True).unstack() points_last_xs = [Point(coord) for coord in xs_subset["geometry"].iloc[-1].coords] points_first_xs = [Point(coord) for coord in xs_subset["geometry"].iloc[0].coords[::-1]] polygon = Polygon(points_first_xs + list(points[0]) + points_last_xs + list(points[1])[::-1]) - logging.info("got polygon") if isinstance(polygon, MultiPolygon): polygons += list(polygon.geoms) else: @@ -1046,7 +1041,7 @@ def units_system(self) -> str | None: def geometry_time(self) -> datetime.datetime | None: if self._geom_attrs == None: self._geom_attrs = self.hdf_object.get_geom_attrs() - return self._geom_attrs.get("Geometry Time").isoformat() + return self._geom_attrs.get("Geometry Time") @property def landcover_date_last_modified(self) -> datetime.datetime | None: @@ -1150,8 +1145,16 @@ def two_d_flow_cell_minimum_size(self) -> int | None: self._2d_flow_attrs = self.hdf_object.get_geom_2d_flow_area_attrs() return int(np.sqrt(self._2d_flow_attrs.get("Cell Minimum Size"))) - def mesh_areas(self, crs=None, return_gdf=False) -> gpd.GeoDataFrame | Polygon | MultiPolygon: + def mesh_areas(self, crs: str = None, return_gdf: bool = False) -> gpd.GeoDataFrame | Polygon | MultiPolygon: + """Retrieves and processes mesh area geometries. + Parameters + ---------- + crs : str, optional + The coordinate reference system (CRS) to set if the mesh areas do not have one. Defaults to None + return_gdf : bool, optional + If True, returns a GeoDataFrame of the mesh areas. If False, returns a unified Polygon or Multipolygon geometry. Defaults to False. + """ mesh_areas = self.hdf_object.mesh_areas() if mesh_areas is None or mesh_areas.empty: raise ValueError("No mesh areas found.") @@ -1174,8 +1177,17 @@ def breaklines(self) -> gpd.GeoDataFrame | None: if breaklines is None or breaklines.empty: raise ValueError("No breaklines found.") - else: - return breaklines + + return breaklines + + @property + def mesh_cells(self) -> gpd.GeoDataFrame | None: + mesh_cells = self.hdf_object.mesh_cell_polygons() + + if mesh_cells is None or mesh_cells.empty: + raise ValueError("No mesh cells found.") + + return mesh_cells @property def bc_lines(self) -> gpd.GeoDataFrame | None: @@ -1183,8 +1195,8 @@ def bc_lines(self) -> gpd.GeoDataFrame | None: if bc_lines is None or bc_lines.empty: raise ValueError("No boundary condition lines found.") - else: - return bc_lines + + return bc_lines @property def landcover_filename(self) -> str | None: @@ -1211,8 +1223,6 @@ def __init__(self, fpath: str, **kwargs): @property def plan_information_base_output_interval(self) -> str | None: - # example property to show pattern: if attributes in which property is found is not loaded, load them - # then use key for the property in the dictionary of attributes to retrieve the property if self._plan_info_attrs == None: self._plan_info_attrs = self.hdf_object.get_plan_info_attrs() return self._plan_info_attrs.get("Base Output Interval") From 5e7a7056d85b63c2de9f69897efe546148536274 Mon Sep 17 00:00:00 2001 From: Stevenray Janke Date: Wed, 29 Jan 2025 16:39:20 -0500 Subject: [PATCH 13/71] Fix thumbnail and geometry, add item time handling --- hecstac/ras/item.py | 119 ++++++++++++++++++++++++-------------------- 1 file changed, 66 insertions(+), 53 deletions(-) diff --git a/hecstac/ras/item.py b/hecstac/ras/item.py index 70c1448..6a709f3 100644 --- a/hecstac/ras/item.py +++ b/hecstac/ras/item.py @@ -8,18 +8,16 @@ from pystac import Item from pystac.extensions.projection import ProjectionExtension from pystac.extensions.storage import StorageExtension -from shapely import Geometry, Polygon, box, simplify, to_geojson, union_all +from shapely import Geometry, Polygon, simplify, to_geojson, union_all from shapely.ops import transform from hecstac.common.path_manager import LocalPathManager from hecstac.ras.parser import ProjectFile - -NULL_DATETIME = datetime.datetime(9999, 9, 9) -NULL_GEOMETRY = Polygon() -NULL_STAC_GEOMETRY = json.loads(to_geojson(NULL_GEOMETRY)) -NULL_BBOX = box(0, 0, 0, 0) -NULL_STAC_BBOX = NULL_BBOX.bounds -PLACEHOLDER_ID = "id" +from hecstac.ras.consts import ( + NULL_DATETIME, + NULL_STAC_GEOMETRY, + NULL_STAC_BBOX, +) from hecstac.common.asset_factory import AssetFactory from hecstac.ras.assets import ( @@ -72,19 +70,21 @@ def __init__(self, ras_project_file, item_id: str, crs: str = None, simplify_geo Path(self.ras_project_file).stem, NULL_STAC_GEOMETRY, NULL_STAC_BBOX, - self._datetime, + NULL_DATETIME, self._properties, href=self._href, ) - # derived_assets = self.add_model_thumbnail() TODO: implement this method + ras_asset_files = self.scan_model_dir() for fpath in ras_asset_files: if fpath and fpath != self._href: - logging.info(f"processing {fpath}") + logging.info(f"Processing asset: {fpath}") self.add_ras_asset(fpath) + # Update geometry and datetime after assets have been added self._geometry + self._datetime def _register_extensions(self) -> None: ProjectionExtension.add_to(self) @@ -103,31 +103,23 @@ def _properties(self) -> None: properties[self.PROJECT_STATUS] = self.pf.project_status properties[self.MODEL_UNITS] = self.pf.project_units - # self.properties[RAS_DATETIME_SOURCE] = self.datetime_source # TODO: once all assets are created, populate associations between assets return properties @property def _geometry(self) -> dict | None: - """ - gets geometry using either list of geom assets or list of hdf assets, perhaps simplified to - a given tolerance to reduce replication of data - (ie item would record simplified geometry used when searching collection, - asset would have more exact geometry representing contents of geom or hdf files) - """ - # if geometry is equal to null placeholder, continue, else return current value + """Parses geometries from 2d hdf files and updates the stac item geometry, simplifying them if needed.""" geometries = [] if self.has_2d: geometries.append(self.parse_2d_geom()) - # if hdf file is not present, get concave hull of cross sections and use as geometry - if self.has_1d: - geometries.append(self.parse_1d_geom()) + # if self.has_1d: + # geometries.append(self.parse_1d_geom()) if len(geometries) == 0: logging.error("No geometry found for RAS item.") - return NULL_STAC_GEOMETRY + return unioned_geometry = union_all(geometries) if self._simplify_geometry: @@ -138,27 +130,43 @@ def _geometry(self) -> dict | None: @property def _datetime(self) -> datetime: - """The datetime for the HMS STAC item.""" - # date = datetime.datetime.strptime(self.pf.basins[0].header.attrs["Last Modified Date"], "%d %B %Y") - # time = datetime.datetime.strptime(self.pf.basins[0].header.attrs["Last Modified Time"], "%H:%M:%S").time() - return datetime.datetime.now() - - @property - def datetime_source(self) -> str: - if self._datetime_source == None: - if self._dts == None: - self.populate() - if len(self._dts) == 0: - self._datetime_source = "processing_time" - else: - self._datetime_source = "model_geometry" - return self._datetime_source - - def add_model_thumbnail(self, layers: list, title: str = "Model_Thumbnail"): + """The datetime for the RAS STAC item.""" + item_datetime = None + + for geom_file in self._geom_files: + if isinstance(geom_file, GeometryHdfAsset): + geom_date = geom_file.hdf_object.geometry_time + if geom_date: + item_datetime = geom_date + self.properties[self.RAS_DATETIME_SOURCE] = "model_geometry" + logging.info(f"Using item datetime from {geom_file.href}") + break + + if item_datetime is None: + logging.warning("Could not extract item datetime from geometry, using item processing time.") + item_datetime = datetime.datetime.now() + self.properties[self.RAS_DATETIME_SOURCE] = "processing_time" + + self.datetime = item_datetime + + def add_model_thumbnails(self, layers: list, title_prefix: str = "Model_Thumbnail"): + """Generates model thumbnail asset for each geometry file. + + Parameters + ---------- + layers : list + List of geometry layers to be included in the plot. Options include 'mesh_areas', 'breaklines', 'bc_lines' + title_prefix : str, optional + Thumbnail title prefix, by default "Model_Thumbnail". + """ for geom in self._geom_files: if isinstance(geom, GeometryHdfAsset): - self.assets["thumbnail"] = geom.thumbnail(layers=layers, title=title, thumbnail_dest=self._href) + self.assets[f"{geom.href[4:]}_thumbnail"] = geom.thumbnail( + layers=layers, title=title_prefix, thumbnail_dest=self._href + ) + + # TODO: Add 1d model thumbnails def add_ras_asset(self, fpath: str = "") -> None: """Add an asset to the HMS STAC item.""" @@ -181,26 +189,29 @@ def add_ras_asset(self, fpath: str = "") -> None: ) self._project = asset elif isinstance(asset, GeometryHdfAsset): - # if crs is None, use the crs from the 2d geom hdf file if it exists. + # if item and asset crs are None, pass and use null geometry if self.crs is None and asset.crs is None: pass + # Use asset crs as item crs if there is no item crs elif self.crs is None and asset.crs is not None: self.crs = asset.crs - self._geom_files.append(asset) + # If item has crs, use it as the asset crs elif self.crs: asset.crs = self.crs - self._geom_files.append(asset) if asset.check_2d: + self._geom_files.append(asset) self.has_2d = True self.properties[self.RAS_HAS_2D] = True - elif isinstance(asset, GeometryAsset): - if asset.geomf.has_1d: - self.has_1d = False - self.properties[self.RAS_HAS_1D] = True - self._geom_files.append(asset) + # elif isinstance(asset, GeometryAsset): + # if asset.geomf.has_1d: + # self.has_1d = False TODO: Implement 1d functionality + # self.properties[self.RAS_HAS_1D] = True + # self._geom_files.append(asset) def _geometry_to_wgs84(self, geom: Geometry) -> Geometry: + """Convert geometry CRS to EPSG:4326 for stac item geometry.""" + pyproj_crs = CRS.from_user_input(self.crs) wgs_crs = CRS.from_authority("EPSG", "4326") if pyproj_crs != wgs_crs: @@ -209,16 +220,15 @@ def _geometry_to_wgs84(self, geom: Geometry) -> Geometry: return geom def parse_1d_geom(self): + """Read 1d geometry from concave hull.""" + logging.info("Creating geometry using 1d text file cross sections") concave_hull_polygons: list[Polygon] = [] for geom_asset in self._geom_files: if isinstance(geom_asset, GeometryAsset): try: - logging.info("Getting concave hull") geom_asset.crs = self.crs - logging.info(geom_asset.crs) concave_hull = geom_asset.geomf.concave_hull - logging.info("Concave hull retrieved") concave_hull = self._geometry_to_wgs84(concave_hull) concave_hull_polygons.append(concave_hull) except ValueError: @@ -227,10 +237,12 @@ def parse_1d_geom(self): return union_all(concave_hull_polygons) def parse_2d_geom(self): - logging.info("Creating 2D geometry elements using hdf file mesh areas") + """Read 2d geometry from hdf file mesh areas.""" + mesh_area_polygons: list[Polygon] = [] for geom_asset in self._geom_files: if isinstance(geom_asset, GeometryHdfAsset): + logging.info(f"Extracting geom from mesh areas in {geom_asset.href}") mesh_areas = self._geometry_to_wgs84(geom_asset.hdf_object.mesh_areas(self.crs)) mesh_area_polygons.append(mesh_areas) @@ -241,6 +253,7 @@ def ensure_projection_schema(self) -> None: ProjectionExtension.ensure_has_extension(self, True) def scan_model_dir(self): + """Find all files in the project folder.""" base_dir = os.path.dirname(self.ras_project_file) files = [] for root, _, filenames in os.walk(base_dir): From 02694d0524ac86407ba437ffdbb0810ef67dc74d Mon Sep 17 00:00:00 2001 From: Stevenray Janke Date: Wed, 29 Jan 2025 16:39:31 -0500 Subject: [PATCH 14/71] Add constants --- hecstac/ras/consts.py | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/hecstac/ras/consts.py b/hecstac/ras/consts.py index 4021bd7..7798ec3 100644 --- a/hecstac/ras/consts.py +++ b/hecstac/ras/consts.py @@ -1,3 +1,15 @@ +import datetime +from shapely import Polygon, to_geojson, box +import json + + SCHEMA_URI = ( "https://raw.githubusercontent.com/fema-ffrd/hecstac/refs/heads/port-ras-stac/hecstac/ras/extension/schema.json" ) + +NULL_DATETIME = datetime.datetime(9999, 9, 9) +NULL_GEOMETRY = Polygon() +NULL_STAC_GEOMETRY = json.loads(to_geojson(NULL_GEOMETRY)) +NULL_BBOX = box(0, 0, 0, 0) +NULL_STAC_BBOX = NULL_BBOX.bounds +PLACEHOLDER_ID = "id" From 58dfe7dee8c284199e12f270726166452c77cc7f Mon Sep 17 00:00:00 2001 From: Stevenray Janke Date: Wed, 29 Jan 2025 17:04:37 -0500 Subject: [PATCH 15/71] Revert event naming, to be changed later --- docs/source/user_guide.rst | 14 +++++++------- hecstac/events/ffrd.py | 8 ++++---- 2 files changed, 11 insertions(+), 11 deletions(-) diff --git a/docs/source/user_guide.rst b/docs/source/user_guide.rst index a3e921e..c920cc3 100644 --- a/docs/source/user_guide.rst +++ b/docs/source/user_guide.rst @@ -17,14 +17,14 @@ have Python already installed and setup: Note that it is highly recommended to create a python `virtual environment -`_ to install, test, and run hecstac. +`_ to install, test, and run hecstac. Workflows --------- -The following snippets provide examples for creating stac items from HEC model data. +The following snippets provide examples for creating stac items from HEC model data. .. code-block:: python @@ -56,14 +56,14 @@ The following snippets provide examples for creating stac items from HEC model d ras_item.save_object(ras_item.pm.item_path(item_id)) -The following snippet provides an example of how to create stac items for an event based simulation. +The following snippet provides an example of how to create stac items for an event based simulation. .. code-block:: python from pystac import Item from hecstac.common.logger import initialize_logger - from hecstac.events.ffrd import EventItem + from hecstac.events.ffrd import FFRDEventItem if __name__ == "__main__": initialize_logger() @@ -79,7 +79,7 @@ The following snippet provides an example of how to create stac items for an eve "/hms-model.met", "/Precip.dss", ] - + # RAS Info ras_source_model_item_path = "//authoritative-ras-model.json" @@ -98,14 +98,14 @@ The following snippet provides an example of how to create stac items for an eve ffrd_event_item_id = f"{realization}-{block_group}-{event_id}" dest_href = f"//{ffrd_event_item_id}.json" - ffrd_event_item = EventItem( + ffrd_event_item = FFRDEventItem( realization=realization, block_group=block_group, event_id=event_id, source_model_items=[ hms_source_model_item, ras_source_model_item - ], + ], hms_simulation_files=hms_simulation_files, ras_simulation_files=ras_simulation_files, ) diff --git a/hecstac/events/ffrd.py b/hecstac/events/ffrd.py index fcbbd23..c67efee 100644 --- a/hecstac/events/ffrd.py +++ b/hecstac/events/ffrd.py @@ -16,10 +16,10 @@ from hecstac.ras.assets import RAS_EXTENSION_MAPPING -class EventItem(Item): - REALIZATION = "montecarlo:realization" - BLOCK_GROUP = "montecarlo:block_group" - MC_EVENT = "montecarlo:event" +class FFRDEventItem(Item): + FFRD_REALIZATION = "FFRD:realization" + FFRD_BLOCK_GROUP = "FFRD:block_group" + FFRD_EVENT = "FFRD:event" def __init__( self, From 031aa105043560521d43ae8761283d6074891dd8 Mon Sep 17 00:00:00 2001 From: Stevenray Janke Date: Wed, 29 Jan 2025 17:14:58 -0500 Subject: [PATCH 16/71] Remove unused function parameter --- hecstac/ras/assets.py | 1 - 1 file changed, 1 deletion(-) diff --git a/hecstac/ras/assets.py b/hecstac/ras/assets.py index 89f48b2..7e61652 100644 --- a/hecstac/ras/assets.py +++ b/hecstac/ras/assets.py @@ -485,7 +485,6 @@ def thumbnail( self, layers: list, title: str = "Model_Thumbnail", - crs="EPSG:4326", thumbnail_dest: str = None, ): """Create a thumbnail figure for a geometry hdf file, including From 3001f6f44ded0bfaa6806bbc5cdd585e69d58f18 Mon Sep 17 00:00:00 2001 From: Stevenray Janke Date: Fri, 31 Jan 2025 11:40:30 -0500 Subject: [PATCH 17/71] Add simple ras item instructions --- README.md | 8 +++++++- new_ras_item.py | 2 +- 2 files changed, 8 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index be9155f..7a4a344 100644 --- a/README.md +++ b/README.md @@ -6,5 +6,11 @@ Utilities for creating STAC items from HEC models -**hecstac** is an open-source Python library designed to mine metadata from HEC model simulations for use in the development of catalogs documenting probabilistic flood studies. This project automates the generation of STAC Items and Assets from HEC-HMS and HEC-RAS model files, enabling improved data and metadata management. +**hecstac** is an open-source Python library designed to mine metadata from HEC model simulations for use in the development of catalogs documenting probabilistic flood studies. This project automates the generation of STAC Items and Assets from HEC-HMS and HEC-RAS model files, enabling improved data and metadata management. +***Testing HEC-RAS model item creation*** + +- Download the HEC-RAS example project data from USACE and place it in your working directory. The data can be downloaded [here](https://github.com/HydrologicEngineeringCenter/hec-downloads/releases/download/1.0.33/Example_Projects_6_6.zip). +- In 'new_ras_item.py', set the ras_project_file to the path of the 2D Muncie project file (ex. ras_project_file = "Example_Projects_6_6/2D Unsteady Flow Hydraulics/Muncie/Muncie.prj"). +- For projects that have projection information within the geometry .hdf files, the CRS info can automatically be detected. The Muncie data lacks that projection info so it must be set by extracting the projection string and setting the CRS in 'new_ras_item.py' to the projection string. The projection can be found in the Muncie/GIS_Data folder in Muncie_IA_Clip.prj. +- Once the CRS and project file location have been set, a new item can be created with 'python -m new_ras_item' in the command line. The new item will be added inside the model directory at the same level as the project file. diff --git a/new_ras_item.py b/new_ras_item.py index b03cc7b..cae3e5b 100644 --- a/new_ras_item.py +++ b/new_ras_item.py @@ -24,7 +24,7 @@ def sanitize_catalog_assets(item: RASModelItem) -> RASModelItem: if __name__ == "__main__": initialize_logger() - ras_project_file = "ElkMiddle/ElkMiddle.prj" + ras_project_file = "Example_Projects_6_6/2D Unsteady Flow Hydraulics/Muncie/Muncie.prj" item_id = Path(ras_project_file).stem ras_item = RASModelItem(ras_project_file, item_id, crs=None) From d06d8479872b5d6a597628792aa0b7de61a6c0b5 Mon Sep 17 00:00:00 2001 From: Stevenray Janke Date: Mon, 3 Feb 2025 15:51:11 -0500 Subject: [PATCH 18/71] Begin RASModelItem refactor --- hecstac/ras/item.py | 86 +++++++++++++++++++++++---------------------- 1 file changed, 44 insertions(+), 42 deletions(-) diff --git a/hecstac/ras/item.py b/hecstac/ras/item.py index 6a709f3..24c5056 100644 --- a/hecstac/ras/item.py +++ b/hecstac/ras/item.py @@ -44,58 +44,54 @@ class RASModelItem(Item): RAS_HAS_2D = "ras:has_2d" RAS_DATETIME_SOURCE = "ras:datetime_source" - def __init__(self, ras_project_file, item_id: str, crs: str = None, simplify_geometry: bool = True): - - self._project = None - self.assets = {} - self.links = [] - self.thumbnail_paths = [] - self.geojson_paths = [] - self.extra_fields = {} - self._geom_files = [] - self.stac_extensions = None - self.pm = LocalPathManager(Path(ras_project_file).parent) - self._href = self.pm.item_path(item_id) - self.crs = crs - self.ras_project_file = ras_project_file - self._simplify_geometry = simplify_geometry - - self.pf = ProjectFile(self.ras_project_file) - - self.factory = AssetFactory(RAS_EXTENSION_MAPPING) - self.has_1d = False - self.has_2d = False - - super().__init__( - Path(self.ras_project_file).stem, + @classmethod + def from_prj(cls, ras_project_file, item_id: str, crs: str = None, simplify_geometry: bool = True): + """Create an item from a RAS .prj file.""" + properties = {"ras_project_file": ras_project_file} + pm = LocalPathManager(Path(ras_project_file).parent) + + href = pm.item_path(item_id) + + stac = cls( + Path(ras_project_file).stem, NULL_STAC_GEOMETRY, NULL_STAC_BBOX, NULL_DATETIME, - self._properties, - href=self._href, + properties, + href=href, ) + stac._href = href + stac.factory = AssetFactory(RAS_EXTENSION_MAPPING) + stac.crs = crs + stac._project = None + stac.pm = pm + stac.pf = ProjectFile(ras_project_file) - ras_asset_files = self.scan_model_dir() + ras_asset_files = stac.scan_model_dir(ras_project_file) for fpath in ras_asset_files: - if fpath and fpath != self._href: + if fpath and fpath != href: logging.info(f"Processing asset: {fpath}") - self.add_ras_asset(fpath) + stac.add_ras_asset(fpath) + + stac.add_geometry(simplify_geometry) + stac.properties.update(stac.add_properties) + stac.datetime = stac._datetime + if stac.crs: + stac.apply_projection_extension(stac.crs) - # Update geometry and datetime after assets have been added - self._geometry - self._datetime + return stac def _register_extensions(self) -> None: ProjectionExtension.add_to(self) StorageExtension.add_to(self) @property - def _properties(self) -> None: + def add_properties(self) -> None: """Properties for the RAS STAC item.""" properties = {} - properties[self.RAS_HAS_1D] = self.has_1d + # properties[self.RAS_HAS_1D] = self.has_1d properties[self.RAS_HAS_2D] = self.has_2d properties[self.PROJECT_TITLE] = self.pf.project_title properties[self.PROJECT_VERSION] = self.pf.ras_version @@ -106,8 +102,7 @@ def _properties(self) -> None: # TODO: once all assets are created, populate associations between assets return properties - @property - def _geometry(self) -> dict | None: + def add_geometry(self, simplify_geometry: bool) -> dict | None: """Parses geometries from 2d hdf files and updates the stac item geometry, simplifying them if needed.""" geometries = [] @@ -122,7 +117,7 @@ def _geometry(self) -> dict | None: return unioned_geometry = union_all(geometries) - if self._simplify_geometry: + if simplify_geometry: unioned_geometry = simplify(unioned_geometry, 0.001) self.geometry = json.loads(to_geojson(unioned_geometry)) @@ -146,8 +141,7 @@ def _datetime(self) -> datetime: logging.warning("Could not extract item datetime from geometry, using item processing time.") item_datetime = datetime.datetime.now() self.properties[self.RAS_DATETIME_SOURCE] = "processing_time" - - self.datetime = item_datetime + return item_datetime def add_model_thumbnails(self, layers: list, title_prefix: str = "Model_Thumbnail"): """Generates model thumbnail asset for each geometry file. @@ -162,7 +156,7 @@ def add_model_thumbnails(self, layers: list, title_prefix: str = "Model_Thumbnai for geom in self._geom_files: if isinstance(geom, GeometryHdfAsset): - self.assets[f"{geom.href[4:]}_thumbnail"] = geom.thumbnail( + self.assets[f"{geom.href}_thumbnail"] = geom.thumbnail( layers=layers, title=title_prefix, thumbnail_dest=self._href ) @@ -200,6 +194,8 @@ def add_ras_asset(self, fpath: str = "") -> None: asset.crs = self.crs if asset.check_2d: + if not getattr(self, "_geom_files", None): + self._geom_files = [] self._geom_files.append(asset) self.has_2d = True self.properties[self.RAS_HAS_2D] = True @@ -252,9 +248,9 @@ def parse_2d_geom(self): def ensure_projection_schema(self) -> None: ProjectionExtension.ensure_has_extension(self, True) - def scan_model_dir(self): + def scan_model_dir(self, ras_project_file): """Find all files in the project folder.""" - base_dir = os.path.dirname(self.ras_project_file) + base_dir = os.path.dirname(ras_project_file) files = [] for root, _, filenames in os.walk(base_dir): depth = root[len(base_dir) :].count(os.sep) @@ -263,3 +259,9 @@ def scan_model_dir(self): for filename in filenames: files.append(os.path.join(root, filename)) return files + + def apply_projection_extension(self, crs: str): + """Apply the projection extension to this item given a CRS.""" + prj_ext = ProjectionExtension.ext(self, add_if_missing=True) + og_crs = CRS(crs) + prj_ext.apply(epsg=og_crs.to_epsg(), wkt2=og_crs.to_wkt()) From d247c1e004f13d4cf1b0d385375f948862107d0f Mon Sep 17 00:00:00 2001 From: Stevenray Janke Date: Mon, 3 Feb 2025 15:52:33 -0500 Subject: [PATCH 19/71] Add RASModelItem import into top level init --- hecstac/__init__.py | 1 + 1 file changed, 1 insertion(+) diff --git a/hecstac/__init__.py b/hecstac/__init__.py index 0466af5..aa7a1b7 100644 --- a/hecstac/__init__.py +++ b/hecstac/__init__.py @@ -5,3 +5,4 @@ """ from hecstac.version import __version__ +from hecstac.ras.item import RASModelItem From 102bfa3aee9a1c1b11c57d0401b4e0c16162d502 Mon Sep 17 00:00:00 2001 From: Stevenray Janke Date: Mon, 3 Feb 2025 16:19:01 -0500 Subject: [PATCH 20/71] Update pyproject dependencies --- pyproject.toml | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 8674fc8..d19c700 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -27,9 +27,13 @@ dependencies = [ "pystac==1.10.0", "rasterio==1.3.10", "requests==2.32.3", - "shapely==2.0.5", + "shapely==2.0.7", "xarray==2024.11.0", - "rioxarray==0.18.1" + "rioxarray==0.18.1", + "mypy-boto3-s3==1.35.93", + "contextily==1.6.2", + "rashdf==0.7.1", + "boto3==1.35.98" ] [project.optional-dependencies] From 854b05346fa03c816add9e757c6cece007cfab2a Mon Sep 17 00:00:00 2001 From: Stevenray Janke Date: Mon, 3 Feb 2025 17:26:43 -0500 Subject: [PATCH 21/71] Fix thumbnail path so href can be absolute --- hecstac/ras/assets.py | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/hecstac/ras/assets.py b/hecstac/ras/assets.py index 7e61652..5221364 100644 --- a/hecstac/ras/assets.py +++ b/hecstac/ras/assets.py @@ -463,8 +463,6 @@ def _plot_bc_lines(self, ax, bc_lines: gpd.GeoDataFrame) -> list[Line2D]: def _add_thumbnail_asset(self, filepath: str) -> None: """Add the thumbnail image as an asset with a relative href.""" - filename = os.path.basename(filepath) - if filepath.startswith("s3://"): media_type = "image/png" else: @@ -473,8 +471,8 @@ def _add_thumbnail_asset(self, filepath: str) -> None: media_type = "image/png" return GenericAsset( - href=filename, - title=filename.split(".")[0], + href=filepath, + title=filepath.split("/")[-1], description="Thumbnail image for the model", media_type=media_type, roles=["thumbnail"], From 2f78096504da9fe0b352e2df6fd69acd33cfabde Mon Sep 17 00:00:00 2001 From: Stevenray Janke Date: Mon, 3 Feb 2025 17:27:22 -0500 Subject: [PATCH 22/71] Add directory arg to thumbnail, clean --- hecstac/ras/item.py | 28 ++++++++++++++-------------- 1 file changed, 14 insertions(+), 14 deletions(-) diff --git a/hecstac/ras/item.py b/hecstac/ras/item.py index 24c5056..b51ac66 100644 --- a/hecstac/ras/item.py +++ b/hecstac/ras/item.py @@ -63,7 +63,6 @@ def from_prj(cls, ras_project_file, item_id: str, crs: str = None, simplify_geom stac._href = href stac.factory = AssetFactory(RAS_EXTENSION_MAPPING) stac.crs = crs - stac._project = None stac.pm = pm stac.pf = ProjectFile(ras_project_file) @@ -74,7 +73,7 @@ def from_prj(cls, ras_project_file, item_id: str, crs: str = None, simplify_geom logging.info(f"Processing asset: {fpath}") stac.add_ras_asset(fpath) - stac.add_geometry(simplify_geometry) + stac.geometry, stac.bbox = stac.add_geometry(simplify_geometry) stac.properties.update(stac.add_properties) stac.datetime = stac._datetime if stac.crs: @@ -120,8 +119,10 @@ def add_geometry(self, simplify_geometry: bool) -> dict | None: if simplify_geometry: unioned_geometry = simplify(unioned_geometry, 0.001) - self.geometry = json.loads(to_geojson(unioned_geometry)) - self.bbox = unioned_geometry.bounds + geometry = json.loads(to_geojson(unioned_geometry)) + bbox = unioned_geometry.bounds + + return geometry, bbox @property def _datetime(self) -> datetime: @@ -143,7 +144,7 @@ def _datetime(self) -> datetime: self.properties[self.RAS_DATETIME_SOURCE] = "processing_time" return item_datetime - def add_model_thumbnails(self, layers: list, title_prefix: str = "Model_Thumbnail"): + def add_model_thumbnails(self, layers: list, title_prefix: str = "Model_Thumbnail", thumbnail_dir=None): """Generates model thumbnail asset for each geometry file. Parameters @@ -153,17 +154,21 @@ def add_model_thumbnails(self, layers: list, title_prefix: str = "Model_Thumbnai title_prefix : str, optional Thumbnail title prefix, by default "Model_Thumbnail". """ + if thumbnail_dir: + thumbnail_dest = thumbnail_dir + else: + thumbnail_dest = self._href for geom in self._geom_files: if isinstance(geom, GeometryHdfAsset): self.assets[f"{geom.href}_thumbnail"] = geom.thumbnail( - layers=layers, title=title_prefix, thumbnail_dest=self._href + layers=layers, title=title_prefix, thumbnail_dest=thumbnail_dest ) # TODO: Add 1d model thumbnails def add_ras_asset(self, fpath: str = "") -> None: - """Add an asset to the HMS STAC item.""" + """Add an asset to the RAS STAC item.""" if not os.path.exists(fpath): logging.warning(f"File not found: {fpath}") return @@ -176,13 +181,8 @@ def add_ras_asset(self, fpath: str = "") -> None: if asset: self.add_asset(asset.title, asset) - if isinstance(asset, ProjectAsset): - if self._project is not None: - logging.error( - f"Only one project asset is allowed. Found {str(asset)} when {str(self._project)} was already set." - ) - self._project = asset - elif isinstance(asset, GeometryHdfAsset): + + if isinstance(asset, GeometryHdfAsset): # if item and asset crs are None, pass and use null geometry if self.crs is None and asset.crs is None: pass From 7a4afc44512e905c2b3945f89ba6e7cada6c06e7 Mon Sep 17 00:00:00 2001 From: Stevenray Janke Date: Tue, 4 Feb 2025 10:02:17 -0500 Subject: [PATCH 23/71] Minor linting changes --- hecstac/ras/utils.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/hecstac/ras/utils.py b/hecstac/ras/utils.py index 24627c6..a898a1f 100644 --- a/hecstac/ras/utils.py +++ b/hecstac/ras/utils.py @@ -236,7 +236,8 @@ def multithreading_enabled(func): ndarrays to False. NB: multithreading also requires the GIL to be released, which is done in - the C extension (ufuncs.c).""" + the C extension (ufuncs.c). + """ @wraps(func) def wrapped(*args, **kwargs): @@ -273,7 +274,7 @@ def reverse(geometry, **kwargs): **kwargs See :ref:`NumPy ufunc docs ` for other keyword arguments. - See also + See Also -------- is_ccw : Checks if a Geometry is clockwise. @@ -287,5 +288,4 @@ def reverse(geometry, **kwargs): >>> reverse(None) is None True """ - return lib.reverse(geometry, **kwargs) From df6b30ad28e5936318063d9ffd6c93a639369d0f Mon Sep 17 00:00:00 2001 From: Stevenray Janke Date: Tue, 4 Feb 2025 10:02:52 -0500 Subject: [PATCH 24/71] Fix so boundary locations arent required --- hecstac/ras/parser.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/hecstac/ras/parser.py b/hecstac/ras/parser.py index afb8e9b..18a5f9a 100644 --- a/hecstac/ras/parser.py +++ b/hecstac/ras/parser.py @@ -750,7 +750,7 @@ def short_identifier(self) -> str: @property def breach_locations(self) -> dict: """ - example file line: + Example file line: Breach Loc= , , ,True,HH_DamEmbankment """ breach_dict = {} @@ -980,11 +980,11 @@ def flow_title(self) -> str: @property def boundary_locations(self) -> list: """ - example file line: + Example file line: Boundary Location= , , , , ,Perimeter 1 , ,PugetSound_Ocean_Boundary , """ boundary_dict = [] - matches = search_contents(self.file_lines, "Boundary Location", expect_one=False) + matches = search_contents(self.file_lines, "Boundary Location", expect_one=False, require_one=False) for line in matches: parts = line.split(",") if len(parts) >= 7: From fb3b28dd338b7b887a6ceab09ed9d1a7b6cc26d0 Mon Sep 17 00:00:00 2001 From: Stevenray Janke Date: Tue, 4 Feb 2025 10:03:16 -0500 Subject: [PATCH 25/71] Minor linting fixes --- hecstac/ras/errors.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/hecstac/ras/errors.py b/hecstac/ras/errors.py index 6e28793..50ea99c 100644 --- a/hecstac/ras/errors.py +++ b/hecstac/ras/errors.py @@ -1,10 +1,10 @@ class GeometryAssetInvalidCRSError(Exception): - "Invalid crs provided to geometry asset" + """Invalid crs provided to geometry asset""" class GeometryAssetMissingCRSError(Exception): - "Required crs is missing from geometry asset definition" + """Required crs is missing from geometry asset definition""" class GeometryAssetNoXSError(Exception): - "1D geometry asset has no cross sections; cross sections are required to calculate the goemetry of the asset" + """1D geometry asset has no cross sections; cross sections are required to calculate the goemetry of the asset""" From 23097339ba7b12540ffed77105f4b4a7f8912517 Mon Sep 17 00:00:00 2001 From: Stevenray Janke Date: Tue, 4 Feb 2025 10:03:49 -0500 Subject: [PATCH 26/71] Minor linting changes --- hecstac/hms/item.py | 3 ++- hecstac/hms/s3_utils.py | 12 ++++++++---- hecstac/ras/assets.py | 2 -- 3 files changed, 10 insertions(+), 7 deletions(-) diff --git a/hecstac/hms/item.py b/hecstac/hms/item.py index 50a2021..b0b694e 100644 --- a/hecstac/hms/item.py +++ b/hecstac/hms/item.py @@ -168,7 +168,8 @@ def add_hms_asset(self, fpath: str) -> None: def make_thumbnail(self, gdfs: dict): """Create a png from the geodataframes (values of the dictionary). - The dictionary keys are used to label the layers in the legend.""" + The dictionary keys are used to label the layers in the legend. + """ cdict = { "Subbasin": "black", "Reach": "blue", diff --git a/hecstac/hms/s3_utils.py b/hecstac/hms/s3_utils.py index 023c98f..f481fb5 100644 --- a/hecstac/hms/s3_utils.py +++ b/hecstac/hms/s3_utils.py @@ -63,10 +63,12 @@ def split_s3_key(s3_path: str) -> tuple[str, str]: """ This function splits an S3 path into the bucket name and the key. - Parameters: + Parameters + ---------- s3_path (str): The S3 path to split. It should be in the format 's3://bucket/key'. - Returns: + Returns + ------- tuple: A tuple containing the bucket name and the key. If the S3 path does not contain a key, the second element of the tuple will be None. @@ -112,10 +114,12 @@ def get_basic_object_metadata(obj: ObjectSummary) -> dict: """ This function retrieves basic metadata of an AWS S3 object. - Parameters: + Parameters + ---------- obj (ObjectSummary): The AWS S3 object. - Returns: + Returns + ------- dict: A dictionary with the size, ETag, last modified date, storage platform, region, and storage tier of the object. """ diff --git a/hecstac/ras/assets.py b/hecstac/ras/assets.py index 5221364..bb204a4 100644 --- a/hecstac/ras/assets.py +++ b/hecstac/ras/assets.py @@ -462,7 +462,6 @@ def _plot_bc_lines(self, ax, bc_lines: gpd.GeoDataFrame) -> list[Line2D]: def _add_thumbnail_asset(self, filepath: str) -> None: """Add the thumbnail image as an asset with a relative href.""" - if filepath.startswith("s3://"): media_type = "image/png" else: @@ -497,7 +496,6 @@ def thumbnail( title : str, optional Title of the figure, by default "Model Thumbnail". """ - fig, ax = plt.subplots(figsize=(12, 12)) legend_handles = [] From f12de793a0efe6502101527af65934ead7f1cb55 Mon Sep 17 00:00:00 2001 From: Stevenray Janke Date: Tue, 4 Feb 2025 10:04:10 -0500 Subject: [PATCH 27/71] Formatting + linting fixes --- hecstac/ras/item.py | 16 +++++----------- 1 file changed, 5 insertions(+), 11 deletions(-) diff --git a/hecstac/ras/item.py b/hecstac/ras/item.py index b51ac66..9a2941e 100644 --- a/hecstac/ras/item.py +++ b/hecstac/ras/item.py @@ -65,6 +65,9 @@ def from_prj(cls, ras_project_file, item_id: str, crs: str = None, simplify_geom stac.crs = crs stac.pm = pm stac.pf = ProjectFile(ras_project_file) + stac.has_2d = False + stac.has_1d = False + stac._geom_files = [] ras_asset_files = stac.scan_model_dir(ras_project_file) @@ -81,14 +84,9 @@ def from_prj(cls, ras_project_file, item_id: str, crs: str = None, simplify_geom return stac - def _register_extensions(self) -> None: - ProjectionExtension.add_to(self) - StorageExtension.add_to(self) - @property def add_properties(self) -> None: """Properties for the RAS STAC item.""" - properties = {} # properties[self.RAS_HAS_1D] = self.has_1d properties[self.RAS_HAS_2D] = self.has_2d @@ -113,7 +111,7 @@ def add_geometry(self, simplify_geometry: bool) -> dict | None: if len(geometries) == 0: logging.error("No geometry found for RAS item.") - return + return NULL_STAC_GEOMETRY, NULL_STAC_BBOX unioned_geometry = union_all(geometries) if simplify_geometry: @@ -194,8 +192,7 @@ def add_ras_asset(self, fpath: str = "") -> None: asset.crs = self.crs if asset.check_2d: - if not getattr(self, "_geom_files", None): - self._geom_files = [] + self._geom_files = [] self._geom_files.append(asset) self.has_2d = True self.properties[self.RAS_HAS_2D] = True @@ -207,7 +204,6 @@ def add_ras_asset(self, fpath: str = "") -> None: def _geometry_to_wgs84(self, geom: Geometry) -> Geometry: """Convert geometry CRS to EPSG:4326 for stac item geometry.""" - pyproj_crs = CRS.from_user_input(self.crs) wgs_crs = CRS.from_authority("EPSG", "4326") if pyproj_crs != wgs_crs: @@ -217,7 +213,6 @@ def _geometry_to_wgs84(self, geom: Geometry) -> Geometry: def parse_1d_geom(self): """Read 1d geometry from concave hull.""" - logging.info("Creating geometry using 1d text file cross sections") concave_hull_polygons: list[Polygon] = [] for geom_asset in self._geom_files: @@ -234,7 +229,6 @@ def parse_1d_geom(self): def parse_2d_geom(self): """Read 2d geometry from hdf file mesh areas.""" - mesh_area_polygons: list[Polygon] = [] for geom_asset in self._geom_files: if isinstance(geom_asset, GeometryHdfAsset): From 84509eb4b45e974b43500d7920764d29586e0764 Mon Sep 17 00:00:00 2001 From: Stevenray Janke Date: Tue, 4 Feb 2025 10:16:10 -0500 Subject: [PATCH 28/71] Move root logger to module level --- hecstac/{common => events}/logger.py | 0 hecstac/hms/logger.py | 32 ++++++++++++++++++++++++++++ hecstac/ras/logger.py | 32 ++++++++++++++++++++++++++++ new_ffrd_event_item.py | 2 +- new_hms_item.py | 2 +- 5 files changed, 66 insertions(+), 2 deletions(-) rename hecstac/{common => events}/logger.py (100%) create mode 100644 hecstac/hms/logger.py create mode 100644 hecstac/ras/logger.py diff --git a/hecstac/common/logger.py b/hecstac/events/logger.py similarity index 100% rename from hecstac/common/logger.py rename to hecstac/events/logger.py diff --git a/hecstac/hms/logger.py b/hecstac/hms/logger.py new file mode 100644 index 0000000..b9bed7d --- /dev/null +++ b/hecstac/hms/logger.py @@ -0,0 +1,32 @@ +"""Logging utility and setup.""" + +import logging +import sys + +SUPPRESS_LOGS = ["boto3", "botocore", "geopandas", "fiona", "rasterio", "pyogrio", "xarray", "shapely", "matplotlib"] + + +def initialize_logger(json_logging: bool = False, level: int = logging.INFO): + datefmt = "%Y-%m-%dT%H:%M:%SZ" + if json_logging: + for module in SUPPRESS_LOGS: + logging.getLogger(module).setLevel(logging.WARNING) + + class FlushStreamHandler(logging.StreamHandler): + def emit(self, record): + super().emit(record) + self.flush() + + handler = FlushStreamHandler(sys.stdout) + + logging.basicConfig( + level=level, + handlers=[handler], + format="""{"time": "%(asctime)s" , "level": "%(levelname)s", "msg": "%(message)s"}""", + datefmt=datefmt, + ) + else: + for package in SUPPRESS_LOGS: + logging.getLogger(package).setLevel(logging.ERROR) + logging.basicConfig(level=level, format="%(asctime)s | %(levelname)s | %(message)s", datefmt=datefmt) + # boto3.set_stream_logger(name="botocore.credentials", level=logging.ERROR) diff --git a/hecstac/ras/logger.py b/hecstac/ras/logger.py new file mode 100644 index 0000000..b9bed7d --- /dev/null +++ b/hecstac/ras/logger.py @@ -0,0 +1,32 @@ +"""Logging utility and setup.""" + +import logging +import sys + +SUPPRESS_LOGS = ["boto3", "botocore", "geopandas", "fiona", "rasterio", "pyogrio", "xarray", "shapely", "matplotlib"] + + +def initialize_logger(json_logging: bool = False, level: int = logging.INFO): + datefmt = "%Y-%m-%dT%H:%M:%SZ" + if json_logging: + for module in SUPPRESS_LOGS: + logging.getLogger(module).setLevel(logging.WARNING) + + class FlushStreamHandler(logging.StreamHandler): + def emit(self, record): + super().emit(record) + self.flush() + + handler = FlushStreamHandler(sys.stdout) + + logging.basicConfig( + level=level, + handlers=[handler], + format="""{"time": "%(asctime)s" , "level": "%(levelname)s", "msg": "%(message)s"}""", + datefmt=datefmt, + ) + else: + for package in SUPPRESS_LOGS: + logging.getLogger(package).setLevel(logging.ERROR) + logging.basicConfig(level=level, format="%(asctime)s | %(levelname)s | %(message)s", datefmt=datefmt) + # boto3.set_stream_logger(name="botocore.credentials", level=logging.ERROR) diff --git a/new_ffrd_event_item.py b/new_ffrd_event_item.py index 8a01258..fcdceca 100644 --- a/new_ffrd_event_item.py +++ b/new_ffrd_event_item.py @@ -1,6 +1,6 @@ from pystac import Item -from hecstac.common.logger import initialize_logger +from hecstac.events.logger import initialize_logger from hecstac.events.ffrd import EventItem if __name__ == "__main__": diff --git a/new_hms_item.py b/new_hms_item.py index 6be539f..a50f260 100644 --- a/new_hms_item.py +++ b/new_hms_item.py @@ -1,6 +1,6 @@ from pathlib import Path -from hecstac.common.logger import initialize_logger +from hecstac.hms.logger import initialize_logger from hecstac.hms.item import HMSModelItem From aa7e9d2756c4f2b746ee7ea9feb9cfa1dce7dab3 Mon Sep 17 00:00:00 2001 From: Stevenray Janke Date: Tue, 4 Feb 2025 14:17:54 -0500 Subject: [PATCH 29/71] Fix issue with item.clone() where description was set and in the kwargs when initializing GenericAsset class. --- hecstac/ras/assets.py | 168 ++++++++++++++++++++++-------------------- 1 file changed, 90 insertions(+), 78 deletions(-) diff --git a/hecstac/ras/assets.py b/hecstac/ras/assets.py index bb204a4..f8a8bea 100644 --- a/hecstac/ras/assets.py +++ b/hecstac/ras/assets.py @@ -112,8 +112,8 @@ class ProjectAsset(GenericAsset): regex_parse_str = r".+\.prj$" def __init__(self, href: str, *args, **kwargs): - roles = ["project-file", "ras-file"] - description = kwargs.get("description", "The HEC-RAS project file.") + roles = kwargs.pop("roles", []) + ["project-file", "ras-file"] + description = kwargs.pop("description", "The HEC-RAS project file.") super().__init__(href, roles=roles, description=description, *args, **kwargs) @@ -139,8 +139,8 @@ class PlanAsset(GenericAsset): regex_parse_str = r".+\.p\d{2}$" def __init__(self, href: str, **kwargs): - roles = kwargs.get("roles", []) + ["plan-file", "ras-file"] - description = kwargs.get( + roles = kwargs.pop("roles", []) + ["plan-file", "ras-file"] + description = kwargs.pop( "description", "The plan file which contains a list of associated input files and all simulation options.", ) @@ -171,8 +171,8 @@ class GeometryAsset(GenericAsset): def __init__(self, href: str, crs: str = None, **kwargs): # self.pyproj_crs = self.validate_crs(crs) self.crs = crs - roles = kwargs.get("roles", []) + ["geometry-file", "ras-file"] - description = kwargs.get( + roles = kwargs.pop("roles", []) + ["geometry-file", "ras-file"] + description = kwargs.pop( "description", "The geometry file which contains cross-sectional, 2D, hydraulic structures, and other geometric data", ) @@ -207,8 +207,8 @@ class SteadyFlowAsset(GenericAsset): regex_parse_str = r".+\.f\d{2}$" def __init__(self, href: str, **kwargs): - roles = kwargs.get("roles", []) + ["steady-flow-file", "ras-file"] - description = kwargs.get( + roles = kwargs.pop("roles", []) + ["steady-flow-file", "ras-file"] + description = kwargs.pop( "description", "Steady Flow file which contains profile information, flow data, and boundary conditions.", ) @@ -235,8 +235,8 @@ class QuasiUnsteadyFlowAsset(GenericAsset): regex_parse_str = r".+\.q\d{2}$" def __init__(self, href: str, **kwargs): - roles = kwargs.get("roles", []) + ["quasi-unsteady-flow-file", "ras-file"] - description = kwargs.get("description", "Quasi-Unsteady Flow file.") + roles = kwargs.pop("roles", []) + ["quasi-unsteady-flow-file", "ras-file"] + description = kwargs.pop("description", "Quasi-Unsteady Flow file.") super().__init__(href, roles=roles, description=description, **kwargs) @@ -257,8 +257,8 @@ class UnsteadyFlowAsset(GenericAsset): regex_parse_str = r".+\.u\d{2}$" def __init__(self, href: str, **kwargs): - roles = kwargs.get("roles", []) + ["unsteady-flow-file", "ras-file"] - description = kwargs.get( + roles = kwargs.pop("roles", []) + ["unsteady-flow-file", "ras-file"] + description = kwargs.pop( "description", "The unsteady file contains hydrographs, initial conditions, and any flow options.", ) @@ -284,8 +284,8 @@ class PlanHdfAsset(GenericAsset): regex_parse_str = r".+\.p\d{2}\.hdf$" def __init__(self, href: str, **kwargs): - roles = kwargs.get("roles", []) + ["ras-file"] - description = kwargs.get("description", "The HEC-RAS plan HDF file.") + roles = kwargs.pop("roles", []) + ["ras-file"] + description = kwargs.pop("description", "The HEC-RAS plan HDF file.") super().__init__(href, roles=roles, description=description, **kwargs) @@ -344,8 +344,8 @@ class GeometryHdfAsset(GenericAsset): regex_parse_str = r".+\.g\d{2}\.hdf$" def __init__(self, href: str, crs: str = None, **kwargs): - roles = kwargs.get("roles", []) + ["geometry-hdf-file"] - description = kwargs.get("description", "The HEC-RAS geometry HDF file.") + roles = kwargs.pop("roles", []) + ["geometry-hdf-file"] + description = kwargs.pop("description", "The HEC-RAS geometry HDF file.") super().__init__(href, roles=roles, description=description, **kwargs) self.hdf_object = GeometryHDFFile(self.href) @@ -552,8 +552,11 @@ class RunFileAsset(GenericAsset): regex_parse_str = r".+\.r\d{2}$" def __init__(self, href: str, *args, **kwargs): - roles = ["run-file", "ras-file", MediaType.TEXT] - description = "Run file for steady flow analysis which contains all the necessary input data required for the RAS computational engine." + roles = kwargs.pop("roles", ["run-file", "ras-file", MediaType.TEXT]) + description = kwargs.pop( + "description", + "Run file for steady flow analysis which contains all the necessary input data required for the RAS computational engine.", + ) super().__init__(href, roles=roles, description=description, *args, **kwargs) @@ -563,8 +566,8 @@ class ComputationalLevelOutputAsset(GenericAsset): regex_parse_str = r".+\.hyd\d{2}$" def __init__(self, href: str, *args, **kwargs): - roles = ["computational-level-output-file", "ras-file", MediaType.TEXT] - description = "Detailed Computational Level output file." + roles = kwargs.pop("roles", ["computational-level-output-file", "ras-file", MediaType.TEXT]) + description = kwargs.pop("description", "Detailed Computational Level output file.") super().__init__(href, roles=roles, description=description, *args, **kwargs) @@ -574,8 +577,11 @@ class GeometricPreprocessorAsset(GenericAsset): regex_parse_str = r".+\.c\d{2}$" def __init__(self, href: str, *args, **kwargs): - roles = ["geometric-preprocessor", "ras-file", MediaType.TEXT] - description = "Geometric Pre-Processor output file containing hydraulic properties, rating curves, and more." + roles = kwargs.pop("roles", ["geometric-preprocessor", "ras-file", MediaType.TEXT]) + description = kwargs.pop( + "description", + "Geometric Pre-Processor output file containing hydraulic properties, rating curves, and more.", + ) super().__init__(href, roles=roles, description=description, *args, **kwargs) @@ -585,8 +591,8 @@ class BoundaryConditionAsset(GenericAsset): regex_parse_str = r".+\.b\d{2}$" def __init__(self, href: str, *args, **kwargs): - roles = ["boundary-condition-file", "ras-file", MediaType.TEXT] - description = "Boundary Condition file." + roles = kwargs.pop("roles", ["boundary-condition-file", "ras-file", MediaType.TEXT]) + description = kwargs.pop("description", "Boundary Condition file.") super().__init__(href, roles=roles, description=description, *args, **kwargs) @@ -596,8 +602,8 @@ class UnsteadyFlowLogAsset(GenericAsset): regex_parse_str = r".+\.bco\d{2}$" def __init__(self, href: str, *args, **kwargs): - roles = ["unsteady-flow-log-file", "ras-file", MediaType.TEXT] - description = "Unsteady Flow Log output file." + roles = kwargs.pop("roles", ["unsteady-flow-log-file", "ras-file", MediaType.TEXT]) + description = kwargs.pop("description", "Unsteady Flow Log output file.") super().__init__(href, roles=roles, description=description, *args, **kwargs) @@ -607,8 +613,10 @@ class SedimentDataAsset(GenericAsset): regex_parse_str = r".+\.s\d{2}$" def __init__(self, href: str, *args, **kwargs): - roles = ["sediment-data-file", "ras-file", MediaType.TEXT] - description = "Sediment data file containing flow data, boundary conditions, and sediment data." + roles = kwargs.pop("roles", ["sediment-data-file", "ras-file", MediaType.TEXT]) + description = kwargs.pop( + "description", "Sediment data file containing flow data, boundary conditions, and sediment data." + ) super().__init__(href, roles=roles, description=description, *args, **kwargs) @@ -618,8 +626,8 @@ class HydraulicDesignAsset(GenericAsset): regex_parse_str = r".+\.h\d{2}$" def __init__(self, href: str, *args, **kwargs): - roles = ["hydraulic-design-file", "ras-file", MediaType.TEXT] - description = "Hydraulic Design data file." + roles = kwargs.pop("roles", ["hydraulic-design-file", "ras-file", MediaType.TEXT]) + description = kwargs.pop("description", "Hydraulic Design data file.") super().__init__(href, roles=roles, description=description, *args, **kwargs) @@ -629,8 +637,10 @@ class WaterQualityAsset(GenericAsset): regex_parse_str = r".+\.w\d{2}$" def __init__(self, href: str, *args, **kwargs): - roles = ["water-quality-file", "ras-file", MediaType.TEXT] - description = "Water Quality file containing temperature boundary conditions and meteorological data." + roles = kwargs.pop("roles", ["water-quality-file", "ras-file", MediaType.TEXT]) + description = kwargs.pop( + "description", "Water Quality file containing temperature boundary conditions and meteorological data." + ) super().__init__(href, roles=roles, description=description, *args, **kwargs) @@ -640,8 +650,8 @@ class SedimentTransportCapacityAsset(GenericAsset): regex_parse_str = r".+\.SedCap\d{2}$" def __init__(self, href: str, *args, **kwargs): - roles = ["sediment-transport-capacity-file", "ras-file", MediaType.TEXT] - description = "Sediment Transport Capacity data." + roles = kwargs.pop("roles", ["sediment-transport-capacity-file", "ras-file", MediaType.TEXT]) + description = kwargs.pop("description", "Sediment Transport Capacity data.") super().__init__(href, roles=roles, description=description, *args, **kwargs) @@ -651,8 +661,8 @@ class XSOutputAsset(GenericAsset): regex_parse_str = r".+\.SedXS\d{2}$" def __init__(self, href: str, *args, **kwargs): - roles = ["xs-output-file", "ras-file", MediaType.TEXT] - description = "Cross section output file." + roles = kwargs.pop("roles", ["xs-output-file", "ras-file", MediaType.TEXT]) + description = kwargs.pop("description", "Cross section output file.") super().__init__(href, roles=roles, description=description, *args, **kwargs) @@ -662,8 +672,8 @@ class XSOutputHeaderAsset(GenericAsset): regex_parse_str = r".+\.SedHeadXS\d{2}$" def __init__(self, href: str, *args, **kwargs): - roles = ["xs-output-header-file", "ras-file", MediaType.TEXT] - description = "Header file for the cross section output." + roles = kwargs.pop("roles", ["xs-output-header-file", "ras-file", MediaType.TEXT]) + description = kwargs.pop("description", "Header file for the cross section output.") super().__init__(href, roles=roles, description=description, *args, **kwargs) @@ -673,8 +683,8 @@ class WaterQualityRestartAsset(GenericAsset): regex_parse_str = r".+\.wqrst\d{2}$" def __init__(self, href: str, *args, **kwargs): - roles = ["water-quality-restart-file", "ras-file", MediaType.TEXT] - description = "The water quality restart file." + roles = kwargs.pop("roles", ["water-quality-restart-file", "ras-file", MediaType.TEXT]) + description = kwargs.pop("description", "The water quality restart file.") super().__init__(href, roles=roles, description=description, *args, **kwargs) @@ -684,8 +694,8 @@ class SedimentOutputAsset(GenericAsset): regex_parse_str = r".+\.sed$" def __init__(self, href: str, *args, **kwargs): - roles = ["sediment-output-file", "ras-file", MediaType.TEXT] - description = "Detailed sediment output file." + roles = kwargs.pop("roles", ["sediment-output-file", "ras-file", MediaType.TEXT]) + description = kwargs.pop("description", "Detailed sediment output file.") super().__init__(href, roles=roles, description=description, *args, **kwargs) @@ -695,8 +705,8 @@ class BinaryLogAsset(GenericAsset): regex_parse_str = r".+\.blf$" def __init__(self, href: str, *args, **kwargs): - roles = ["binary-log-file", "ras-file", MediaType.TEXT] - description = "Binary Log file." + roles = kwargs.pop("roles", ["binary-log-file", "ras-file", MediaType.TEXT]) + description = kwargs.pop("description", "Binary Log file.") super().__init__(href, roles=roles, description=description, *args, **kwargs) @@ -706,8 +716,8 @@ class DSSAsset(GenericAsset): regex_parse_str = r".+\.dss$" def __init__(self, href: str, *args, **kwargs): - roles = ["ras-dss", "ras-file", MediaType.TEXT] - description = "The DSS file contains results and other simulation information." + roles = kwargs.pop("roles", ["ras-dss", "ras-file", MediaType.TEXT]) + description = kwargs.pop("description", "The DSS file contains results and other simulation information.") super().__init__(href, roles=roles, description=description, *args, **kwargs) @@ -717,8 +727,8 @@ class LogAsset(GenericAsset): regex_parse_str = r".+\.log$" def __init__(self, href: str, *args, **kwargs): - roles = ["ras-log", "ras-file", MediaType.TEXT] - description = "The log file contains information related to simulation processes." + roles = kwargs.pop("roles", ["ras-log", "ras-file", MediaType.TEXT]) + description = kwargs.pop("description", "The log file contains information related to simulation processes.") super().__init__(href, roles=roles, description=description, *args, **kwargs) @@ -728,8 +738,8 @@ class RestartAsset(GenericAsset): regex_parse_str = r".+\.rst$" def __init__(self, href: str, *args, **kwargs): - roles = ["restart-file", "ras-file", MediaType.TEXT] - description = "Restart file for resuming simulation runs." + roles = kwargs.pop("roles", ["restart-file", "ras-file", MediaType.TEXT]) + description = kwargs.pop("description", "Restart file for resuming simulation runs.") super().__init__(href, roles=roles, description=description, *args, **kwargs) @@ -739,8 +749,8 @@ class SiamInputAsset(GenericAsset): regex_parse_str = r".+\.SiamInput$" def __init__(self, href: str, *args, **kwargs): - roles = ["siam-input-file", "ras-file", MediaType.TEXT] - description = "SIAM Input Data file." + roles = kwargs.pop("roles", ["siam-input-file", "ras-file", MediaType.TEXT]) + description = kwargs.pop("description", "SIAM Input Data file.") super().__init__(href, roles=roles, description=description, *args, **kwargs) @@ -750,8 +760,8 @@ class SiamOutputAsset(GenericAsset): regex_parse_str = r".+\.SiamOutput$" def __init__(self, href: str, *args, **kwargs): - roles = ["siam-output-file", "ras-file", MediaType.TEXT] - description = "SIAM Output Data file." + roles = kwargs.pop("roles", ["siam-output-file", "ras-file", MediaType.TEXT]) + description = kwargs.pop("description", "SIAM Output Data file.") super().__init__(href, roles=roles, description=description, *args, **kwargs) @@ -761,8 +771,8 @@ class WaterQualityLogAsset(GenericAsset): regex_parse_str = r".+\.bco$" def __init__(self, href: str, *args, **kwargs): - roles = ["water-quality-log", "ras-file", MediaType.TEXT] - description = "Water quality log file." + roles = kwargs.pop("roles", ["water-quality-log", "ras-file", MediaType.TEXT]) + description = kwargs.pop("description", "Water quality log file.") super().__init__(href, roles=roles, description=description, *args, **kwargs) @@ -772,8 +782,8 @@ class ColorScalesAsset(GenericAsset): regex_parse_str = r".+\.color-scales$" def __init__(self, href: str, *args, **kwargs): - roles = ["color-scales", "ras-file", MediaType.TEXT] - description = "File that contains the water quality color scale." + roles = kwargs.pop("roles", ["color-scales", "ras-file", MediaType.TEXT]) + description = kwargs.pop("description", "File that contains the water quality color scale.") super().__init__(href, roles=roles, description=description, *args, **kwargs) @@ -783,8 +793,10 @@ class ComputationalMessageAsset(GenericAsset): regex_parse_str = r".+\.comp-msgs.txt$" def __init__(self, href: str, *args, **kwargs): - roles = ["computational-message-file", "ras-file", MediaType.TEXT] - description = "Computational Message text file which contains messages from the computation process." + roles = kwargs.pop("roles", ["computational-message-file", "ras-file", MediaType.TEXT]) + description = kwargs.pop( + "description", "Computational Message text file which contains messages from the computation process." + ) super().__init__(href, roles=roles, description=description, *args, **kwargs) @@ -794,8 +806,8 @@ class UnsteadyRunFileAsset(GenericAsset): regex_parse_str = r".+\.x\d{2}$" def __init__(self, href: str, *args, **kwargs): - roles = ["run-file", "ras-file", MediaType.TEXT] - description = "Run file for Unsteady Flow simulations." + roles = kwargs.pop("roles", ["run-file", "ras-file", MediaType.TEXT]) + description = kwargs.pop("description", "Run file for Unsteady Flow simulations.") super().__init__(href, roles=roles, description=description, *args, **kwargs) @@ -805,8 +817,8 @@ class OutputFileAsset(GenericAsset): regex_parse_str = r".+\.o\d{2}$" def __init__(self, href: str, *args, **kwargs): - roles = ["output-file", "ras-file", MediaType.TEXT] - description = "Output RAS file which contains all computed results." + roles = kwargs.pop("roles", ["output-file", "ras-file", MediaType.TEXT]) + description = kwargs.pop("description", "Output RAS file which contains all computed results.") super().__init__(href, roles=roles, description=description, *args, **kwargs) @@ -816,8 +828,8 @@ class InitialConditionsFileAsset(GenericAsset): regex_parse_str = r".+\.IC\.O\d{2}$" def __init__(self, href: str, *args, **kwargs): - roles = ["initial-conditions-file", "ras-file", MediaType.TEXT] - description = "Initial conditions file for unsteady flow plan." + roles = kwargs.pop("roles", ["initial-conditions-file", "ras-file", MediaType.TEXT]) + description = kwargs.pop("description", "Initial conditions file for unsteady flow plan.") super().__init__(href, roles=roles, description=description, *args, **kwargs) @@ -827,8 +839,8 @@ class PlanRestartFileAsset(GenericAsset): regex_parse_str = r".+\.p\d{2}\.rst$" def __init__(self, href: str, *args, **kwargs): - roles = ["restart-file", "ras-file", MediaType.TEXT] - description = "Restart file for unsteady flow plan." + roles = kwargs.pop("roles", ["restart-file", "ras-file", MediaType.TEXT]) + description = kwargs.pop("description", "Restart file for unsteady flow plan.") super().__init__(href, roles=roles, description=description, *args, **kwargs) @@ -838,8 +850,8 @@ class RasMapperFileAsset(GenericAsset): regex_parse_str = r".+\.rasmap$" def __init__(self, href: str, *args, **kwargs): - roles = ["ras-mapper-file", "ras-file", MediaType.TEXT] - description = "RAS Mapper file." + roles = kwargs.pop("roles", ["ras-mapper-file", "ras-file", MediaType.TEXT]) + description = kwargs.pop("description", "RAS Mapper file.") super().__init__(href, roles=roles, description=description, *args, **kwargs) @@ -849,8 +861,8 @@ class RasMapperBackupFileAsset(GenericAsset): regex_parse_str = r".+\.rasmap\.backup$" def __init__(self, href: str, *args, **kwargs): - roles = ["ras-mapper-file", "ras-file", MediaType.TEXT] - description = "Backup RAS Mapper file." + roles = kwargs.pop("roles", ["ras-mapper-file", "ras-file", MediaType.TEXT]) + description = kwargs.pop("description", "Backup RAS Mapper file.") super().__init__(href, roles=roles, description=description, *args, **kwargs) @@ -860,8 +872,8 @@ class RasMapperOriginalFileAsset(GenericAsset): regex_parse_str = r".+\.rasmap\.original$" def __init__(self, href: str, *args, **kwargs): - roles = ["ras-mapper-file", "ras-file", MediaType.TEXT] - description = "Original RAS Mapper file." + roles = kwargs.pop("roles", ["ras-mapper-file", "ras-file", MediaType.TEXT]) + description = kwargs.pop("description", "Original RAS Mapper file.") super().__init__(href, roles=roles, description=description, *args, **kwargs) @@ -871,8 +883,8 @@ class MiscTextFileAsset(GenericAsset): regex_parse_str = r".+\.txt$" def __init__(self, href: str, *args, **kwargs): - roles = [MediaType.TEXT] - description = "Miscellaneous text file." + roles = kwargs.pop("roles", [MediaType.TEXT]) + description = kwargs.pop("description", "Miscellaneous text file.") super().__init__(href, roles=roles, description=description, *args, **kwargs) @@ -882,8 +894,8 @@ class MiscXMLFileAsset(GenericAsset): regex_parse_str = r".+\.xml$" def __init__(self, href: str, *args, **kwargs): - roles = [MediaType.XML] - description = "Miscellaneous XML file." + roles = kwargs.pop("roles", [MediaType.XML]) + description = kwargs.pop("description", "Miscellaneous XML file.") super().__init__(href, roles=roles, description=description, *args, **kwargs) From f4be62b306ebc5d6262bf9741b2bc27f3f747efc Mon Sep 17 00:00:00 2001 From: Stevenray Janke Date: Tue, 4 Feb 2025 14:26:22 -0500 Subject: [PATCH 30/71] Organize --- hecstac/ras/item.py | 17 +++++++++-------- 1 file changed, 9 insertions(+), 8 deletions(-) diff --git a/hecstac/ras/item.py b/hecstac/ras/item.py index 9a2941e..8118585 100644 --- a/hecstac/ras/item.py +++ b/hecstac/ras/item.py @@ -47,10 +47,19 @@ class RASModelItem(Item): @classmethod def from_prj(cls, ras_project_file, item_id: str, crs: str = None, simplify_geometry: bool = True): """Create an item from a RAS .prj file.""" + stac.pf = ProjectFile(ras_project_file) + stac.has_2d = False + stac.has_1d = False + stac._geom_files = [] + stac.crs = crs + stac.factory = AssetFactory(RAS_EXTENSION_MAPPING) + properties = {"ras_project_file": ras_project_file} pm = LocalPathManager(Path(ras_project_file).parent) + stac.pm = pm href = pm.item_path(item_id) + stac._href = href stac = cls( Path(ras_project_file).stem, @@ -60,14 +69,6 @@ def from_prj(cls, ras_project_file, item_id: str, crs: str = None, simplify_geom properties, href=href, ) - stac._href = href - stac.factory = AssetFactory(RAS_EXTENSION_MAPPING) - stac.crs = crs - stac.pm = pm - stac.pf = ProjectFile(ras_project_file) - stac.has_2d = False - stac.has_1d = False - stac._geom_files = [] ras_asset_files = stac.scan_model_dir(ras_project_file) From cf908631ff67d30d50f54c348af94633edc1811e Mon Sep 17 00:00:00 2001 From: Stevenray Janke Date: Tue, 4 Feb 2025 16:53:08 -0500 Subject: [PATCH 31/71] Add doc strings/fix linting errors for ras --- hecstac/common/__init__.py | 1 + hecstac/common/asset_factory.py | 12 ++- hecstac/common/path_manager.py | 12 ++- hecstac/common/schemas.py | 2 + hecstac/ras/__init__.py | 1 + hecstac/ras/assets.py | 21 +++--- hecstac/ras/consts.py | 2 + hecstac/ras/errors.py | 9 ++- hecstac/ras/item.py | 11 +-- hecstac/ras/logger.py | 1 + hecstac/ras/parser.py | 128 +++++++++++++++++++++++++++----- hecstac/ras/utils.py | 103 +------------------------ new_ffrd_event_item.py | 2 + new_hms_item.py | 6 +- new_ras_item.py | 13 ++-- 15 files changed, 164 insertions(+), 160 deletions(-) diff --git a/hecstac/common/__init__.py b/hecstac/common/__init__.py index e69de29..0f9954d 100644 --- a/hecstac/common/__init__.py +++ b/hecstac/common/__init__.py @@ -0,0 +1 @@ +"""Common scripts.""" diff --git a/hecstac/common/asset_factory.py b/hecstac/common/asset_factory.py index 0aaad59..d98b335 100644 --- a/hecstac/common/asset_factory.py +++ b/hecstac/common/asset_factory.py @@ -1,3 +1,5 @@ +"""Create instances of assets.""" + import logging from pathlib import Path from typing import Dict, Type @@ -8,7 +10,7 @@ class GenericAsset(Asset): - """Generic Asset.""" + """Provides a base structure for assets.""" def __init__(self, href: str, roles=None, description=None, *args, **kwargs): super().__init__(href, *args, **kwargs) @@ -23,9 +25,11 @@ def name_from_suffix(self, suffix: str) -> str: return f"{self.stem}.{suffix}" def __repr__(self): + """Return string representation of the GenericAsset instance.""" return f"<{self.__class__.__name__} name={self.name}>" def __str__(self): + """Return string representation of assets name.""" return f"{self.name}" @@ -33,14 +37,13 @@ class AssetFactory: """Factory for creating HEC asset instances based on file extensions.""" def __init__(self, extension_to_asset: Dict[str, Type[GenericAsset]]): - """ - Initialize the AssetFactory with a mapping of file extensions to asset types and metadata. - """ + """Initialize the AssetFactory with a mapping of file extensions to asset types and metadata.""" self.extension_to_asset = extension_to_asset def create_hms_asset(self, fpath: str, item_type: str = "model") -> Asset: """ Create an asset instance based on the file extension. + item_type: str The type of item to create. This is used to determine the asset class. @@ -60,6 +63,7 @@ def create_hms_asset(self, fpath: str, item_type: str = "model") -> Asset: return check_storage_extension(asset) def create_ras_asset(self, fpath: str): + """Create an asset instance based on the file extension.""" logging.debug(f"Creating asset for {fpath}") for pattern, asset_class in self.extension_to_asset.items(): if pattern.match(fpath): diff --git a/hecstac/common/path_manager.py b/hecstac/common/path_manager.py index fb49792..69666fc 100644 --- a/hecstac/common/path_manager.py +++ b/hecstac/common/path_manager.py @@ -1,29 +1,33 @@ +"""Path manager.""" + from pathlib import Path class LocalPathManager: - """ - Builds consistent paths for STAC items and collections assuming a top level local catalog - """ + """Builds consistent paths for STAC items and collections assuming a top level local catalog.""" def __init__(self, model_root_dir: str): self._model_root_dir = model_root_dir @property def model_root_dir(self) -> str: + """Model root directory.""" return str(self._model_root_dir) @property def model_parent_dir(self) -> str: + """Model parent directory.""" return str(Path(self._model_root_dir).parent) @property def item_dir(self) -> str: - """Duplicate of model_root, added for clarity in the calling code""" + """Duplicate of model_root, added for clarity in the calling code.""" return self.model_root_dir def item_path(self, item_id: str) -> str: + """Item path.""" return f"{self._model_root_dir}/{item_id}.json" def derived_item_asset(self, filename: str) -> str: + """Derive item asset path.""" return f"{self._model_root_dir}/{filename}" diff --git a/hecstac/common/schemas.py b/hecstac/common/schemas.py index a2c89bf..279afa7 100644 --- a/hecstac/common/schemas.py +++ b/hecstac/common/schemas.py @@ -1,3 +1,5 @@ +"""Schema parsing.""" + # TODO: update these, add imports, etc. # def extract_schema_definition(definition_name: str) -> dict[str, Any]: # """Extract asset specific schema from ras extension schema""" diff --git a/hecstac/ras/__init__.py b/hecstac/ras/__init__.py index e69de29..02ba329 100644 --- a/hecstac/ras/__init__.py +++ b/hecstac/ras/__init__.py @@ -0,0 +1 @@ +"""HEC-RAS STAC Item creation module.""" diff --git a/hecstac/ras/assets.py b/hecstac/ras/assets.py index f8a8bea..a41a8de 100644 --- a/hecstac/ras/assets.py +++ b/hecstac/ras/assets.py @@ -1,3 +1,5 @@ +"""Asset instances of HEC-RAS model files.""" + import logging import os import re @@ -388,9 +390,7 @@ def check_2d(self): return False def _plot_mesh_areas(self, ax, mesh_polygons: gpd.GeoDataFrame) -> list[Line2D]: - """ - Plots mesh areas on the given axes. - """ + """Plot mesh areas on the given axes.""" mesh_polygons.plot( ax=ax, edgecolor="silver", @@ -412,9 +412,7 @@ def _plot_mesh_areas(self, ax, mesh_polygons: gpd.GeoDataFrame) -> list[Line2D]: return legend_handle def _plot_breaklines(self, ax, breaklines: gpd.GeoDataFrame) -> list[Line2D]: - """ - Plots breaklines on the given axes. - """ + """Plot breaklines on the given axes.""" breaklines.plot(ax=ax, edgecolor="red", linestyle="-", alpha=0.3, label="Breaklines") legend_handle = [ Line2D( @@ -430,9 +428,7 @@ def _plot_breaklines(self, ax, breaklines: gpd.GeoDataFrame) -> list[Line2D]: return legend_handle def _plot_bc_lines(self, ax, bc_lines: gpd.GeoDataFrame) -> list[Line2D]: - """ - Plots boundary condition lines on the given axes. - """ + """Plot boundary condition lines on the given axes.""" legend_handles = [ Line2D([0], [0], color="none", linestyle="None", label="BC Lines"), ] @@ -484,9 +480,8 @@ def thumbnail( title: str = "Model_Thumbnail", thumbnail_dest: str = None, ): - """Create a thumbnail figure for a geometry hdf file, including - various geospatial layers such as USGS gages, mesh areas, - breaklines, and boundary condition (BC) lines. + """ + Create a thumbnail figure for a geometry hdf file, includingvarious geospatial layers such as USGS gages, mesh areas, breaklines, and boundary condition (BC) lines. Parameters ---------- @@ -495,6 +490,8 @@ def thumbnail( Options include "usgs_gages", "mesh_areas", "breaklines", and "bc_lines". title : str, optional Title of the figure, by default "Model Thumbnail". + thumbnail_dest : str, optional + Directory for created thumbnails. If None then thumbnails will be exported to same level as the item. """ fig, ax = plt.subplots(figsize=(12, 12)) legend_handles = [] diff --git a/hecstac/ras/consts.py b/hecstac/ras/consts.py index 7798ec3..ea5a05e 100644 --- a/hecstac/ras/consts.py +++ b/hecstac/ras/consts.py @@ -1,3 +1,5 @@ +"""Constants.""" + import datetime from shapely import Polygon, to_geojson, box import json diff --git a/hecstac/ras/errors.py b/hecstac/ras/errors.py index 50ea99c..fc70548 100644 --- a/hecstac/ras/errors.py +++ b/hecstac/ras/errors.py @@ -1,10 +1,13 @@ +"""Errors for the ras module.""" + + class GeometryAssetInvalidCRSError(Exception): - """Invalid crs provided to geometry asset""" + """Invalid crs provided to geometry asset.""" class GeometryAssetMissingCRSError(Exception): - """Required crs is missing from geometry asset definition""" + """Required crs is missing from geometry asset definition.""" class GeometryAssetNoXSError(Exception): - """1D geometry asset has no cross sections; cross sections are required to calculate the goemetry of the asset""" + """1D geometry asset has no cross sections; cross sections are required to calculate the goemetry of the asset.""" diff --git a/hecstac/ras/item.py b/hecstac/ras/item.py index 8118585..6f359e0 100644 --- a/hecstac/ras/item.py +++ b/hecstac/ras/item.py @@ -1,3 +1,5 @@ +"""HEC-RAS STAC Item creation class.""" + import datetime import json import logging @@ -101,7 +103,7 @@ def add_properties(self) -> None: return properties def add_geometry(self, simplify_geometry: bool) -> dict | None: - """Parses geometries from 2d hdf files and updates the stac item geometry, simplifying them if needed.""" + """Parse geometries from 2d hdf files and updates the stac item geometry, simplifying them if needed.""" geometries = [] if self.has_2d: @@ -144,7 +146,7 @@ def _datetime(self) -> datetime: return item_datetime def add_model_thumbnails(self, layers: list, title_prefix: str = "Model_Thumbnail", thumbnail_dir=None): - """Generates model thumbnail asset for each geometry file. + """Generate model thumbnail asset for each geometry file. Parameters ---------- @@ -152,6 +154,8 @@ def add_model_thumbnails(self, layers: list, title_prefix: str = "Model_Thumbnai List of geometry layers to be included in the plot. Options include 'mesh_areas', 'breaklines', 'bc_lines' title_prefix : str, optional Thumbnail title prefix, by default "Model_Thumbnail". + thumbnail_dir : str, optional + Directory for created thumbnails. If None then thumbnails will be exported to same level as the item. """ if thumbnail_dir: thumbnail_dest = thumbnail_dir @@ -240,9 +244,6 @@ def parse_2d_geom(self): return union_all(mesh_area_polygons) - def ensure_projection_schema(self) -> None: - ProjectionExtension.ensure_has_extension(self, True) - def scan_model_dir(self, ras_project_file): """Find all files in the project folder.""" base_dir = os.path.dirname(ras_project_file) diff --git a/hecstac/ras/logger.py b/hecstac/ras/logger.py index b9bed7d..2e5a3d7 100644 --- a/hecstac/ras/logger.py +++ b/hecstac/ras/logger.py @@ -7,6 +7,7 @@ def initialize_logger(json_logging: bool = False, level: int = logging.INFO): + """Initialize the ras logger.""" datefmt = "%Y-%m-%dT%H:%M:%SZ" if json_logging: for module in SUPPRESS_LOGS: diff --git a/hecstac/ras/parser.py b/hecstac/ras/parser.py index 18a5f9a..d6e44b0 100644 --- a/hecstac/ras/parser.py +++ b/hecstac/ras/parser.py @@ -1,3 +1,5 @@ +"""Contains classes and methods to parse HEC-RAS files.""" + import datetime import logging import math @@ -31,6 +33,7 @@ def name_from_suffix(fpath: str, suffix: str) -> str: class River: + """HEC-RAS River.""" def __init__(self, river: str, reaches: list[str] = []): self.river = river @@ -193,6 +196,7 @@ def subdivisions(self) -> tuple[list[float], list[float]]: @property def is_interpolated(self) -> bool: + """Check if xs is interpolated.""" if self._is_interpolated == None: self._is_interpolated = "*" in self.split_xs_header(1) return self._is_interpolated @@ -284,6 +288,8 @@ def get_mannings_discharge(self, wse: float, slope: float, units: str) -> float: class StructureType(Enum): + """Structure types.""" + XS = 1 CULVERT = 2 BRIDGE = 3 @@ -293,7 +299,7 @@ class StructureType(Enum): class Structure: - """Structure.""" + """HEC-RAS Structures.""" def __init__( self, @@ -626,6 +632,7 @@ def gdf(self): class StorageArea: + """HEC-RAS StorageArea.""" def __init__(self, ras_data: list[str], crs: str): self.crs = crs @@ -634,6 +641,7 @@ def __init__(self, ras_data: list[str], crs: str): class Connection: + """HEC-RAS Connection.""" def __init__(self, ras_data: list[str], crs: str): self.crs = crs @@ -652,24 +660,29 @@ def __init__(self, fpath): @property def project_title(self) -> str: + """Return the project title.""" return search_contents(self.file_lines, "Proj Title") @property def project_description(self) -> str: + """Return the model description.""" return search_contents(self.file_lines, "Model Description", token=":", require_one=False) @property def project_status(self) -> str: + """Return the model status.""" return search_contents(self.file_lines, "Status of Model", token=":", require_one=False) @property def project_units(self) -> str | None: + """Return the project units.""" for line in self.file_lines: if "Units" in line: return " ".join(line.split(" ")[:-1]) @property def plan_current(self) -> str | None: + """Return the current plan.""" try: suffix = search_contents(self.file_lines, "Current Plan", expect_one=True, require_one=False) return self.name_from_suffix(suffix) @@ -679,6 +692,7 @@ def plan_current(self) -> str | None: @property def ras_version(self) -> str | None: + """Return the ras version.""" version = search_contents(self.file_lines, "Program Version", token="=", expect_one=False, require_one=False) if version == []: version = search_contents( @@ -692,26 +706,31 @@ def ras_version(self) -> str | None: @property def plan_files(self) -> list[str]: + """Return the plan files.""" suffixes = search_contents(self.file_lines, "Plan File", expect_one=False, require_one=False) return [name_from_suffix(self.fpath, i) for i in suffixes] @property def geometry_files(self) -> list[str]: + """Return the geometry files.""" suffixes = search_contents(self.file_lines, "Geom File", expect_one=False, require_one=False) return [name_from_suffix(self.fpath, i) for i in suffixes] @property def steady_flow_files(self) -> list[str]: + """Return the flow files.""" suffixes = search_contents(self.file_lines, "Flow File", expect_one=False, require_one=False) return [name_from_suffix(self.fpath, i) for i in suffixes] @property def quasi_unsteady_flow_files(self) -> list[str]: + """Return the quasisteady flow files.""" suffixes = search_contents(self.file_lines, "QuasiSteady File", expect_one=False, require_one=False) return [name_from_suffix(self.fpath, i) for i in suffixes] @property def unsteady_flow_files(self) -> list[str]: + """Return the unsteady flow files.""" suffixes = search_contents(self.file_lines, "Unsteady File", expect_one=False, require_one=False) return [name_from_suffix(self.fpath, i) for i in suffixes] @@ -727,29 +746,36 @@ def __init__(self, fpath): @property def plan_title(self) -> str: + """Return plan title.""" return search_contents(self.file_lines, "Plan Title") @property def plan_version(self) -> str: + """Return program version.""" return search_contents(self.file_lines, "Program Version") @property def geometry_file(self) -> str: + """Return geometry file.""" suffix = search_contents(self.file_lines, "Geom File", expect_one=True) return name_from_suffix(self.fpath, suffix) @property def flow_file(self) -> str: + """Return flow file.""" suffix = search_contents(self.file_lines, "Flow File", expect_one=True) return name_from_suffix(self.fpath, suffix) @property def short_identifier(self) -> str: + """Return short identifier.""" return search_contents(self.file_lines, "Short Identifier", expect_one=True) @property def breach_locations(self) -> dict: """ + Return breach locations. + Example file line: Breach Loc= , , ,True,HH_DamEmbankment """ @@ -776,10 +802,12 @@ def __init__(self, fpath, crs): @property def geom_title(self) -> str: + """Return geometry title.""" return search_contents(self.file_lines, "Geom Title") @property def geom_version(self) -> str: + """Return program version.""" return search_contents(self.file_lines, "Program Version") @property @@ -837,7 +865,7 @@ def connections(self) -> dict[str, "Connection"]: @property def datetimes(self) -> list[datetime.datetime]: - """Get the latest node last updated entry for this geometry""" + """Get the latest node last updated entry for this geometry.""" dts = search_contents(self.file_lines, "Node Last Edited Time", expect_one=False) if len(dts) >= 1: try: @@ -849,7 +877,7 @@ def datetimes(self) -> list[datetime.datetime]: @property def has_2d(self) -> bool: - """Check if RAS geometry has any 2D areas""" + """Check if RAS geometry has any 2D areas.""" for line in self.file_lines: if line.startswith("Storage Area Is2D=") and int(line[len("Storage Area Is2D=") :].strip()) in (1, -1): # RAS mostly uses "-1" to indicate True and "0" to indicate False. Checking for "1" also here. @@ -858,7 +886,7 @@ def has_2d(self) -> bool: @property def has_1d(self) -> bool: - """Check if RAS geometry has any 1D components""" + """Check if RAS geometry has any 1D components.""" return len(self.cross_sections) > 0 @property @@ -938,6 +966,7 @@ def get_subtype_gdf(self, subtype: str) -> gpd.GeoDataFrame: ) # TODO: may need to add some logic here for empty dicts def iter_labeled_gdfs(self) -> Iterator[tuple[str, gpd.GeoDataFrame]]: + """Return gdf and associated property.""" for property in self.PROPERTIES_WITH_GDF: gdf = self.get_subtype_gdf(property) yield property, gdf @@ -958,10 +987,12 @@ def __init__(self, fpath): @property def flow_title(self) -> str: + """Return flow title.""" return search_contents(self.file_lines, "Flow Title") @property def n_profiles(self) -> int: + """Return number of profiles.""" return int(search_contents(self.file_lines, "Number of Profiles")) @@ -975,11 +1006,14 @@ def __init__(self, fpath): @property def flow_title(self) -> str: + """Return flow title.""" return search_contents(self.file_lines, "Flow Title") @property def boundary_locations(self) -> list: """ + Return boundary locations. + Example file line: Boundary Location= , , , , ,Perimeter 1 , ,PugetSound_Ocean_Boundary , """ @@ -996,6 +1030,7 @@ def boundary_locations(self) -> list: @property def reference_lines(self): + """Return reference lines.""" return search_contents( self.file_lines, "Observed Rating Curve=Name=Ref Line", token=":", expect_one=False, require_one=False ) @@ -1014,7 +1049,7 @@ class QuasiUnsteadyFlowFile: class RASHDFFile: - """Base class for HDF assets (Plan and Geometry HDF files).""" + """Base class for parsing HDF assets (Plan and Geometry HDF files).""" def __init__(self, fpath, hdf_constructor): self.fpath = fpath @@ -1027,126 +1062,146 @@ def __init__(self, fpath, hdf_constructor): @property def file_version(self) -> str | None: + """Return File Version.""" if self._root_attrs == None: self._root_attrs = self.hdf_object.get_root_attrs() return self._root_attrs.get("File Version") @property def units_system(self) -> str | None: + """Return Units System.""" if self._root_attrs == None: self._root_attrs = self.hdf_object.get_root_attrs() return self._root_attrs.get("Units System") @property def geometry_time(self) -> datetime.datetime | None: + """Return Geometry Time.""" if self._geom_attrs == None: self._geom_attrs = self.hdf_object.get_geom_attrs() return self._geom_attrs.get("Geometry Time") @property def landcover_date_last_modified(self) -> datetime.datetime | None: + """Return Land Cover Date Last Modified.""" if self._geom_attrs == None: self._geom_attrs = self.hdf_object.get_geom_attrs() return self._geom_attrs.get("Land Cover Date Last Modified") @property def landcover_filename(self) -> str | None: + """Return Land Cover Filename.""" if self._geom_attrs == None: self._geom_attrs = self.hdf_object.get_geom_attrs() return self._geom_attrs.get("Land Cover Filename") @property def landcover_layername(self) -> str | None: + """Return Land Cover Layername.""" if self._geom_attrs == None: self._geom_attrs = self.hdf_object.get_geom_attrs() return self._geom_attrs.get("Land Cover Layername") @property def rasmapperlibdll_date(self) -> datetime.datetime | None: + """Return RasMapperLib.dll Date.""" if self._geom_attrs == None: self._geom_attrs = self.hdf_object.get_geom_attrs() return self._geom_attrs.get("RasMapperLib.dll Date").isoformat() @property def si_units(self) -> bool | None: + """Return SI Units.""" if self._geom_attrs == None: self._geom_attrs = self.hdf_object.get_geom_attrs() return self._geom_attrs.get("SI Units") @property def terrain_file_date(self) -> datetime.datetime | None: + """Return Terrain File Date.""" if self._geom_attrs == None: self._geom_attrs = self.hdf_object.get_geom_attrs() return self._geom_attrs.get("Terrain File Date").isoformat() @property def terrain_filename(self) -> str | None: + """Return Terrain Filename.""" if self._geom_attrs == None: self._geom_attrs = self.hdf_object.get_geom_attrs() return self._geom_attrs.get("Terrain Filename") @property def terrain_layername(self) -> str | None: + """Return Terrain Layername.""" if self._geom_attrs == None: self._geom_attrs = self.hdf_object.get_geom_attrs() return self._geom_attrs.get("Terrain Layername") @property def geometry_version(self) -> str | None: + """Return Version.""" if self._geom_attrs == None: self._geom_attrs = self.hdf_object.get_geom_attrs() return self._geom_attrs.get("Version") @property def bridges_culverts(self) -> int | None: + """Return Bridge/Culvert Count.""" if self._structures_attrs == None: self._structures_attrs = self.hdf_object.get_geom_structures_attrs() return self._structures_attrs.get("Bridge/Culvert Count") @property def connections(self) -> int | None: + """Return Connection Count.""" if self._structures_attrs == None: self._structures_attrs = self.hdf_object.get_geom_structures_attrs() return self._structures_attrs.get("Connection Count") @property def inline_structures(self) -> int | None: + """Return Inline Structure Count.""" if self._structures_attrs == None: self._structures_attrs = self.hdf_object.get_geom_structures_attrs() return self._structures_attrs.get("Inline Structure Count") @property def lateral_structures(self) -> int | None: + """Return Lateral Structure Count.""" if self._structures_attrs == None: self._structures_attrs = self.hdf_object.get_geom_structures_attrs() return self._structures_attrs.get("Lateral Structure Count") @property def two_d_flow_cell_average_size(self) -> float | None: + """Return Cell Average Size.""" if self._2d_flow_attrs == None: self._2d_flow_attrs = self.hdf_object.get_geom_2d_flow_area_attrs() return int(np.sqrt(self._2d_flow_attrs.get("Cell Average Size"))) @property def two_d_flow_cell_maximum_index(self) -> int | None: + """Return Cell Maximum Index.""" if self._2d_flow_attrs == None: self._2d_flow_attrs = self.hdf_object.get_geom_2d_flow_area_attrs() return self._2d_flow_attrs.get("Cell Maximum Index") @property def two_d_flow_cell_maximum_size(self) -> int | None: + """Return Cell Maximum Size.""" if self._2d_flow_attrs == None: self._2d_flow_attrs = self.hdf_object.get_geom_2d_flow_area_attrs() return int(np.sqrt(self._2d_flow_attrs.get("Cell Maximum Size"))) @property def two_d_flow_cell_minimum_size(self) -> int | None: + """Return Cell Minimum Size.""" if self._2d_flow_attrs == None: self._2d_flow_attrs = self.hdf_object.get_geom_2d_flow_area_attrs() return int(np.sqrt(self._2d_flow_attrs.get("Cell Minimum Size"))) def mesh_areas(self, crs: str = None, return_gdf: bool = False) -> gpd.GeoDataFrame | Polygon | MultiPolygon: - """Retrieves and processes mesh area geometries. + """Retrieve and process mesh area geometries. Parameters ---------- @@ -1173,6 +1228,7 @@ def mesh_areas(self, crs: str = None, return_gdf: bool = False) -> gpd.GeoDataFr @property def breaklines(self) -> gpd.GeoDataFrame | None: + """Return breaklines.""" breaklines = self.hdf_object.breaklines() if breaklines is None or breaklines.empty: @@ -1182,6 +1238,7 @@ def breaklines(self) -> gpd.GeoDataFrame | None: @property def mesh_cells(self) -> gpd.GeoDataFrame | None: + """Return mesh cell polygons.""" mesh_cells = self.hdf_object.mesh_cell_polygons() if mesh_cells is None or mesh_cells.empty: @@ -1191,6 +1248,7 @@ def mesh_cells(self) -> gpd.GeoDataFrame | None: @property def bc_lines(self) -> gpd.GeoDataFrame | None: + """Return boundary condition lines.""" bc_lines = self.hdf_object.bc_lines() if bc_lines is None or bc_lines.empty: @@ -1198,20 +1256,9 @@ def bc_lines(self) -> gpd.GeoDataFrame | None: return bc_lines - @property - def landcover_filename(self) -> str | None: - # broken example property which would give a filename to use when linking assets together - if self._geom_attrs == None: - self._geom_attrs = self.hdf_object.get_attrs("geom_or_something") - return self._geom_attrs.get("land_cover_filename") - - def associate_related_assets(self, asset_dict: dict[str, Asset]) -> None: - if self.landcover_filename: - landcover_asset = asset_dict[self.parent.joinpath(self.landcover_filename).resolve()] - self.extra_fields["ras:landcover_file"] = landcover_asset.href - class PlanHDFFile(RASHDFFile): + """Class to parse data from Plan HDF files.""" def __init__(self, fpath: str, **kwargs): super().__init__(fpath, RasPlanHdf, **kwargs) @@ -1223,234 +1270,273 @@ def __init__(self, fpath: str, **kwargs): @property def plan_information_base_output_interval(self) -> str | None: + """Return Base Output Interval.""" if self._plan_info_attrs == None: self._plan_info_attrs = self.hdf_object.get_plan_info_attrs() return self._plan_info_attrs.get("Base Output Interval") @property def plan_information_computation_time_step_base(self): + """Return Computation Time Step Base.""" if self._plan_info_attrs == None: self._plan_info_attrs = self.hdf_object.get_plan_info_attrs() return self._plan_info_attrs.get("Computation Time Step Base") @property def plan_information_flow_filename(self): + """Return Flow Filename.""" if self._plan_info_attrs == None: self._plan_info_attrs = self.hdf_object.get_plan_info_attrs() return self._plan_info_attrs.get("Flow Filename") @property def plan_information_geometry_filename(self): + """Return Geometry Filename.""" if self._plan_info_attrs == None: self._plan_info_attrs = self.hdf_object.get_plan_info_attrs() return self._plan_info_attrs.get("Geometry Filename") @property def plan_information_plan_filename(self): + """Return Plan Filename.""" if self._plan_info_attrs == None: self._plan_info_attrs = self.hdf_object.get_plan_info_attrs() return self._plan_info_attrs.get("Plan Filename") @property def plan_information_plan_name(self): + """Return Plan Name.""" if self._plan_info_attrs == None: self._plan_info_attrs = self.hdf_object.get_plan_info_attrs() return self._plan_info_attrs.get("Plan Name") @property def plan_information_project_filename(self): + """Return Project Filename.""" if self._plan_info_attrs == None: self._plan_info_attrs = self.hdf_object.get_plan_info_attrs() return self._plan_info_attrs.get("Project Filename") @property def plan_information_project_title(self): + """Return Project Title.""" if self._plan_info_attrs == None: self._plan_info_attrs = self.hdf_object.get_plan_info_attrs() return self._plan_info_attrs.get("Project Title") @property def plan_information_simulation_end_time(self): + """Return Simulation End Time.""" if self._plan_info_attrs == None: self._plan_info_attrs = self.hdf_object.get_plan_info_attrs() return self._plan_info_attrs.get("Simulation End Time").isoformat() @property def plan_information_simulation_start_time(self): + """Return Simulation Start Time.""" if self._plan_info_attrs == None: self._plan_info_attrs = self.hdf_object.get_plan_info_attrs() return self._plan_info_attrs.get("Simulation Start Time").isoformat() @property def plan_parameters_1d_flow_tolerance(self): + """Return 1D Flow Tolerance.""" if self._plan_parameters_attrs == None: self._plan_parameters_attrs = self.hdf_object.get_plan_param_attrs() return self._plan_parameters_attrs.get("1D Flow Tolerance") @property def plan_parameters_1d_maximum_iterations(self): + """Return 1D Maximum Iterations.""" if self._plan_parameters_attrs == None: self._plan_parameters_attrs = self.hdf_object.get_plan_param_attrs() return self._plan_parameters_attrs.get("1D Maximum Iterations") @property def plan_parameters_1d_maximum_iterations_without_improvement(self): + """Return 1D Maximum Iterations Without Improvement.""" if self._plan_parameters_attrs == None: self._plan_parameters_attrs = self.hdf_object.get_plan_param_attrs() return self._plan_parameters_attrs.get("1D Maximum Iterations Without Improvement") @property def plan_parameters_1d_maximum_water_surface_error_to_abort(self): + """Return 1D Maximum Water Surface Error To Abort.""" if self._plan_parameters_attrs == None: self._plan_parameters_attrs = self.hdf_object.get_plan_param_attrs() return self._plan_parameters_attrs.get("1D Maximum Water Surface Error To Abort") @property def plan_parameters_1d_storage_area_elevation_tolerance(self): + """Return 1D Storage Area Elevation Tolerance.""" if self._plan_parameters_attrs == None: self._plan_parameters_attrs = self.hdf_object.get_plan_param_attrs() return self._plan_parameters_attrs.get("1D Storage Area Elevation Tolerance") @property def plan_parameters_1d_theta(self): + """Return 1D Theta.""" if self._plan_parameters_attrs == None: self._plan_parameters_attrs = self.hdf_object.get_plan_param_attrs() return self._plan_parameters_attrs.get("1D Theta") @property def plan_parameters_1d_theta_warmup(self): + """Return 1D Theta Warmup.""" if self._plan_parameters_attrs == None: self._plan_parameters_attrs = self.hdf_object.get_plan_param_attrs() return self._plan_parameters_attrs.get("1D Theta Warmup") @property def plan_parameters_1d_water_surface_elevation_tolerance(self): + """Return 1D Water Surface Elevation Tolerance.""" if self._plan_parameters_attrs == None: self._plan_parameters_attrs = self.hdf_object.get_plan_param_attrs() return self._plan_parameters_attrs.get("1D Water Surface Elevation Tolerance") @property def plan_parameters_1d2d_gate_flow_submergence_decay_exponent(self): + """Return 1D-2D Gate Flow Submergence Decay Exponent.""" if self._plan_parameters_attrs == None: self._plan_parameters_attrs = self.hdf_object.get_plan_param_attrs() return self._plan_parameters_attrs.get("1D-2D Gate Flow Submergence Decay Exponent") @property def plan_parameters_1d2d_is_stablity_factor(self): + """Return 1D-2D IS Stablity Factor.""" if self._plan_parameters_attrs == None: self._plan_parameters_attrs = self.hdf_object.get_plan_param_attrs() return self._plan_parameters_attrs.get("1D-2D IS Stablity Factor") @property def plan_parameters_1d2d_ls_stablity_factor(self): + """Return 1D-2D LS Stablity Factor.""" if self._plan_parameters_attrs == None: self._plan_parameters_attrs = self.hdf_object.get_plan_param_attrs() return self._plan_parameters_attrs.get("1D-2D LS Stablity Factor") @property def plan_parameters_1d2d_maximum_number_of_time_slices(self): + """Return 1D-2D Maximum Number of Time Slices.""" if self._plan_parameters_attrs == None: self._plan_parameters_attrs = self.hdf_object.get_plan_param_attrs() return self._plan_parameters_attrs.get("1D-2D Maximum Number of Time Slices") @property def plan_parameters_1d2d_minimum_time_step_for_slicinghours(self): + """Return 1D-2D Minimum Time Step for Slicing(hours).""" if self._plan_parameters_attrs == None: self._plan_parameters_attrs = self.hdf_object.get_plan_param_attrs() return self._plan_parameters_attrs.get("1D-2D Minimum Time Step for Slicing(hours)") @property def plan_parameters_1d2d_number_of_warmup_steps(self): + """Return 1D-2D Number of Warmup Steps.""" if self._plan_parameters_attrs == None: self._plan_parameters_attrs = self.hdf_object.get_plan_param_attrs() return self._plan_parameters_attrs.get("1D-2D Number of Warmup Steps") @property def plan_parameters_1d2d_warmup_time_step_hours(self): + """Return 1D-2D Warmup Time Step (hours).""" if self._plan_parameters_attrs == None: self._plan_parameters_attrs = self.hdf_object.get_plan_param_attrs() return self._plan_parameters_attrs.get("1D-2D Warmup Time Step (hours)") @property def plan_parameters_1d2d_weir_flow_submergence_decay_exponent(self): + """Return 1D-2D Weir Flow Submergence Decay Exponent.""" if self._plan_parameters_attrs == None: self._plan_parameters_attrs = self.hdf_object.get_plan_param_attrs() return self._plan_parameters_attrs.get("1D-2D Weir Flow Submergence Decay Exponent") @property def plan_parameters_1d2d_maxiter(self): + """Return 1D2D MaxIter.""" if self._plan_parameters_attrs == None: self._plan_parameters_attrs = self.hdf_object.get_plan_param_attrs() return self._plan_parameters_attrs.get("1D2D MaxIter") @property def plan_parameters_2d_equation_set(self): + """Return 2D Equation Set.""" if self._plan_parameters_attrs == None: self._plan_parameters_attrs = self.hdf_object.get_plan_param_attrs() return self._plan_parameters_attrs.get("2D Equation Set") @property def plan_parameters_2d_names(self): + """Return 2D Names.""" if self._plan_parameters_attrs == None: self._plan_parameters_attrs = self.hdf_object.get_plan_param_attrs() return self._plan_parameters_attrs.get("2D Names") @property def plan_parameters_2d_volume_tolerance(self): + """Return 2D Volume Tolerance.""" if self._plan_parameters_attrs == None: self._plan_parameters_attrs = self.hdf_object.get_plan_param_attrs() return self._plan_parameters_attrs.get("2D Volume Tolerance") @property def plan_parameters_2d_water_surface_tolerance(self): + """Return 2D Water Surface Tolerance.""" if self._plan_parameters_attrs == None: self._plan_parameters_attrs = self.hdf_object.get_plan_param_attrs() return self._plan_parameters_attrs.get("2D Water Surface Tolerance") @property def meteorology_dss_filename(self): + """Return meteorology precip DSS Filename.""" if self._meteorology_attrs == None: self._meteorology_attrs = self.hdf_object.get_meteorology_precip_attrs() return self._meteorology_attrs.get("DSS Filename") @property def meteorology_dss_pathname(self): + """Return meteorology precip DSS Pathname.""" if self._meteorology_attrs == None: self._meteorology_attrs = self.hdf_object.get_meteorology_precip_attrs() return self._meteorology_attrs.get("DSS Pathname") @property def meteorology_data_type(self): + """Return meteorology precip Data Type.""" if self._meteorology_attrs == None: self._meteorology_attrs = self.hdf_object.get_meteorology_precip_attrs() return self._meteorology_attrs.get("Data Type") @property def meteorology_mode(self): + """Return meteorology precip Mode.""" if self._meteorology_attrs == None: self._meteorology_attrs = self.hdf_object.get_meteorology_precip_attrs() return self._meteorology_attrs.get("Mode") @property def meteorology_raster_cellsize(self): + """Return meteorology precip Raster Cellsize.""" if self._meteorology_attrs == None: self._meteorology_attrs = self.hdf_object.get_meteorology_precip_attrs() return self._meteorology_attrs.get("Raster Cellsize") @property def meteorology_source(self): + """Return meteorology precip Source.""" if self._meteorology_attrs == None: self._meteorology_attrs = self.hdf_object.get_meteorology_precip_attrs() return self._meteorology_attrs.get("Source") @property def meteorology_units(self): + """Return meteorology precip units.""" if self._meteorology_attrs == None: self._meteorology_attrs = self.hdf_object.get_meteorology_precip_attrs() return self._meteorology_attrs.get("Units") class GeometryHDFFile(RASHDFFile): + """Class to parse data from Geometry HDF files.""" def __init__(self, fpath: str, **kwargs): super().__init__(fpath, RasGeomHdf, **kwargs) @@ -1462,15 +1548,17 @@ def __init__(self, fpath: str, **kwargs): @property def projection(self): + """Return geometry projection.""" return self.hdf_object.projection() @property def cross_sections(self) -> int | None: - pass + """Return geometry cross sections.""" + return self.hdf_object.cross_sections() @property def reference_lines(self) -> gpd.GeoDataFrame | None: - + """Return geometry reference lines.""" ref_lines = self.hdf_object.reference_lines() if ref_lines is None or ref_lines.empty: diff --git a/hecstac/ras/utils.py b/hecstac/ras/utils.py index a898a1f..70d01d5 100644 --- a/hecstac/ras/utils.py +++ b/hecstac/ras/utils.py @@ -1,3 +1,5 @@ +"""Utility functions for the hecstac ras module.""" + import logging import os from functools import wraps @@ -188,104 +190,3 @@ def validate_point(geom): raise IndexError(f"expected point at xs-river intersection got: {type(geom)} | {geom}") else: raise TypeError(f"expected point at xs-river intersection got: {type(geom)} | {geom}") - - -class requires_geos: - def __init__(self, version): - if version.count(".") != 2: - raise ValueError("Version must be .. format") - self.version = tuple(int(x) for x in version.split(".")) - - def __call__(self, func): - is_compatible = lib.geos_version >= self.version - is_doc_build = os.environ.get("SPHINX_DOC_BUILD") == "1" # set in docs/conf.py - if is_compatible and not is_doc_build: - return func # return directly, do not change the docstring - - msg = "'{}' requires at least GEOS {}.{}.{}.".format(func.__name__, *self.version) - if is_compatible: - - @wraps(func) - def wrapped(*args, **kwargs): - return func(*args, **kwargs) - - else: - - @wraps(func) - def wrapped(*args, **kwargs): - raise UnsupportedGEOSVersionError(msg) - - doc = wrapped.__doc__ - if doc: - # Insert the message at the first double newline - position = doc.find("\n\n") + 2 - # Figure out the indentation level - indent = 0 - while True: - if doc[position + indent] == " ": - indent += 1 - else: - break - wrapped.__doc__ = doc.replace("\n\n", "\n\n{}.. note:: {}\n\n".format(" " * indent, msg), 1) - - return wrapped - - -def multithreading_enabled(func): - """Prepare multithreading by setting the writable flags of object type - ndarrays to False. - - NB: multithreading also requires the GIL to be released, which is done in - the C extension (ufuncs.c). - """ - - @wraps(func) - def wrapped(*args, **kwargs): - array_args = [arg for arg in args if isinstance(arg, np.ndarray) and arg.dtype == object] + [ - arg - for name, arg in kwargs.items() - if name not in {"where", "out"} and isinstance(arg, np.ndarray) and arg.dtype == object - ] - old_flags = [arr.flags.writeable for arr in array_args] - try: - for arr in array_args: - arr.flags.writeable = False - return func(*args, **kwargs) - finally: - for arr, old_flag in zip(array_args, old_flags): - arr.flags.writeable = old_flag - - return wrapped - - -@requires_geos("3.7.0") -@multithreading_enabled -def reverse(geometry, **kwargs): - """Returns a copy of a Geometry with the order of coordinates reversed. - - If a Geometry is a polygon with interior rings, the interior rings are also - reversed. - - Points are unchanged. None is returned where Geometry is None. - - Parameters - ---------- - geometry : Geometry or array_like - **kwargs - See :ref:`NumPy ufunc docs ` for other keyword arguments. - - See Also - -------- - is_ccw : Checks if a Geometry is clockwise. - - Examples - -------- - >>> from shapely import LineString, Polygon - >>> reverse(LineString([(0, 0), (1, 2)])) - - >>> reverse(Polygon([(0, 0), (1, 0), (1, 1), (0, 1), (0, 0)])) - - >>> reverse(None) is None - True - """ - return lib.reverse(geometry, **kwargs) diff --git a/new_ffrd_event_item.py b/new_ffrd_event_item.py index fcdceca..7e22397 100644 --- a/new_ffrd_event_item.py +++ b/new_ffrd_event_item.py @@ -1,3 +1,5 @@ +"""Creates a STAC Item from an event.""" + from pystac import Item from hecstac.events.logger import initialize_logger diff --git a/new_hms_item.py b/new_hms_item.py index a50f260..322aff4 100644 --- a/new_hms_item.py +++ b/new_hms_item.py @@ -1,3 +1,5 @@ +"""Creates a STAC Item from a HEC-HMS model .prj file.""" + from pathlib import Path from hecstac.hms.logger import initialize_logger @@ -5,9 +7,7 @@ def sanitize_catalog_assets(item: HMSModelItem) -> HMSModelItem: - """ - Forces the asset paths in the catalog relative to item root. - """ + """Force the asset paths in the catalog relative to item root.""" for asset in item.assets.values(): if item.pm.model_root_dir in asset.href: asset.href = asset.href.replace(item.pm.item_dir, ".") diff --git a/new_ras_item.py b/new_ras_item.py index cae3e5b..ac66c69 100644 --- a/new_ras_item.py +++ b/new_ras_item.py @@ -1,13 +1,13 @@ +"""Creates a STAC Item from a HEC-RAS model .prj file.""" + import logging from pathlib import Path -from hecstac.common.logger import initialize_logger -from hecstac.ras.item import RASModelItem +from hecstac.ras.logger import initialize_logger +from hecstac import RASModelItem def sanitize_catalog_assets(item: RASModelItem) -> RASModelItem: - """ - Forces the asset paths in the catalog to be relative to the item root. - """ + """Force the asset paths in the catalog to be relative to the item root.""" item_dir = Path(item.pm.item_dir).resolve() for _, asset in item.assets.items(): @@ -30,8 +30,5 @@ def sanitize_catalog_assets(item: RASModelItem) -> RASModelItem: ras_item = RASModelItem(ras_project_file, item_id, crs=None) ras_item = sanitize_catalog_assets(ras_item) # ras_item.add_model_thumbnails(["mesh_areas", "breaklines", "bc_lines"]) - fs = ras_item.scan_model_dir() - - ras_item.add_ras_asset() ras_item.save_object(ras_item.pm.item_path(item_id)) logging.info(f"Saved {ras_item.pm.item_path(item_id)}") From 63197a05912a6a63bf060ca1693deaed4fda5f64 Mon Sep 17 00:00:00 2001 From: Stevenray Janke Date: Tue, 4 Feb 2025 20:30:08 -0500 Subject: [PATCH 32/71] Move vars --- hecstac/ras/item.py | 17 +++++++++-------- 1 file changed, 9 insertions(+), 8 deletions(-) diff --git a/hecstac/ras/item.py b/hecstac/ras/item.py index 6f359e0..34e66a0 100644 --- a/hecstac/ras/item.py +++ b/hecstac/ras/item.py @@ -49,19 +49,11 @@ class RASModelItem(Item): @classmethod def from_prj(cls, ras_project_file, item_id: str, crs: str = None, simplify_geometry: bool = True): """Create an item from a RAS .prj file.""" - stac.pf = ProjectFile(ras_project_file) - stac.has_2d = False - stac.has_1d = False - stac._geom_files = [] - stac.crs = crs - stac.factory = AssetFactory(RAS_EXTENSION_MAPPING) properties = {"ras_project_file": ras_project_file} pm = LocalPathManager(Path(ras_project_file).parent) - stac.pm = pm href = pm.item_path(item_id) - stac._href = href stac = cls( Path(ras_project_file).stem, @@ -72,6 +64,15 @@ def from_prj(cls, ras_project_file, item_id: str, crs: str = None, simplify_geom href=href, ) + stac.pf = ProjectFile(ras_project_file) + stac.has_2d = False + stac.has_1d = False + stac._geom_files = [] + stac.crs = crs + stac.factory = AssetFactory(RAS_EXTENSION_MAPPING) + stac.pm = pm + stac._href = href + ras_asset_files = stac.scan_model_dir(ras_project_file) for fpath in ras_asset_files: From be44ab54a5535ac7721673d088da2752929444ec Mon Sep 17 00:00:00 2001 From: sclaw Date: Wed, 5 Feb 2025 09:56:01 -0500 Subject: [PATCH 33/71] initial draft refactor --- hecstac/common/asset_factory.py | 14 ++ hecstac/ras/assets.py | 18 ++- hecstac/ras/item.py | 238 ++++++++++++++++++-------------- 3 files changed, 159 insertions(+), 111 deletions(-) diff --git a/hecstac/common/asset_factory.py b/hecstac/common/asset_factory.py index 0aaad59..783ad0a 100644 --- a/hecstac/common/asset_factory.py +++ b/hecstac/common/asset_factory.py @@ -2,6 +2,8 @@ from pathlib import Path from typing import Dict, Type +import pystac +from pyproj import CRS from pystac import Asset from hecstac.hms.s3_utils import check_storage_extension @@ -22,6 +24,18 @@ def name_from_suffix(self, suffix: str) -> str: """Generate a name by appending a suffix to the file stem.""" return f"{self.stem}.{suffix}" + @property + def crs(self) -> CRS: + """Get the authority code for the model CRS.""" + try: + wkt2 = self.ext.proj.wkt2 + if wkt2 is None: + return + else: + return CRS(wkt2) + except pystac.errors.ExtensionNotImplemented: + return None + def __repr__(self): return f"<{self.__class__.__name__} name={self.name}>" diff --git a/hecstac/ras/assets.py b/hecstac/ras/assets.py index f8a8bea..05b6476 100644 --- a/hecstac/ras/assets.py +++ b/hecstac/ras/assets.py @@ -6,9 +6,11 @@ import geopandas as gpd import matplotlib.pyplot as plt from matplotlib.lines import Line2D -from pystac import MediaType from pyproj import CRS from pyproj.exceptions import CRSError +from pystac import MediaType +from pystac.extensions.projection import ProjectionExtension + from hecstac.common.asset_factory import GenericAsset from hecstac.ras.parser import ( GeometryFile, @@ -168,9 +170,8 @@ class GeometryAsset(GenericAsset): regex_parse_str = r".+\.g\d{2}$" PROPERTIES_WITH_GDF = ["reaches", "junctions", "cross_sections", "structures"] - def __init__(self, href: str, crs: str = None, **kwargs): + def __init__(self, href: str, **kwargs): # self.pyproj_crs = self.validate_crs(crs) - self.crs = crs roles = kwargs.pop("roles", []) + ["geometry-file", "ras-file"] description = kwargs.pop( "description", @@ -343,13 +344,12 @@ class GeometryHdfAsset(GenericAsset): regex_parse_str = r".+\.g\d{2}\.hdf$" - def __init__(self, href: str, crs: str = None, **kwargs): + def __init__(self, href: str, **kwargs): roles = kwargs.pop("roles", []) + ["geometry-hdf-file"] description = kwargs.pop("description", "The HEC-RAS geometry HDF file.") super().__init__(href, roles=roles, description=description, **kwargs) self.hdf_object = GeometryHDFFile(self.href) - self.crs = crs self.has_2d = None if self.crs is None: try: @@ -373,6 +373,14 @@ def __init__(self, href: str, crs: str = None, **kwargs): if value } + # @GenericAsset.crs.setter + # def crs(self, crs): + # """Add projection extension.""" + # if crs is not None: + # prj_ext = ProjectionExtension.ext(self, add_if_missing=True) + # crs = CRS(crs) + # prj_ext.apply(epsg=crs.to_epsg(), wkt2=crs.to_wkt()) + @property def check_2d(self): """Check if the geometry asset has 2d geometry, if yes then return True and set has_2d to True.""" diff --git a/hecstac/ras/item.py b/hecstac/ras/item.py index 8118585..65d4f75 100644 --- a/hecstac/ras/item.py +++ b/hecstac/ras/item.py @@ -2,30 +2,36 @@ import json import logging import os +from collections import UserDict +from functools import lru_cache from pathlib import Path +import pystac +import pystac.errors from pyproj import CRS, Transformer from pystac import Item from pystac.extensions.projection import ProjectionExtension from pystac.extensions.storage import StorageExtension +from rashdf import RasGeomHdf from shapely import Geometry, Polygon, simplify, to_geojson, union_all +from shapely.geometry import shape from shapely.ops import transform -from hecstac.common.path_manager import LocalPathManager -from hecstac.ras.parser import ProjectFile -from hecstac.ras.consts import ( - NULL_DATETIME, - NULL_STAC_GEOMETRY, - NULL_STAC_BBOX, -) - from hecstac.common.asset_factory import AssetFactory +from hecstac.common.geometry import reproject_to_wgs84 +from hecstac.common.path_manager import LocalPathManager from hecstac.ras.assets import ( RAS_EXTENSION_MAPPING, GeometryAsset, GeometryHdfAsset, ProjectAsset, ) +from hecstac.ras.consts import ( + NULL_DATETIME, + NULL_STAC_BBOX, + NULL_STAC_GEOMETRY, +) +from hecstac.ras.parser import ProjectFile class RASModelItem(Item): @@ -44,64 +50,90 @@ class RASModelItem(Item): RAS_HAS_2D = "ras:has_2d" RAS_DATETIME_SOURCE = "ras:datetime_source" + def __init__(self, *args, **kwargs): + """Add a few default properties to the base class.""" + super().__init__(*args, **kwargs) + self.simplify_geometry = True + @classmethod def from_prj(cls, ras_project_file, item_id: str, crs: str = None, simplify_geometry: bool = True): """Create an item from a RAS .prj file.""" - stac.pf = ProjectFile(ras_project_file) - stac.has_2d = False - stac.has_1d = False - stac._geom_files = [] - stac.crs = crs - stac.factory = AssetFactory(RAS_EXTENSION_MAPPING) - - properties = {"ras_project_file": ras_project_file} pm = LocalPathManager(Path(ras_project_file).parent) - stac.pm = pm - href = pm.item_path(item_id) - stac._href = href stac = cls( Path(ras_project_file).stem, NULL_STAC_GEOMETRY, NULL_STAC_BBOX, NULL_DATETIME, - properties, + {"ras_project_file": ras_project_file}, href=href, ) + stac.crs + if crs: + stac.crs = crs + stac.simplify_geometry = simplify_geometry - ras_asset_files = stac.scan_model_dir(ras_project_file) - - for fpath in ras_asset_files: + for fpath in stac.scan_model_dir(): if fpath and fpath != href: logging.info(f"Processing asset: {fpath}") stac.add_ras_asset(fpath) + return stac - stac.geometry, stac.bbox = stac.add_geometry(simplify_geometry) - stac.properties.update(stac.add_properties) - stac.datetime = stac._datetime - if stac.crs: - stac.apply_projection_extension(stac.crs) + @property + def ras_project_file(self) -> str: + """Get the path to the HEC-RAS .prj file.""" + return self._properties.get("ras_project_file") - return stac + @property + @lru_cache + def factory(self) -> AssetFactory: + """Return AssetFactory for this item.""" + return AssetFactory(RAS_EXTENSION_MAPPING) @property - def add_properties(self) -> None: - """Properties for the RAS STAC item.""" - properties = {} - # properties[self.RAS_HAS_1D] = self.has_1d - properties[self.RAS_HAS_2D] = self.has_2d - properties[self.PROJECT_TITLE] = self.pf.project_title - properties[self.PROJECT_VERSION] = self.pf.ras_version - properties[self.PROJECT_DESCRIPTION] = self.pf.project_description - properties[self.PROJECT_STATUS] = self.pf.project_status - properties[self.MODEL_UNITS] = self.pf.project_units + @lru_cache + def pf(self) -> ProjectFile: + """Get a ProjectFile instance for the RAS Model .prj file.""" + return ProjectFile(self.ras_project_file) - # TODO: once all assets are created, populate associations between assets - return properties + @property + def has_2d(self) -> bool: + """Whether any geometry file has 2D elements.""" + return any([a.check_2d() for a in self.geometry_assets]) + + @property + def has_1d(self) -> bool: + """Whether any geometry file has 2D elements.""" + return any([a.check_1d() for a in self.geometry_assets]) + + @property + def geometry_assets(self) -> list[RasGeomHdf]: + """Return any RasGeomHdf in assets.""" + return [a for a in self.assets.values() if isinstance(a, RasGeomHdf)] + + @property + def crs(self) -> CRS: + """Get the authority code for the model CRS.""" + try: + return CRS(self.ext.proj.wkt2) + except pystac.errors.ExtensionNotImplemented: + return None + + @crs.setter + def crs(self, crs): + """Apply the projection extension to this item given a CRS.""" + prj_ext = ProjectionExtension.ext(self, add_if_missing=True) + crs = CRS(crs) + prj_ext.apply(epsg=crs.to_epsg(), wkt2=crs.to_wkt()) + + @property + def geometry(self) -> dict: + """Return footprint of model as a geojson.""" + if self.crs is None: + logging.warning("Geometry requested for model with no spatial reference.") + return NULL_STAC_GEOMETRY - def add_geometry(self, simplify_geometry: bool) -> dict | None: - """Parses geometries from 2d hdf files and updates the stac item geometry, simplifying them if needed.""" geometries = [] if self.has_2d: @@ -112,23 +144,48 @@ def add_geometry(self, simplify_geometry: bool) -> dict | None: if len(geometries) == 0: logging.error("No geometry found for RAS item.") - return NULL_STAC_GEOMETRY, NULL_STAC_BBOX + return NULL_STAC_GEOMETRY unioned_geometry = union_all(geometries) - if simplify_geometry: + if self.simplify_geometry: unioned_geometry = simplify(unioned_geometry, 0.001) - geometry = json.loads(to_geojson(unioned_geometry)) - bbox = unioned_geometry.bounds + unioned_geometry = reproject_to_wgs84(unioned_geometry, self.crs) + return json.loads(to_geojson(unioned_geometry)) + + @property + def bbox(self) -> list[float]: + """Get the bounding box of the model geometry.""" + return shape(self.geometry).bounds + + @property + def properties(self) -> None: + """Properties for the RAS STAC item.""" + if self.ras_project_file is None: + return self._properties + properties = self._properties + # properties[self.RAS_HAS_1D] = self.has_1d + properties[self.RAS_HAS_2D] = self.has_2d + properties[self.PROJECT_TITLE] = self.pf.project_title + properties[self.PROJECT_VERSION] = self.pf.ras_version + properties[self.PROJECT_DESCRIPTION] = self.pf.project_description + properties[self.PROJECT_STATUS] = self.pf.project_status + properties[self.MODEL_UNITS] = self.pf.project_units + + # TODO: once all assets are created, populate associations between assets + return properties - return geometry, bbox + @properties.setter + def properties(self, properties: dict): + """Set properties.""" + self._properties = properties @property - def _datetime(self) -> datetime: + def datetime(self) -> datetime: """The datetime for the RAS STAC item.""" item_datetime = None - for geom_file in self._geom_files: + for geom_file in self.geometry_assets: if isinstance(geom_file, GeometryHdfAsset): geom_date = geom_file.hdf_object.geometry_time if geom_date: @@ -156,9 +213,9 @@ def add_model_thumbnails(self, layers: list, title_prefix: str = "Model_Thumbnai if thumbnail_dir: thumbnail_dest = thumbnail_dir else: - thumbnail_dest = self._href + thumbnail_dest = self.self_href - for geom in self._geom_files: + for geom in self.geometry_assets: if isinstance(geom, GeometryHdfAsset): self.assets[f"{geom.href}_thumbnail"] = geom.thumbnail( layers=layers, title=title_prefix, thumbnail_dest=thumbnail_dest @@ -173,44 +230,11 @@ def add_ras_asset(self, fpath: str = "") -> None: return try: asset = self.factory.create_ras_asset(fpath) + self.add_asset(asset.title, asset) + # TODO: add feature to set crs from geom hdf logging.debug(f"Adding asset {str(asset)}") except TypeError as e: logging.error(f"Error creating asset for {fpath}: {e}") - return - - if asset: - self.add_asset(asset.title, asset) - - if isinstance(asset, GeometryHdfAsset): - # if item and asset crs are None, pass and use null geometry - if self.crs is None and asset.crs is None: - pass - # Use asset crs as item crs if there is no item crs - elif self.crs is None and asset.crs is not None: - self.crs = asset.crs - # If item has crs, use it as the asset crs - elif self.crs: - asset.crs = self.crs - - if asset.check_2d: - self._geom_files = [] - self._geom_files.append(asset) - self.has_2d = True - self.properties[self.RAS_HAS_2D] = True - # elif isinstance(asset, GeometryAsset): - # if asset.geomf.has_1d: - # self.has_1d = False TODO: Implement 1d functionality - # self.properties[self.RAS_HAS_1D] = True - # self._geom_files.append(asset) - - def _geometry_to_wgs84(self, geom: Geometry) -> Geometry: - """Convert geometry CRS to EPSG:4326 for stac item geometry.""" - pyproj_crs = CRS.from_user_input(self.crs) - wgs_crs = CRS.from_authority("EPSG", "4326") - if pyproj_crs != wgs_crs: - transformer = Transformer.from_crs(pyproj_crs, wgs_crs, True) - return transform(transformer.transform, geom) - return geom def parse_1d_geom(self): """Read 1d geometry from concave hull.""" @@ -240,23 +264,25 @@ def parse_2d_geom(self): return union_all(mesh_area_polygons) - def ensure_projection_schema(self) -> None: - ProjectionExtension.ensure_has_extension(self, True) - - def scan_model_dir(self, ras_project_file): + def scan_model_dir(self): """Find all files in the project folder.""" - base_dir = os.path.dirname(ras_project_file) - files = [] - for root, _, filenames in os.walk(base_dir): - depth = root[len(base_dir) :].count(os.sep) - if depth > 1: - break - for filename in filenames: - files.append(os.path.join(root, filename)) - return files - - def apply_projection_extension(self, crs: str): - """Apply the projection extension to this item given a CRS.""" - prj_ext = ProjectionExtension.ext(self, add_if_missing=True) - og_crs = CRS(crs) - prj_ext.apply(epsg=og_crs.to_epsg(), wkt2=og_crs.to_wkt()) + parent = Path(self.ras_project_file).parent + stem = Path(self.ras_project_file).name.split(".")[0] + return [str(i) for i in parent.glob(f"{stem}*")] + + ### Some properties are dynamically generated. Ignore external updates ### + + @geometry.setter + def geometry(self, *args, **kwargs): + """Ignore.""" + pass + + @bbox.setter + def bbox(self, *args, **kwargs): + """Ignore.""" + pass + + @datetime.setter + def datetime(self, *args, **kwargs): + """Ignore.""" + pass From 35144341ee0aca8729f3c9ff5e3a6c3dcc4de446 Mon Sep 17 00:00:00 2001 From: sclaw Date: Wed, 5 Feb 2025 10:17:14 -0500 Subject: [PATCH 34/71] update paths to posix --- hecstac/ras/item.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/hecstac/ras/item.py b/hecstac/ras/item.py index 65d4f75..d7e1872 100644 --- a/hecstac/ras/item.py +++ b/hecstac/ras/item.py @@ -268,7 +268,7 @@ def scan_model_dir(self): """Find all files in the project folder.""" parent = Path(self.ras_project_file).parent stem = Path(self.ras_project_file).name.split(".")[0] - return [str(i) for i in parent.glob(f"{stem}*")] + return [str(i.as_posix()) for i in parent.glob(f"{stem}*")] ### Some properties are dynamically generated. Ignore external updates ### From eb291e9226db392a0f5bfe8603d613c3ff6c9b70 Mon Sep 17 00:00:00 2001 From: sclaw Date: Wed, 5 Feb 2025 10:17:29 -0500 Subject: [PATCH 35/71] update null geom to square at 0, 0 --- hecstac/ras/consts.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/hecstac/ras/consts.py b/hecstac/ras/consts.py index 7798ec3..2c43a1e 100644 --- a/hecstac/ras/consts.py +++ b/hecstac/ras/consts.py @@ -1,15 +1,15 @@ import datetime -from shapely import Polygon, to_geojson, box import json +from shapely import Polygon, box, to_geojson SCHEMA_URI = ( "https://raw.githubusercontent.com/fema-ffrd/hecstac/refs/heads/port-ras-stac/hecstac/ras/extension/schema.json" ) NULL_DATETIME = datetime.datetime(9999, 9, 9) -NULL_GEOMETRY = Polygon() +NULL_GEOMETRY = Polygon([(0, 0), (0, 1), (1, 1), (1, 0)]) NULL_STAC_GEOMETRY = json.loads(to_geojson(NULL_GEOMETRY)) -NULL_BBOX = box(0, 0, 0, 0) +NULL_BBOX = box(0, 0, 1, 1) NULL_STAC_BBOX = NULL_BBOX.bounds PLACEHOLDER_ID = "id" From d54bce96884b19df14475974805b912bbf18f416 Mon Sep 17 00:00:00 2001 From: Stevenray Janke Date: Wed, 5 Feb 2025 10:23:19 -0500 Subject: [PATCH 36/71] Check if .prj is a project asset or not and assign it accordingly --- hecstac/common/asset_factory.py | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/hecstac/common/asset_factory.py b/hecstac/common/asset_factory.py index d98b335..706efde 100644 --- a/hecstac/common/asset_factory.py +++ b/hecstac/common/asset_factory.py @@ -9,6 +9,16 @@ from hecstac.hms.s3_utils import check_storage_extension +def is_ras_prj(url: str) -> bool: + """Check if a file is a HEC-RAS project file.""" + with open(url) as f: + file_str = f.read() + if "Proj Title" in file_str.split("\n")[0]: + return True + else: + return False + + class GenericAsset(Asset): """Provides a base structure for assets.""" @@ -65,6 +75,14 @@ def create_hms_asset(self, fpath: str, item_type: str = "model") -> Asset: def create_ras_asset(self, fpath: str): """Create an asset instance based on the file extension.""" logging.debug(f"Creating asset for {fpath}") + from hecstac.ras.assets import ProjectAsset + + if fpath.lower().endswith(".prj"): + if is_ras_prj(fpath): + return ProjectAsset(href=fpath, title=Path(fpath).name) + else: + return GenericAsset(href=fpath, title=Path(fpath).name) + for pattern, asset_class in self.extension_to_asset.items(): if pattern.match(fpath): logging.debug(f"Matched {pattern} for {Path(fpath).name}: {asset_class}") From 70a5df906cd3bceb4a6a55146adcefc8e63d602f Mon Sep 17 00:00:00 2001 From: sclaw Date: Wed, 5 Feb 2025 10:23:23 -0500 Subject: [PATCH 37/71] add option to pull crs from geom hdf --- hecstac/ras/assets.py | 6 ------ hecstac/ras/item.py | 3 ++- 2 files changed, 2 insertions(+), 7 deletions(-) diff --git a/hecstac/ras/assets.py b/hecstac/ras/assets.py index 05b6476..7ff2520 100644 --- a/hecstac/ras/assets.py +++ b/hecstac/ras/assets.py @@ -351,12 +351,6 @@ def __init__(self, href: str, **kwargs): super().__init__(href, roles=roles, description=description, **kwargs) self.hdf_object = GeometryHDFFile(self.href) self.has_2d = None - if self.crs is None: - try: - self.crs = CRS.from_user_input(self.hdf_object.projection) - logging.info(f"crs has been set to {self.crs}") - except CRSError: - logging.warning(f"Could not extract crs from {self.href}") self.extra_fields = { key: value diff --git a/hecstac/ras/item.py b/hecstac/ras/item.py index d7e1872..40d7fd2 100644 --- a/hecstac/ras/item.py +++ b/hecstac/ras/item.py @@ -231,7 +231,8 @@ def add_ras_asset(self, fpath: str = "") -> None: try: asset = self.factory.create_ras_asset(fpath) self.add_asset(asset.title, asset) - # TODO: add feature to set crs from geom hdf + if self.crs is None and isinstance(asset, GeometryHdfAsset) and asset.hdf_object.projection is not None: + self.crs = asset.hdf_object.projection logging.debug(f"Adding asset {str(asset)}") except TypeError as e: logging.error(f"Error creating asset for {fpath}: {e}") From 016d106666c7a86feaeddc09f28e5dc645da8ecd Mon Sep 17 00:00:00 2001 From: sclaw Date: Wed, 5 Feb 2025 11:04:56 -0500 Subject: [PATCH 38/71] add lru_cache to project file --- hecstac/ras/parser.py | 16 ++++++++++++++-- 1 file changed, 14 insertions(+), 2 deletions(-) diff --git a/hecstac/ras/parser.py b/hecstac/ras/parser.py index 18a5f9a..18736e9 100644 --- a/hecstac/ras/parser.py +++ b/hecstac/ras/parser.py @@ -3,6 +3,7 @@ import math from collections import defaultdict from enum import Enum +from functools import lru_cache from pathlib import Path from typing import Iterator @@ -651,33 +652,39 @@ def __init__(self, fpath): self.file_lines = f.readlines() @property + @lru_cache def project_title(self) -> str: return search_contents(self.file_lines, "Proj Title") @property + @lru_cache def project_description(self) -> str: return search_contents(self.file_lines, "Model Description", token=":", require_one=False) @property + @lru_cache def project_status(self) -> str: return search_contents(self.file_lines, "Status of Model", token=":", require_one=False) @property + @lru_cache def project_units(self) -> str | None: for line in self.file_lines: if "Units" in line: return " ".join(line.split(" ")[:-1]) @property + @lru_cache def plan_current(self) -> str | None: try: - suffix = search_contents(self.file_lines, "Current Plan", expect_one=True, require_one=False) - return self.name_from_suffix(suffix) + suffix = search_contents(self.file_lines, "Current Plan", expect_one=True, require_one=False).strip() + return name_from_suffix(self.fpath, suffix) except Exception: logging.warning("Ras model has no current plan") return None @property + @lru_cache def ras_version(self) -> str | None: version = search_contents(self.file_lines, "Program Version", token="=", expect_one=False, require_one=False) if version == []: @@ -691,26 +698,31 @@ def ras_version(self) -> str | None: return version[0] @property + @lru_cache def plan_files(self) -> list[str]: suffixes = search_contents(self.file_lines, "Plan File", expect_one=False, require_one=False) return [name_from_suffix(self.fpath, i) for i in suffixes] @property + @lru_cache def geometry_files(self) -> list[str]: suffixes = search_contents(self.file_lines, "Geom File", expect_one=False, require_one=False) return [name_from_suffix(self.fpath, i) for i in suffixes] @property + @lru_cache def steady_flow_files(self) -> list[str]: suffixes = search_contents(self.file_lines, "Flow File", expect_one=False, require_one=False) return [name_from_suffix(self.fpath, i) for i in suffixes] @property + @lru_cache def quasi_unsteady_flow_files(self) -> list[str]: suffixes = search_contents(self.file_lines, "QuasiSteady File", expect_one=False, require_one=False) return [name_from_suffix(self.fpath, i) for i in suffixes] @property + @lru_cache def unsteady_flow_files(self) -> list[str]: suffixes = search_contents(self.file_lines, "Unsteady File", expect_one=False, require_one=False) return [name_from_suffix(self.fpath, i) for i in suffixes] From 34312b731a582e128b4e254299ea6d222625ea55 Mon Sep 17 00:00:00 2001 From: sclaw Date: Wed, 5 Feb 2025 11:18:34 -0500 Subject: [PATCH 39/71] simplify geometry parsing --- hecstac/ras/assets.py | 15 +++++++++++++++ hecstac/ras/item.py | 45 ++++--------------------------------------- 2 files changed, 19 insertions(+), 41 deletions(-) diff --git a/hecstac/ras/assets.py b/hecstac/ras/assets.py index 7ff2520..0c7b1dc 100644 --- a/hecstac/ras/assets.py +++ b/hecstac/ras/assets.py @@ -1,15 +1,18 @@ import logging import os import re +from functools import lru_cache import contextily as ctx import geopandas as gpd import matplotlib.pyplot as plt from matplotlib.lines import Line2D +from pandas import lreshape from pyproj import CRS from pyproj.exceptions import CRSError from pystac import MediaType from pystac.extensions.projection import ProjectionExtension +from shapely import MultiPolygon, Polygon from hecstac.common.asset_factory import GenericAsset from hecstac.ras.parser import ( @@ -201,6 +204,12 @@ def __init__(self, href: str, **kwargs): if value } + @property + @lru_cache + def geometry(self, crs: CRS) -> Polygon | MultiPolygon: + """Retrieves concave hull of cross-sections.""" + return gpd.GeoDataFrame() # TODO: fill this in. + class SteadyFlowAsset(GenericAsset): """HEC-RAS Steady Flow file asset.""" @@ -389,6 +398,12 @@ def check_2d(self): self.has_2d = False return False + @property + @lru_cache + def geometry(self, crs: CRS) -> Polygon | MultiPolygon: + """Retrieves concave hull of cross-sections.""" + return self.hdf_object.mesh_areas(crs) + def _plot_mesh_areas(self, ax, mesh_polygons: gpd.GeoDataFrame) -> list[Line2D]: """ Plots mesh areas on the given axes. diff --git a/hecstac/ras/item.py b/hecstac/ras/item.py index 40d7fd2..f0417b3 100644 --- a/hecstac/ras/item.py +++ b/hecstac/ras/item.py @@ -110,7 +110,7 @@ def has_1d(self) -> bool: @property def geometry_assets(self) -> list[RasGeomHdf]: """Return any RasGeomHdf in assets.""" - return [a for a in self.assets.values() if isinstance(a, RasGeomHdf)] + return [a for a in self.assets.values() if isinstance(a, (RasGeomHdf, GeometryAsset))] @property def crs(self) -> CRS: @@ -133,24 +133,15 @@ def geometry(self) -> dict: if self.crs is None: logging.warning("Geometry requested for model with no spatial reference.") return NULL_STAC_GEOMETRY - - geometries = [] - - if self.has_2d: - geometries.append(self.parse_2d_geom()) - - # if self.has_1d: - # geometries.append(self.parse_1d_geom()) - - if len(geometries) == 0: + if len(self.geometry_assets) == 0: logging.error("No geometry found for RAS item.") return NULL_STAC_GEOMETRY + crs = CRS.from_authority("EPSG", "4326") + geometries = [i.geometry(crs) for i in self.geometry_assets] unioned_geometry = union_all(geometries) if self.simplify_geometry: unioned_geometry = simplify(unioned_geometry, 0.001) - - unioned_geometry = reproject_to_wgs84(unioned_geometry, self.crs) return json.loads(to_geojson(unioned_geometry)) @property @@ -237,34 +228,6 @@ def add_ras_asset(self, fpath: str = "") -> None: except TypeError as e: logging.error(f"Error creating asset for {fpath}: {e}") - def parse_1d_geom(self): - """Read 1d geometry from concave hull.""" - logging.info("Creating geometry using 1d text file cross sections") - concave_hull_polygons: list[Polygon] = [] - for geom_asset in self._geom_files: - if isinstance(geom_asset, GeometryAsset): - try: - geom_asset.crs = self.crs - concave_hull = geom_asset.geomf.concave_hull - concave_hull = self._geometry_to_wgs84(concave_hull) - concave_hull_polygons.append(concave_hull) - except ValueError: - logging.warning(f"Could not extract geometry from {geom_asset.href}") - - return union_all(concave_hull_polygons) - - def parse_2d_geom(self): - """Read 2d geometry from hdf file mesh areas.""" - mesh_area_polygons: list[Polygon] = [] - for geom_asset in self._geom_files: - if isinstance(geom_asset, GeometryHdfAsset): - logging.info(f"Extracting geom from mesh areas in {geom_asset.href}") - - mesh_areas = self._geometry_to_wgs84(geom_asset.hdf_object.mesh_areas(self.crs)) - mesh_area_polygons.append(mesh_areas) - - return union_all(mesh_area_polygons) - def scan_model_dir(self): """Find all files in the project folder.""" parent = Path(self.ras_project_file).parent From 61b8c4cf1166a1423c3be7f978d0fdb53c5253d2 Mon Sep 17 00:00:00 2001 From: sclaw Date: Wed, 5 Feb 2025 11:35:35 -0500 Subject: [PATCH 40/71] debug new geometry approach --- hecstac/ras/assets.py | 45 ++++++++++++++++++++++++++++++++++++------- hecstac/ras/item.py | 9 ++++----- 2 files changed, 42 insertions(+), 12 deletions(-) diff --git a/hecstac/ras/assets.py b/hecstac/ras/assets.py index 0c7b1dc..b5d2867 100644 --- a/hecstac/ras/assets.py +++ b/hecstac/ras/assets.py @@ -15,6 +15,7 @@ from shapely import MultiPolygon, Polygon from hecstac.common.asset_factory import GenericAsset +from hecstac.common.geometry import reproject_to_wgs84 from hecstac.ras.parser import ( GeometryFile, GeometryHDFFile, @@ -206,9 +207,28 @@ def __init__(self, href: str, **kwargs): @property @lru_cache - def geometry(self, crs: CRS) -> Polygon | MultiPolygon: + def geometry(self) -> Polygon | MultiPolygon: """Retrieves concave hull of cross-sections.""" - return gpd.GeoDataFrame() # TODO: fill this in. + return Polygon([(0, 0), (0, 1), (1, 1), (1, 0)]) # TODO: fill this in. + + @property + @lru_cache + def has_1d(self) -> bool: + """Check if geometry has any river centerlines.""" + return False # TODO: implement + + @property + @lru_cache + def has_2d(self) -> bool: + """Check if geometry has any 2D areas.""" + return False # TODO: implement + + @property + @lru_cache + def geometry_wgs84(self) -> Polygon | MultiPolygon: + """Reproject geometry to wgs84.""" + # TODO: this could be generalized to be a function that takes argument for CRS. + return reproject_to_wgs84(self.geometry, self.crs) class SteadyFlowAsset(GenericAsset): @@ -359,7 +379,6 @@ def __init__(self, href: str, **kwargs): super().__init__(href, roles=roles, description=description, **kwargs) self.hdf_object = GeometryHDFFile(self.href) - self.has_2d = None self.extra_fields = { key: value @@ -385,25 +404,37 @@ def __init__(self, href: str, **kwargs): # prj_ext.apply(epsg=crs.to_epsg(), wkt2=crs.to_wkt()) @property - def check_2d(self): - """Check if the geometry asset has 2d geometry, if yes then return True and set has_2d to True.""" + @lru_cache + def has_2d(self) -> bool: + """Check if the geometry asset has 2d geometry.""" try: logging.debug(f"reading mesh areas using crs {self.crs}...") if self.hdf_object.mesh_areas(self.crs): - self.has_2d = True return True except ValueError: logging.warning(f"No mesh areas found for {self.href}") - self.has_2d = False return False + @property + @lru_cache + def has_1d(self) -> bool: + """Check if the geometry asset has 2d geometry.""" + return False # TODO: implement + @property @lru_cache def geometry(self, crs: CRS) -> Polygon | MultiPolygon: """Retrieves concave hull of cross-sections.""" return self.hdf_object.mesh_areas(crs) + @property + @lru_cache + def geometry_wgs84(self) -> Polygon | MultiPolygon: + """Reproject geometry to wgs84.""" + # TODO: this could be generalized to be a function that takes argument for CRS. + return reproject_to_wgs84(self.geometry, self.crs) + def _plot_mesh_areas(self, ax, mesh_polygons: gpd.GeoDataFrame) -> list[Line2D]: """ Plots mesh areas on the given axes. diff --git a/hecstac/ras/item.py b/hecstac/ras/item.py index f0417b3..726b5f9 100644 --- a/hecstac/ras/item.py +++ b/hecstac/ras/item.py @@ -100,15 +100,15 @@ def pf(self) -> ProjectFile: @property def has_2d(self) -> bool: """Whether any geometry file has 2D elements.""" - return any([a.check_2d() for a in self.geometry_assets]) + return any([a.has_2d for a in self.geometry_assets]) @property def has_1d(self) -> bool: """Whether any geometry file has 2D elements.""" - return any([a.check_1d() for a in self.geometry_assets]) + return any([a.has_1d for a in self.geometry_assets]) @property - def geometry_assets(self) -> list[RasGeomHdf]: + def geometry_assets(self) -> list[RasGeomHdf | GeometryAsset]: """Return any RasGeomHdf in assets.""" return [a for a in self.assets.values() if isinstance(a, (RasGeomHdf, GeometryAsset))] @@ -137,8 +137,7 @@ def geometry(self) -> dict: logging.error("No geometry found for RAS item.") return NULL_STAC_GEOMETRY - crs = CRS.from_authority("EPSG", "4326") - geometries = [i.geometry(crs) for i in self.geometry_assets] + geometries = [i.geometry_wgs84 for i in self.geometry_assets] unioned_geometry = union_all(geometries) if self.simplify_geometry: unioned_geometry = simplify(unioned_geometry, 0.001) From 102ab863c044e9fefaf7e240b6d2232b4c575032 Mon Sep 17 00:00:00 2001 From: sclaw Date: Wed, 5 Feb 2025 12:08:30 -0500 Subject: [PATCH 41/71] refactor asset addition --- hecstac/common/asset_factory.py | 7 +++++++ hecstac/common/path_manager.py | 4 ++-- hecstac/ras/item.py | 35 +++++++++++---------------------- hecstac/ras/utils.py | 10 ++++++++++ 4 files changed, 30 insertions(+), 26 deletions(-) diff --git a/hecstac/common/asset_factory.py b/hecstac/common/asset_factory.py index 783ad0a..de90b22 100644 --- a/hecstac/common/asset_factory.py +++ b/hecstac/common/asset_factory.py @@ -82,3 +82,10 @@ def create_ras_asset(self, fpath: str): logging.warning(f"Unable to pattern match asset for file {fpath}") return GenericAsset(href=fpath, title=Path(fpath).name) + + def asset_from_dict(self, asset: Asset): + fpath = asset.href + for pattern, asset_class in self.extension_to_asset.items(): + if pattern.match(fpath): + logging.debug(f"Matched {pattern} for {Path(fpath).name}: {asset_class}") + return asset_class.from_dict(asset.to_dict()) diff --git a/hecstac/common/path_manager.py b/hecstac/common/path_manager.py index fb49792..fa4843b 100644 --- a/hecstac/common/path_manager.py +++ b/hecstac/common/path_manager.py @@ -23,7 +23,7 @@ def item_dir(self) -> str: return self.model_root_dir def item_path(self, item_id: str) -> str: - return f"{self._model_root_dir}/{item_id}.json" + return str(Path(self._model_root_dir) / f"{item_id}.json") def derived_item_asset(self, filename: str) -> str: - return f"{self._model_root_dir}/{filename}" + return str(Path(self._model_root_dir) / filename) diff --git a/hecstac/ras/item.py b/hecstac/ras/item.py index 726b5f9..aaf2deb 100644 --- a/hecstac/ras/item.py +++ b/hecstac/ras/item.py @@ -5,11 +5,12 @@ from collections import UserDict from functools import lru_cache from pathlib import Path +from typing import Optional import pystac import pystac.errors from pyproj import CRS, Transformer -from pystac import Item +from pystac import Asset, Item from pystac.extensions.projection import ProjectionExtension from pystac.extensions.storage import StorageExtension from rashdf import RasGeomHdf @@ -32,6 +33,7 @@ NULL_STAC_GEOMETRY, ) from hecstac.ras.parser import ProjectFile +from hecstac.ras.utils import find_model_files class RASModelItem(Item): @@ -60,6 +62,7 @@ def from_prj(cls, ras_project_file, item_id: str, crs: str = None, simplify_geom """Create an item from a RAS .prj file.""" pm = LocalPathManager(Path(ras_project_file).parent) href = pm.item_path(item_id) + assets = {Path(i).name: Asset(i, Path(i).name) for i in find_model_files(ras_project_file)} stac = cls( Path(ras_project_file).stem, @@ -68,16 +71,13 @@ def from_prj(cls, ras_project_file, item_id: str, crs: str = None, simplify_geom NULL_DATETIME, {"ras_project_file": ras_project_file}, href=href, + assets=assets, ) stac.crs if crs: stac.crs = crs stac.simplify_geometry = simplify_geometry - for fpath in stac.scan_model_dir(): - if fpath and fpath != href: - logging.info(f"Processing asset: {fpath}") - stac.add_ras_asset(fpath) return stac @property @@ -213,25 +213,12 @@ def add_model_thumbnails(self, layers: list, title_prefix: str = "Model_Thumbnai # TODO: Add 1d model thumbnails - def add_ras_asset(self, fpath: str = "") -> None: - """Add an asset to the RAS STAC item.""" - if not os.path.exists(fpath): - logging.warning(f"File not found: {fpath}") - return - try: - asset = self.factory.create_ras_asset(fpath) - self.add_asset(asset.title, asset) - if self.crs is None and isinstance(asset, GeometryHdfAsset) and asset.hdf_object.projection is not None: - self.crs = asset.hdf_object.projection - logging.debug(f"Adding asset {str(asset)}") - except TypeError as e: - logging.error(f"Error creating asset for {fpath}: {e}") - - def scan_model_dir(self): - """Find all files in the project folder.""" - parent = Path(self.ras_project_file).parent - stem = Path(self.ras_project_file).name.split(".")[0] - return [str(i.as_posix()) for i in parent.glob(f"{stem}*")] + def add_asset(self, key, asset): + """Subclass asset then add.""" + asset = self.factory.asset_from_dict(asset) + if self.crs is None and isinstance(asset, GeometryHdfAsset) and asset.hdf_object.projection is not None: + self.crs = asset.hdf_object.projection + return super().add_asset(key, asset) ### Some properties are dynamically generated. Ignore external updates ### diff --git a/hecstac/ras/utils.py b/hecstac/ras/utils.py index a898a1f..6ecd447 100644 --- a/hecstac/ras/utils.py +++ b/hecstac/ras/utils.py @@ -1,6 +1,8 @@ import logging import os from functools import wraps +from pathlib import Path +from typing import Optional import geopandas as gpd import numpy as np @@ -9,6 +11,14 @@ from shapely.geometry import LineString, MultiPoint, Point +def find_model_files(ras_prj: str) -> list[str]: + """Find all files with same base name.""" + ras_prj = Path(ras_prj) + parent = ras_prj.parent + stem = Path(ras_prj).name.split(".")[0] + return [str(i.as_posix()) for i in parent.glob(f"{stem}*") if i != ras_prj] + + def is_ras_prj(url: str) -> bool: """Check if a file is a HEC-RAS project file.""" with open(url) as f: From 34206c9b9e91640fa82f7ec3afbc636525674ea2 Mon Sep 17 00:00:00 2001 From: sclaw Date: Wed, 5 Feb 2025 14:42:18 -0500 Subject: [PATCH 42/71] incremental refactor assets --- hecstac/common/asset_factory.py | 46 +++-- hecstac/ras/assets.py | 290 +++++++++++++------------------- hecstac/ras/item.py | 10 +- 3 files changed, 158 insertions(+), 188 deletions(-) diff --git a/hecstac/common/asset_factory.py b/hecstac/common/asset_factory.py index de90b22..a9734d4 100644 --- a/hecstac/common/asset_factory.py +++ b/hecstac/common/asset_factory.py @@ -12,13 +12,41 @@ class GenericAsset(Asset): """Generic Asset.""" - def __init__(self, href: str, roles=None, description=None, *args, **kwargs): - super().__init__(href, *args, **kwargs) - self.href = href - self.name = Path(href).name - self.stem = Path(href).stem - self.roles = roles or [] - self.description = description or "" + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + if self.description is None: + self.description = self.__description__ + self._roles = [] + self._extra_fields = {} + + @property + def roles(self) -> list[str]: + """Return roles with enforced values.""" + roles = self._roles + for i in self.__roles__: + if i not in roles: + roles.append(i) + return roles + + @roles.setter + def roles(self, roles: list): + self._roles = roles + + @property + def extra_fields(self): + """Return extra fields.""" + # boilerplate here, but overwritten in subclasses + return self._extra_fields + + @extra_fields.setter + def extra_fields(self, extra_fields: dict): + """Set user-defined extra fields.""" + self._extra_fields = extra_fields + + @property + def file(self): + """Return class to access asset file contents.""" + return self.__file_class__(self.href) def name_from_suffix(self, suffix: str) -> str: """Generate a name by appending a suffix to the file stem.""" @@ -27,14 +55,12 @@ def name_from_suffix(self, suffix: str) -> str: @property def crs(self) -> CRS: """Get the authority code for the model CRS.""" - try: + if self.ext.has("proj"): wkt2 = self.ext.proj.wkt2 if wkt2 is None: return else: return CRS(wkt2) - except pystac.errors.ExtensionNotImplemented: - return None def __repr__(self): return f"<{self.__class__.__name__} name={self.name}>" diff --git a/hecstac/ras/assets.py b/hecstac/ras/assets.py index b5d2867..41eab6a 100644 --- a/hecstac/ras/assets.py +++ b/hecstac/ras/assets.py @@ -116,94 +116,73 @@ class ProjectAsset(GenericAsset): """HEC-RAS Project file asset.""" regex_parse_str = r".+\.prj$" - - def __init__(self, href: str, *args, **kwargs): - roles = kwargs.pop("roles", []) + ["project-file", "ras-file"] - description = kwargs.pop("description", "The HEC-RAS project file.") - - super().__init__(href, roles=roles, description=description, *args, **kwargs) - - self.href = href - self.pf = ProjectFile(self.href) - self.extra_fields = { - key: value - for key, value in { - CURRENT_PLAN: self.pf.plan_current, - PLAN_FILES: self.pf.plan_files, - GEOMETRY_FILES: self.pf.geometry_files, - STEADY_FLOW_FILES: self.pf.steady_flow_files, - QUASI_UNSTEADY_FLOW_FILES: self.pf.quasi_unsteady_flow_files, - UNSTEADY_FLOW_FILES: self.pf.unsteady_flow_files, - }.items() - if value - } + __roles__ = ["project-file", "ras-file"] + __description__ = "The HEC-RAS project file." + __file_class__ = ProjectFile + + @GenericAsset.extra_fields.getter + def extra_fields(self) -> dict: + """Return extra fields with added dynamic keys/values.""" + self._extra_fields[CURRENT_PLAN] = self.file.plan_current + self._extra_fields[PLAN_FILES] = self.file.plan_files + self._extra_fields[GEOMETRY_FILES] = self.file.geometry_files + self._extra_fields[STEADY_FLOW_FILES] = self.file.steady_flow_files + self._extra_fields[QUASI_UNSTEADY_FLOW_FILES] = self.file.quasi_unsteady_flow_files + self._extra_fields[UNSTEADY_FLOW_FILES] = self.file.unsteady_flow_files + return self._extra_fields class PlanAsset(GenericAsset): """HEC-RAS Plan file asset.""" regex_parse_str = r".+\.p\d{2}$" + __roles__ = ["plan-file", "ras-file"] + __description__ = "The plan file which contains a list of associated input files and all simulation options." + __file_class__ = PlanFile - def __init__(self, href: str, **kwargs): - roles = kwargs.pop("roles", []) + ["plan-file", "ras-file"] - description = kwargs.pop( - "description", - "The plan file which contains a list of associated input files and all simulation options.", - ) - - super().__init__(href, roles=roles, description=description, **kwargs) - - self.href = href - self.planf = PlanFile(self.href) - self.extra_fields = { - key: value - for key, value in { - TITLE: self.planf.plan_title, - VERSION: self.planf.plan_version, - GEOMETRY_FILE: self.planf.geometry_file, - FLOW_FILE: self.planf.flow_file, - BREACH_LOCATIONS: self.planf.breach_locations, - }.items() - if value - } + @GenericAsset.extra_fields.getter + def extra_fields(self) -> dict: + """Return extra fields with added dynamic keys/values.""" + self._extra_fields[TITLE] = self.file.plan_title + self._extra_fields[VERSION] = self.file.plan_version + self._extra_fields[GEOMETRY_FILE] = self.file.geometry_file + self._extra_fields[FLOW_FILE] = self.file.flow_file + self._extra_fields[BREACH_LOCATIONS] = self.file.breach_locations + return self._extra_fields class GeometryAsset(GenericAsset): """HEC-RAS Geometry file asset.""" regex_parse_str = r".+\.g\d{2}$" + __roles__ = ["geometry-file", "ras-file"] + __description__ = ( + "The geometry file which contains cross-sectional, 2D, hydraulic structures, and other geometric data." + ) + __file_class__ = GeometryFile PROPERTIES_WITH_GDF = ["reaches", "junctions", "cross_sections", "structures"] - def __init__(self, href: str, **kwargs): - # self.pyproj_crs = self.validate_crs(crs) - roles = kwargs.pop("roles", []) + ["geometry-file", "ras-file"] - description = kwargs.pop( - "description", - "The geometry file which contains cross-sectional, 2D, hydraulic structures, and other geometric data", - ) - - super().__init__(href, roles=roles, description=description, **kwargs) + @GenericAsset.extra_fields.getter + def extra_fields(self) -> dict: + """Return extra fields with added dynamic keys/values.""" + self._extra_fields[TITLE] = self.file.geom_title + self._extra_fields[VERSION] = self.file.geom_version + self._extra_fields[HAS_1D] = self.file.has_1d + self._extra_fields[HAS_2D] = self.file.has_2d + # self._extra_fields[RIVERS] = self.file.rivers + # self._extra_fields[REACHES] = self.file.reaches + # self._extra_fields[JUNCTIONS] = self.file.junctions + # self._extra_fields[CROSS_SECTIONS] = self.file.cross_sections + # self._extra_fields[STRUCTURES] = self.file.structures + # self._extra_fields[STORAGE_AREAS] = self.file.storage_areas + # self._extra_fields[CONNECTIONS] = self.file.connections + # self._extra_fields[BREACH_LOCATIONS] = self.file.breach_locations + return self._extra_fields - self.href = href - self.geomf = GeometryFile(self.href, self.crs) - self.extra_fields = { - key: value - for key, value in { - TITLE: self.geomf.geom_title, - VERSION: self.geomf.geom_version, - HAS_1D: self.geomf.has_1d, - HAS_2D: self.geomf.has_2d, - # RIVERS: self.geomf.rivers, - # REACHES: self.geomf.reaches, - # JUNCTIONS: self.geomf.junctions, - # CROSS_SECTIONS: self.geomf.cross_sections, - # STRUCTURES: self.geomf.structures, - # STORAGE_AREAS: self.geomf.storage_areas, #TODO: fix this - # CONNECTIONS: self.geomf.connections,#TODO: fix this - # BREACH_LOCATIONS: self.planf.breach_locations, - }.items() - if value - } + @property + def file(self): + """Return class to access asset file contents.""" + return self.__file_class__(self.href, self.owner.crs) @property @lru_cache @@ -235,26 +214,16 @@ class SteadyFlowAsset(GenericAsset): """HEC-RAS Steady Flow file asset.""" regex_parse_str = r".+\.f\d{2}$" + __roles__ = ["steady-flow-file", "ras-file"] + __description__ = "Steady Flow file which contains profile information, flow data, and boundary conditions." + __file_class__ = SteadyFlowFile - def __init__(self, href: str, **kwargs): - roles = kwargs.pop("roles", []) + ["steady-flow-file", "ras-file"] - description = kwargs.pop( - "description", - "Steady Flow file which contains profile information, flow data, and boundary conditions.", - ) - - super().__init__(href, roles=roles, description=description, **kwargs) - - self.href = href - self.flowf = SteadyFlowFile(self.href) - self.extra_fields = { - key: value - for key, value in { - TITLE: self.flowf.flow_title, - N_PROFILES: self.flowf.n_profiles, - }.items() - if value - } + @GenericAsset.extra_fields.getter + def extra_fields(self) -> dict: + """Return extra fields with added dynamic keys/values.""" + self._extra_fields[TITLE] = self.file.flow_title + self._extra_fields[N_PROFILES] = self.file.n_profiles + return self._extra_fields class QuasiUnsteadyFlowAsset(GenericAsset): @@ -263,49 +232,43 @@ class QuasiUnsteadyFlowAsset(GenericAsset): # TODO: implement this class regex_parse_str = r".+\.q\d{2}$" - - def __init__(self, href: str, **kwargs): - roles = kwargs.pop("roles", []) + ["quasi-unsteady-flow-file", "ras-file"] - description = kwargs.pop("description", "Quasi-Unsteady Flow file.") - - super().__init__(href, roles=roles, description=description, **kwargs) - - self.href = href - self.flowf = QuasiUnsteadyFlowFile(self.href) - self.extra_fields = { - key: value - for key, value in { - TITLE: self.flowf.flow_title, - }.items() - if value - } + __roles__ = ["quasi-unsteady-flow-file", "ras-file"] + __description__ = "Quasi-Unsteady Flow file." + __file_class__ = QuasiUnsteadyFlowFile + + @GenericAsset.extra_fields.getter + def extra_fields(self) -> dict: + """Return extra fields with added dynamic keys/values.""" + self._extra_fields[TITLE] = self.file.flow_title + self._extra_fields[VERSION] = self.file.geom_version + self._extra_fields[HAS_1D] = self.file.has_1d + self._extra_fields[HAS_2D] = self.file.has_2d + self._extra_fields[RIVERS] = self.file.rivers + self._extra_fields[REACHES] = self.file.reaches + self._extra_fields[JUNCTIONS] = self.file.junctions + self._extra_fields[CROSS_SECTIONS] = self.file.cross_sections + self._extra_fields[STRUCTURES] = self.file.structures + self._extra_fields[STORAGE_AREAS] = self.file.storage_areas + self._extra_fields[CONNECTIONS] = self.file.connections + self._extra_fields[BREACH_LOCATIONS] = self.file.breach_locations + return self._extra_fields class UnsteadyFlowAsset(GenericAsset): """HEC-RAS Unsteady Flow file asset.""" regex_parse_str = r".+\.u\d{2}$" + __roles__ = ["unsteady-flow-file", "ras-file"] + __description__ = "The unsteady file contains hydrographs, initial conditions, and any flow options." + __file_class__ = UnsteadyFlowFile - def __init__(self, href: str, **kwargs): - roles = kwargs.pop("roles", []) + ["unsteady-flow-file", "ras-file"] - description = kwargs.pop( - "description", - "The unsteady file contains hydrographs, initial conditions, and any flow options.", - ) - - super().__init__(href, roles=roles, description=description, **kwargs) - - self.href = href - self.flowf = UnsteadyFlowFile(self.href) - self.extra_fields = { - key: value - for key, value in { - TITLE: self.flowf.flow_title, - BOUNDARY_LOCATIONS: self.flowf.boundary_locations, - REFERENCE_LINES: self.flowf.reference_lines, - }.items() - if value - } + @GenericAsset.extra_fields.getter + def extra_fields(self) -> dict: + """Return extra fields with added dynamic keys/values.""" + self._extra_fields[TITLE] = self.file.flow_title + self._extra_fields[BOUNDARY_LOCATIONS] = self.file.boundary_locations + self._extra_fields[REFERENCE_LINES] = self.file.reference_lines + return self._extra_fields class PlanHdfAsset(GenericAsset): @@ -372,36 +335,24 @@ class GeometryHdfAsset(GenericAsset): """HEC-RAS Geometry HDF file asset.""" regex_parse_str = r".+\.g\d{2}\.hdf$" + __roles__ = ["geometry-hdf-file"] + __description__ = "The HEC-RAS geometry HDF file." + __file_class__ = GeometryHDFFile + + @GenericAsset.extra_fields.getter + def extra_fields(self) -> dict: + """Return extra fields with added dynamic keys/values.""" + self._extra_fields[VERSION] = self.file.file_version + self._extra_fields[UNITS] = self.file.units_system + self._extra_fields[PROJECTION] = self.owner.crs.to_wkt() + self._extra_fields[UNITS] = self.file.units_system + return self._extra_fields - def __init__(self, href: str, **kwargs): - roles = kwargs.pop("roles", []) + ["geometry-hdf-file"] - description = kwargs.pop("description", "The HEC-RAS geometry HDF file.") - - super().__init__(href, roles=roles, description=description, **kwargs) - self.hdf_object = GeometryHDFFile(self.href) - - self.extra_fields = { - key: value - for key, value in { - VERSION: self.hdf_object.file_version, - UNITS: self.hdf_object.units_system, - PROJECTION: self.crs.to_wkt() if self.crs is not None else None, - REFERENCE_LINES: ( - list(self.hdf_object.reference_lines["refln_name"]) - if self.hdf_object.reference_lines is not None and not self.hdf_object.reference_lines.empty - else None - ), - }.items() - if value - } - - # @GenericAsset.crs.setter - # def crs(self, crs): - # """Add projection extension.""" - # if crs is not None: - # prj_ext = ProjectionExtension.ext(self, add_if_missing=True) - # crs = CRS(crs) - # prj_ext.apply(epsg=crs.to_epsg(), wkt2=crs.to_wkt()) + @property + def reference_lines(self) -> list[gpd.GeoDataFrame] | None: + """Docstring.""" # TODO: fill out + if self.hdf_object.reference_lines is not None and not self.hdf_object.reference_lines.empty: + return list(self.file.reference_lines["refln_name"]) @property @lru_cache @@ -623,14 +574,9 @@ class GeometricPreprocessorAsset(GenericAsset): """Geometric Pre-Processor asset.""" regex_parse_str = r".+\.c\d{2}$" - - def __init__(self, href: str, *args, **kwargs): - roles = kwargs.pop("roles", ["geometric-preprocessor", "ras-file", MediaType.TEXT]) - description = kwargs.pop( - "description", - "Geometric Pre-Processor output file containing hydraulic properties, rating curves, and more.", - ) - super().__init__(href, roles=roles, description=description, *args, **kwargs) + __roles__ = ["geometric-preprocessor", "ras-file", MediaType.TEXT] + __description__ = "Geometric Pre-Processor output file containing hydraulic properties, rating curves, and more." + __file_class__ = None # TODO: make a generic parent for these. class BoundaryConditionAsset(GenericAsset): @@ -762,11 +708,9 @@ class DSSAsset(GenericAsset): """DSS asset.""" regex_parse_str = r".+\.dss$" - - def __init__(self, href: str, *args, **kwargs): - roles = kwargs.pop("roles", ["ras-dss", "ras-file", MediaType.TEXT]) - description = kwargs.pop("description", "The DSS file contains results and other simulation information.") - super().__init__(href, roles=roles, description=description, *args, **kwargs) + __roles__ = ["ras-dss", "ras-file", MediaType.TEXT] + __description__ = "The DSS file contains results and other simulation information." + __file_class__ = None class LogAsset(GenericAsset): @@ -896,11 +840,9 @@ class RasMapperFileAsset(GenericAsset): """RAS Mapper file asset.""" regex_parse_str = r".+\.rasmap$" - - def __init__(self, href: str, *args, **kwargs): - roles = kwargs.pop("roles", ["ras-mapper-file", "ras-file", MediaType.TEXT]) - description = kwargs.pop("description", "RAS Mapper file.") - super().__init__(href, roles=roles, description=description, *args, **kwargs) + __roles__ = ["ras-mapper-file", "ras-file", MediaType.TEXT] + __description__ = "RAS Mapper file." + __file_class__ = None class RasMapperBackupFileAsset(GenericAsset): diff --git a/hecstac/ras/item.py b/hecstac/ras/item.py index aaf2deb..1342734 100644 --- a/hecstac/ras/item.py +++ b/hecstac/ras/item.py @@ -215,10 +215,12 @@ def add_model_thumbnails(self, layers: list, title_prefix: str = "Model_Thumbnai def add_asset(self, key, asset): """Subclass asset then add.""" - asset = self.factory.asset_from_dict(asset) - if self.crs is None and isinstance(asset, GeometryHdfAsset) and asset.hdf_object.projection is not None: - self.crs = asset.hdf_object.projection - return super().add_asset(key, asset) + subclass = self.factory.asset_from_dict(asset) + if subclass is None: + return + if self.crs is None and isinstance(asset, GeometryHdfAsset) and asset.file.projection is not None: + self.crs = subclass.file.projection + return super().add_asset(key, subclass) ### Some properties are dynamically generated. Ignore external updates ### From 46b147a4308d80357fbc53f63b374c23335703c7 Mon Sep 17 00:00:00 2001 From: Stevenray Janke Date: Wed, 5 Feb 2025 15:42:22 -0500 Subject: [PATCH 43/71] Reformat/add dynamic properties --- hecstac/ras/item.py | 149 +++++++++++++++++++++++++++++++------------- pyproject.toml | 7 ++- 2 files changed, 111 insertions(+), 45 deletions(-) diff --git a/hecstac/ras/item.py b/hecstac/ras/item.py index 34e66a0..ed9d706 100644 --- a/hecstac/ras/item.py +++ b/hecstac/ras/item.py @@ -5,11 +5,13 @@ import logging import os from pathlib import Path - +from functools import lru_cache from pyproj import CRS, Transformer from pystac import Item +from rashdf import RasGeomHdf from pystac.extensions.projection import ProjectionExtension from pystac.extensions.storage import StorageExtension +from shapely.geometry import shape from shapely import Geometry, Polygon, simplify, to_geojson, union_all from shapely.ops import transform @@ -48,9 +50,26 @@ class RASModelItem(Item): @classmethod def from_prj(cls, ras_project_file, item_id: str, crs: str = None, simplify_geometry: bool = True): - """Create an item from a RAS .prj file.""" + """ + Create a STAC item from a HEC-RAS .prj file. + + Parameters + ---------- + ras_project_file : str + Path to the HEC-RAS project file (.prj). + item_id : str + Unique item id for the STAC item. + crs : str, optional + Coordinate reference system (CRS) to apply to the item. If None, the CRS will be extracted from the geometry .hdf file. + simplify_geometry : bool, optional + Whether to simplify geometry. Defaults to True. + + Returns + ---------- + stac : RASModelItem + An instance of the class representing the STAC item. - properties = {"ras_project_file": ras_project_file} + """ pm = LocalPathManager(Path(ras_project_file).parent) href = pm.item_path(item_id) @@ -60,38 +79,62 @@ def from_prj(cls, ras_project_file, item_id: str, crs: str = None, simplify_geom NULL_STAC_GEOMETRY, NULL_STAC_BBOX, NULL_DATETIME, - properties, + {"ras_project_file": ras_project_file}, href=href, ) - stac.pf = ProjectFile(ras_project_file) - stac.has_2d = False - stac.has_1d = False - stac._geom_files = [] stac.crs = crs - stac.factory = AssetFactory(RAS_EXTENSION_MAPPING) stac.pm = pm - stac._href = href - - ras_asset_files = stac.scan_model_dir(ras_project_file) + stac.simplify_geometry = simplify_geometry - for fpath in ras_asset_files: + for fpath in stac.scan_model_dir(ras_project_file): if fpath and fpath != href: logging.info(f"Processing asset: {fpath}") stac.add_ras_asset(fpath) - stac.geometry, stac.bbox = stac.add_geometry(simplify_geometry) - stac.properties.update(stac.add_properties) - stac.datetime = stac._datetime if stac.crs: stac.apply_projection_extension(stac.crs) return stac @property - def add_properties(self) -> None: + def ras_project_file(self) -> str: + """Get the path to the HEC-RAS .prj file.""" + return self._properties.get("ras_project_file") + + @property + @lru_cache + def factory(self) -> AssetFactory: + """Return AssetFactory for this item.""" + return AssetFactory(RAS_EXTENSION_MAPPING) + + @property + @lru_cache + def pf(self) -> ProjectFile: + """Get a ProjectFile instance for the RAS Model .prj file.""" + return ProjectFile(self.ras_project_file) + + @property + def has_2d(self) -> bool: + """Whether any geometry file has 2D elements.""" + return any([a.check_2d for a in self.geometry_assets]) + + @property + def has_1d(self) -> bool: + """Whether any geometry file has 2D elements.""" + return any([a.check_1d for a in self.geometry_assets]) + + @property + def geometry_assets(self) -> list[RasGeomHdf]: + """Return any RasGeomHdf in assets.""" + return [a for a in self.assets.values() if isinstance(a, GeometryHdfAsset)] + + @property + def properties(self) -> None: """Properties for the RAS STAC item.""" - properties = {} + if self.ras_project_file is None: + return self._properties + properties = self._properties # properties[self.RAS_HAS_1D] = self.has_1d properties[self.RAS_HAS_2D] = self.has_2d properties[self.PROJECT_TITLE] = self.pf.project_title @@ -103,7 +146,13 @@ def add_properties(self) -> None: # TODO: once all assets are created, populate associations between assets return properties - def add_geometry(self, simplify_geometry: bool) -> dict | None: + @properties.setter + def properties(self, properties: dict): + """Set properties.""" + self._properties = properties + + @property + def geometry(self) -> dict | None: """Parse geometries from 2d hdf files and updates the stac item geometry, simplifying them if needed.""" geometries = [] @@ -118,20 +167,23 @@ def add_geometry(self, simplify_geometry: bool) -> dict | None: return NULL_STAC_GEOMETRY, NULL_STAC_BBOX unioned_geometry = union_all(geometries) - if simplify_geometry: - unioned_geometry = simplify(unioned_geometry, 0.001) - geometry = json.loads(to_geojson(unioned_geometry)) - bbox = unioned_geometry.bounds + unioned_geometry = self.reproject_to_wgs84(unioned_geometry, self.crs) + if self.simplify_geometry: + unioned_geometry = simplify(unioned_geometry, 0.001) + return json.loads(to_geojson(unioned_geometry)) - return geometry, bbox + @property + def bbox(self) -> list[float]: + """Get the bounding box of the model geometry.""" + return shape(self.geometry).bounds @property - def _datetime(self) -> datetime: + def datetime(self) -> datetime: """The datetime for the RAS STAC item.""" item_datetime = None - for geom_file in self._geom_files: + for geom_file in self.geometry_assets: if isinstance(geom_file, GeometryHdfAsset): geom_date = geom_file.hdf_object.geometry_time if geom_date: @@ -161,10 +213,10 @@ def add_model_thumbnails(self, layers: list, title_prefix: str = "Model_Thumbnai if thumbnail_dir: thumbnail_dest = thumbnail_dir else: - thumbnail_dest = self._href + thumbnail_dest = self.self_href - for geom in self._geom_files: - if isinstance(geom, GeometryHdfAsset): + for geom in self.geometry_assets: + if isinstance(geom, GeometryHdfAsset) and geom.has_2d: self.assets[f"{geom.href}_thumbnail"] = geom.thumbnail( layers=layers, title=title_prefix, thumbnail_dest=thumbnail_dest ) @@ -196,21 +248,15 @@ def add_ras_asset(self, fpath: str = "") -> None: # If item has crs, use it as the asset crs elif self.crs: asset.crs = self.crs - - if asset.check_2d: - self._geom_files = [] - self._geom_files.append(asset) - self.has_2d = True - self.properties[self.RAS_HAS_2D] = True # elif isinstance(asset, GeometryAsset): # if asset.geomf.has_1d: # self.has_1d = False TODO: Implement 1d functionality # self.properties[self.RAS_HAS_1D] = True - # self._geom_files.append(asset) + # self.geometry_assets.append(asset) - def _geometry_to_wgs84(self, geom: Geometry) -> Geometry: + def reproject_to_wgs84(self, geom: Geometry, crs) -> Geometry: """Convert geometry CRS to EPSG:4326 for stac item geometry.""" - pyproj_crs = CRS.from_user_input(self.crs) + pyproj_crs = CRS.from_user_input(crs) wgs_crs = CRS.from_authority("EPSG", "4326") if pyproj_crs != wgs_crs: transformer = Transformer.from_crs(pyproj_crs, wgs_crs, True) @@ -221,12 +267,11 @@ def parse_1d_geom(self): """Read 1d geometry from concave hull.""" logging.info("Creating geometry using 1d text file cross sections") concave_hull_polygons: list[Polygon] = [] - for geom_asset in self._geom_files: + for geom_asset in self.geometry_assets: if isinstance(geom_asset, GeometryAsset): try: geom_asset.crs = self.crs concave_hull = geom_asset.geomf.concave_hull - concave_hull = self._geometry_to_wgs84(concave_hull) concave_hull_polygons.append(concave_hull) except ValueError: logging.warning(f"Could not extract geometry from {geom_asset.href}") @@ -236,12 +281,13 @@ def parse_1d_geom(self): def parse_2d_geom(self): """Read 2d geometry from hdf file mesh areas.""" mesh_area_polygons: list[Polygon] = [] - for geom_asset in self._geom_files: + for geom_asset in self.geometry_assets: if isinstance(geom_asset, GeometryHdfAsset): - logging.info(f"Extracting geom from mesh areas in {geom_asset.href}") + if geom_asset.has_2d: + logging.info(f"Extracting geom from mesh areas in {geom_asset.href}") - mesh_areas = self._geometry_to_wgs84(geom_asset.hdf_object.mesh_areas(self.crs)) - mesh_area_polygons.append(mesh_areas) + mesh_areas = geom_asset.hdf_object.mesh_areas(self.crs) + mesh_area_polygons.append(mesh_areas) return union_all(mesh_area_polygons) @@ -262,3 +308,18 @@ def apply_projection_extension(self, crs: str): prj_ext = ProjectionExtension.ext(self, add_if_missing=True) og_crs = CRS(crs) prj_ext.apply(epsg=og_crs.to_epsg(), wkt2=og_crs.to_wkt()) + + @datetime.setter + def datetime(self, *args, **kwargs): + """Ignore.""" + pass + + @geometry.setter + def geometry(self, *args, **kwargs): + """Ignore.""" + pass + + @bbox.setter + def bbox(self, *args, **kwargs): + """Ignore.""" + pass diff --git a/pyproject.toml b/pyproject.toml index d19c700..259438d 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -23,7 +23,7 @@ classifiers = [ dependencies = [ "fiona==1.9.6", "geopandas==1.0.1", - "matplotlib==3.9.0", + "matplotlib==3.7.3", "pystac==1.10.0", "rasterio==1.3.10", "requests==2.32.3", @@ -59,6 +59,11 @@ line-length = 120 [tool.ruff.lint.per-file-ignores] "tests/**" = ["D"] "docs/**" = ["D"] +"hecstac/hms/**" = ["D"] +"server.py" = ["D"] +"hecstac/utils/**" = ["D"] +"hecstac/events/**" = ["D"] + [tool.setuptools.packages.find] where = ["."] From 104c834e6357c12df3ae5e6fcdecef5df9a802f4 Mon Sep 17 00:00:00 2001 From: Stevenray Janke Date: Wed, 5 Feb 2025 15:43:43 -0500 Subject: [PATCH 44/71] linting fix --- hecstac/ras/item.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/hecstac/ras/item.py b/hecstac/ras/item.py index ed9d706..2b06bcf 100644 --- a/hecstac/ras/item.py +++ b/hecstac/ras/item.py @@ -65,10 +65,9 @@ def from_prj(cls, ras_project_file, item_id: str, crs: str = None, simplify_geom Whether to simplify geometry. Defaults to True. Returns - ---------- + ------- stac : RASModelItem An instance of the class representing the STAC item. - """ pm = LocalPathManager(Path(ras_project_file).parent) From 7a7e876956a93b37521159fb01b112f497600b3f Mon Sep 17 00:00:00 2001 From: sclaw Date: Thu, 6 Feb 2025 08:17:12 -0500 Subject: [PATCH 45/71] add todo --- hecstac/ras/assets.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/hecstac/ras/assets.py b/hecstac/ras/assets.py index 41eab6a..38ff800 100644 --- a/hecstac/ras/assets.py +++ b/hecstac/ras/assets.py @@ -112,7 +112,7 @@ METEOROLOGY_UNITS = "ras:meteorology_units" -class ProjectAsset(GenericAsset): +class ProjectAsset(GenericAsset): # TODO: add super class PrjAsset that subplasses to Project and Projection """HEC-RAS Project file asset.""" regex_parse_str = r".+\.prj$" From a2a7cd87bc02d3eaf21791c9274e6c30d6244363 Mon Sep 17 00:00:00 2001 From: sclaw Date: Thu, 6 Feb 2025 08:57:24 -0500 Subject: [PATCH 46/71] post merge cleanup --- hecstac/ras/item.py | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/hecstac/ras/item.py b/hecstac/ras/item.py index 5cd39d9..9474bf1 100644 --- a/hecstac/ras/item.py +++ b/hecstac/ras/item.py @@ -76,7 +76,7 @@ def from_prj(cls, ras_project_file, item_id: str, crs: str = None, simplify_geom Whether to simplify geometry. Defaults to True. Returns - ---------- + ------- stac : RASModelItem An instance of the class representing the STAC item. @@ -86,14 +86,12 @@ def from_prj(cls, ras_project_file, item_id: str, crs: str = None, simplify_geom href = pm.item_path(item_id) assets = {Path(i).name: Asset(i, Path(i).name) for i in find_model_files(ras_project_file)} - stac = cls( Path(ras_project_file).stem, NULL_STAC_GEOMETRY, NULL_STAC_BBOX, NULL_DATETIME, {"ras_project_file": ras_project_file}, - {"ras_project_file": ras_project_file}, href=href, assets=assets, ) @@ -195,12 +193,10 @@ def properties(self, properties: dict): self._properties = properties @property - def datetime(self) -> datetime: def datetime(self) -> datetime: """The datetime for the RAS STAC item.""" item_datetime = None - for geom_file in self.geometry_assets: for geom_file in self.geometry_assets: if isinstance(geom_file, GeometryHdfAsset): geom_date = geom_file.hdf_object.geometry_time From d87937a23df056f93dc9a5c708c72fc953a7a0cc Mon Sep 17 00:00:00 2001 From: sclaw Date: Thu, 6 Feb 2025 09:15:29 -0500 Subject: [PATCH 47/71] finish asset refactor --- hecstac/ras/assets.py | 233 +++++++++++++++--------------------------- 1 file changed, 84 insertions(+), 149 deletions(-) diff --git a/hecstac/ras/assets.py b/hecstac/ras/assets.py index 1948424..9b742c6 100644 --- a/hecstac/ras/assets.py +++ b/hecstac/ras/assets.py @@ -546,25 +546,18 @@ class RunFileAsset(GenericAsset): """Run file asset for steady flow analysis.""" regex_parse_str = r".+\.r\d{2}$" - - def __init__(self, href: str, *args, **kwargs): - roles = kwargs.pop("roles", ["run-file", "ras-file", MediaType.TEXT]) - description = kwargs.pop( - "description", - "Run file for steady flow analysis which contains all the necessary input data required for the RAS computational engine.", - ) - super().__init__(href, roles=roles, description=description, *args, **kwargs) + __roles__ = ["run-file", "ras-file", MediaType.TEXT] + __description__ = "Run file for steady flow analysis which contains all the necessary input data required for the RAS computational engine." + __file_class__ = None class ComputationalLevelOutputAsset(GenericAsset): """Computational Level Output asset.""" regex_parse_str = r".+\.hyd\d{2}$" - - def __init__(self, href: str, *args, **kwargs): - roles = kwargs.pop("roles", ["computational-level-output-file", "ras-file", MediaType.TEXT]) - description = kwargs.pop("description", "Detailed Computational Level output file.") - super().__init__(href, roles=roles, description=description, *args, **kwargs) + __roles__ = ["computational-level-output-file", "ras-file", MediaType.TEXT] + __description__ = "Detailed Computational Level output file." + __file_class__ = None class GeometricPreprocessorAsset(GenericAsset): @@ -580,125 +573,99 @@ class BoundaryConditionAsset(GenericAsset): """Boundary Condition asset.""" regex_parse_str = r".+\.b\d{2}$" - - def __init__(self, href: str, *args, **kwargs): - roles = kwargs.pop("roles", ["boundary-condition-file", "ras-file", MediaType.TEXT]) - description = kwargs.pop("description", "Boundary Condition file.") - super().__init__(href, roles=roles, description=description, *args, **kwargs) + __roles__ = ["boundary-condition-file", "ras-file", MediaType.TEXT] + __description__ = "Boundary Condition file." + __file_class__ = None class UnsteadyFlowLogAsset(GenericAsset): """Unsteady Flow Log asset.""" regex_parse_str = r".+\.bco\d{2}$" - - def __init__(self, href: str, *args, **kwargs): - roles = kwargs.pop("roles", ["unsteady-flow-log-file", "ras-file", MediaType.TEXT]) - description = kwargs.pop("description", "Unsteady Flow Log output file.") - super().__init__(href, roles=roles, description=description, *args, **kwargs) + __roles__ = ["unsteady-flow-log-file", "ras-file", MediaType.TEXT] + __description__ = "Unsteady Flow Log output file." + __file_class__ = None class SedimentDataAsset(GenericAsset): """Sediment Data asset.""" regex_parse_str = r".+\.s\d{2}$" - - def __init__(self, href: str, *args, **kwargs): - roles = kwargs.pop("roles", ["sediment-data-file", "ras-file", MediaType.TEXT]) - description = kwargs.pop( - "description", "Sediment data file containing flow data, boundary conditions, and sediment data." - ) - super().__init__(href, roles=roles, description=description, *args, **kwargs) + __roles__ = ["sediment-data-file", "ras-file", MediaType.TEXT] + __description__ = "Sediment data file containing flow data, boundary conditions, and sediment data." + __file_class__ = None class HydraulicDesignAsset(GenericAsset): """Hydraulic Design asset.""" regex_parse_str = r".+\.h\d{2}$" - - def __init__(self, href: str, *args, **kwargs): - roles = kwargs.pop("roles", ["hydraulic-design-file", "ras-file", MediaType.TEXT]) - description = kwargs.pop("description", "Hydraulic Design data file.") - super().__init__(href, roles=roles, description=description, *args, **kwargs) + __roles__ = ["hydraulic-design-file", "ras-file", MediaType.TEXT] + __description__ = "Hydraulic Design data file." + __file_class__ = None class WaterQualityAsset(GenericAsset): """Water Quality asset.""" regex_parse_str = r".+\.w\d{2}$" - - def __init__(self, href: str, *args, **kwargs): - roles = kwargs.pop("roles", ["water-quality-file", "ras-file", MediaType.TEXT]) - description = kwargs.pop( - "description", "Water Quality file containing temperature boundary conditions and meteorological data." - ) - super().__init__(href, roles=roles, description=description, *args, **kwargs) + __roles__ = ["water-quality-file", "ras-file", MediaType.TEXT] + __description__ = "Water Quality file containing temperature boundary conditions and meteorological data." + __file_class__ = None class SedimentTransportCapacityAsset(GenericAsset): """Sediment Transport Capacity asset.""" regex_parse_str = r".+\.SedCap\d{2}$" - - def __init__(self, href: str, *args, **kwargs): - roles = kwargs.pop("roles", ["sediment-transport-capacity-file", "ras-file", MediaType.TEXT]) - description = kwargs.pop("description", "Sediment Transport Capacity data.") - super().__init__(href, roles=roles, description=description, *args, **kwargs) + __roles__ = ["sediment-transport-capacity-file", "ras-file", MediaType.TEXT] + __description__ = "Sediment Transport Capacity data." + __file_class__ = None class XSOutputAsset(GenericAsset): """Cross Section Output asset.""" regex_parse_str = r".+\.SedXS\d{2}$" - - def __init__(self, href: str, *args, **kwargs): - roles = kwargs.pop("roles", ["xs-output-file", "ras-file", MediaType.TEXT]) - description = kwargs.pop("description", "Cross section output file.") - super().__init__(href, roles=roles, description=description, *args, **kwargs) + __roles__ = ["xs-output-file", "ras-file", MediaType.TEXT] + __description__ = "Cross section output file." + __file_class__ = None class XSOutputHeaderAsset(GenericAsset): """Cross Section Output Header asset.""" regex_parse_str = r".+\.SedHeadXS\d{2}$" - - def __init__(self, href: str, *args, **kwargs): - roles = kwargs.pop("roles", ["xs-output-header-file", "ras-file", MediaType.TEXT]) - description = kwargs.pop("description", "Header file for the cross section output.") - super().__init__(href, roles=roles, description=description, *args, **kwargs) + __roles__ = ["xs-output-header-file", "ras-file", MediaType.TEXT] + __description__ = "Header file for the cross section output." + __file_class__ = None class WaterQualityRestartAsset(GenericAsset): """Water Quality Restart asset.""" regex_parse_str = r".+\.wqrst\d{2}$" - - def __init__(self, href: str, *args, **kwargs): - roles = kwargs.pop("roles", ["water-quality-restart-file", "ras-file", MediaType.TEXT]) - description = kwargs.pop("description", "The water quality restart file.") - super().__init__(href, roles=roles, description=description, *args, **kwargs) + __roles__ = ["water-quality-restart-file", "ras-file", MediaType.TEXT] + __description__ = "The water quality restart file." + __file_class__ = None class SedimentOutputAsset(GenericAsset): """Sediment Output asset.""" regex_parse_str = r".+\.sed$" - - def __init__(self, href: str, *args, **kwargs): - roles = kwargs.pop("roles", ["sediment-output-file", "ras-file", MediaType.TEXT]) - description = kwargs.pop("description", "Detailed sediment output file.") - super().__init__(href, roles=roles, description=description, *args, **kwargs) + __roles__ = ["sediment-output-file", "ras-file", MediaType.TEXT] + __description__ = "Detailed sediment output file." + __file_class__ = None class BinaryLogAsset(GenericAsset): """Binary Log asset.""" regex_parse_str = r".+\.blf$" - - def __init__(self, href: str, *args, **kwargs): - roles = kwargs.pop("roles", ["binary-log-file", "ras-file", MediaType.TEXT]) - description = kwargs.pop("description", "Binary Log file.") - super().__init__(href, roles=roles, description=description, *args, **kwargs) + __roles__ = ["binary-log-file", "ras-file", MediaType.TEXT] + __description__ = "Binary Log file." + __file_class__ = None class DSSAsset(GenericAsset): @@ -714,123 +681,99 @@ class LogAsset(GenericAsset): """Log asset.""" regex_parse_str = r".+\.log$" - - def __init__(self, href: str, *args, **kwargs): - roles = kwargs.pop("roles", ["ras-log", "ras-file", MediaType.TEXT]) - description = kwargs.pop("description", "The log file contains information related to simulation processes.") - super().__init__(href, roles=roles, description=description, *args, **kwargs) + __roles__ = ["ras-log", "ras-file", MediaType.TEXT] + __description__ = "The log file contains information related to simulation processes." + __file_class__ = None class RestartAsset(GenericAsset): """Restart file asset.""" regex_parse_str = r".+\.rst$" - - def __init__(self, href: str, *args, **kwargs): - roles = kwargs.pop("roles", ["restart-file", "ras-file", MediaType.TEXT]) - description = kwargs.pop("description", "Restart file for resuming simulation runs.") - super().__init__(href, roles=roles, description=description, *args, **kwargs) + __roles__ = ["restart-file", "ras-file", MediaType.TEXT] + __description__ = "Restart file for resuming simulation runs." + __file_class__ = None class SiamInputAsset(GenericAsset): """SIAM Input Data file asset.""" regex_parse_str = r".+\.SiamInput$" - - def __init__(self, href: str, *args, **kwargs): - roles = kwargs.pop("roles", ["siam-input-file", "ras-file", MediaType.TEXT]) - description = kwargs.pop("description", "SIAM Input Data file.") - super().__init__(href, roles=roles, description=description, *args, **kwargs) + __roles__ = ["siam-input-file", "ras-file", MediaType.TEXT] + __description__ = "SIAM Input Data file." + __file_class__ = None class SiamOutputAsset(GenericAsset): """SIAM Output Data file asset.""" regex_parse_str = r".+\.SiamOutput$" - - def __init__(self, href: str, *args, **kwargs): - roles = kwargs.pop("roles", ["siam-output-file", "ras-file", MediaType.TEXT]) - description = kwargs.pop("description", "SIAM Output Data file.") - super().__init__(href, roles=roles, description=description, *args, **kwargs) + __roles__ = ["siam-output-file", "ras-file", MediaType.TEXT] + __description__ = "SIAM Output Data file." + __file_class__ = None class WaterQualityLogAsset(GenericAsset): """Water Quality Log file asset.""" regex_parse_str = r".+\.bco$" - - def __init__(self, href: str, *args, **kwargs): - roles = kwargs.pop("roles", ["water-quality-log", "ras-file", MediaType.TEXT]) - description = kwargs.pop("description", "Water quality log file.") - super().__init__(href, roles=roles, description=description, *args, **kwargs) + __roles__ = ["water-quality-log", "ras-file", MediaType.TEXT] + __description__ = "Water quality log file." + __file_class__ = None class ColorScalesAsset(GenericAsset): """Color Scales file asset.""" regex_parse_str = r".+\.color-scales$" - - def __init__(self, href: str, *args, **kwargs): - roles = kwargs.pop("roles", ["color-scales", "ras-file", MediaType.TEXT]) - description = kwargs.pop("description", "File that contains the water quality color scale.") - super().__init__(href, roles=roles, description=description, *args, **kwargs) + __roles__ = ["color-scales", "ras-file", MediaType.TEXT] + __description__ = "File that contains the water quality color scale." + __file_class__ = None class ComputationalMessageAsset(GenericAsset): """Computational Message file asset.""" regex_parse_str = r".+\.comp-msgs.txt$" - - def __init__(self, href: str, *args, **kwargs): - roles = kwargs.pop("roles", ["computational-message-file", "ras-file", MediaType.TEXT]) - description = kwargs.pop( - "description", "Computational Message text file which contains messages from the computation process." - ) - super().__init__(href, roles=roles, description=description, *args, **kwargs) + __roles__ = ["computational-message-file", "ras-file", MediaType.TEXT] + __description__ = "Computational Message text file which contains messages from the computation process." + __file_class__ = None class UnsteadyRunFileAsset(GenericAsset): """Run file for Unsteady Flow asset.""" regex_parse_str = r".+\.x\d{2}$" - - def __init__(self, href: str, *args, **kwargs): - roles = kwargs.pop("roles", ["run-file", "ras-file", MediaType.TEXT]) - description = kwargs.pop("description", "Run file for Unsteady Flow simulations.") - super().__init__(href, roles=roles, description=description, *args, **kwargs) + __roles__ = ["run-file", "ras-file", MediaType.TEXT] + __description__ = "Run file for Unsteady Flow simulations." + __file_class__ = None class OutputFileAsset(GenericAsset): """Output RAS file asset.""" regex_parse_str = r".+\.o\d{2}$" - - def __init__(self, href: str, *args, **kwargs): - roles = kwargs.pop("roles", ["output-file", "ras-file", MediaType.TEXT]) - description = kwargs.pop("description", "Output RAS file which contains all computed results.") - super().__init__(href, roles=roles, description=description, *args, **kwargs) + __roles__ = ["output-file", "ras-file", MediaType.TEXT] + __description__ = "Output RAS file which contains all computed results." + __file_class__ = None class InitialConditionsFileAsset(GenericAsset): """Initial Conditions file asset.""" regex_parse_str = r".+\.IC\.O\d{2}$" - - def __init__(self, href: str, *args, **kwargs): - roles = kwargs.pop("roles", ["initial-conditions-file", "ras-file", MediaType.TEXT]) - description = kwargs.pop("description", "Initial conditions file for unsteady flow plan.") - super().__init__(href, roles=roles, description=description, *args, **kwargs) + __roles__ = ["initial-conditions-file", "ras-file", MediaType.TEXT] + __description__ = "Initial conditions file for unsteady flow plan." + __file_class__ = None class PlanRestartFileAsset(GenericAsset): """Restart file for Unsteady Flow Plan asset.""" regex_parse_str = r".+\.p\d{2}\.rst$" - - def __init__(self, href: str, *args, **kwargs): - roles = kwargs.pop("roles", ["restart-file", "ras-file", MediaType.TEXT]) - description = kwargs.pop("description", "Restart file for unsteady flow plan.") - super().__init__(href, roles=roles, description=description, *args, **kwargs) + __roles__ = ["restart-file", "ras-file", MediaType.TEXT] + __description__ = "Restart file for unsteady flow plan." + __file_class__ = None class RasMapperFileAsset(GenericAsset): @@ -846,44 +789,36 @@ class RasMapperBackupFileAsset(GenericAsset): """Backup RAS Mapper file asset.""" regex_parse_str = r".+\.rasmap\.backup$" - - def __init__(self, href: str, *args, **kwargs): - roles = kwargs.pop("roles", ["ras-mapper-file", "ras-file", MediaType.TEXT]) - description = kwargs.pop("description", "Backup RAS Mapper file.") - super().__init__(href, roles=roles, description=description, *args, **kwargs) + __roles__ = ["ras-mapper-file", "ras-file", MediaType.TEXT] + __description__ = "Backup RAS Mapper file." + __file_class__ = None class RasMapperOriginalFileAsset(GenericAsset): """Original RAS Mapper file asset.""" regex_parse_str = r".+\.rasmap\.original$" - - def __init__(self, href: str, *args, **kwargs): - roles = kwargs.pop("roles", ["ras-mapper-file", "ras-file", MediaType.TEXT]) - description = kwargs.pop("description", "Original RAS Mapper file.") - super().__init__(href, roles=roles, description=description, *args, **kwargs) + __roles__ = ["ras-mapper-file", "ras-file", MediaType.TEXT] + __description__ = "Original RAS Mapper file." + __file_class__ = None class MiscTextFileAsset(GenericAsset): """Miscellaneous Text file asset.""" regex_parse_str = r".+\.txt$" - - def __init__(self, href: str, *args, **kwargs): - roles = kwargs.pop("roles", [MediaType.TEXT]) - description = kwargs.pop("description", "Miscellaneous text file.") - super().__init__(href, roles=roles, description=description, *args, **kwargs) + __roles__ = [MediaType.TEXT] + __description__ = "Miscellaneous text file." + __file_class__ = None class MiscXMLFileAsset(GenericAsset): """Miscellaneous XML file asset.""" regex_parse_str = r".+\.xml$" - - def __init__(self, href: str, *args, **kwargs): - roles = kwargs.pop("roles", [MediaType.XML]) - description = kwargs.pop("description", "Miscellaneous XML file.") - super().__init__(href, roles=roles, description=description, *args, **kwargs) + __roles__ = [MediaType.XML] + __description__ = "Miscellaneous XML file." + __file_class__ = None RAS_ASSET_CLASSES = [ From 3bfc1caafefe03173a579c482e577dc464f6c37d Mon Sep 17 00:00:00 2001 From: sclaw Date: Thu, 6 Feb 2025 09:52:14 -0500 Subject: [PATCH 48/71] add handling for projection and project .prj assets --- hecstac/ras/assets.py | 155 +++++++++++++++++++++++++++--------------- hecstac/ras/utils.py | 2 +- 2 files changed, 100 insertions(+), 57 deletions(-) diff --git a/hecstac/ras/assets.py b/hecstac/ras/assets.py index 9b742c6..ddfd8dd 100644 --- a/hecstac/ras/assets.py +++ b/hecstac/ras/assets.py @@ -28,6 +28,7 @@ SteadyFlowFile, UnsteadyFlowFile, ) +from hecstac.ras.utils import is_ras_prj CURRENT_PLAN = "ras:current_plan" PLAN_SHORT_ID = "ras:short_plan_id" @@ -114,10 +115,34 @@ METEOROLOGY_UNITS = "ras:meteorology_units" -class ProjectAsset(GenericAsset): # TODO: add super class PrjAsset that subplasses to Project and Projection - """HEC-RAS Project file asset.""" +class PrjAsset(GenericAsset): + """A helper class to delegate .prj files into RAS project or Projection file classes.""" regex_parse_str = r".+\.prj$" + + def __new__(cls, *args, **kwargs): + """Delegate to Project or Projection asset.""" + if cls is PrjAsset: # Ensuring we don't instantiate Parent directly + href = kwargs.get("href") or args[0] + is_ras = is_ras_prj(href) + if is_ras: + return ProjectAsset(*args, **kwargs) + else: + return ProjectionAsset(*args, **kwargs) + return super().__new__(cls) + + +class ProjectionAsset(GenericAsset): + """A geospatial projection file.""" + + __roles__ = ["projection-file", MediaType.TEXT] + __description__ = "A geospatial projection file." + __file_class__ = None + + +class ProjectAsset(GenericAsset): + """HEC-RAS Project file asset.""" + __roles__ = ["project-file", "ras-file"] __description__ = "The HEC-RAS project file." __file_class__ = ProjectFile @@ -277,60 +302,78 @@ class PlanHdfAsset(GenericAsset): """HEC-RAS Plan HDF file asset.""" regex_parse_str = r".+\.p\d{2}\.hdf$" + __roles__ = ["ras-file"] + __description__ = "The HEC-RAS plan HDF file." + __file_class__ = PlanHDFFile - def __init__(self, href: str, **kwargs): - roles = kwargs.pop("roles", []) + ["ras-file"] - description = kwargs.pop("description", "The HEC-RAS plan HDF file.") - - super().__init__(href, roles=roles, description=description, **kwargs) - - self.hdf_object = PlanHDFFile(self.href) - self.extra_fields = { - key: value - for key, value in { - VERSION: self.hdf_object.file_version, - UNITS: self.hdf_object.units_system, - PLAN_INFORMATION_BASE_OUTPUT_INTERVAL: self.hdf_object.plan_information_base_output_interval, - PLAN_INFORMATION_COMPUTATION_TIME_STEP_BASE: self.hdf_object.plan_information_computation_time_step_base, - PLAN_INFORMATION_FLOW_FILENAME: self.hdf_object.plan_information_flow_filename, - PLAN_INFORMATION_GEOMETRY_FILENAME: self.hdf_object.plan_information_geometry_filename, - PLAN_INFORMATION_PLAN_FILENAME: self.hdf_object.plan_information_plan_filename, - PLAN_INFORMATION_PLAN_NAME: self.hdf_object.plan_information_plan_name, - PLAN_INFORMATION_PROJECT_FILENAME: self.hdf_object.plan_information_project_filename, - PLAN_INFORMATION_PROJECT_TITLE: self.hdf_object.plan_information_project_title, - PLAN_INFORMATION_SIMULATION_END_TIME: self.hdf_object.plan_information_simulation_end_time, - PLAN_INFORMATION_SIMULATION_START_TIME: self.hdf_object.plan_information_simulation_start_time, - PLAN_PARAMETERS_1D_FLOW_TOLERANCE: self.hdf_object.plan_parameters_1d_flow_tolerance, - PLAN_PARAMETERS_1D_MAXIMUM_ITERATIONS: self.hdf_object.plan_parameters_1d_maximum_iterations, - PLAN_PARAMETERS_1D_MAXIMUM_ITERATIONS_WITHOUT_IMPROVEMENT: self.hdf_object.plan_parameters_1d_maximum_iterations_without_improvement, - PLAN_PARAMETERS_1D_MAXIMUM_WATER_SURFACE_ERROR_TO_ABORT: self.hdf_object.plan_parameters_1d_maximum_water_surface_error_to_abort, - PLAN_PARAMETERS_1D_STORAGE_AREA_ELEVATION_TOLERANCE: self.hdf_object.plan_parameters_1d_storage_area_elevation_tolerance, - PLAN_PARAMETERS_1D_THETA: self.hdf_object.plan_parameters_1d_theta, - PLAN_PARAMETERS_1D_THETA_WARMUP: self.hdf_object.plan_parameters_1d_theta_warmup, - PLAN_PARAMETERS_1D_WATER_SURFACE_ELEVATION_TOLERANCE: self.hdf_object.plan_parameters_1d_water_surface_elevation_tolerance, - PLAN_PARAMETERS_1D2D_GATE_FLOW_SUBMERGENCE_DECAY_EXPONENT: self.hdf_object.plan_parameters_1d2d_gate_flow_submergence_decay_exponent, - PLAN_PARAMETERS_1D2D_IS_STABLITY_FACTOR: self.hdf_object.plan_parameters_1d2d_is_stablity_factor, - PLAN_PARAMETERS_1D2D_LS_STABLITY_FACTOR: self.hdf_object.plan_parameters_1d2d_ls_stablity_factor, - PLAN_PARAMETERS_1D2D_MAXIMUM_NUMBER_OF_TIME_SLICES: self.hdf_object.plan_parameters_1d2d_maximum_number_of_time_slices, - PLAN_PARAMETERS_1D2D_MINIMUM_TIME_STEP_FOR_SLICINGHOURS: self.hdf_object.plan_parameters_1d2d_minimum_time_step_for_slicinghours, - PLAN_PARAMETERS_1D2D_NUMBER_OF_WARMUP_STEPS: self.hdf_object.plan_parameters_1d2d_number_of_warmup_steps, - PLAN_PARAMETERS_1D2D_WARMUP_TIME_STEP_HOURS: self.hdf_object.plan_parameters_1d2d_warmup_time_step_hours, - PLAN_PARAMETERS_1D2D_WEIR_FLOW_SUBMERGENCE_DECAY_EXPONENT: self.hdf_object.plan_parameters_1d2d_weir_flow_submergence_decay_exponent, - PLAN_PARAMETERS_1D2D_MAXITER: self.hdf_object.plan_parameters_1d2d_maxiter, - PLAN_PARAMETERS_2D_EQUATION_SET: self.hdf_object.plan_parameters_2d_equation_set, - PLAN_PARAMETERS_2D_NAMES: self.hdf_object.plan_parameters_2d_names, - PLAN_PARAMETERS_2D_VOLUME_TOLERANCE: self.hdf_object.plan_parameters_2d_volume_tolerance, - PLAN_PARAMETERS_2D_WATER_SURFACE_TOLERANCE: self.hdf_object.plan_parameters_2d_water_surface_tolerance, - METEOROLOGY_DSS_FILENAME: self.hdf_object.meteorology_dss_filename, - METEOROLOGY_DSS_PATHNAME: self.hdf_object.meteorology_dss_pathname, - METEOROLOGY_DATA_TYPE: self.hdf_object.meteorology_data_type, - METEOROLOGY_MODE: self.hdf_object.meteorology_mode, - METEOROLOGY_RASTER_CELLSIZE: self.hdf_object.meteorology_raster_cellsize, - METEOROLOGY_SOURCE: self.hdf_object.meteorology_source, - METEOROLOGY_UNITS: self.hdf_object.meteorology_units, - }.items() - if value - } + @GenericAsset.extra_fields.getter + def extra_fields(self) -> dict: + """Return extra fields with added dynamic keys/values.""" + self._extra_fields[VERSION] = self.file.flow_title + self._extra_fields[UNITS] = self.file.units_system + self._extra_fields[PLAN_INFORMATION_BASE_OUTPUT_INTERVAL] = self.file.plan_information_base_output_interval + self._extra_fields[PLAN_INFORMATION_COMPUTATION_TIME_STEP_BASE] = ( + self.file.plan_information_computation_time_step_base + ) + self._extra_fields[PLAN_INFORMATION_FLOW_FILENAME] = self.file.plan_information_flow_filename + self._extra_fields[PLAN_INFORMATION_GEOMETRY_FILENAME] = self.file.plan_information_geometry_filename + self._extra_fields[PLAN_INFORMATION_PLAN_FILENAME] = self.file.plan_information_plan_filename + self._extra_fields[PLAN_INFORMATION_PLAN_NAME] = self.file.plan_information_plan_name + self._extra_fields[PLAN_INFORMATION_PROJECT_FILENAME] = self.file.plan_information_project_filename + self._extra_fields[PLAN_INFORMATION_PROJECT_TITLE] = self.file.plan_information_project_title + self._extra_fields[PLAN_INFORMATION_SIMULATION_END_TIME] = self.file.plan_information_simulation_end_time + self._extra_fields[PLAN_INFORMATION_SIMULATION_START_TIME] = self.file.plan_information_simulation_start_time + self._extra_fields[PLAN_PARAMETERS_1D_FLOW_TOLERANCE] = self.file.plan_parameters_1d_flow_tolerance + self._extra_fields[PLAN_PARAMETERS_1D_MAXIMUM_ITERATIONS] = self.file.plan_parameters_1d_maximum_iterations + self._extra_fields[PLAN_PARAMETERS_1D_MAXIMUM_ITERATIONS_WITHOUT_IMPROVEMENT] = ( + self.file.plan_parameters_1d_maximum_iterations_without_improvement + ) + self._extra_fields[PLAN_PARAMETERS_1D_MAXIMUM_WATER_SURFACE_ERROR_TO_ABORT] = ( + self.file.plan_parameters_1d_maximum_water_surface_error_to_abort + ) + self._extra_fields[PLAN_PARAMETERS_1D_STORAGE_AREA_ELEVATION_TOLERANCE] = ( + self.file.plan_parameters_1d_storage_area_elevation_tolerance + ) + self._extra_fields[PLAN_PARAMETERS_1D_THETA] = self.file.plan_parameters_1d_theta + self._extra_fields[PLAN_PARAMETERS_1D_THETA_WARMUP] = self.file.plan_parameters_1d_theta_warmup + self._extra_fields[PLAN_PARAMETERS_1D_WATER_SURFACE_ELEVATION_TOLERANCE] = ( + self.file.plan_parameters_1d_water_surface_elevation_tolerance + ) + self._extra_fields[PLAN_PARAMETERS_1D2D_GATE_FLOW_SUBMERGENCE_DECAY_EXPONENT] = ( + self.file.plan_parameters_1d2d_gate_flow_submergence_decay_exponent + ) + self._extra_fields[PLAN_PARAMETERS_1D2D_IS_STABLITY_FACTOR] = self.file.plan_parameters_1d2d_is_stablity_factor + self._extra_fields[PLAN_PARAMETERS_1D2D_LS_STABLITY_FACTOR] = self.file.plan_parameters_1d2d_ls_stablity_factor + self._extra_fields[PLAN_PARAMETERS_1D2D_MAXIMUM_NUMBER_OF_TIME_SLICES] = ( + self.file.plan_parameters_1d2d_maximum_number_of_time_slices + ) + self._extra_fields[PLAN_PARAMETERS_1D2D_MINIMUM_TIME_STEP_FOR_SLICINGHOURS] = ( + self.file.plan_parameters_1d2d_minimum_time_step_for_slicinghours + ) + self._extra_fields[PLAN_PARAMETERS_1D2D_NUMBER_OF_WARMUP_STEPS] = ( + self.file.plan_parameters_1d2d_number_of_warmup_steps + ) + self._extra_fields[PLAN_PARAMETERS_1D2D_WARMUP_TIME_STEP_HOURS] = ( + self.file.plan_parameters_1d2d_warmup_time_step_hours + ) + self._extra_fields[PLAN_PARAMETERS_1D2D_WEIR_FLOW_SUBMERGENCE_DECAY_EXPONENT] = ( + self.file.plan_parameters_1d2d_weir_flow_submergence_decay_exponent + ) + self._extra_fields[PLAN_PARAMETERS_1D2D_MAXITER] = self.file.plan_parameters_1d2d_maxiter + self._extra_fields[PLAN_PARAMETERS_2D_EQUATION_SET] = self.file.plan_parameters_2d_equation_set + self._extra_fields[PLAN_PARAMETERS_2D_NAMES] = self.file.plan_parameters_2d_names + self._extra_fields[PLAN_PARAMETERS_2D_VOLUME_TOLERANCE] = self.file.plan_parameters_2d_volume_tolerance + self._extra_fields[PLAN_PARAMETERS_2D_WATER_SURFACE_TOLERANCE] = ( + self.file.plan_parameters_2d_water_surface_tolerance + ) + self._extra_fields[METEOROLOGY_DSS_FILENAME] = self.file.meteorology_dss_filename + self._extra_fields[METEOROLOGY_DSS_PATHNAME] = self.file.meteorology_dss_pathname + self._extra_fields[METEOROLOGY_DATA_TYPE] = self.file.meteorology_data_type + self._extra_fields[METEOROLOGY_MODE] = self.file.meteorology_mode + self._extra_fields[METEOROLOGY_RASTER_CELLSIZE] = self.file.meteorology_raster_cellsize + self._extra_fields[METEOROLOGY_SOURCE] = self.file.meteorology_source + self._extra_fields[METEOROLOGY_UNITS] = self.file.meteorology_units + return self._extra_fields class GeometryHdfAsset(GenericAsset): @@ -822,7 +865,7 @@ class MiscXMLFileAsset(GenericAsset): RAS_ASSET_CLASSES = [ - ProjectAsset, + PrjAsset, PlanAsset, GeometryAsset, SteadyFlowAsset, diff --git a/hecstac/ras/utils.py b/hecstac/ras/utils.py index d66a6c0..9c1ba4b 100644 --- a/hecstac/ras/utils.py +++ b/hecstac/ras/utils.py @@ -18,7 +18,7 @@ def find_model_files(ras_prj: str) -> list[str]: ras_prj = Path(ras_prj) parent = ras_prj.parent stem = Path(ras_prj).name.split(".")[0] - return [str(i.as_posix()) for i in parent.glob(f"{stem}*") if i != ras_prj] + return [str(i.as_posix()) for i in parent.glob(f"{stem}*")] def is_ras_prj(url: str) -> bool: From ad14f59ec8d34ea42412147d7f2e375a9b1ba7b4 Mon Sep 17 00:00:00 2001 From: sclaw Date: Thu, 6 Feb 2025 10:17:24 -0500 Subject: [PATCH 49/71] change logging to module level instead of root --- hecstac/common/asset_factory.py | 10 ++++++---- hecstac/events/ffrd.py | 8 +++++--- hecstac/events/logger.py | 2 +- hecstac/hms/item.py | 20 +++++++++++--------- hecstac/hms/parser.py | 10 ++++++---- hecstac/ras/assets.py | 8 +++++--- hecstac/ras/item.py | 10 ++++++---- hecstac/ras/parser.py | 12 +++++++----- hecstac/ras/utils.py | 4 +++- new_ras_item.py | 7 +++++-- 10 files changed, 55 insertions(+), 36 deletions(-) diff --git a/hecstac/common/asset_factory.py b/hecstac/common/asset_factory.py index 0c8e104..0935e26 100644 --- a/hecstac/common/asset_factory.py +++ b/hecstac/common/asset_factory.py @@ -10,6 +10,8 @@ from hecstac.hms.s3_utils import check_storage_extension +logger = logging.getLogger(__name__) + def is_ras_prj(url: str) -> bool: """Check if a file is a HEC-RAS project file.""" @@ -114,7 +116,7 @@ def create_hms_asset(self, fpath: str, item_type: str = "model") -> Asset: def create_ras_asset(self, fpath: str): """Create an asset instance based on the file extension.""" - logging.debug(f"Creating asset for {fpath}") + logger.debug(f"Creating asset for {fpath}") from hecstac.ras.assets import ProjectAsset if fpath.lower().endswith(".prj"): @@ -125,15 +127,15 @@ def create_ras_asset(self, fpath: str): for pattern, asset_class in self.extension_to_asset.items(): if pattern.match(fpath): - logging.debug(f"Matched {pattern} for {Path(fpath).name}: {asset_class}") + logger.debug(f"Matched {pattern} for {Path(fpath).name}: {asset_class}") return asset_class(href=fpath, title=Path(fpath).name) - logging.warning(f"Unable to pattern match asset for file {fpath}") + logger.warning(f"Unable to pattern match asset for file {fpath}") return GenericAsset(href=fpath, title=Path(fpath).name) def asset_from_dict(self, asset: Asset): fpath = asset.href for pattern, asset_class in self.extension_to_asset.items(): if pattern.match(fpath): - logging.debug(f"Matched {pattern} for {Path(fpath).name}: {asset_class}") + logger.debug(f"Matched {pattern} for {Path(fpath).name}: {asset_class}") return asset_class.from_dict(asset.to_dict()) diff --git a/hecstac/events/ffrd.py b/hecstac/events/ffrd.py index c67efee..65f7f6c 100644 --- a/hecstac/events/ffrd.py +++ b/hecstac/events/ffrd.py @@ -15,6 +15,8 @@ from hecstac.hms.assets import HMS_EXTENSION_MAPPING from hecstac.ras.assets import RAS_EXTENSION_MAPPING +logger = logging.getLogger(__name__) + class FFRDEventItem(Item): FFRD_REALIZATION = "FFRD:realization" @@ -66,7 +68,7 @@ def _register_extensions(self) -> None: def _add_model_links(self) -> None: """Add links to the model items.""" for item in self.source_model_items: - logging.info(f"Adding link from source model item: {item.id}") + logger.info(f"Adding link from source model item: {item.id}") link = Link( rel="derived_from", target=item, @@ -116,7 +118,7 @@ def _bbox(self) -> list[float]: def add_hms_asset(self, fpath: str, item_type: str = "event") -> None: """Add an asset to the FFRD Event STAC item.""" if os.path.exists(fpath): - logging.info(f"Adding asset: {fpath}") + logger.info(f"Adding asset: {fpath}") asset = self.hms_factory.create_hms_asset(fpath, item_type=item_type) if asset is not None: self.add_asset(asset.title, asset) @@ -124,7 +126,7 @@ def add_hms_asset(self, fpath: str, item_type: str = "event") -> None: def add_ras_asset(self, fpath: str) -> None: """Add an asset to the FFRD Event STAC item.""" if os.path.exists(fpath): - logging.info(f"Adding asset: {fpath}") + logger.info(f"Adding asset: {fpath}") asset = self.ras_factory.create_ras_asset(fpath) if asset is not None: self.add_asset(asset.title, asset) diff --git a/hecstac/events/logger.py b/hecstac/events/logger.py index b9bed7d..8b03ca0 100644 --- a/hecstac/events/logger.py +++ b/hecstac/events/logger.py @@ -6,7 +6,7 @@ SUPPRESS_LOGS = ["boto3", "botocore", "geopandas", "fiona", "rasterio", "pyogrio", "xarray", "shapely", "matplotlib"] -def initialize_logger(json_logging: bool = False, level: int = logging.INFO): +def initialize_logger(json_logging: bool = False, level: int = logger.INFO): datefmt = "%Y-%m-%dT%H:%M:%SZ" if json_logging: for module in SUPPRESS_LOGS: diff --git a/hecstac/hms/item.py b/hecstac/hms/item.py index b0b694e..3f3e7c5 100644 --- a/hecstac/hms/item.py +++ b/hecstac/hms/item.py @@ -18,6 +18,8 @@ from hecstac.hms.assets import HMS_EXTENSION_MAPPING, ProjectAsset from hecstac.hms.parser import BasinFile, ProjectFile +logger = logging.getLogger(__name__) + class HMSModelItem(Item): """An object representation of a HEC-HMS model.""" @@ -84,7 +86,7 @@ def _properties(self): properties["proj:code"] = self.pf.basins[0].epsg if self.pf.basins[0].epsg: - logging.warning("No EPSG code found in basin file.") + logger.warning("No EPSG code found in basin file.") properties["proj:wkt"] = self.pf.basins[0].wkt properties[SUMMARY] = self.pf.file_counts return properties @@ -121,7 +123,7 @@ def _check_files_exists(self, files: list[str]): for file in files: if not os.path.exists(file): - logging.warning(f"File not found {file}") + logger.warning(f"File not found {file}") def make_thumbnails(self, basins: list[BasinFile], overwrite: bool = False): """Create a png for each basin. Optionally overwrite existing files.""" @@ -129,9 +131,9 @@ def make_thumbnails(self, basins: list[BasinFile], overwrite: bool = False): thumbnail_path = self.pm.derived_item_asset(f"{bf.name}.png".replace(" ", "_").replace("-", "_")) if not overwrite and os.path.exists(thumbnail_path): - logging.info(f"Thumbnail for basin `{bf.name}` already exists. Skipping creation.") + logger.info(f"Thumbnail for basin `{bf.name}` already exists. Skipping creation.") else: - logging.info(f"{'Overwriting' if overwrite else 'Creating'} thumbnail for basin `{bf.name}`") + logger.info(f"{'Overwriting' if overwrite else 'Creating'} thumbnail for basin `{bf.name}`") fig = self.make_thumbnail(bf.hms_schematic_2_gdfs) fig.savefig(thumbnail_path) fig.clf() @@ -140,14 +142,14 @@ def make_thumbnails(self, basins: list[BasinFile], overwrite: bool = False): def write_element_geojsons(self, basins: list[BasinFile], overwrite: bool = False): """Write the HMS elements (Subbasins, Juctions, Reaches, etc.) to geojson.""" for element_type in basins.elements.element_types: - logging.debug(f"Checking if geojson for {element_type} exists") + logger.debug(f"Checking if geojson for {element_type} exists") path = self.pm.derived_item_asset(f"{element_type}.geojson") if not overwrite and os.path.exists(path): - logging.info(f"Geojson for {element_type} already exists. Skipping creation.") + logger.info(f"Geojson for {element_type} already exists. Skipping creation.") else: - logging.info(f"Creating geojson for {element_type}") + logger.info(f"Creating geojson for {element_type}") gdf = self.pf.basins[0].feature_2_gdf(element_type).to_crs(4326) - logging.debug(gdf.columns) + logger.debug(gdf.columns) keep_columns = ["name", "geometry", "Last Modified Date", "Last Modified Time", "Number Subreaches"] gdf = gdf[[col for col in keep_columns if col in gdf.columns]] gdf.to_file(path) @@ -161,7 +163,7 @@ def add_hms_asset(self, fpath: str) -> None: self.add_asset(asset.title, asset) if isinstance(asset, ProjectAsset): if self._project is not None: - logging.error( + logger.error( f"Only one project asset is allowed. Found {str(asset)} when {str(self._project)} was already set." ) self._project = asset diff --git a/hecstac/hms/parser.py b/hecstac/hms/parser.py index 2762be3..62a2ff1 100644 --- a/hecstac/hms/parser.py +++ b/hecstac/hms/parser.py @@ -43,6 +43,8 @@ Temperature, ) +logger = logging.getLogger(__name__) + class BaseTextFile(ABC): def __init__(self, path: str, client=None, bucket=None): @@ -65,7 +67,7 @@ def read_content(self): response = self.client.get_object(Bucket=self.bucket, Key=self.path) self.content = response["Body"].read().decode() except Exception as e: - logging.error(e) + logger.error(e) raise FileNotFoundError(f"could not find {self.path} locally nor on s3") def parse_header(self): @@ -218,7 +220,7 @@ def assert_uniform_version(self): @property def files(self): - # logging.info(f"other paths {[i.path for i in [self.terrain, self.run, self.grid, self.gage, self.pdata] if i]}") + # logger.info(f"other paths {[i.path for i in [self.terrain, self.run, self.grid, self.gage, self.pdata] if i]}") return ( [self.path] @@ -581,7 +583,7 @@ def junction_connection_lines(self) -> gpd.GeoDataFrame: for junction in self.junctions: us_point = junction.geom if "Downstream" not in junction.attrs: - logging.warning(f"Warning no downstream element for junction {junction.name}") + logger.warning(f"Warning no downstream element for junction {junction.name}") continue ds_element = self.elements[junction.attrs["Downstream"]] if ds_element in self.reaches: @@ -800,7 +802,7 @@ def __init__(self, path: str, client=None, bucket=None): response = client.get_object(Bucket=bucket, Key=path) self.content = response["Body"].read().decode() except Exception as e: - logging.info(f" {e}: No Paired Data File found: creating empty Paired Data File") + logger.info(f" {e}: No Paired Data File found: creating empty Paired Data File") self.create_pdata(path) super().__init__(path, client=client, bucket=bucket) self.elements = ElementSet() diff --git a/hecstac/ras/assets.py b/hecstac/ras/assets.py index ddfd8dd..92b2b06 100644 --- a/hecstac/ras/assets.py +++ b/hecstac/ras/assets.py @@ -30,6 +30,8 @@ ) from hecstac.ras.utils import is_ras_prj +logger = logging.getLogger(__name__) + CURRENT_PLAN = "ras:current_plan" PLAN_SHORT_ID = "ras:short_plan_id" TITLE = "ras:title" @@ -404,12 +406,12 @@ def reference_lines(self) -> list[gpd.GeoDataFrame] | None: def has_2d(self) -> bool: """Check if the geometry asset has 2d geometry.""" try: - logging.debug(f"reading mesh areas using crs {self.crs}...") + logger.debug(f"reading mesh areas using crs {self.crs}...") if self.hdf_object.mesh_areas(self.crs): return True except ValueError: - logging.warning(f"No mesh areas found for {self.href}") + logger.warning(f"No mesh areas found for {self.href}") return False @property @@ -553,7 +555,7 @@ def thumbnail( bc_lines_data_geo = bc_lines_data.set_crs(self.crs) legend_handles += self._plot_bc_lines(ax, bc_lines_data_geo) except Exception as e: - logging.warning(f"Warning: Failed to process layer '{layer}' for {self.href}: {e}") + logger.warning(f"Warning: Failed to process layer '{layer}' for {self.href}: {e}") # Add OpenStreetMap basemap ctx.add_basemap( diff --git a/hecstac/ras/item.py b/hecstac/ras/item.py index 9474bf1..5a87162 100644 --- a/hecstac/ras/item.py +++ b/hecstac/ras/item.py @@ -37,6 +37,8 @@ from hecstac.ras.parser import ProjectFile from hecstac.ras.utils import find_model_files +logger = logging.getLogger(__name__) + class RASModelItem(Item): """An object representation of a HEC-RAS model.""" @@ -153,10 +155,10 @@ def crs(self, crs): def geometry(self) -> dict: """Return footprint of model as a geojson.""" if self.crs is None: - logging.warning("Geometry requested for model with no spatial reference.") + logger.warning("Geometry requested for model with no spatial reference.") return NULL_STAC_GEOMETRY if len(self.geometry_assets) == 0: - logging.error("No geometry found for RAS item.") + logger.error("No geometry found for RAS item.") return NULL_STAC_GEOMETRY geometries = [i.geometry_wgs84 for i in self.geometry_assets] @@ -203,11 +205,11 @@ def datetime(self) -> datetime: if geom_date: item_datetime = geom_date self.properties[self.RAS_DATETIME_SOURCE] = "model_geometry" - logging.info(f"Using item datetime from {geom_file.href}") + logger.info(f"Using item datetime from {geom_file.href}") break if item_datetime is None: - logging.warning("Could not extract item datetime from geometry, using item processing time.") + logger.warning("Could not extract item datetime from geometry, using item processing time.") item_datetime = datetime.datetime.now() self.properties[self.RAS_DATETIME_SOURCE] = "processing_time" return item_datetime diff --git a/hecstac/ras/parser.py b/hecstac/ras/parser.py index 553efb3..66af068 100644 --- a/hecstac/ras/parser.py +++ b/hecstac/ras/parser.py @@ -27,6 +27,8 @@ text_block_from_start_str_to_empty_line, ) +logger = logging.getLogger(__name__) + def name_from_suffix(fpath: str, suffix: str) -> str: """Generate a name by appending a suffix to the file stem.""" @@ -693,7 +695,7 @@ def plan_current(self) -> str | None: suffix = search_contents(self.file_lines, "Current Plan", expect_one=True, require_one=False).strip() return name_from_suffix(self.fpath, suffix) except Exception: - logging.warning("Ras model has no current plan") + logger.warning("Ras model has no current plan") return None @property @@ -706,7 +708,7 @@ def ras_version(self) -> str | None: self.file_lines, "Program and Version", token=":", expect_one=False, require_one=False ) if version == []: - logging.warning("Unable to parse project version") + logger.warning("Unable to parse project version") return "N/A" else: return version[0] @@ -798,7 +800,7 @@ def breach_locations(self) -> dict: if len(parts) >= 4: key = parts[4].strip() breach_dict[key] = eval(parts[3].strip()) - logging.debug(f"breach_dict {breach_dict}") + logger.debug(f"breach_dict {breach_dict}") return breach_dict @@ -1037,7 +1039,7 @@ def boundary_locations(self) -> list: flow_area = parts[5].strip() bc_line = parts[7].strip() boundary_dict.append({flow_area: bc_line}) - logging.debug(f"boundary_dict:{boundary_dict}") + logger.debug(f"boundary_dict:{boundary_dict}") return boundary_dict @property @@ -1574,6 +1576,6 @@ def reference_lines(self) -> gpd.GeoDataFrame | None: ref_lines = self.hdf_object.reference_lines() if ref_lines is None or ref_lines.empty: - logging.warning("No reference lines found.") + logger.warning("No reference lines found.") else: return ref_lines diff --git a/hecstac/ras/utils.py b/hecstac/ras/utils.py index 9c1ba4b..c564c72 100644 --- a/hecstac/ras/utils.py +++ b/hecstac/ras/utils.py @@ -12,6 +12,8 @@ from shapely.errors import UnsupportedGEOSVersionError from shapely.geometry import LineString, MultiPoint, Point +logger = logging.getLogger(__name__) + def find_model_files(ras_prj: str) -> list[str]: """Find all files with same base name.""" @@ -181,7 +183,7 @@ def check_xs_direction(cross_sections: gpd.GeoDataFrame, reach: LineString): river_reach_rs.append(xs["river_reach_rs"]) except IndexError as e: - logging.debug( + logger.debug( f"cross section does not intersect river-reach: {xs['river']} {xs['reach']} {xs['river_station']}: error: {e}" ) continue diff --git a/new_ras_item.py b/new_ras_item.py index ac66c69..d361426 100644 --- a/new_ras_item.py +++ b/new_ras_item.py @@ -2,8 +2,11 @@ import logging from pathlib import Path -from hecstac.ras.logger import initialize_logger + from hecstac import RASModelItem +from hecstac.ras.logger import initialize_logger + +logger = logging.getLogger(__name__) def sanitize_catalog_assets(item: RASModelItem) -> RASModelItem: @@ -31,4 +34,4 @@ def sanitize_catalog_assets(item: RASModelItem) -> RASModelItem: ras_item = sanitize_catalog_assets(ras_item) # ras_item.add_model_thumbnails(["mesh_areas", "breaklines", "bc_lines"]) ras_item.save_object(ras_item.pm.item_path(item_id)) - logging.info(f"Saved {ras_item.pm.item_path(item_id)}") + logger.info(f"Saved {ras_item.pm.item_path(item_id)}") From f04792484f19a4653abb0797fbf1cfaa07bebabd Mon Sep 17 00:00:00 2001 From: sclaw Date: Thu, 6 Feb 2025 13:03:37 -0500 Subject: [PATCH 50/71] update logging --- hecstac/events/logger.py | 32 -------------------------------- hecstac/hms/logger.py | 32 -------------------------------- hecstac/ras/logger.py | 33 --------------------------------- 3 files changed, 97 deletions(-) delete mode 100644 hecstac/events/logger.py delete mode 100644 hecstac/hms/logger.py delete mode 100644 hecstac/ras/logger.py diff --git a/hecstac/events/logger.py b/hecstac/events/logger.py deleted file mode 100644 index 8b03ca0..0000000 --- a/hecstac/events/logger.py +++ /dev/null @@ -1,32 +0,0 @@ -"""Logging utility and setup.""" - -import logging -import sys - -SUPPRESS_LOGS = ["boto3", "botocore", "geopandas", "fiona", "rasterio", "pyogrio", "xarray", "shapely", "matplotlib"] - - -def initialize_logger(json_logging: bool = False, level: int = logger.INFO): - datefmt = "%Y-%m-%dT%H:%M:%SZ" - if json_logging: - for module in SUPPRESS_LOGS: - logging.getLogger(module).setLevel(logging.WARNING) - - class FlushStreamHandler(logging.StreamHandler): - def emit(self, record): - super().emit(record) - self.flush() - - handler = FlushStreamHandler(sys.stdout) - - logging.basicConfig( - level=level, - handlers=[handler], - format="""{"time": "%(asctime)s" , "level": "%(levelname)s", "msg": "%(message)s"}""", - datefmt=datefmt, - ) - else: - for package in SUPPRESS_LOGS: - logging.getLogger(package).setLevel(logging.ERROR) - logging.basicConfig(level=level, format="%(asctime)s | %(levelname)s | %(message)s", datefmt=datefmt) - # boto3.set_stream_logger(name="botocore.credentials", level=logging.ERROR) diff --git a/hecstac/hms/logger.py b/hecstac/hms/logger.py deleted file mode 100644 index b9bed7d..0000000 --- a/hecstac/hms/logger.py +++ /dev/null @@ -1,32 +0,0 @@ -"""Logging utility and setup.""" - -import logging -import sys - -SUPPRESS_LOGS = ["boto3", "botocore", "geopandas", "fiona", "rasterio", "pyogrio", "xarray", "shapely", "matplotlib"] - - -def initialize_logger(json_logging: bool = False, level: int = logging.INFO): - datefmt = "%Y-%m-%dT%H:%M:%SZ" - if json_logging: - for module in SUPPRESS_LOGS: - logging.getLogger(module).setLevel(logging.WARNING) - - class FlushStreamHandler(logging.StreamHandler): - def emit(self, record): - super().emit(record) - self.flush() - - handler = FlushStreamHandler(sys.stdout) - - logging.basicConfig( - level=level, - handlers=[handler], - format="""{"time": "%(asctime)s" , "level": "%(levelname)s", "msg": "%(message)s"}""", - datefmt=datefmt, - ) - else: - for package in SUPPRESS_LOGS: - logging.getLogger(package).setLevel(logging.ERROR) - logging.basicConfig(level=level, format="%(asctime)s | %(levelname)s | %(message)s", datefmt=datefmt) - # boto3.set_stream_logger(name="botocore.credentials", level=logging.ERROR) diff --git a/hecstac/ras/logger.py b/hecstac/ras/logger.py deleted file mode 100644 index 2e5a3d7..0000000 --- a/hecstac/ras/logger.py +++ /dev/null @@ -1,33 +0,0 @@ -"""Logging utility and setup.""" - -import logging -import sys - -SUPPRESS_LOGS = ["boto3", "botocore", "geopandas", "fiona", "rasterio", "pyogrio", "xarray", "shapely", "matplotlib"] - - -def initialize_logger(json_logging: bool = False, level: int = logging.INFO): - """Initialize the ras logger.""" - datefmt = "%Y-%m-%dT%H:%M:%SZ" - if json_logging: - for module in SUPPRESS_LOGS: - logging.getLogger(module).setLevel(logging.WARNING) - - class FlushStreamHandler(logging.StreamHandler): - def emit(self, record): - super().emit(record) - self.flush() - - handler = FlushStreamHandler(sys.stdout) - - logging.basicConfig( - level=level, - handlers=[handler], - format="""{"time": "%(asctime)s" , "level": "%(levelname)s", "msg": "%(message)s"}""", - datefmt=datefmt, - ) - else: - for package in SUPPRESS_LOGS: - logging.getLogger(package).setLevel(logging.ERROR) - logging.basicConfig(level=level, format="%(asctime)s | %(levelname)s | %(message)s", datefmt=datefmt) - # boto3.set_stream_logger(name="botocore.credentials", level=logging.ERROR) From b51bbe21dec54905c7ee05046cf59e12c2e99840 Mon Sep 17 00:00:00 2001 From: sclaw Date: Thu, 6 Feb 2025 13:05:24 -0500 Subject: [PATCH 51/71] commit forgotten files --- hecstac/common/geometry.py | 13 +++++++++++++ hecstac/common/logger.py | 35 +++++++++++++++++++++++++++++++++++ 2 files changed, 48 insertions(+) create mode 100644 hecstac/common/geometry.py create mode 100644 hecstac/common/logger.py diff --git a/hecstac/common/geometry.py b/hecstac/common/geometry.py new file mode 100644 index 0000000..537a543 --- /dev/null +++ b/hecstac/common/geometry.py @@ -0,0 +1,13 @@ +from pyproj import CRS, Transformer +from shapely import Geometry +from shapely.ops import transform + + +def reproject_to_wgs84(geom: Geometry, crs: str) -> Geometry: + """Convert geometry CRS to EPSG:4326 for stac item geometry.""" + pyproj_crs = CRS.from_user_input(crs) + wgs_crs = CRS.from_authority("EPSG", "4326") + if pyproj_crs != wgs_crs: + transformer = Transformer.from_crs(pyproj_crs, wgs_crs, True) + return transform(transformer.transform, geom) + return geom diff --git a/hecstac/common/logger.py b/hecstac/common/logger.py new file mode 100644 index 0000000..afa99e9 --- /dev/null +++ b/hecstac/common/logger.py @@ -0,0 +1,35 @@ +"""Logging utility and setup.""" + +import logging +import sys +from re import L + +SUPPRESS_LOGS = ["boto3", "botocore", "geopandas", "fiona", "rasterio", "pyogrio", "xarray", "shapely", "matplotlib"] + + +def initialize_logger(json_logging: bool = False, level: int = logging.INFO): + """Initialize the ras logger.""" + logger = logging.getLogger("hecstac") + logger.setLevel(level) + if json_logging: + for module in SUPPRESS_LOGS: + logging.getLogger(module).setLevel(logging.WARNING) + + class FlushStreamHandler(logging.StreamHandler): + def emit(self, record): + super().emit(record) + self.flush() + + handler = FlushStreamHandler(sys.stdout) + + handler.setLevel(level) + + datefmt = "%Y-%m-%dT%H:%M:%SZ" + fmt = """{"time": "%(asctime)s" , "level": "%(levelname)s", "msg": "%(message)s"}""" + formatter = logging.Formatter(fmt=fmt, datefmt=datefmt) + handler.setFormatter(formatter) + + logger.addHandler(handler) + else: + for package in SUPPRESS_LOGS: + logging.getLogger(package).setLevel(logging.ERROR) From 09ff6014b25740d4279e38002fb25804ed82a524 Mon Sep 17 00:00:00 2001 From: sclaw Date: Thu, 6 Feb 2025 13:06:53 -0500 Subject: [PATCH 52/71] cleanup --- hecstac/ras/item.py | 1 - 1 file changed, 1 deletion(-) diff --git a/hecstac/ras/item.py b/hecstac/ras/item.py index 5a87162..63ee344 100644 --- a/hecstac/ras/item.py +++ b/hecstac/ras/item.py @@ -97,7 +97,6 @@ def from_prj(cls, ras_project_file, item_id: str, crs: str = None, simplify_geom href=href, assets=assets, ) - stac.crs if crs: stac.crs = crs stac.simplify_geometry = simplify_geometry From 553fd136c81dcd7c25f876a8b077262e0da96906 Mon Sep 17 00:00:00 2001 From: sclaw Date: Thu, 6 Feb 2025 14:51:33 -0500 Subject: [PATCH 53/71] update CRS methods to allow GeometryFile to function without CRS. Asset applies it's own CRS later. --- hecstac/common/asset_factory.py | 13 +- hecstac/ras/assets.py | 62 ++++------ hecstac/ras/parser.py | 204 ++++++++++++++++++++------------ hecstac/ras/utils.py | 103 ++++++++++++++++ 4 files changed, 261 insertions(+), 121 deletions(-) diff --git a/hecstac/common/asset_factory.py b/hecstac/common/asset_factory.py index 0935e26..5f01915 100644 --- a/hecstac/common/asset_factory.py +++ b/hecstac/common/asset_factory.py @@ -2,7 +2,7 @@ import logging from pathlib import Path -from typing import Dict, Type +from typing import ClassVar, Dict, Generic, Type, TypeVar import pystac from pyproj import CRS @@ -12,6 +12,8 @@ logger = logging.getLogger(__name__) +T = TypeVar("T") # Generic for asset file accessor classes + def is_ras_prj(url: str) -> bool: """Check if a file is a HEC-RAS project file.""" @@ -23,9 +25,14 @@ def is_ras_prj(url: str) -> bool: return False -class GenericAsset(Asset): +class GenericAsset(Asset, Generic[T]): """Provides a base structure for assets.""" + regex_parse_str: str + __roles__: list[str] + __description__: str + __file_class__: T + def __init__(self, *args, **kwargs): super().__init__(*args, **kwargs) if self.description is None: @@ -58,7 +65,7 @@ def extra_fields(self, extra_fields: dict): self._extra_fields = extra_fields @property - def file(self): + def file(self) -> T: """Return class to access asset file contents.""" return self.__file_class__(self.href) diff --git a/hecstac/ras/assets.py b/hecstac/ras/assets.py index 92b2b06..2777a5a 100644 --- a/hecstac/ras/assets.py +++ b/hecstac/ras/assets.py @@ -142,7 +142,7 @@ class ProjectionAsset(GenericAsset): __file_class__ = None -class ProjectAsset(GenericAsset): +class ProjectAsset(GenericAsset[ProjectFile]): """HEC-RAS Project file asset.""" __roles__ = ["project-file", "ras-file"] @@ -161,7 +161,7 @@ def extra_fields(self) -> dict: return self._extra_fields -class PlanAsset(GenericAsset): +class PlanAsset(GenericAsset[PlanFile]): """HEC-RAS Plan file asset.""" regex_parse_str = r".+\.p\d{2}$" @@ -180,7 +180,7 @@ def extra_fields(self) -> dict: return self._extra_fields -class GeometryAsset(GenericAsset): +class GeometryAsset(GenericAsset[GeometryFile]): """HEC-RAS Geometry file asset.""" regex_parse_str = r".+\.g\d{2}$" @@ -198,38 +198,32 @@ def extra_fields(self) -> dict: self._extra_fields[VERSION] = self.file.geom_version self._extra_fields[HAS_1D] = self.file.has_1d self._extra_fields[HAS_2D] = self.file.has_2d - # self._extra_fields[RIVERS] = self.file.rivers - # self._extra_fields[REACHES] = self.file.reaches - # self._extra_fields[JUNCTIONS] = self.file.junctions - # self._extra_fields[CROSS_SECTIONS] = self.file.cross_sections - # self._extra_fields[STRUCTURES] = self.file.structures - # self._extra_fields[STORAGE_AREAS] = self.file.storage_areas - # self._extra_fields[CONNECTIONS] = self.file.connections - # self._extra_fields[BREACH_LOCATIONS] = self.file.breach_locations + self._extra_fields[RIVERS] = list(self.file.rivers.keys()) + self._extra_fields[REACHES] = list(self.file.reaches.keys()) + self._extra_fields[JUNCTIONS] = list(self.file.junctions.keys()) + self._extra_fields[CROSS_SECTIONS] = list(self.file.cross_sections.keys()) + self._extra_fields[STRUCTURES] = list(self.file.structures.keys()) + self._extra_fields[STORAGE_AREAS] = list(self.file.storage_areas.keys()) + self._extra_fields[CONNECTIONS] = list(self.file.connections.keys()) return self._extra_fields - @property - def file(self): - """Return class to access asset file contents.""" - return self.__file_class__(self.href, self.owner.crs) - @property @lru_cache def geometry(self) -> Polygon | MultiPolygon: """Retrieves concave hull of cross-sections.""" - return Polygon([(0, 0), (0, 1), (1, 1), (1, 0)]) # TODO: fill this in. + return self.file.concave_hull @property @lru_cache def has_1d(self) -> bool: """Check if geometry has any river centerlines.""" - return False # TODO: implement + return self.file.has_1d @property @lru_cache def has_2d(self) -> bool: """Check if geometry has any 2D areas.""" - return False # TODO: implement + return self.file.has_2d @property @lru_cache @@ -239,7 +233,7 @@ def geometry_wgs84(self) -> Polygon | MultiPolygon: return reproject_to_wgs84(self.geometry, self.crs) -class SteadyFlowAsset(GenericAsset): +class SteadyFlowAsset(GenericAsset[SteadyFlowFile]): """HEC-RAS Steady Flow file asset.""" regex_parse_str = r".+\.f\d{2}$" @@ -255,7 +249,7 @@ def extra_fields(self) -> dict: return self._extra_fields -class QuasiUnsteadyFlowAsset(GenericAsset): +class QuasiUnsteadyFlowAsset(GenericAsset[QuasiUnsteadyFlowFile]): """HEC-RAS Quasi-Unsteady Flow file asset.""" # TODO: implement this class @@ -269,21 +263,10 @@ class QuasiUnsteadyFlowAsset(GenericAsset): def extra_fields(self) -> dict: """Return extra fields with added dynamic keys/values.""" self._extra_fields[TITLE] = self.file.flow_title - self._extra_fields[VERSION] = self.file.geom_version - self._extra_fields[HAS_1D] = self.file.has_1d - self._extra_fields[HAS_2D] = self.file.has_2d - self._extra_fields[RIVERS] = self.file.rivers - self._extra_fields[REACHES] = self.file.reaches - self._extra_fields[JUNCTIONS] = self.file.junctions - self._extra_fields[CROSS_SECTIONS] = self.file.cross_sections - self._extra_fields[STRUCTURES] = self.file.structures - self._extra_fields[STORAGE_AREAS] = self.file.storage_areas - self._extra_fields[CONNECTIONS] = self.file.connections - self._extra_fields[BREACH_LOCATIONS] = self.file.breach_locations return self._extra_fields -class UnsteadyFlowAsset(GenericAsset): +class UnsteadyFlowAsset(GenericAsset[UnsteadyFlowFile]): """HEC-RAS Unsteady Flow file asset.""" regex_parse_str = r".+\.u\d{2}$" @@ -300,7 +283,7 @@ def extra_fields(self) -> dict: return self._extra_fields -class PlanHdfAsset(GenericAsset): +class PlanHdfAsset(GenericAsset[PlanHDFFile]): """HEC-RAS Plan HDF file asset.""" regex_parse_str = r".+\.p\d{2}\.hdf$" @@ -378,7 +361,7 @@ def extra_fields(self) -> dict: return self._extra_fields -class GeometryHdfAsset(GenericAsset): +class GeometryHdfAsset(GenericAsset[GeometryHDFFile]): """HEC-RAS Geometry HDF file asset.""" regex_parse_str = r".+\.g\d{2}\.hdf$" @@ -391,14 +374,13 @@ def extra_fields(self) -> dict: """Return extra fields with added dynamic keys/values.""" self._extra_fields[VERSION] = self.file.file_version self._extra_fields[UNITS] = self.file.units_system - self._extra_fields[PROJECTION] = self.owner.crs.to_wkt() - self._extra_fields[UNITS] = self.file.units_system + self._extra_fields[REFERENCE_LINES] = self.file.reference_lines return self._extra_fields @property def reference_lines(self) -> list[gpd.GeoDataFrame] | None: """Docstring.""" # TODO: fill out - if self.hdf_object.reference_lines is not None and not self.hdf_object.reference_lines.empty: + if self.file.reference_lines is not None and not self.file.reference_lines.empty: return list(self.file.reference_lines["refln_name"]) @property @@ -408,7 +390,7 @@ def has_2d(self) -> bool: try: logger.debug(f"reading mesh areas using crs {self.crs}...") - if self.hdf_object.mesh_areas(self.crs): + if self.file.mesh_areas(self.crs): return True except ValueError: logger.warning(f"No mesh areas found for {self.href}") @@ -424,7 +406,7 @@ def has_1d(self) -> bool: @lru_cache def geometry(self, crs: CRS) -> Polygon | MultiPolygon: """Retrieves concave hull of cross-sections.""" - return self.hdf_object.mesh_areas(crs) + return self.file.mesh_areas(crs) @property @lru_cache diff --git a/hecstac/ras/parser.py b/hecstac/ras/parser.py index 66af068..e3e7225 100644 --- a/hecstac/ras/parser.py +++ b/hecstac/ras/parser.py @@ -15,12 +15,14 @@ from pyproj.exceptions import CRSError from pystac import Asset from rashdf import RasGeomHdf, RasPlanHdf -from shapely import LineString, MultiPolygon, Point, Polygon, make_valid, union_all +from shapely import GeometryCollection, LineString, MultiPolygon, Point, Polygon, make_valid, union_all from shapely.ops import unary_union from hecstac.ras.utils import ( + check_xs_direction, data_pairs_from_text_block, delimited_pairs_to_lists, + reverse, search_contents, text_block_from_start_end_str, text_block_from_start_str_length, @@ -46,9 +48,8 @@ def __init__(self, river: str, reaches: list[str] = []): class XS: """HEC-RAS Cross Section.""" - def __init__(self, ras_data: list[str], river_reach: str, river: str, reach: str, crs: str): + def __init__(self, ras_data: list[str], river_reach: str, river: str, reach: str): self.ras_data = ras_data - self.crs = crs self.river = river self.reach = reach self.river_reach = river_reach @@ -165,7 +166,6 @@ def gdf(self) -> gpd.GeoDataFrame: "number_of_coords": [self.number_of_coords], # "coords": [self.coords], }, - crs=self.crs, geometry="geometry", ) @@ -310,11 +310,9 @@ def __init__( river_reach: str, river: str, reach: str, - crs: str, us_xs: XS, ): self.ras_data = ras_data - self.crs = crs self.river = river self.reach = reach self.river_reach = river_reach @@ -392,7 +390,6 @@ def gdf(self) -> gpd.GeoDataFrame: "width": [self.width], "ras_data": ["\n".join(self.ras_data)], }, - crs=self.crs, geometry="geometry", ) @@ -400,10 +397,9 @@ def gdf(self) -> gpd.GeoDataFrame: class Reach: """HEC-RAS River Reach.""" - def __init__(self, ras_data: list[str], river_reach: str, crs: str): + def __init__(self, ras_data: list[str], river_reach: str): reach_lines = text_block_from_start_end_str(f"River Reach={river_reach}", ["River Reach"], ras_data, -1) self.ras_data = reach_lines - self.crs = crs self.river_reach = river_reach self.river = river_reach.split(",")[0].rstrip() self.reach = river_reach.split(",")[1].rstrip() @@ -469,7 +465,7 @@ def cross_sections(self) -> dict[str, "XS"]: ["Type RM Length L Ch R", "River Reach"], self.ras_data, ) - cross_section = XS(xs_lines, self.river_reach, self.river, self.reach, self.crs) + cross_section = XS(xs_lines, self.river_reach, self.river, self.reach) cross_sections[cross_section.river_reach_rs] = cross_section return cross_sections @@ -486,7 +482,7 @@ def structures(self) -> dict[str, "Structure"]: ["Type RM Length L Ch R", "River Reach"], self.ras_data, ) - cross_section = XS(xs_lines, self.river_reach, self.river, self.reach, self.crs) + cross_section = XS(xs_lines, self.river_reach, self.river, self.reach) continue elif int(type) in [2, 3, 4, 5, 6]: # culvert or bridge or multiple openeing structure_lines = text_block_from_start_end_str( @@ -505,7 +501,6 @@ def structures(self) -> dict[str, "Structure"]: self.river_reach, self.river, self.reach, - self.crs, cross_section, ) structures[structure.river_reach_rs] = structure @@ -525,7 +520,6 @@ def gdf(self) -> gpd.GeoDataFrame: # "coords": [self.coords], "ras_data": ["\n".join(self.ras_data)], }, - crs=self.crs, geometry="geometry", ) @@ -543,8 +537,7 @@ def structures_gdf(self) -> gpd.GeoDataFrame: class Junction: """HEC-RAS Junction.""" - def __init__(self, ras_data: list[str], junct: str, crs: str): - self.crs = crs + def __init__(self, ras_data: list[str], junct: str): self.name = junct self.ras_data = text_block_from_start_str_to_empty_line(f"Junct Name={junct}", ras_data) @@ -630,15 +623,13 @@ def gdf(self): "ras_data": ["\n".join(self.ras_data)], }, geometry="geometry", - crs=self.crs, ) class StorageArea: """HEC-RAS StorageArea.""" - def __init__(self, ras_data: list[str], crs: str): - self.crs = crs + def __init__(self, ras_data: list[str]): self.ras_data = ras_data # TODO: Implement this @@ -646,8 +637,7 @@ def __init__(self, ras_data: list[str], crs: str): class Connection: """HEC-RAS Connection.""" - def __init__(self, ras_data: list[str], crs: str): - self.crs = crs + def __init__(self, ras_data: list[str]): self.ras_data = ras_data # TODO: Implement this @@ -807,12 +797,11 @@ def breach_locations(self) -> dict: class GeometryFile: """HEC-RAS Geometry file asset.""" - def __init__(self, fpath, crs): + def __init__(self, fpath): # TODO: Compare with HMS implementation self.fpath = fpath - self.crs = crs with open(fpath, "r") as f: - self.file_lines = f.readlines() + self.file_lines = f.read().splitlines() @property def geom_title(self) -> str: @@ -825,7 +814,33 @@ def geom_version(self) -> str: return search_contents(self.file_lines, "Program Version") @property - def rivers(self) -> dict[str, "River"]: + def datetimes(self) -> list[datetime.datetime]: + """Get the latest node last updated entry for this geometry.""" + dts = search_contents(self.file_lines, "Node Last Edited Time", expect_one=False, require_one=False) + if len(dts) >= 1: + try: + return [datetime.datetime.strptime(d, "%b/%d/%Y %H:%M:%S") for d in dts] + except ValueError: + return [] + else: + return [] + + @property + def has_2d(self) -> bool: + """Check if RAS geometry has any 2D areas.""" + for line in self.file_lines: + if line.startswith("Storage Area Is2D=") and int(line[len("Storage Area Is2D=") :].strip()) in (1, -1): + # RAS mostly uses "-1" to indicate True and "0" to indicate False. Checking for "1" also here. + return True + return False + + @property + def has_1d(self) -> bool: + """Check if RAS geometry has any 1D components.""" + return len(self.cross_sections) > 0 + + @property + def rivers(self) -> dict[str, River]: """A dictionary of river_name: River (class) for the rivers contained in the HEC-RAS geometry file.""" tmp_rivers = defaultdict(list) for reach in self.reaches.values(): # First, group all reaches into their respective rivers @@ -838,19 +853,19 @@ def rivers(self) -> dict[str, "River"]: return tmp_rivers @property - def reaches(self) -> dict[str, "Reach"]: + def reaches(self) -> dict[str, Reach]: """A dictionary of the reaches contained in the HEC-RAS geometry file.""" river_reaches = search_contents(self.file_lines, "River Reach", expect_one=False, require_one=False) - return {river_reach: Reach(self.file_lines, river_reach, self.crs) for river_reach in river_reaches} + return {river_reach: Reach(self.file_lines, river_reach) for river_reach in river_reaches} @property - def junctions(self) -> dict[str, "Junction"]: + def junctions(self) -> dict[str, Junction]: """A dictionary of the junctions contained in the HEC-RAS geometry file.""" - juncts = search_contents(self.file_lines, "Junct Name", expect_one=False) - return {junction: Junction(self.file_lines, junction, self.crs) for junction in juncts} + juncts = search_contents(self.file_lines, "Junct Name", expect_one=False, require_one=False) + return {junction: Junction(self.file_lines, junction) for junction in juncts} @property - def cross_sections(self) -> dict[str, "XS"]: + def cross_sections(self) -> dict[str, XS]: """A dictionary of all the cross sections contained in the HEC-RAS geometry file.""" cross_sections = {} for reach in self.reaches.values(): @@ -858,7 +873,7 @@ def cross_sections(self) -> dict[str, "XS"]: return cross_sections @property - def structures(self) -> dict[str, "Structure"]: + def structures(self) -> dict[str, Structure]: """A dictionary of the structures contained in the HEC-RAS geometry file.""" structures = {} for reach in self.reaches.values(): @@ -866,67 +881,100 @@ def structures(self) -> dict[str, "Structure"]: return structures @property - def storage_areas(self) -> dict[str, "StorageArea"]: + def storage_areas(self) -> dict[str, StorageArea]: """A dictionary of the storage areas contained in the HEC-RAS geometry file.""" - areas = search_contents(self.file_lines, "Storage Area", expect_one=False) - return {a: StorageArea(a, self.crs) for a in areas} + areas = search_contents(self.file_lines, "Storage Area", expect_one=False, require_one=False) + return {a: StorageArea(a) for a in areas} @property - def connections(self) -> dict[str, "Connection"]: + def connections(self) -> dict[str, Connection]: """A dictionary of the SA/2D connections contained in the HEC-RAS geometry file.""" - connections = search_contents(self.file_lines, "Connection", expect_one=False) - return {c: Connection(c, self.crs) for c in connections} + connections = search_contents(self.file_lines, "Connection", expect_one=False, require_one=False) + return {c: Connection(c) for c in connections} @property - def datetimes(self) -> list[datetime.datetime]: - """Get the latest node last updated entry for this geometry.""" - dts = search_contents(self.file_lines, "Node Last Edited Time", expect_one=False) - if len(dts) >= 1: - try: - return [datetime.datetime.strptime(d, "%b/%d/%Y %H:%M:%S") for d in dts] - except ValueError: - return [] - else: - return [] + def reach_gdf(self): + """A GeodataFrame of the reaches contained in the HEC-RAS geometry file.""" + return gpd.GeoDataFrame(pd.concat([reach.gdf for reach in self.reaches.values()], ignore_index=True)) @property - def has_2d(self) -> bool: - """Check if RAS geometry has any 2D areas.""" - for line in self.file_lines: - if line.startswith("Storage Area Is2D=") and int(line[len("Storage Area Is2D=") :].strip()) in (1, -1): - # RAS mostly uses "-1" to indicate True and "0" to indicate False. Checking for "1" also here. - return True - return False + def junction_gdf(self): + """A GeodataFrame of the junctions contained in the HEC-RAS geometry file.""" + if self.junctions: + return gpd.GeoDataFrame( + pd.concat( + [junction.gdf for junction in self.junctions.values()], + ignore_index=True, + ) + ) @property - def has_1d(self) -> bool: - """Check if RAS geometry has any 1D components.""" - return len(self.cross_sections) > 0 + def xs_gdf(self) -> gpd.GeoDataFrame: + """Geodataframe of all cross sections in the geometry text file.""" + xs_gdf = pd.concat([xs.gdf for xs in self.cross_sections.values()], ignore_index=True) + + subsets = [] + for _, reach in self.reach_gdf.iterrows(): + subset_xs = xs_gdf.loc[xs_gdf["river_reach"] == reach["river_reach"]].copy() + not_reversed_xs = check_xs_direction(subset_xs, reach.geometry) + subset_xs["geometry"] = subset_xs.apply( + lambda row: ( + row.geometry + if row["river_reach_rs"] in list(not_reversed_xs["river_reach_rs"]) + else reverse(row.geometry) + ), + axis=1, + ) + subsets.append(subset_xs) + return gpd.GeoDataFrame(pd.concat(subsets)) @property - def concave_hull(self) -> Polygon: + def structures_gdf(self) -> gpd.GeoDataFrame: + """Geodataframe of all structures in the geometry text file.""" + return gpd.GeoDataFrame(pd.concat([structure.gdf for structure in self.structures.values()], ignore_index=True)) + + @property + @lru_cache + def concave_hull(self): """Compute and return the concave hull (polygon) for cross sections.""" polygons = [] - if self.cross_sections: - xs_gdf = pd.concat([xs.gdf for xs in self.cross_sections.values()], ignore_index=True) - for river_reach in xs_gdf["river_reach"].unique(): - xs_subset: gpd.GeoSeries = xs_gdf[xs_gdf["river_reach"] == river_reach] - points = xs_subset.boundary.explode(index_parts=True).unstack() - points_last_xs = [Point(coord) for coord in xs_subset["geometry"].iloc[-1].coords] - points_first_xs = [Point(coord) for coord in xs_subset["geometry"].iloc[0].coords[::-1]] - polygon = Polygon(points_first_xs + list(points[0]) + points_last_xs + list(points[1])[::-1]) - if isinstance(polygon, MultiPolygon): - polygons += list(polygon.geoms) - else: - polygons.append(polygon) - if len(self.junctions) > 0: - for junction in self.junctions.values(): - for _, j in junction.gdf.iterrows(): - polygons.append(self.junction_hull(xs_gdf, j)) - out_hull = union_all([make_valid(p) for p in polygons]) - return out_hull - else: - raise ValueError(f"No cross sections found for {self.fpath}. Cannot calculate geometry") + xs_df = self.xs_gdf # shorthand + assert not all( + [i.is_empty for i in xs_df.geometry] + ), "No valid cross-sections found. Possibly non-georeferenced model" + assert len(xs_df) > 1, "Only one valid cross-section found." + for river_reach in xs_df["river_reach"].unique(): + xs_subset = xs_df[xs_df["river_reach"] == river_reach] + points = xs_subset.boundary.explode(index_parts=True).unstack() + points_last_xs = [Point(coord) for coord in xs_subset["geometry"].iloc[-1].coords] + points_first_xs = [Point(coord) for coord in xs_subset["geometry"].iloc[0].coords[::-1]] + polygon = Polygon(points_first_xs + list(points[0]) + points_last_xs + list(points[1])[::-1]) + if isinstance(polygon, MultiPolygon): + polygons += list(polygon.geoms) + else: + polygons.append(polygon) + if self.junction_gdf is not None: + for _, j in self.junction_gdf.iterrows(): + polygons.append(self.junction_hull(j)) + out_hull = self.clean_polygons(polygons) + return out_hull + + def clean_polygons(self, polygons: list) -> list: + """Make polygons valid and remove geometry collections.""" + all_valid = [] + for p in polygons: + valid = make_valid(p) + if isinstance(valid, GeometryCollection): + polys = [] + for i in valid.geoms: + if isinstance(i, MultiPolygon): + polys.extend([j for j in i.geoms]) + elif isinstance(i, Polygon): + polys.append(i) + all_valid.extend(polys) + else: + all_valid.append(valid) + return union_all(all_valid) def junction_hull(self, xs_gdf: gpd.GeoDataFrame, junction: gpd.GeoSeries) -> Polygon: """Compute and return the concave hull (polygon) for a juction.""" diff --git a/hecstac/ras/utils.py b/hecstac/ras/utils.py index c564c72..b3925c5 100644 --- a/hecstac/ras/utils.py +++ b/hecstac/ras/utils.py @@ -202,3 +202,106 @@ def validate_point(geom): raise IndexError(f"expected point at xs-river intersection got: {type(geom)} | {geom}") else: raise TypeError(f"expected point at xs-river intersection got: {type(geom)} | {geom}") + + +class requires_geos: + """Unsure.""" + + def __init__(self, version): + if version.count(".") != 2: + raise ValueError("Version must be .. format") + self.version = tuple(int(x) for x in version.split(".")) + + def __call__(self, func): + is_compatible = lib.geos_version >= self.version + is_doc_build = os.environ.get("SPHINX_DOC_BUILD") == "1" # set in docs/conf.py + if is_compatible and not is_doc_build: + return func # return directly, do not change the docstring + + msg = "'{}' requires at least GEOS {}.{}.{}.".format(func.__name__, *self.version) + if is_compatible: + + @wraps(func) + def wrapped(*args, **kwargs): + return func(*args, **kwargs) + + else: + + @wraps(func) + def wrapped(*args, **kwargs): + raise UnsupportedGEOSVersionError(msg) + + doc = wrapped.__doc__ + if doc: + # Insert the message at the first double newline + position = doc.find("\n\n") + 2 + # Figure out the indentation level + indent = 0 + while True: + if doc[position + indent] == " ": + indent += 1 + else: + break + wrapped.__doc__ = doc.replace("\n\n", "\n\n{}.. note:: {}\n\n".format(" " * indent, msg), 1) + + return wrapped + + +def multithreading_enabled(func): + """ + Prepare multithreading by setting the writable flags of object type ndarrays to False. + + NB: multithreading also requires the GIL to be released, which is done in + the C extension (ufuncs.c). + """ + + @wraps(func) + def wrapped(*args, **kwargs): + array_args = [arg for arg in args if isinstance(arg, np.ndarray) and arg.dtype == object] + [ + arg + for name, arg in kwargs.items() + if name not in {"where", "out"} and isinstance(arg, np.ndarray) and arg.dtype == object + ] + old_flags = [arr.flags.writeable for arr in array_args] + try: + for arr in array_args: + arr.flags.writeable = False + return func(*args, **kwargs) + finally: + for arr, old_flag in zip(array_args, old_flags): + arr.flags.writeable = old_flag + + return wrapped + + +@requires_geos("3.7.0") +@multithreading_enabled +def reverse(geometry, **kwargs): + """Returns a copy of a Geometry with the order of coordinates reversed. + + If a Geometry is a polygon with interior rings, the interior rings are also + reversed. + + Points are unchanged. None is returned where Geometry is None. + + Parameters + ---------- + geometry : Geometry or array_like + **kwargs + See :ref:`NumPy ufunc docs ` for other keyword arguments. + + See Also + -------- + is_ccw : Checks if a Geometry is clockwise. + + Examples + -------- + >>> from shapely import LineString, Polygon + >>> reverse(LineString([(0, 0), (1, 2)])) + + >>> reverse(Polygon([(0, 0), (1, 0), (1, 1), (0, 1), (0, 0)])) + + >>> reverse(None) is None + True + """ + return lib.reverse(geometry, **kwargs) From 44616ebd0bf0acd3e42f8fcea1e2623b21b54272 Mon Sep 17 00:00:00 2001 From: sclaw Date: Thu, 6 Feb 2025 15:29:49 -0500 Subject: [PATCH 54/71] debug --- hecstac/common/asset_factory.py | 7 ++++--- hecstac/ras/assets.py | 11 ++++------- hecstac/ras/item.py | 4 ++-- hecstac/ras/parser.py | 7 ++----- 4 files changed, 12 insertions(+), 17 deletions(-) diff --git a/hecstac/common/asset_factory.py b/hecstac/common/asset_factory.py index 5f01915..302b058 100644 --- a/hecstac/common/asset_factory.py +++ b/hecstac/common/asset_factory.py @@ -28,9 +28,9 @@ def is_ras_prj(url: str) -> bool: class GenericAsset(Asset, Generic[T]): """Provides a base structure for assets.""" - regex_parse_str: str - __roles__: list[str] - __description__: str + regex_parse_str: str = r"" + __roles__: list[str] = [] + __description__: str = "" __file_class__: T def __init__(self, *args, **kwargs): @@ -146,3 +146,4 @@ def asset_from_dict(self, asset: Asset): if pattern.match(fpath): logger.debug(f"Matched {pattern} for {Path(fpath).name}: {asset_class}") return asset_class.from_dict(asset.to_dict()) + return asset diff --git a/hecstac/ras/assets.py b/hecstac/ras/assets.py index 2777a5a..1beba67 100644 --- a/hecstac/ras/assets.py +++ b/hecstac/ras/assets.py @@ -294,7 +294,7 @@ class PlanHdfAsset(GenericAsset[PlanHDFFile]): @GenericAsset.extra_fields.getter def extra_fields(self) -> dict: """Return extra fields with added dynamic keys/values.""" - self._extra_fields[VERSION] = self.file.flow_title + self._extra_fields[VERSION] = self.file.file_version self._extra_fields[UNITS] = self.file.units_system self._extra_fields[PLAN_INFORMATION_BASE_OUTPUT_INTERVAL] = self.file.plan_information_base_output_interval self._extra_fields[PLAN_INFORMATION_COMPUTATION_TIME_STEP_BASE] = ( @@ -388,12 +388,9 @@ def reference_lines(self) -> list[gpd.GeoDataFrame] | None: def has_2d(self) -> bool: """Check if the geometry asset has 2d geometry.""" try: - logger.debug(f"reading mesh areas using crs {self.crs}...") - - if self.file.mesh_areas(self.crs): + if self.file.mesh_areas(): return True except ValueError: - logger.warning(f"No mesh areas found for {self.href}") return False @property @@ -404,9 +401,9 @@ def has_1d(self) -> bool: @property @lru_cache - def geometry(self, crs: CRS) -> Polygon | MultiPolygon: + def geometry(self) -> Polygon | MultiPolygon: """Retrieves concave hull of cross-sections.""" - return self.file.mesh_areas(crs) + return self.file.mesh_areas(self.crs) @property @lru_cache diff --git a/hecstac/ras/item.py b/hecstac/ras/item.py index 63ee344..85c61d3 100644 --- a/hecstac/ras/item.py +++ b/hecstac/ras/item.py @@ -133,7 +133,7 @@ def has_1d(self) -> bool: @property def geometry_assets(self) -> list[RasGeomHdf | GeometryAsset]: """Return any RasGeomHdf in assets.""" - return [a for a in self.assets.values() if isinstance(a, (RasGeomHdf, GeometryAsset))] + return [a for a in self.assets.values() if isinstance(a, (GeometryHdfAsset, GeometryAsset))] @property def crs(self) -> CRS: @@ -200,7 +200,7 @@ def datetime(self) -> datetime: for geom_file in self.geometry_assets: if isinstance(geom_file, GeometryHdfAsset): - geom_date = geom_file.hdf_object.geometry_time + geom_date = geom_file.file.geometry_time if geom_date: item_datetime = geom_date self.properties[self.RAS_DATETIME_SOURCE] = "model_geometry" diff --git a/hecstac/ras/parser.py b/hecstac/ras/parser.py index e3e7225..e75a6d7 100644 --- a/hecstac/ras/parser.py +++ b/hecstac/ras/parser.py @@ -15,7 +15,7 @@ from pyproj.exceptions import CRSError from pystac import Asset from rashdf import RasGeomHdf, RasPlanHdf -from shapely import GeometryCollection, LineString, MultiPolygon, Point, Polygon, make_valid, union_all +from shapely import GeometryCollection, LineString, MultiPolygon, Point, Polygon, empty, make_valid, union_all from shapely.ops import unary_union from hecstac.ras.utils import ( @@ -1274,14 +1274,11 @@ def mesh_areas(self, crs: str = None, return_gdf: bool = False) -> gpd.GeoDataFr """ mesh_areas = self.hdf_object.mesh_areas() if mesh_areas is None or mesh_areas.empty: - raise ValueError("No mesh areas found.") + return Polygon() if mesh_areas.crs is None and crs is not None: mesh_areas = mesh_areas.set_crs(crs) - elif mesh_areas.crs is None and crs is None: - raise CRSError("Mesh areas have no CRS and have none to be set to") - if return_gdf: return mesh_areas else: From 05729295e5147db995777c4d5edd22883095883c Mon Sep 17 00:00:00 2001 From: sclaw Date: Thu, 6 Feb 2025 16:03:41 -0500 Subject: [PATCH 55/71] update and debuug datetime --- hecstac/ras/item.py | 52 ++++++++++++++++++++++++++----------------- hecstac/ras/parser.py | 2 +- 2 files changed, 33 insertions(+), 21 deletions(-) diff --git a/hecstac/ras/item.py b/hecstac/ras/item.py index 85c61d3..c0db2d1 100644 --- a/hecstac/ras/item.py +++ b/hecstac/ras/item.py @@ -15,6 +15,7 @@ from pystac import Asset, Item from pystac.extensions.projection import ProjectionExtension from pystac.extensions.storage import StorageExtension +from pystac.utils import datetime_to_str from rashdf import RasGeomHdf from shapely import Geometry, Polygon, simplify, to_geojson, union_all from shapely.geometry import shape @@ -131,7 +132,7 @@ def has_1d(self) -> bool: return any([a.has_1d for a in self.geometry_assets]) @property - def geometry_assets(self) -> list[RasGeomHdf | GeometryAsset]: + def geometry_assets(self) -> list[GeometryHdfAsset | GeometryAsset]: """Return any RasGeomHdf in assets.""" return [a for a in self.assets.values() if isinstance(a, (GeometryHdfAsset, GeometryAsset))] @@ -177,14 +178,17 @@ def properties(self) -> None: if self.ras_project_file is None: return self._properties properties = self._properties - # properties[self.RAS_HAS_1D] = self.has_1d + properties[self.RAS_HAS_1D] = self.has_1d properties[self.RAS_HAS_2D] = self.has_2d properties[self.PROJECT_TITLE] = self.pf.project_title properties[self.PROJECT_VERSION] = self.pf.ras_version properties[self.PROJECT_DESCRIPTION] = self.pf.project_description properties[self.PROJECT_STATUS] = self.pf.project_status properties[self.MODEL_UNITS] = self.pf.project_units - + if self.datetime is not None: + properties["datetime"] = datetime_to_str(self.datetime) + else: + properties["datetime"] = None # TODO: once all assets are created, populate associations between assets return properties @@ -194,24 +198,32 @@ def properties(self, properties: dict): self._properties = properties @property - def datetime(self) -> datetime: - """The datetime for the RAS STAC item.""" - item_datetime = None - - for geom_file in self.geometry_assets: - if isinstance(geom_file, GeometryHdfAsset): - geom_date = geom_file.file.geometry_time - if geom_date: - item_datetime = geom_date - self.properties[self.RAS_DATETIME_SOURCE] = "model_geometry" - logger.info(f"Using item datetime from {geom_file.href}") - break - - if item_datetime is None: + def datetime(self) -> datetime.datetime | None: + """Parse datetime from model geometry and return result.""" + datetimes = [] + for i in self.geometry_assets: + i = i.file.geometry_time + if i is None: + continue + elif isinstance(i, list): + datetimes.extend([j for j in i if j is not None]) + elif isinstance(i, datetime.datetime): + datetimes.append(i) + + datetimes = list(set(datetimes)) + if len(datetimes) > 1: + self._properties["start_datetime"] = datetime_to_str(min(datetimes)) + self._properties["end_datetime"] = datetime_to_str(max(datetimes)) + self._properties[self.RAS_DATETIME_SOURCE] = "model_geometry" + item_time = None + elif len(datetimes) == 1: + item_time = datetimes[0] + self._properties[self.RAS_DATETIME_SOURCE] = "model_geometry" + else: logger.warning("Could not extract item datetime from geometry, using item processing time.") - item_datetime = datetime.datetime.now() - self.properties[self.RAS_DATETIME_SOURCE] = "processing_time" - return item_datetime + item_time = datetime.datetime.now() + self._properties[self.RAS_DATETIME_SOURCE] = "processing_time" + return item_time def add_model_thumbnails(self, layers: list, title_prefix: str = "Model_Thumbnail", thumbnail_dir=None): """Generate model thumbnail asset for each geometry file. diff --git a/hecstac/ras/parser.py b/hecstac/ras/parser.py index e75a6d7..8e50234 100644 --- a/hecstac/ras/parser.py +++ b/hecstac/ras/parser.py @@ -814,7 +814,7 @@ def geom_version(self) -> str: return search_contents(self.file_lines, "Program Version") @property - def datetimes(self) -> list[datetime.datetime]: + def geometry_time(self) -> list[datetime.datetime]: """Get the latest node last updated entry for this geometry.""" dts = search_contents(self.file_lines, "Node Last Edited Time", expect_one=False, require_one=False) if len(dts) >= 1: From 56297594e663d620a4b311dc24937096382606ea Mon Sep 17 00:00:00 2001 From: sclaw Date: Thu, 6 Feb 2025 16:11:09 -0500 Subject: [PATCH 56/71] fill interior polygon holes --- hecstac/ras/item.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/hecstac/ras/item.py b/hecstac/ras/item.py index c0db2d1..37e70ed 100644 --- a/hecstac/ras/item.py +++ b/hecstac/ras/item.py @@ -165,6 +165,8 @@ def geometry(self) -> dict: unioned_geometry = union_all(geometries) if self.simplify_geometry: unioned_geometry = simplify(unioned_geometry, 0.001) + if unioned_geometry.interiors: + unioned_geometry = Polygon(list(unioned_geometry.exterior.coords)) return json.loads(to_geojson(unioned_geometry)) @property From 2b5fc344b36db655c74ca04e882bfed85b32cb13 Mon Sep 17 00:00:00 2001 From: sclaw Date: Thu, 6 Feb 2025 16:28:21 -0500 Subject: [PATCH 57/71] cleanup --- hecstac/common/asset_factory.py | 32 +------------------------------- hecstac/ras/assets.py | 16 ++++++++++------ hecstac/ras/item.py | 12 ++---------- hecstac/ras/parser.py | 4 +--- hecstac/ras/utils.py | 1 - 5 files changed, 14 insertions(+), 51 deletions(-) diff --git a/hecstac/common/asset_factory.py b/hecstac/common/asset_factory.py index 302b058..c6449f8 100644 --- a/hecstac/common/asset_factory.py +++ b/hecstac/common/asset_factory.py @@ -2,9 +2,8 @@ import logging from pathlib import Path -from typing import ClassVar, Dict, Generic, Type, TypeVar +from typing import Dict, Generic, Type, TypeVar -import pystac from pyproj import CRS from pystac import Asset @@ -15,16 +14,6 @@ T = TypeVar("T") # Generic for asset file accessor classes -def is_ras_prj(url: str) -> bool: - """Check if a file is a HEC-RAS project file.""" - with open(url) as f: - file_str = f.read() - if "Proj Title" in file_str.split("\n")[0]: - return True - else: - return False - - class GenericAsset(Asset, Generic[T]): """Provides a base structure for assets.""" @@ -121,25 +110,6 @@ def create_hms_asset(self, fpath: str, item_type: str = "model") -> Asset: asset.title = Path(fpath).name return check_storage_extension(asset) - def create_ras_asset(self, fpath: str): - """Create an asset instance based on the file extension.""" - logger.debug(f"Creating asset for {fpath}") - from hecstac.ras.assets import ProjectAsset - - if fpath.lower().endswith(".prj"): - if is_ras_prj(fpath): - return ProjectAsset(href=fpath, title=Path(fpath).name) - else: - return GenericAsset(href=fpath, title=Path(fpath).name) - - for pattern, asset_class in self.extension_to_asset.items(): - if pattern.match(fpath): - logger.debug(f"Matched {pattern} for {Path(fpath).name}: {asset_class}") - return asset_class(href=fpath, title=Path(fpath).name) - - logger.warning(f"Unable to pattern match asset for file {fpath}") - return GenericAsset(href=fpath, title=Path(fpath).name) - def asset_from_dict(self, asset: Asset): fpath = asset.href for pattern, asset_class in self.extension_to_asset.items(): diff --git a/hecstac/ras/assets.py b/hecstac/ras/assets.py index 1beba67..c96f273 100644 --- a/hecstac/ras/assets.py +++ b/hecstac/ras/assets.py @@ -9,15 +9,12 @@ import geopandas as gpd import matplotlib.pyplot as plt from matplotlib.lines import Line2D -from pandas import lreshape -from pyproj import CRS -from pyproj.exceptions import CRSError from pystac import MediaType -from pystac.extensions.projection import ProjectionExtension from shapely import MultiPolygon, Polygon from hecstac.common.asset_factory import GenericAsset from hecstac.common.geometry import reproject_to_wgs84 +from hecstac.ras.consts import NULL_GEOMETRY from hecstac.ras.parser import ( GeometryFile, GeometryHDFFile, @@ -230,7 +227,10 @@ def has_2d(self) -> bool: def geometry_wgs84(self) -> Polygon | MultiPolygon: """Reproject geometry to wgs84.""" # TODO: this could be generalized to be a function that takes argument for CRS. - return reproject_to_wgs84(self.geometry, self.crs) + if self.crs is None: + return NULL_GEOMETRY + else: + return reproject_to_wgs84(self.geometry, self.crs) class SteadyFlowAsset(GenericAsset[SteadyFlowFile]): @@ -378,6 +378,7 @@ def extra_fields(self) -> dict: return self._extra_fields @property + @lru_cache def reference_lines(self) -> list[gpd.GeoDataFrame] | None: """Docstring.""" # TODO: fill out if self.file.reference_lines is not None and not self.file.reference_lines.empty: @@ -410,7 +411,10 @@ def geometry(self) -> Polygon | MultiPolygon: def geometry_wgs84(self) -> Polygon | MultiPolygon: """Reproject geometry to wgs84.""" # TODO: this could be generalized to be a function that takes argument for CRS. - return reproject_to_wgs84(self.geometry, self.crs) + if self.crs is None: + return NULL_GEOMETRY + else: + return reproject_to_wgs84(self.geometry, self.crs) def _plot_mesh_areas(self, ax, mesh_polygons: gpd.GeoDataFrame) -> list[Line2D]: """Plot mesh areas on the given axes.""" diff --git a/hecstac/ras/item.py b/hecstac/ras/item.py index 37e70ed..d875333 100644 --- a/hecstac/ras/item.py +++ b/hecstac/ras/item.py @@ -3,32 +3,24 @@ import datetime import json import logging -import os -from collections import UserDict from functools import lru_cache from pathlib import Path -from typing import Optional import pystac import pystac.errors -from pyproj import CRS, Transformer +from pyproj import CRS from pystac import Asset, Item from pystac.extensions.projection import ProjectionExtension -from pystac.extensions.storage import StorageExtension from pystac.utils import datetime_to_str -from rashdf import RasGeomHdf -from shapely import Geometry, Polygon, simplify, to_geojson, union_all +from shapely import Polygon, simplify, to_geojson, union_all from shapely.geometry import shape -from shapely.ops import transform from hecstac.common.asset_factory import AssetFactory -from hecstac.common.geometry import reproject_to_wgs84 from hecstac.common.path_manager import LocalPathManager from hecstac.ras.assets import ( RAS_EXTENSION_MAPPING, GeometryAsset, GeometryHdfAsset, - ProjectAsset, ) from hecstac.ras.consts import ( NULL_DATETIME, diff --git a/hecstac/ras/parser.py b/hecstac/ras/parser.py index 8e50234..9068955 100644 --- a/hecstac/ras/parser.py +++ b/hecstac/ras/parser.py @@ -12,10 +12,8 @@ import geopandas as gpd import numpy as np import pandas as pd -from pyproj.exceptions import CRSError -from pystac import Asset from rashdf import RasGeomHdf, RasPlanHdf -from shapely import GeometryCollection, LineString, MultiPolygon, Point, Polygon, empty, make_valid, union_all +from shapely import GeometryCollection, LineString, MultiPolygon, Point, Polygon, make_valid, union_all from shapely.ops import unary_union from hecstac.ras.utils import ( diff --git a/hecstac/ras/utils.py b/hecstac/ras/utils.py index b3925c5..e3c2aff 100644 --- a/hecstac/ras/utils.py +++ b/hecstac/ras/utils.py @@ -4,7 +4,6 @@ import os from functools import wraps from pathlib import Path -from typing import Optional import geopandas as gpd import numpy as np From d4ae67d972d5acd502c374cb128d68b4932da247 Mon Sep 17 00:00:00 2001 From: Stevenray Janke Date: Fri, 7 Feb 2025 09:38:26 -0500 Subject: [PATCH 58/71] Remove unused files --- hecstac/utils/generate_schema_markdown.py | 385 ---------------------- hecstac/utils/placeholders.py | 11 - hecstac/utils/s3_utils.py | 73 ---- 3 files changed, 469 deletions(-) delete mode 100644 hecstac/utils/generate_schema_markdown.py delete mode 100644 hecstac/utils/placeholders.py delete mode 100644 hecstac/utils/s3_utils.py diff --git a/hecstac/utils/generate_schema_markdown.py b/hecstac/utils/generate_schema_markdown.py deleted file mode 100644 index 93ed5e4..0000000 --- a/hecstac/utils/generate_schema_markdown.py +++ /dev/null @@ -1,385 +0,0 @@ -import json -import re -from dataclasses import dataclass, field -from typing import Any, Iterator - -import jsonschema -import requests - -# Define schema which defines expected structure for extensions schemas -META_SCHEMA = { - "$schema": "http://json-schema.org/draft-07/schema#", - "required": ["$schema", "$id", "title", "description", "oneOf", "definitions"], - "properties": { - "$schema": {"type": "string"}, - "$id": {"type": "string"}, - "title": {"type": "string"}, - "description": {"type": "string"}, - "oneOf": {"type": "array", "items": {"type": "object"}}, - "definitions": {"type": "object", "required": ["stac_extensions", "require_any_field", "fields"]}, - }, -} - -ASSET_SPECIFIC_META_SCHEMA = { - "$schema": "http://json-schema.org/draft-07/schema#", - "required": ["$schema", "$id", "title", "description", "oneOf", "definitions"], - "properties": { - "$schema": {"type": "string"}, - "$id": {"type": "string"}, - "title": {"type": "string"}, - "description": {"type": "string"}, - "oneOf": {"type": "array", "items": {"type": "object"}}, - "definitions": {"type": "object", "required": ["stac_extensions", "require_any_field", "fields", "assets"]}, - }, -} - - -@dataclass -class Field: - field_name: str - type: str | list[str] - description: str - required: bool | None - table_description: str = field(init=False) - type_str: str = field(init=False) - - def __post_init__(self) -> None: - if self.required: - self.table_description = f"**REQUIRED** {self.description}" - else: - self.table_description = self.description - if isinstance(self.type, list): - modified_type_list = [self.modify_link(t) for t in self.type] - self.type_str = " \| ".join(modified_type_list) - else: - self.type_str = self.modify_link(self.type) - - @staticmethod - def modify_link(potential_link: str) -> str: - # modify definitions link to internal markdown link, else returns input without modification - if "#/definitions/" in potential_link: - internal_link = potential_link.replace("#/definitions/", "#") - link_name = potential_link.replace("#/definitions/", "") - link_name = link_name.replace("_", " ") - linked = f"[{link_name}]({internal_link})" - return linked - else: - return potential_link - - -@dataclass -class FieldUsability: - catalog: bool - collection_properties: bool - collection_item_assets: bool - item_properties: bool - item_assets: bool - links: bool - catalog_str: str = field(init=False) - collection_properties_str: str = field(init=False) - collection_item_assets_str: str = field(init=False) - item_properties_str: str = field(init=False) - item_assets_str: str = field(init=False) - links_str: str = field(init=False) - - def __post_init__(self): - self.catalog_str = " " - if self.catalog: - self.catalog_str = "x" - self.collection_properties_str = " " - if self.collection_properties: - self.collection_properties_str = "x" - self.collection_item_assets_str = " " - if self.collection_item_assets: - self.collection_item_assets_str = "x" - self.item_properties_str = " " - if self.item_properties: - self.item_properties_str = "x" - self.item_assets_str = " " - if self.item_assets: - self.item_assets_str = "x" - self.links_str = " " - if self.links: - self.links_str = "x" - - -# Class for generating Markdown documents documenting extensions in such a way that they fall in line with template markdown file (https://github.com/stac-extensions/template/blob/main/README.md) with alterations where necessary or to reduce required manual input -# intention is to have properties which pull out headings and subheadings from schema -class ExtensionSchema: - def __init__(self, schema_url: str, field_usability: FieldUsability, prefix: str | None = None) -> None: - # reads schema url - self.identifier = schema_url - self.field_usability = field_usability - self._schema_str = read_schema_text_from_url(schema_url) - self.schema = json.loads(self._schema_str) - self.validate_schema() - self._prefix = prefix - self._not_common_definition_names: list[str] = ["fields", "stac_extensions", "require_any_field"] - self._required_item_properties: list[str] | None = None - - def validate_schema(self) -> None: - jsonschema.validate(self.schema, META_SCHEMA) - - @property - def title(self) -> str: - return self.schema["title"] - - @property - def prefix(self) -> str: - if self._prefix == None: - prefix_pattern = re.compile(r"^\^\(\?\!(.+):\)$") - prefix_exclusivity_pattern = list(self.schema["definitions"]["fields"]["patternProperties"].keys())[0] - match = prefix_pattern.match(prefix_exclusivity_pattern) - self._prefix = match.group(1) - return self._prefix - - @property - def item_property_definitions(self) -> list[Field]: - field_list: list[Field] = [] - definition_schema = {"type": "object", "required": ["type", "description"]} - for definition_key, definition_value in self.schema["definitions"]["fields"]["properties"].items(): - jsonschema.validate(definition_value, definition_schema, jsonschema.Draft7Validator) - field = Field( - definition_key, - definition_value["type"], - definition_value["description"], - definition_key in self.required_item_properties, - ) - field_list.append(field) - return field_list - - @property - def common_definitions(self) -> list[Field]: - field_list: list[Field] = [] - definition_schema = {"type": "object", "required": ["type", "description"]} - for definition_key, definition_value in self.schema["definitions"].items(): - if definition_key not in self._not_common_definition_names: - jsonschema.validate(definition_value, definition_schema, jsonschema.Draft7Validator) - field = Field(definition_key, definition_value["type"], definition_value["description"], None) - field_list.append(field) - return field_list - - def get_item_meta_schema(self) -> dict[str, Any]: - for stac_type_subschema in self.schema["oneOf"]: - if "type" not in stac_type_subschema: - for subschema in stac_type_subschema["allOf"]: - if "properties" in subschema: - meta_properties = subschema["properties"] - type_definition = meta_properties["type"] - if type_definition == {"const": "Feature"}: - return meta_properties - raise ValueError("Item meta schema was not found in expected location") - - @property - def required_item_properties(self) -> list[str]: - if self._required_item_properties == None: - self.validate_item_metadata_schema() - return self._required_item_properties - - def to_markdown(self, path: str | None) -> str: - # parses schema to markdown and saves to path - markdown_str = "" - # write title - markdown_str += f"# {self.title} Extension\n\n" - # write overall metadata (title, identifier, prefix) - markdown_str += f"- **Title:** {self.title}\n" - markdown_str += f"- **Identifier:** {self.identifier}\n" - markdown_str += f"- **Field Name Prefix:** {self.prefix}\n\n" - # write short description for extension - markdown_str += f"The {self.title} Extension is an extension to the [SpatioTemporal Asset Catalog](https://github.com/radiantearth/stac-spec) (STAC) specification. The purpose of the extension is to introduce vocabulary useful in describing RAS models as STAC items and assets.\n\n" - # summarize field usability - markdown_str += f"## Fields\n\nThe fields in the table below can be used in these parts of STAC documents:\n\n" - markdown_str += f"- [{self.field_usability.catalog_str}] Catalogs\n" - markdown_str += f"- [{self.field_usability.collection_properties_str}] Collection Properties\n" - markdown_str += f"- [{self.field_usability.collection_item_assets_str}] Collection Item Assets\n" - markdown_str += f"- [{self.field_usability.item_properties_str}] Item Properties\n" - markdown_str += f"- [{self.field_usability.item_assets_str}] Item Assets\n" - markdown_str += f"- [{self.field_usability.links_str}] Links\n\n" - # overview of fields - fields_table = self._fields_to_table(self.item_property_definitions) - markdown_str += fields_table - markdown_str += "\n\n### Additional Field Information\n\n" - # common definitions - for field in self.common_definitions: - subsection = self._field_to_subsection(field) - markdown_str += subsection - if path: - with open(path, "w") as f: - f.write(markdown_str) - return markdown_str - - @staticmethod - def _field_to_subsection(field: Field) -> str: - subsection = f"\n#### {field.field_name}\n\n" - if field.type_str == "object": - raise ValueError - subsection += f"- Type: {field.type_str}\n" - subsection += f"- Description: {field.description}\n" - return subsection - - @staticmethod - def _fields_to_table(fields: list[Field]) -> str: - max_field_name_length = len("Field Name") - max_type_length = len("Type") - max_description_length = len("Description") - for field in fields: - if len(field.field_name) > max_field_name_length: - max_field_name_length = len(field.field_name) - if len(field.type_str) > max_type_length: - max_type_length = len(field.type_str) - if len(field.table_description) > max_description_length: - max_description_length = len(field.table_description) - field_name_header = "Field Name".ljust(max_field_name_length) - type_header = "Type".ljust(max_type_length) - description_header = "Description".ljust(max_description_length) - table = f"| {field_name_header} | {type_header} | {description_header} |" - table += f"\n| {'-' * len(field_name_header)} | {'-' * len(type_header)} | {'-' * len(description_header)} |" - for field in fields: - row = f"\n| {field.field_name.ljust(max_field_name_length)} | {field.type_str.ljust(max_type_length)} | {field.table_description.ljust(max_description_length)} |" - table += row - return table - - def validate_item_metadata_schema(self) -> None: - # validates that the inner schema used to validate the structure of a stac item is structured as expected (has required properties and a ref to definitions/fields or alternatively just a ref to definitions/fields if no properties are required) - # also populates _required_item_properties - definitions_fields_referenced = False - required_property_names = None - meta_properties = self.get_item_meta_schema() - item_properties_schema = meta_properties["properties"] - if "allOf" in item_properties_schema: - subschemas = item_properties_schema["allOf"] - if len(subschemas) != 2: - raise ValueError(f"Expected 2 subschemas in item properties meta schema, got {len(subschemas)}") - for subschema in item_properties_schema["allOf"]: - if "$ref" in subschema: - reference = subschema["$ref"] - if reference != "#/definitions/fields": - raise ValueError( - f"Expected definitions/fields to hold all definitions for properties in item, got {reference}" - ) - definitions_fields_referenced = True - elif "required" in subschema: - required_property_names = subschema["required"] - else: - raise ValueError(f"Subschema found with neither $ref nor required: {subschema}") - else: - required_property_names = [] - if "$ref" not in item_properties_schema: - raise ValueError( - "Expected definitions/fields to hold all definitions for properties in item, instead $ref was not used" - ) - reference = item_properties_schema["$ref"] - if reference != "#/definitions/fields": - raise ValueError( - f"Expected definitions/fields to hold all definitions for properties in item, got {reference}" - ) - definitions_fields_referenced = True - if definitions_fields_referenced == False: - raise ValueError("Reference to definitions/fields never found") - if required_property_names == None: - raise TypeError("Required property names was neither set to an empty list nor pulled from schema") - self._required_item_properties = required_property_names - - -class ExtensionSchemaAssetSpecific(ExtensionSchema): - def __init__(self, schema_url, field_usability: FieldUsability, prefix=None): - super().__init__(schema_url, field_usability, prefix) - self.pattern_definition_dict = self.get_pattern_definition_dict() - self._not_common_definition_names.append("assets") - self._not_common_definition_names.extend(self.pattern_definition_dict.values()) - - def get_pattern_definition_dict(self) -> dict[str, str]: - pattern_to_definition_name_dict = {} - self.validate_asset_metadata_schema() - asset_definitions = self.schema["definitions"]["assets"] - for pattern, ref in asset_definitions["patternProperties"].items(): - pattern_to_definition_name_dict[pattern] = ref["$ref"].replace("#/definitions/", "", 1) - return pattern_to_definition_name_dict - - def validate_asset_metadata_schema(self) -> None: - meta_properties = self.get_item_meta_schema() - asset_properties_schema = meta_properties["assets"] - if "$ref" not in asset_properties_schema: - raise ValueError( - "Expected definitions/assets to hold all definitions for properties expected in assets, instead $ref was not used" - ) - reference = asset_properties_schema["$ref"] - if reference != "#/definitions/assets": - raise ValueError( - f"Expected definitions/assets to hold all definitions for properties expected in assets, got {reference}" - ) - - @property - def asset_definitions(self) -> Iterator[tuple[str, list[Field]]]: - no_ref_definition_schema = {"type": "object", "required": ["type", "description"]} - with_ref_definition_schema = { - "type": "object", - "required": ["allOf"], - "properties": { - "allOf": { - "type": "array", - "items": { - "oneOf": [ - {"type": "object", "required": ["description"]}, - {"type": "object", "required": ["$ref"]}, - ] - }, - } - }, - } - for pattern, definition_name in self.pattern_definition_dict.items(): - field_list: list[Field] = [] - meta_definition = self.schema["definitions"][definition_name] - required_properties: list[str] = meta_definition.get("required", []) - for asset_property_definition_name, asset_property_definition_value in meta_definition[ - "properties" - ].items(): - try: - jsonschema.validate( - asset_property_definition_value, no_ref_definition_schema, jsonschema.Draft7Validator - ) - field = Field( - asset_property_definition_name, - asset_property_definition_value["type"], - asset_property_definition_value["description"], - asset_property_definition_name in required_properties, - ) - field_list.append(field) - except jsonschema.ValidationError: - jsonschema.validate( - asset_property_definition_value, with_ref_definition_schema, jsonschema.Draft7Validator - ) - asset_property_definition_description = None - asset_property_definition_type = None - for all_of_property in asset_property_definition_value["allOf"]: - if "description" in all_of_property: - asset_property_definition_description = all_of_property["description"] - elif "$ref" in all_of_property: - asset_property_definition_type = all_of_property["$ref"] - field = Field( - asset_property_definition_name, - asset_property_definition_type, - asset_property_definition_description, - asset_property_definition_name in required_properties, - ) - field_list.append(field) - yield pattern, field_list - - def to_markdown(self, path: str | None) -> str: - markdown_str = super().to_markdown(None) - markdown_str += "\n\n## Asset Properties with Pattern Matching\n\n" - markdown_str += "This section describes the pattern used to match against the asset title along with the expected structure of that asset\n" - for pattern, field_list in self.asset_definitions: - markdown_str += f"\n### {pattern}\n\n" - properties_table = self._fields_to_table(field_list) - markdown_str += f"{properties_table}\n" - if path: - with open(path, "w") as f: - f.write(markdown_str) - return markdown_str - - -def read_schema_text_from_url(url: str) -> str: - with requests.get(url) as resp: - schema = resp.text - return schema diff --git a/hecstac/utils/placeholders.py b/hecstac/utils/placeholders.py deleted file mode 100644 index 9a333bf..0000000 --- a/hecstac/utils/placeholders.py +++ /dev/null @@ -1,11 +0,0 @@ -import datetime -import json - -from shapely import Polygon, box, to_geojson - -NULL_DATETIME = datetime.datetime(9999, 9, 9) -NULL_GEOMETRY = Polygon() -NULL_STAC_GEOMETRY = json.loads(to_geojson(NULL_GEOMETRY)) -NULL_BBOX = box(0, 0, 0, 0) -NULL_STAC_BBOX = NULL_BBOX.bounds -PLACEHOLDER_ID = "id" diff --git a/hecstac/utils/s3_utils.py b/hecstac/utils/s3_utils.py deleted file mode 100644 index 1adcd74..0000000 --- a/hecstac/utils/s3_utils.py +++ /dev/null @@ -1,73 +0,0 @@ -from urllib.parse import urlparse -import os -import boto3 -from mypy_boto3_s3 import S3ServiceResource -from pystac.stac_io import DefaultStacIO - - -class S3StacIO(DefaultStacIO): - def __init__(self, headers=None): - super().__init__(headers) - self.session = boto3.Session() - self.s3: S3ServiceResource = self.session.resource("s3") - - def read_text(self, source: str, *_, **__) -> str: - parsed = urlparse(url=source) - if parsed.scheme == "s3": - bucket = parsed.netloc - key = parsed.path[1:] - obj = self.s3.Object(bucket, key) - data_encoded: bytes = obj.get()["Body"].read() - data_decoded = data_encoded.decode() - return data_decoded - else: - return super().read_text(source) - - def write_text(self, dest: str, txt, *_, **__) -> None: - parsed = urlparse(url=dest) - if parsed.scheme == "s3": - bucket = parsed.netloc - key = parsed.path[1:] - obj = self.s3.Object(bucket, key) - obj.put(Body=txt, ContentEncoding="utf-8") - else: - return super().write_text(dest, txt, *_, **__) - - -def split_s3_path(s3_path: str) -> tuple[str, str]: - """Split an S3 path into the bucket name and the key. - - Parameters - ---------- - s3_path (str): The S3 path to split. It should be in the format 's3://bucket/key'. - - Returns - ------- - tuple: A tuple containing the bucket name and the key. If the S3 path does not contain a key, the second element - of the tuple will be None. - """ - if not s3_path.startswith("s3://"): - raise ValueError(f"s3_path does not start with s3://: {s3_path}") - bucket, _, key = s3_path[5:].partition("/") - if not key: - raise ValueError(f"s3_path contains bucket only, no key: {s3_path}") - return bucket, key - - -def init_s3_resources(): - """Create a Boto3 session using AWS credentials from environment variables and creates both an S3 client and S3 resource for interacting with AWS S3.""" - session = boto3.Session( - aws_access_key_id=os.environ["AWS_ACCESS_KEY_ID"], - aws_secret_access_key=os.environ["AWS_SECRET_ACCESS_KEY"], - ) - - s3_client = session.client("s3") - s3_resource = session.resource("s3") - return session, s3_client, s3_resource - - -def save_bytes_s3(byte_obj: bytes, s3_key: str) -> None: - """Save bytes to S3.""" - _, s3_client, _ = init_s3_resources() - bucket, key = split_s3_path(s3_key) - s3_client.put_object(Body=byte_obj, Bucket=bucket, Key=key) From 04818bdebd49d74cf5053ac9d06d48ac9c8f5dcb Mon Sep 17 00:00:00 2001 From: Stevenray Janke Date: Fri, 7 Feb 2025 09:38:58 -0500 Subject: [PATCH 59/71] Fix hms file parsing/linting fixes --- hecstac/hms/parser.py | 135 ++++++++++++++++++++++++++++++++++++------ 1 file changed, 118 insertions(+), 17 deletions(-) diff --git a/hecstac/hms/parser.py b/hecstac/hms/parser.py index 62a2ff1..6033479 100644 --- a/hecstac/hms/parser.py +++ b/hecstac/hms/parser.py @@ -1,3 +1,5 @@ +"""HEC-HMS file parsing classes.""" + from __future__ import annotations import logging @@ -47,6 +49,8 @@ class BaseTextFile(ABC): + """Base class for text files.""" + def __init__(self, path: str, client=None, bucket=None): self.path: str = path self.directory: str = os.path.dirname(self.path) @@ -59,9 +63,14 @@ def __init__(self, path: str, client=None, bucket=None): self.parse_header() def read_content(self): + """Read contents of text file.""" if os.path.exists(self.path): - with open(self.path) as f: - self.content = f.read() + try: + with open(self.path, encoding="utf-8") as f: + self.content = f.read() + except UnicodeDecodeError: + with open(self.path, encoding="cp1252") as f: + self.content = f.read() else: try: response = self.client.get_object(Bucket=self.bucket, Key=self.path) @@ -71,7 +80,7 @@ def read_content(self): raise FileNotFoundError(f"could not find {self.path} locally nor on s3") def parse_header(self): - """Scan the file down to the first instance of 'End:' and save each colon-separated keyval pair as attrs dict""" + """Scan the file down to the first instance of 'End:' and save each colon-separated keyval pair as attrs dict.""" lines = self.content.splitlines() if not lines[0].startswith( ( @@ -91,6 +100,8 @@ def parse_header(self): class ProjectFile(BaseTextFile): + """Class for parsing HEC-HMS project files.""" + def __init__( self, path: str, @@ -125,15 +136,18 @@ def __repr__(self): @property @lru_cache def name(self): + """Extract name from project file.""" lines = self.content.splitlines() if not lines[0].startswith("Project: "): raise ValueError(f"unexpected first line: {lines[0]}") return lines[0][len("Project: ") :] def combine_stem_ext(self, ext: str) -> str: + """Combine stem and extension.""" return f"{self.stem}.{ext}" def scan_for_terrain_run_grid_gage_pdata(self): + """Scan for terrain, run, grid, gage, and pdata files.""" for ext in ["terrain", "run", "grid", "gage", "pdata"]: path = self.combine_stem_ext(ext) if os.path.exists(path): @@ -149,6 +163,7 @@ def scan_for_terrain_run_grid_gage_pdata(self): self.pdata = PairedDataFile(path) def scan_for_basins_mets_controls(self): + """Scan for basin, meteorology, and control files.""" lines = self.content.splitlines() i = -1 while True: @@ -183,6 +198,7 @@ def scan_for_basins_mets_controls(self): @property def file_counts(self): + """Return file counts.""" return { "Basins": len(self.basins), "Controls": len(self.controls), @@ -196,6 +212,7 @@ def file_counts(self): } def assert_uniform_version(self): + """Assert uniform version.""" errors = [] version = self.attrs["Version"] for basin in self.basins: @@ -219,9 +236,8 @@ def assert_uniform_version(self): @property def files(self): - - # logger.info(f"other paths {[i.path for i in [self.terrain, self.run, self.grid, self.gage, self.pdata] if i]}") - + """Return associated files.""" + # logging.info(f"other paths {[i.path for i in [self.terrain, self.run, self.grid, self.gage, self.pdata] if i]}") return ( [self.path] + [basin.path for basin in self.basins] @@ -235,21 +251,28 @@ def files(self): @property def dss_files(self): + """Return dss files.""" files = set( [gage.attrs["Variant"]["Variant-1"]["DSS File Name"] for gage in self.gage.elements.elements.values()] - + [ - grid.attrs["Variant"]["Variant-1"]["DSS File Name"] - for grid in self.grid.elements.elements.values() - if "Variant" in grid.attrs - ] + [pdata.attrs["DSS File"] for pdata in self.pdata.elements.elements.values()] ) + if self.grid: + files.update( + [ + grid.attrs["Variant"]["Variant-1"]["DSS File Name"] + for grid in self.grid.elements.elements.values() + if "Variant" in grid.attrs + ] + ) + else: + logging.warning("No grid file to extract dss files from.") files = [str(Path(f.replace("\\", "/"))) for f in files] return self.absolute_paths(files) @property def result_files(self): + """Return result files.""" files = set( [i[1].attrs["Log File"] for i in self.run.elements] + [i[1].attrs["DSS File"] for i in self.run.elements] @@ -260,25 +283,41 @@ def result_files(self): return self.absolute_paths(set(files)) def absolute_paths(self, paths): + """Return absolute path.""" return [os.path.join(self.directory, path) for path in paths] @property def rasters(self): + """Return raster files.""" files = [] + if self.terrain: for terrain in self.terrain.layers: - files += [os.path.join(terrain["raster_dir"], f) for f in os.listdir(terrain["raster_dir"])] - files += [grid.attrs["Filename"] for grid in self.grid.elements.elements.values() if "Filename" in grid.attrs] + raster_dir = terrain.get("raster_dir", "").strip() + if raster_dir and os.path.exists(raster_dir): + files += [os.path.join(raster_dir, f) for f in os.listdir(raster_dir)] + else: + logging.warning(f"Skipping missing raster directory: {raster_dir}") + + if self.grid is None: + logging.warning("No grid file, skipping grid rasters.") + else: + files += [ + grid.attrs["Filename"] for grid in self.grid.elements.elements.values() if "Filename" in grid.attrs + ] files = [str(Path(f.replace("\\", "/"))) for f in files] return self.absolute_paths(set(files)) @property @lru_cache def sqlitedbs(self): + """Return SQLite database.""" return [SqliteDB(basin.sqlite_path) for basin in self.basins] class BasinFile(BaseTextFile): + """Class for parsing HEC-HMS basin files.""" + def __init__( self, path: str, @@ -315,25 +354,30 @@ def __repr__(self): @property def wkt(self): + """Return wkt representation of the CRS.""" for line in self.spatial_properties.content.splitlines(): if "Coordinate System: " in line: return line.split(": ")[1] @property def crs(self): + """Return the CRS.""" return CRS(self.wkt) @property def epsg(self): + """Return the EPSG code.""" return self.crs.to_epsg() def parse_name(self): + """Parse basin name.""" lines = self.content.splitlines() if not lines[0].startswith("Basin: "): raise ValueError(f"unexpected first line: {lines[0]}") self.name = lines[0][len("Basin: ") :] def scan_for_headers_and_footers(self): + """Scan for basin headers and footers.""" lines = self.content.splitlines() for i, line in enumerate(lines): if line.startswith("Basin: "): @@ -353,6 +397,7 @@ def scan_for_headers_and_footers(self): self.computation_points = ComputationPoints(content) def identify_sqlite(self): + """Identify SQLite.""" for line in self.content.splitlines(): if ".sqlite" in line: return line.split("File: ")[1] @@ -360,6 +405,7 @@ def identify_sqlite(self): @property @lru_cache def elements(self): + """Return basin elements.""" elements = ElementSet() if self.read_geom: sqlite = SqliteDB( @@ -435,62 +481,75 @@ def elements(self): @property @lru_cache def subbasins(self): + """Return subbasin elements.""" return self.elements.get_element_type("Subbasin") @property @lru_cache def reaches(self): + """Return reach elements.""" return self.elements.get_element_type("Reach") @property @lru_cache def junctions(self): + """Return junction elements.""" return self.elements.get_element_type("Junction") @property @lru_cache def reservoirs(self): + """Return reservoir elements.""" return self.elements.get_element_type("Reservoir") @property @lru_cache def diversions(self): + """Return diversion elements.""" return self.elements.get_element_type("Diversion") @property @lru_cache def sinks(self): + """Return sink elements.""" return self.elements.get_element_type("Sink") @property @lru_cache def sources(self): + """Return source elements.""" return self.elements.get_element_type("Source") @property @lru_cache def gages(self): + """Return gages.""" return self.elements.gages @property @lru_cache def drainage_area(self): + """Return drainage areas..""" return sum([subbasin.geom.area for subbasin in self.subbasins]) @property @lru_cache def reach_miles(self): + """Return reach lengths in miles..""" return sum([reach.geom.length for reach in self.reaches]) @property @lru_cache def basin_geom(self): + """Return basin geometry.""" return utils.remove_holes(self.feature_2_gdf("Subbasin").make_valid().to_crs(4326).union_all()) def bbox(self, crs): + """Return basin bounding box.""" return self.feature_2_gdf("Subbasin").to_crs(crs).total_bounds def feature_2_gdf(self, element_type: str) -> gpd.GeoDataFrame: + """Convert feature to GeoDataFrame.""" gdf_list = [] for e in self.elements.get_element_type(element_type): gdf_list.append( @@ -506,6 +565,7 @@ def feature_2_gdf(self, element_type: str) -> gpd.GeoDataFrame: @property @lru_cache def observation_points_gdf(self): + """Return GeoDataFrame of observation points.""" gdf_list = [] for name, element in self.elements: if "Observed Hydrograph Gage" in element.attrs.keys(): @@ -558,6 +618,7 @@ def observation_points_gdf(self): return gdf def subbasin_connection_lines(self) -> gpd.GeoDataFrame: + """Return GeoDataframe of subbasin connection lines.""" df_list = [] for subbasin in self.subbasins: us_point = subbasin.geom.centroid @@ -579,6 +640,7 @@ def subbasin_connection_lines(self) -> gpd.GeoDataFrame: return gdf def junction_connection_lines(self) -> gpd.GeoDataFrame: + """Return GeoDataframe of junction connection lines.""" df_list = [] for junction in self.junctions: us_point = junction.geom @@ -617,6 +679,7 @@ def junction_connection_lines(self) -> gpd.GeoDataFrame: @property @lru_cache def hms_schematic_2_gdfs(self) -> dict[gpd.GeoDataFrame]: + """Convert HMS schematics to GeoDataframe.""" element_gdfs = {} for element_type in [ "Reach", @@ -635,6 +698,7 @@ def hms_schematic_2_gdfs(self) -> dict[gpd.GeoDataFrame]: return element_gdfs def subbasin_bc_lines(self): + """Return subbasin boundary condition lines.""" df_list = [] for _, row in self.subbasin_connection_lines().iterrows(): geom = row.geometry @@ -656,6 +720,8 @@ def subbasin_bc_lines(self): class MetFile(BaseTextFile): + """Class for parsing HEC-HMS meteorology files.""" + def __init__(self, path: str, client=None, bucket=None): if not path.endswith(".met"): raise ValueError(f"invalid extension for Meteorology file: {path}") @@ -670,12 +736,14 @@ def __repr__(self): @property @lru_cache def name(self): + """Return meteorology name.""" lines = self.content.splitlines() if not lines[0].startswith("Meteorology: "): raise ValueError(f"unexpected first line: {lines[0]}") return lines[0][len("Meteorology: ") :] def scan_for_elements(self): + """Scan for meteorology elements.""" elements = ElementSet() lines = self.content.splitlines() for i, line in enumerate(lines): @@ -702,6 +770,8 @@ def scan_for_elements(self): class ControlFile(BaseTextFile): + """Class for parsing HEC-HMS control files.""" + def __init__(self, path: str, client=None, bucket=None): if not path.endswith(".control"): raise ValueError(f"invalid extension for Control file: {path}") @@ -714,6 +784,7 @@ def __repr__(self): @property @lru_cache def name(self): + """Return control name.""" lines = self.content.splitlines() if not lines[0].startswith("Control: "): raise ValueError(f"unexpected first line: {lines[0]}") @@ -721,36 +792,43 @@ def name(self): class TerrainFile(BaseTextFile): + """Class for parsing HEC-HMS terrain files.""" + def __init__(self, path: str, client=None, bucket=None): if not path.endswith(".terrain"): - raise ValueError(f"invalid extension for Terrain file: {path}") + raise ValueError(f"Invalid extension for Terrain file: {path}") super().__init__(path, client=client, bucket=bucket) self.layers = [] found_first = False - name, raster_path, vert_units = "", "", "" + name, raster_path, raster_dir, vert_units = "", "", "", "" + for line in self.content.splitlines(): if not found_first: if line.startswith("Terrain Data: "): found_first = True else: continue + if line == "End:": self.layers.append( { "name": name, "raster_path": raster_path, - "raster_dir": os.path.dirname(raster_path), + "raster_dir": raster_dir, "vert_units": vert_units, } ) - name, raster_path, vert_units = "", "", "" + name, raster_path, raster_dir, vert_units = "", "", "", "" elif line.startswith("Terrain Data: "): name = line[len("Terrain Data: ") :] elif line.startswith(" Elevation File Name: "): raster_path_raw = line[len(" Elevation File Name: ") :] raster_path = os.path.join(os.path.dirname(self.path), raster_path_raw.replace("\\", os.sep)) + elif line.startswith(" Terrain Directory: "): + raster_dir_raw = line[len(" Terrain Directory: ") :] + raster_dir = os.path.join(os.path.dirname(self.path), raster_dir_raw.replace("\\", os.sep)) elif line.startswith(" Vertical Units: "): vert_units = line[len(" Vertical Units: ") :] @@ -761,10 +839,13 @@ def __repr__(self): @property @lru_cache def name(self): + """Return name.""" return None class RunFile(BaseTextFile): + """Class for parsing HEC-HMS run files.""" + def __init__(self, path: str, client=None, bucket=None): if not path.endswith(".run"): raise ValueError(f"invalid extension for Run file: {path}") @@ -775,6 +856,7 @@ def __repr__(self): return f"HMSRunFile({self.path})" def runs(self): + """Retrieve all runs.""" runs = ElementSet() lines = self.content.splitlines() i = -1 @@ -790,10 +872,13 @@ def runs(self): @property def elements(self): + """Return run elements.""" return self.runs() class PairedDataFile(BaseTextFile): + """Class for parsing HEC-HMS paired data files.""" + def __init__(self, path: str, client=None, bucket=None): if not path.endswith(".pdata"): raise ValueError(f"invalid extension for Paired Data file: {path}") @@ -815,12 +900,14 @@ def __repr__(self): @property @lru_cache def name(self): + """Return paired data manager.""" lines = self.content.splitlines() if not lines[0].startswith("Paired Data Manager: "): raise ValueError(f"unexpected first line: {lines[0]}") return lines[0][len("Paired Data Manager: ") :] def scan_for_tables(self): + """Scan for tables.""" lines = self.content.splitlines() for i, line in enumerate(lines): if line.startswith("Table: "): @@ -830,6 +917,7 @@ def scan_for_tables(self): self.elements[f"{name}+{table_type}"] = Table(name, attrs) def scan_for_patterns(self): + """Scan for patterns.""" lines = self.content.splitlines() for i, line in enumerate(lines): if line.startswith("Pattern: "): @@ -840,6 +928,8 @@ def scan_for_patterns(self): class SqliteDB: + """SQLite database class.""" + def __init__(self, path: str, client=None, bucket=None, fiona_aws_session=None): sqlite_file, _ = os.path.splitext(path) path = f"{sqlite_file}.sqlite" @@ -871,6 +961,8 @@ def __init__(self, path: str, client=None, bucket=None, fiona_aws_session=None): class GridFile(BaseTextFile): + """Class for parsing HEC-HMS grid files.""" + def __init__(self, path: str, client=None, bucket=None): if not path.endswith(".grid"): raise ValueError(f"invalid extension for Grid file: {path}") @@ -885,12 +977,14 @@ def __repr__(self): @property @lru_cache def name(self): + """Return grid manager name.""" lines = self.content.splitlines() if not lines[0].startswith("Grid Manager: "): raise ValueError(f"unexpected first line: {lines[0]}") return lines[0][len("Grid Manager: ") :] def scan_for_grids(self): + """Scan for all grids.""" lines = self.content.splitlines() for i, line in enumerate(lines): if line.startswith("Grid: "): @@ -900,6 +994,7 @@ def scan_for_grids(self): self.elements[f"{name}+{grid_type}"] = Grid(f"{name}+{grid_type}", attrs) def remove_grid_type(self, grid_types: list[str]): + """Remove given grid types.""" new_elements = ElementSet() for name, g in self.elements.elements.items(): if g.attrs["Grid Type"] not in grid_types: @@ -909,10 +1004,13 @@ def remove_grid_type(self, grid_types: list[str]): @property @lru_cache def grids(self): + """Return grid elements.""" return self.elements.get_element_type("Grid") class GageFile(BaseTextFile): + """Class for parsing HEC-HMS gage files.""" + def __init__(self, path: str, client=None, bucket=None): if not path.endswith(".gage"): raise ValueError(f"invalid extension for Gage file: {path}") @@ -927,12 +1025,14 @@ def __repr__(self): @property @lru_cache def name(self): + """Return gage manager name.""" lines = self.content.splitlines() if not lines[0].startswith("Gage Manager: "): raise ValueError(f"unexpected first line: {lines[0]}") return lines[0][len("Gage Manager: ") :] def scan_for_gages(self): + """Search for all gages.""" lines = self.content.splitlines() for i, line in enumerate(lines): if line.startswith("Gage: "): @@ -943,4 +1043,5 @@ def scan_for_gages(self): @property @lru_cache def gages(self): + """Return gage elements.""" return self.elements.get_element_type("Gage") From 9026c6cd41ceb7931701e783fe623279652133ed Mon Sep 17 00:00:00 2001 From: Stevenray Janke Date: Fri, 7 Feb 2025 09:40:12 -0500 Subject: [PATCH 60/71] Linting fixes --- hecstac/events/__init__.py | 1 + hecstac/events/ffrd.py | 4 +++ hecstac/hms/__init__.py | 1 + hecstac/hms/assets.py | 4 ++- hecstac/hms/consts.py | 2 ++ hecstac/hms/data_model.py | 61 ++++++++++++++++++++++++++++++-------- hecstac/hms/item.py | 8 ++--- hecstac/hms/s3_utils.py | 8 +++-- hecstac/hms/utils.py | 24 +++++++++++---- hecstac/ras/__init__.py | 2 +- hecstac/ras/item.py | 2 +- hecstac/utils/__init__.py | 1 + pyproject.toml | 3 -- 13 files changed, 92 insertions(+), 29 deletions(-) diff --git a/hecstac/events/__init__.py b/hecstac/events/__init__.py index e69de29..2ab1885 100644 --- a/hecstac/events/__init__.py +++ b/hecstac/events/__init__.py @@ -0,0 +1 @@ +"""HEC event stac items.""" diff --git a/hecstac/events/ffrd.py b/hecstac/events/ffrd.py index 65f7f6c..a9aff0d 100644 --- a/hecstac/events/ffrd.py +++ b/hecstac/events/ffrd.py @@ -1,3 +1,5 @@ +"""Class for event items.""" + import json import logging import os @@ -19,6 +21,8 @@ class FFRDEventItem(Item): + """Class for event items.""" + FFRD_REALIZATION = "FFRD:realization" FFRD_BLOCK_GROUP = "FFRD:block_group" FFRD_EVENT = "FFRD:event" diff --git a/hecstac/hms/__init__.py b/hecstac/hms/__init__.py index e69de29..4240fe8 100644 --- a/hecstac/hms/__init__.py +++ b/hecstac/hms/__init__.py @@ -0,0 +1 @@ +"""HEC-HMS STAC Item module.""" diff --git a/hecstac/hms/assets.py b/hecstac/hms/assets.py index b1dbc0d..64bef43 100644 --- a/hecstac/hms/assets.py +++ b/hecstac/hms/assets.py @@ -1,3 +1,5 @@ +"""HEC-HMS Stac Item asset classes.""" + from pystac import MediaType from hecstac.common.asset_factory import GenericAsset @@ -208,7 +210,7 @@ def __init__(self, href: str, *args, **kwargs): class GridAsset(GenericAsset): - """Grid asset""" + """Grid asset.""" def __init__(self, href: str, *args, **kwargs): roles = ["hms-grid"] diff --git a/hecstac/hms/consts.py b/hecstac/hms/consts.py index 1f4836e..4fd4f48 100644 --- a/hecstac/hms/consts.py +++ b/hecstac/hms/consts.py @@ -1,3 +1,5 @@ +"""HEC-HMS STAC Item constants.""" + GPD_WRITE_ENGINE = "fiona" # Latest default as of 2024-11-11 seems to be "pyogrio" which is causing issues. # 5 spaces, (key), colon, (val), ignoring whitespace before and after key and val, e.g. " Version: 4.10" ATTR_KEYVAL_GROUPER = r"^ (\S.*?)\s*:\s*(.*?)\s*$" diff --git a/hecstac/hms/data_model.py b/hecstac/hms/data_model.py index e3c7eae..9bbb942 100644 --- a/hecstac/hms/data_model.py +++ b/hecstac/hms/data_model.py @@ -1,3 +1,5 @@ +"""HEC-HMS STAC Item data classes.""" + from __future__ import annotations from collections import Counter, OrderedDict @@ -10,7 +12,7 @@ @dataclass class Element: - """Parent class of basin elements (Subbasins, Reaches, etc)""" + """Parent class of basin elements (Subbasins, Reaches, etc).""" name: str attrs: OrderedDict @@ -18,76 +20,84 @@ class Element: @dataclass class BasinHeader: - """Header of .basin""" + """Header of .basin.""" attrs: dict @dataclass class BasinLayerProperties: - """Part of footer of .basin, find via 'Basin Layer Properties:'. - Data is stored as a series of layers rather than a set of attributes, so just storing the raw content for now. - """ + """Part of footer of .basin, find via 'Basin Layer Properties:'. Data is stored as a series of layers rather than a set of attributes, so just storing the raw content for now.""" content: str @dataclass class Control(Element): + """Represents a control element.""" + pass @dataclass class Grid(Element): + """Represents a grid element.""" + pass @dataclass class Precipitation(Element): + """Represents a precipitation element.""" + pass @dataclass class Temperature(Element): + """Represents a temperature element.""" + pass @dataclass class ET(Element): + """Represents a ET element.""" + pass @dataclass class Subbasin_ET(Element): + """Represents a Subbasin_ET element.""" + pass @dataclass class Gage(Element): + """Represents a gage element.""" + pass @dataclass class ComputationPoints: - """Part of footer of .basin, find via 'Computation Points:'. - Data has some complex attributes with nested end-flags, so just storing raw content for now. - """ + """Part of footer of .basin, find via 'Computation Points:'. Data has some complex attributes with nested end-flags, so just storing raw content for now.""" content: str @dataclass class BasinSpatialProperties: - """Part of footer of .basin, find via 'Basin Spatial Properties:'. - Data has some complex attributes with nested end-flags, so just storing raw content for now. - """ + """Part of footer of .basin, find via 'Basin Spatial Properties:'. Data has some complex attributes with nested end-flags, so just storing raw content for now.""" content: str @dataclass class BasinSchematicProperties: - """Part of footer of .basin, find via 'Basin Schematic Properties:'""" + """Part of footer of .basin, find via 'Basin Schematic Properties:'.""" attrs: dict @@ -102,21 +112,29 @@ class Run: @dataclass class Subbasin(Element): + """Represents a Subbasin element.""" + geom: Polygon = None @dataclass class Table(Element): + """Represents a Table element.""" + pass @dataclass class Pattern(Element): + """Represents a Pattern element.""" + pass @dataclass class Reach(Element): + """Represents a Reach element.""" + geom: LineString = None slope: float = ( None # assumed units of the coordinate system is the same as what is used for the project.. need to confirm this assumption @@ -125,26 +143,36 @@ class Reach(Element): @dataclass class Junction(Element): + """Represents a Junction element.""" + geom: Point = None @dataclass class Sink(Element): + """Represents a Sink element.""" + geom: Point = None @dataclass class Reservoir(Element): + """Represents a Reservoir element.""" + geom: Point = None @dataclass class Source(Element): + """Represents a Source element.""" + geom: Point = None @dataclass class Diversion(Element): + """Represents a Diversion element.""" + geom: Point = None @@ -156,18 +184,23 @@ def __init__(self): self.index_ = 0 def __setitem__(self, key, item): + """Add an element to the set.""" utils.add_no_duplicate(self.elements, key, item) def __getitem__(self, key): + """Retrieve an element by name.""" return self.elements[key] def __len__(self): + """Return the number of elements.""" return len(self.elements) def __iter__(self): + """Iterate over elements.""" return iter(self.elements.items()) def subset(self, element_type: Element): + """Retrieve a subset of elements of a given type.""" element_subset = ElementSet() for element in self.elements.values(): if isinstance(element, element_type): @@ -175,6 +208,7 @@ def subset(self, element_type: Element): return element_subset def get_element_type(self, element_type): + """Retrieve elements of a specific type by name.""" element_list = [] for element in self.elements.values(): if type(element).__name__ == element_type: @@ -183,6 +217,7 @@ def get_element_type(self, element_type): @property def element_types(self) -> list: + """Get a list of unique element types.""" types = [] for element in self.elements.values(): types.append(type(element).__name__) @@ -190,6 +225,7 @@ def element_types(self) -> list: @property def element_counts(self) -> dict: + """Get a count of each element type.""" types = [] for element in self.elements.values(): types.append(type(element).__name__) @@ -197,6 +233,7 @@ def element_counts(self) -> dict: @property def gages(self): + """Retrieve gage elements with their observed hydrograph gage names.""" gages = {} for name, element in self.elements.items(): if "Observed Hydrograph Gage" in element.attrs.keys(): diff --git a/hecstac/hms/item.py b/hecstac/hms/item.py index 3f3e7c5..1180775 100644 --- a/hecstac/hms/item.py +++ b/hecstac/hms/item.py @@ -1,3 +1,5 @@ +"""HEC-RAS STAC Item class.""" + import json import logging import os @@ -88,7 +90,7 @@ def _properties(self): if self.pf.basins[0].epsg: logger.warning("No EPSG code found in basin file.") properties["proj:wkt"] = self.pf.basins[0].wkt - properties[SUMMARY] = self.pf.file_counts + properties[self.SUMMARY] = self.pf.file_counts return properties @property @@ -169,9 +171,7 @@ def add_hms_asset(self, fpath: str) -> None: self._project = asset def make_thumbnail(self, gdfs: dict): - """Create a png from the geodataframes (values of the dictionary). - The dictionary keys are used to label the layers in the legend. - """ + """Create a png from the geodataframes (values of the dictionary). The dictionary keys are used to label the layers in the legend.""" cdict = { "Subbasin": "black", "Reach": "blue", diff --git a/hecstac/hms/s3_utils.py b/hecstac/hms/s3_utils.py index f481fb5..1b41943 100644 --- a/hecstac/hms/s3_utils.py +++ b/hecstac/hms/s3_utils.py @@ -1,3 +1,5 @@ +"""AWS S3 utlity functions.""" + import os from pathlib import Path @@ -29,6 +31,7 @@ def file_location(file: str | Path) -> str: def list_keys(s3_client, bucket, prefix, suffix=""): + """List s3 keys in a given bucket and prefix.""" keys = [] kwargs = {"Bucket": bucket, "Prefix": prefix} while True: @@ -61,7 +64,7 @@ def get_metadata(key: str) -> str: def split_s3_key(s3_path: str) -> tuple[str, str]: """ - This function splits an S3 path into the bucket name and the key. + Split an S3 path into the bucket name and the key. Parameters ---------- @@ -87,6 +90,7 @@ def split_s3_key(s3_path: str) -> tuple[str, str]: def init_s3_resources(minio_mode: bool = False): + """Initialize s3 resources.""" if minio_mode: session = boto3.Session( aws_access_key_id=os.environ.get("MINIO_ACCESS_KEY_ID"), @@ -112,7 +116,7 @@ def init_s3_resources(minio_mode: bool = False): def get_basic_object_metadata(obj: ObjectSummary) -> dict: """ - This function retrieves basic metadata of an AWS S3 object. + Retrieve basic metadata of an AWS S3 object. Parameters ---------- diff --git a/hecstac/hms/utils.py b/hecstac/hms/utils.py index 89b81c5..9892b35 100644 --- a/hecstac/hms/utils.py +++ b/hecstac/hms/utils.py @@ -1,3 +1,5 @@ +"""HEC-HMS STAC Item utlity functions.""" + from __future__ import annotations import re @@ -24,6 +26,7 @@ def add_no_duplicate(d: dict, key, val): def get_lines_until_end_sentinel(lines: list[str]) -> list[str]: + """Retrieve all lines until the End point.""" lines_found = [] for line in lines: if line in ["End:", "End Computation Point: "]: @@ -35,6 +38,7 @@ def get_lines_until_end_sentinel(lines: list[str]) -> list[str]: def handle_special_cases(key, val): + """Handle special cases.""" if key == "Groundwater Layer": key = key + val elif "Groundwater Layer" in key: @@ -47,7 +51,7 @@ def handle_special_cases(key, val): def parse_attrs(lines: list[str]) -> OrderedDict: - """Scan the lines down to the first instance of 'End:' and return dict containing all of the colon-separated keyval pair""" + """Scan the lines down to the first instance of 'End:' and return dict containing all of the colon-separated keyval pair.""" attrs = {} for line in lines: if line == "End:": @@ -89,6 +93,7 @@ def parse_attrs(lines: list[str]) -> OrderedDict: def remove_holes(geom): + """Remove holes in the geometry.""" if isinstance(geom, Polygon): return Polygon(geom.exterior) elif isinstance(geom, MultiPolygon): @@ -105,6 +110,7 @@ def remove_holes(geom): def attrs2list(attrs: OrderedDict) -> list[str]: + """Convert dictionary of attributes to a list.""" content = [] for key, val in attrs.items(): if not isinstance(val, str): @@ -130,7 +136,7 @@ def attrs2list(attrs: OrderedDict) -> list[str]: def insert_after_key(dic: dict, insert_key: str, new_key: str, new_val: str) -> OrderedDict: - # recreate the dictionary to insert key-val after the occurance of the insert_key if key-val doesn't exist yet in the dictionary + """Recreate the dictionary to insert key-val after the occurance of the insert_key if key-val doesn't exist yet in the dictionary.""" new_dic = {} for key, val in dic.items(): if key == new_key: @@ -159,41 +165,49 @@ def search_contents(lines: list, search_string: str, token: str = "=", expect_on class StacPathManager: - """ - Builds consistent paths for STAC items and collections assuming a top level local catalog - """ + """Build consistent paths for STAC items and collections assuming a top level local catalog.""" def __init__(self, local_catalog_dir: str): self._catalog_dir = local_catalog_dir @property def catalog_dir(self): + """Return the catalog directory.""" return self._catalog_dir @property def catalog_file(self): + """Return the catalog file path.""" return f"{self._catalog_dir}/catalog.json" def catalog_item(self, item_id: str) -> str: + """Return the catalog item file path.""" return f"{self.catalog_dir}/{item_id}/{item_id}.json" def catalog_asset(self, item_id: str, asset_dir: str = "hydro_domains") -> str: + """Return the catalog asset file path.""" return f"{self.catalog_dir}/{asset_dir}/{item_id}.json" def collection_file(self, collection_id: str) -> str: + """Return the collection file path.""" return f"{self.catalog_dir}/{collection_id}/collection.json" def collection_dir(self, collection_id: str) -> str: + """Return the collection directory.""" return f"{self.catalog_dir}/{collection_id}" def collection_asset(self, collection_id: str, filename: str) -> str: + """Return the collection asset filepath.""" return f"{self.catalog_dir}/{collection_id}/{filename}" def collection_item_dir(self, collection_id: str, item_id: str) -> str: + """Return the collection item directory.""" return f"{self.catalog_dir}/{collection_id}/{item_id}" def collection_item(self, collection_id: str, item_id: str) -> str: + """Return the collection item filepath.""" return f"{self.catalog_dir}/{collection_id}/{item_id}/{item_id}.json" def collection_item_asset(self, collection_id: str, item_id: str, filename: str) -> str: + """Return the collection item asset filepath.""" return f"{self.catalog_dir}/{collection_id}/{item_id}/{filename}" diff --git a/hecstac/ras/__init__.py b/hecstac/ras/__init__.py index 02ba329..19b0aea 100644 --- a/hecstac/ras/__init__.py +++ b/hecstac/ras/__init__.py @@ -1 +1 @@ -"""HEC-RAS STAC Item creation module.""" +"""HEC-RAS STAC Item module.""" diff --git a/hecstac/ras/item.py b/hecstac/ras/item.py index 20d28ca..9e7b4a1 100644 --- a/hecstac/ras/item.py +++ b/hecstac/ras/item.py @@ -1,4 +1,4 @@ -"""HEC-RAS STAC Item creation class.""" +"""HEC-RAS STAC Item class.""" import datetime import json diff --git a/hecstac/utils/__init__.py b/hecstac/utils/__init__.py index e69de29..ddf8d81 100644 --- a/hecstac/utils/__init__.py +++ b/hecstac/utils/__init__.py @@ -0,0 +1 @@ +"""Utlity scripts for hecstac.""" diff --git a/pyproject.toml b/pyproject.toml index 259438d..1bef4ac 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -59,10 +59,7 @@ line-length = 120 [tool.ruff.lint.per-file-ignores] "tests/**" = ["D"] "docs/**" = ["D"] -"hecstac/hms/**" = ["D"] "server.py" = ["D"] -"hecstac/utils/**" = ["D"] -"hecstac/events/**" = ["D"] [tool.setuptools.packages.find] From c4d02d5f1f1d30444292063be9cf4557eb76e4c2 Mon Sep 17 00:00:00 2001 From: Stevenray Janke Date: Fri, 7 Feb 2025 15:45:34 -0500 Subject: [PATCH 61/71] Bug fixes --- hecstac/common/asset_factory.py | 1 + hecstac/ras/assets.py | 8 ++++---- hecstac/ras/item.py | 16 ++++++++++++---- 3 files changed, 17 insertions(+), 8 deletions(-) diff --git a/hecstac/common/asset_factory.py b/hecstac/common/asset_factory.py index c6449f8..ba93244 100644 --- a/hecstac/common/asset_factory.py +++ b/hecstac/common/asset_factory.py @@ -28,6 +28,7 @@ def __init__(self, *args, **kwargs): self.description = self.__description__ self._roles = [] self._extra_fields = {} + self.name = Path(self.href).name @property def roles(self) -> list[str]: diff --git a/hecstac/ras/assets.py b/hecstac/ras/assets.py index c96f273..21a2904 100644 --- a/hecstac/ras/assets.py +++ b/hecstac/ras/assets.py @@ -374,7 +374,7 @@ def extra_fields(self) -> dict: """Return extra fields with added dynamic keys/values.""" self._extra_fields[VERSION] = self.file.file_version self._extra_fields[UNITS] = self.file.units_system - self._extra_fields[REFERENCE_LINES] = self.file.reference_lines + self._extra_fields[REFERENCE_LINES] = self.reference_lines return self._extra_fields @property @@ -526,15 +526,15 @@ def thumbnail( for layer in layers: try: if layer == "mesh_areas": - mesh_areas_data = self.hdf_object.mesh_cells + mesh_areas_data = self.file.mesh_cells mesh_areas_geo = mesh_areas_data.set_crs(self.crs) legend_handles += self._plot_mesh_areas(ax, mesh_areas_geo) elif layer == "breaklines": - breaklines_data = self.hdf_object.breaklines + breaklines_data = self.file.breaklines breaklines_data_geo = breaklines_data.set_crs(self.crs) legend_handles += self._plot_breaklines(ax, breaklines_data_geo) elif layer == "bc_lines": - bc_lines_data = self.hdf_object.bc_lines + bc_lines_data = self.file.bc_lines bc_lines_data_geo = bc_lines_data.set_crs(self.crs) legend_handles += self._plot_bc_lines(ax, bc_lines_data_geo) except Exception as e: diff --git a/hecstac/ras/item.py b/hecstac/ras/item.py index 9e7b4a1..8e4fd39 100644 --- a/hecstac/ras/item.py +++ b/hecstac/ras/item.py @@ -151,8 +151,16 @@ def geometry(self) -> dict: if len(self.geometry_assets) == 0: logger.error("No geometry found for RAS item.") return NULL_STAC_GEOMETRY + print(self.geometry_assets) + + geometries = [] + for i in self.geometry_assets: + try: + geometries.append(i.geometry_wgs84) + except Exception as e: + logger.warning(f"Could not process geometry from {i.href}") + continue - geometries = [i.geometry_wgs84 for i in self.geometry_assets] unioned_geometry = union_all(geometries) if self.simplify_geometry: unioned_geometry = simplify(unioned_geometry, 0.001) @@ -234,10 +242,9 @@ def add_model_thumbnails(self, layers: list, title_prefix: str = "Model_Thumbnai thumbnail_dest = thumbnail_dir else: thumbnail_dest = self.self_href - thumbnail_dest = self.self_href for geom in self.geometry_assets: - if isinstance(geom, GeometryHdfAsset): + if isinstance(geom, GeometryHdfAsset) and geom.has_2d: self.assets[f"{geom.href}_thumbnail"] = geom.thumbnail( layers=layers, title=title_prefix, thumbnail_dest=thumbnail_dest ) @@ -249,7 +256,8 @@ def add_asset(self, key, asset): subclass = self.factory.asset_from_dict(asset) if subclass is None: return - if self.crs is None and isinstance(asset, GeometryHdfAsset) and asset.file.projection is not None: + if self.crs is None and isinstance(subclass, GeometryHdfAsset) and subclass.file.projection is not None: + print("setting crs") self.crs = subclass.file.projection return super().add_asset(key, subclass) From a4c597023abb5f5e411cf4a5ff41779ff3a2c6b9 Mon Sep 17 00:00:00 2001 From: Stevenray Janke Date: Fri, 7 Feb 2025 15:45:55 -0500 Subject: [PATCH 62/71] Refactor hms assets to align with new GenericAsset class --- hecstac/hms/assets.py | 309 ++++++++++++++++++------------------------ 1 file changed, 133 insertions(+), 176 deletions(-) diff --git a/hecstac/hms/assets.py b/hecstac/hms/assets.py index 64bef43..0b80007 100644 --- a/hecstac/hms/assets.py +++ b/hecstac/hms/assets.py @@ -20,256 +20,213 @@ class GeojsonAsset(GenericAsset): """Geojson asset.""" - def __init__(self, href: str, *args, **kwargs): - roles = ["data"] - media_type = MediaType.GEOJSON - description = "Geojson file." - super().__init__(href, roles=roles, description=description, media_type=media_type, *args, **kwargs) + __roles__ = ["data", MediaType.GEOJSON] + __description__ = "Geojson file." class TiffAsset(GenericAsset): """Tiff Asset.""" - def __init__(self, href: str, *args, **kwargs): - roles = ["data"] - media_type = MediaType.GEOTIFF - description = "Tiff file." - super().__init__(href, roles=roles, description=description, media_type=media_type, *args, **kwargs) + __roles__ = ["data", MediaType.GEOTIFF] + __description__ = "Tiff file." -class ProjectAsset(GenericAsset): +class ProjectAsset(GenericAsset[ProjectFile]): """HEC-HMS Project file asset.""" - def __init__(self, href: str, *args, **kwargs): - - roles = ["hms-project"] - media_type = MediaType.TEXT - description = "The HEC-HMS project file. Summary provied at the item level" - - super().__init__(href, roles=roles, description=description, media_type=media_type, *args, **kwargs) - self.pf = ProjectFile(href, assert_uniform_version=False) + __roles__ = ["hms-project", MediaType.TEXT] + __description__ = "The HEC-HMS project file. Summary provied at the item level" + __file_class__ = ProjectFile class ThumbnailAsset(GenericAsset): """Thumbnail asset.""" - def __init__(self, href: str, *args, **kwargs): - roles = ["thumbnail"] - media_type = MediaType.PNG - description = "Thumbnail" - super().__init__(href, roles=roles, description=description, media_type=media_type, *args, **kwargs) + __roles__ = ["thumbnail", MediaType.PNG] + __description__ = "Thumbnail" -class ModelBasinAsset(GenericAsset): +class ModelBasinAsset(GenericAsset[BasinFile]): """HEC-HMS Basin file asset from authoritative model, containing geometry and other detailed data.""" - def __init__(self, href: str, *args, **kwargs): - roles = ["hms-basin"] - media_type = MediaType.TEXT - description = "Defines the basin geometry and elements for HEC-HMS simulations." - super().__init__( - href, - roles=roles, - description=description, - media_type=media_type, - *args, - **kwargs, - ) - self.bf = BasinFile(href, read_geom=True) - self.extra_fields = { - "hms:title": self.bf.name, - "hms:version": self.bf.header.attrs["Version"], - "hms:description": self.bf.header.attrs.get("Description"), - "hms:unit_system": self.bf.header.attrs["Unit System"], - "hms:gages": self.bf.gages, - "hms:drainage_area_miles": self.bf.drainage_area, - "hms:reach_length_miles": self.bf.reach_miles, - "proj:wkt": self.bf.wkt, - "proj:code": self.bf.epsg, - } | {f"hms_basin:{key}".lower(): val for key, val in self.bf.elements.element_counts.items()} - - -class EventBasinAsset(GenericAsset): + __roles__ = ["hms-basin", MediaType.TEXT] + __description__ = "Defines the basin geometry and elements for HEC-HMS simulations." + __file_class__ = BasinFile + + @GenericAsset.extra_fields.getter + def extra_fields(self): + return { + "hms:title": self.file.name, + "hms:version": self.file.header.attrs["Version"], + "hms:description": self.file.header.attrs.get("Description"), + "hms:unit_system": self.file.header.attrs["Unit System"], + "hms:gages": self.file.gages, + "hms:drainage_area_miles": self.file.drainage_area, + "hms:reach_length_miles": self.file.reach_miles, + "proj:wkt": self.file.wkt, + "proj:code": self.file.epsg, + } | {f"hms_basin:{key}".lower(): val for key, val in self.file.elements.element_counts.items()} + + +class EventBasinAsset(GenericAsset[BasinFile]): """HEC-HMS Basin file asset from event, with limited basin info.""" - def __init__(self, href: str, *args, **kwargs): - roles = ["hms-basin"] - media_type = MediaType.TEXT - description = "Defines the basin geometry and elements for HEC-HMS simulations." - super().__init__( - href, - roles=roles, - description=description, - media_type=media_type, - *args, - **kwargs, - ) - self.bf = BasinFile(href) - self.extra_fields = { - "hms:title": self.bf.name, - "hms:version": self.bf.header.attrs["Version"], - "hms:description": self.bf.header.attrs.get("Description"), - "hms:unit_system": self.bf.header.attrs["Unit System"], + __roles__ = ["hms-basin", MediaType.TEXT] + __description__ = "Defines the basin geometry and elements for HEC-HMS simulations." + __file_class__ = BasinFile + + @GenericAsset.extra_fields.getter + def extra_fields(self): + return { + "hms:title": self.file.name, + "hms:version": self.file.header.attrs["Version"], + "hms:description": self.file.header.attrs.get("Description"), + "hms:unit_system": self.file.header.attrs["Unit System"], } -class RunAsset(GenericAsset): +class RunAsset(GenericAsset[RunFile]): """Run asset.""" - def __init__(self, href: str, *args, **kwargs): - self.rf = RunFile(href) - roles = ["hms-run"] - media_type = MediaType.TEXT - description = "Contains data for HEC-HMS simulations." - super().__init__(href, roles=roles, description=description, media_type=media_type, *args, **kwargs) - self.extra_fields = {"hms:title": self.name} | { - run.name: {f"hms:{key}".lower(): val for key, val in run.attrs.items()} for _, run in self.rf.elements + __file_class__ = RunFile + __roles__ = ["hms-run", MediaType.TEXT] + __description__ = "Contains data for HEC-HMS simulations." + + @GenericAsset.extra_fields.getter + def extra_fields(self): + return {"hms:title": self.name} | { + run.name: {f"hms:{key}".lower(): val for key, val in run.attrs.items()} for _, run in self.file.elements } -class ControlAsset(GenericAsset): +class ControlAsset(GenericAsset[ControlFile]): """HEC-HMS Control file asset.""" - def __init__(self, href: str, *args, **kwargs): - roles = ["hms-control"] - media_type = MediaType.TEXT - description = "Defines time control information for HEC-HMS simulations." - super().__init__( - href, - roles=roles, - description=description, - media_type=media_type, - *args, - **kwargs, - ) - self.cf = ControlFile(href) - self.extra_fields = { - "hms:title": self.cf.name, - **{f"hms:{key}".lower(): val for key, val in self.cf.attrs.items()}, + __roles__ = ["hms-control", MediaType.TEXT] + __description__ = "Defines time control information for HEC-HMS simulations." + __file_class__ = ControlFile + + @GenericAsset.extra_fields.getter + def extra_fields(self): + return { + "hms:title": self.file.name, + **{f"hms:{key}".lower(): val for key, val in self.file.attrs.items()}, } -class MetAsset(GenericAsset): +class MetAsset(GenericAsset[MetFile]): """HEC-HMS Meteorological file asset.""" - def __init__(self, href: str, *args, **kwargs): - roles = ["hms-met"] - media_type = MediaType.TEXT - description = "Contains meteorological data such as precipitation and temperature." - super().__init__( - href, - roles=roles, - description=description, - media_type=media_type, - *args, - **kwargs, - ) - self.mf = MetFile(href) - self.extra_fields = { - "hms:title": self.mf.name, - **{f"hms:{key}".lower(): val for key, val in self.mf.attrs.items()}, + __roles__ = ["hms-met", MediaType.TEXT] + __description__ = "Contains meteorological data such as precipitation and temperature." + __file_class__ = MetFile + + @GenericAsset.extra_fields.getter + def extra_fields(self): + return { + "hms:title": self.file.name, + **{f"hms:{key}".lower(): val for key, val in self.file.attrs.items()}, } class DSSAsset(GenericAsset): """DSS asset.""" - def __init__(self, href: str, *args, **kwargs): - roles = ["hec-dss"] - media_type = "application/octet-stream" - description = "HEC-DSS file." - super().__init__(href, roles=roles, description=description, media_type=media_type, *args, **kwargs) + __roles__ = ["hec-dss", "application/octet-stream"] + __description__ = "HEC-DSS file." - self.extra_fields["hms:title"] = self.name + @GenericAsset.extra_fields.getter + def extra_fields(self): + return {"hms:title": self.name} -class SqliteAsset(GenericAsset): +class SqliteAsset(GenericAsset[SqliteDB]): """HEC-HMS SQLite database asset.""" - def __init__(self, href: str, *args, **kwargs): - roles = ["hms-sqlite"] - media_type = "application/x-sqlite3" - description = "Stores spatial data for HEC-HMS basin files." - super().__init__(href, roles=roles, description=description, media_type=media_type, *args, **kwargs) - self.sqdb = SqliteDB(href) - self.extra_fields = {"hms:title": self.name, "hms:layers": self.sqdb.layers} + __roles__ = ["hms-sqlite", "application/x-sqlite3"] + __description__ = "Stores spatial data for HEC-HMS basin files." + __file_class__ = SqliteDB + + @GenericAsset.extra_fields.getter + def extra_fields(self): + return {"hms:title": self.name, "hms:layers": self.file.layers} -class GageAsset(GenericAsset): +class GageAsset(GenericAsset[GageFile]): """Gage asset.""" - def __init__(self, href: str, *args, **kwargs): - roles = ["hms-gage"] - media_type = MediaType.TEXT - description = "Contains data for HEC-HMS gages." - super().__init__(href, roles=roles, description=description, media_type=media_type, *args, **kwargs) - self.gf = GageFile(href) - self.extra_fields = {"hms:title": self.gf.name, "hms:version": self.gf.attrs["Version"]} | { - f"hms:{gage.name}".lower(): {key: val for key, val in gage.attrs.items()} for gage in self.gf.gages + __roles__ = ["hms-gage", MediaType.TEXT] + __description__ = "Contains data for HEC-HMS gages." + __file_class__ = GageFile + + @GenericAsset.extra_fields.getter + def extra_fields(self): + return {"hms:title": self.file.name, "hms:version": self.file.attrs["Version"]} | { + f"hms:{gage.name}".lower(): {key: val for key, val in gage.attrs.items()} for gage in self.file.gages } -class GridAsset(GenericAsset): +class GridAsset(GenericAsset[GridFile]): """Grid asset.""" - def __init__(self, href: str, *args, **kwargs): - roles = ["hms-grid"] - media_type = MediaType.TEXT - description = "Contains data for HEC-HMS grid files." - super().__init__(href, roles=roles, description=description, media_type=media_type, *args, **kwargs) - self.gf = GridFile(href) - self.extra_fields = ( - {"hms:title": self.gf.name} - | {f"hms:{key}".lower(): val for key, val in self.gf.attrs.items()} - | {f"hms:{grid.name}".lower(): {key: val for key, val in grid.attrs.items()} for grid in self.gf.grids} + __roles__ = ["hms-grid", MediaType.TEXT] + __description__ = "Contains data for HEC-HMS grid files." + __file_class__ = GridFile + + @GenericAsset.extra_fields.getter + def extra_fields(self): + return ( + {"hms:title": self.file.name} + | {f"hms:{key}".lower(): val for key, val in self.file.attrs.items()} + | {f"hms:{grid.name}".lower(): {key: val for key, val in grid.attrs.items()} for grid in self.file.grids} ) class LogAsset(GenericAsset): """Log asset.""" - def __init__(self, href: str, *args, **kwargs): - roles = ["hms-log", "results"] - media_type = MediaType.TEXT - description = "Contains log data for HEC-HMS simulations." - super().__init__(href, roles=roles, description=description, media_type=media_type, *args, **kwargs) - self.extra_fields["hms:title"] = self.name + __roles__ = ["hms-log", "results", MediaType.TEXT] + __description__ = "Contains log data for HEC-HMS simulations." + + @GenericAsset.extra_fields.getter + def extra_fields(self): + return {"hms:title": self.name} class OutAsset(GenericAsset): """Out asset.""" - def __init__(self, href: str, *args, **kwargs): - roles = ["hms-out", "results"] - media_type = MediaType.TEXT - description = "Contains output data for HEC-HMS simulations." - super().__init__(href, roles=roles, description=description, media_type=media_type, *args, **kwargs) - self.extra_fields["hms:title"] = self.name + __roles__ = ["hms-out", "results", MediaType.TEXT] + __description__ = "Contains output data for HEC-HMS simulations." + + @GenericAsset.extra_fields.getter + def extra_fields(self): + return {"hms:title": self.name} -class PdataAsset(GenericAsset): +class PdataAsset(GenericAsset[PairedDataFile]): """Pdata asset.""" - def __init__(self, href: str, *args, **kwargs): - roles = ["hms-pdata"] - media_type = MediaType.TEXT - description = "Contains paired data for HEC-HMS simulations." - super().__init__(href, roles=roles, description=description, media_type=media_type, *args, **kwargs) - self.pd = PairedDataFile(href) - self.extra_fields = {"hms:title": self.pd.name, "hms:version": self.pd.attrs["Version"]} + __roles__ = ["hms-pdata", MediaType.TEXT] + __description__ = "Contains paired data for HEC-HMS simulations." + __file_class__ = PairedDataFile + @GenericAsset.extra_fields.getter + def extra_fields(self): + return {"hms:title": self.file.name, "hms:version": self.file.attrs["Version"]} -class TerrainAsset(GenericAsset): + +class TerrainAsset(GenericAsset[TerrainFile]): """Terrain asset.""" - def __init__(self, href: str, *args, **kwargs): - roles = ["hms-terrain"] - media_type = MediaType.GEOTIFF - description = "Contains terrain data for HEC-HMS simulations." - super().__init__(href, roles=roles, description=description, media_type=media_type, *args, **kwargs) - self.tf = TerrainFile(href) - self.extra_fields = {"hms:title": self.tf.name, "hms:version": self.tf.attrs["Version"]} | { - f"hms:{layer['name']}".lower(): {key: val for key, val in layer.items()} for layer in self.tf.layers + __roles__ = ["hms-terrain", MediaType.GEOTIFF] + __description__ = "Contains terrain data for HEC-HMS simulations." + __file_class__ = TerrainFile + + @GenericAsset.extra_fields.getter + def extra_fields(self): + return {"hms:title": self.file.name, "hms:version": self.file.attrs["Version"]} | { + f"hms:{layer['name']}".lower(): {key: val for key, val in layer.items()} for layer in self.file.layers } From 8142d845032fab9749d1a443ad6626ced83f892b Mon Sep 17 00:00:00 2001 From: Stevenray Janke Date: Fri, 7 Feb 2025 17:34:23 -0500 Subject: [PATCH 63/71] Begin HMS Item refactor --- hecstac/hms/assets.py | 82 +++++++++++---- hecstac/hms/item.py | 225 +++++++++++++++++++++++++++--------------- 2 files changed, 208 insertions(+), 99 deletions(-) diff --git a/hecstac/hms/assets.py b/hecstac/hms/assets.py index 0b80007..c292372 100644 --- a/hecstac/hms/assets.py +++ b/hecstac/hms/assets.py @@ -1,7 +1,7 @@ """HEC-HMS Stac Item asset classes.""" from pystac import MediaType - +import re from hecstac.common.asset_factory import GenericAsset from hecstac.hms.parser import ( BasinFile, @@ -20,6 +20,7 @@ class GeojsonAsset(GenericAsset): """Geojson asset.""" + ext = r".*\.geojson$" __roles__ = ["data", MediaType.GEOJSON] __description__ = "Geojson file." @@ -27,13 +28,15 @@ class GeojsonAsset(GenericAsset): class TiffAsset(GenericAsset): """Tiff Asset.""" + ext = r".*\.tiff$" __roles__ = ["data", MediaType.GEOTIFF] __description__ = "Tiff file." class ProjectAsset(GenericAsset[ProjectFile]): - """HEC-HMS Project file asset.""" + """Project asset.""" + ext = r".*\.hms$" __roles__ = ["hms-project", MediaType.TEXT] __description__ = "The HEC-HMS project file. Summary provied at the item level" __file_class__ = ProjectFile @@ -42,6 +45,7 @@ class ProjectAsset(GenericAsset[ProjectFile]): class ThumbnailAsset(GenericAsset): """Thumbnail asset.""" + ext = r".*\.png$" __roles__ = ["thumbnail", MediaType.PNG] __description__ = "Thumbnail" @@ -49,6 +53,7 @@ class ThumbnailAsset(GenericAsset): class ModelBasinAsset(GenericAsset[BasinFile]): """HEC-HMS Basin file asset from authoritative model, containing geometry and other detailed data.""" + ext = r".*\.basin$" __roles__ = ["hms-basin", MediaType.TEXT] __description__ = "Defines the basin geometry and elements for HEC-HMS simulations." __file_class__ = BasinFile @@ -71,6 +76,7 @@ def extra_fields(self): class EventBasinAsset(GenericAsset[BasinFile]): """HEC-HMS Basin file asset from event, with limited basin info.""" + ext = r".*\.basin$" __roles__ = ["hms-basin", MediaType.TEXT] __description__ = "Defines the basin geometry and elements for HEC-HMS simulations." __file_class__ = BasinFile @@ -88,6 +94,7 @@ def extra_fields(self): class RunAsset(GenericAsset[RunFile]): """Run asset.""" + ext = r".*\.run$" __file_class__ = RunFile __roles__ = ["hms-run", MediaType.TEXT] __description__ = "Contains data for HEC-HMS simulations." @@ -102,6 +109,7 @@ def extra_fields(self): class ControlAsset(GenericAsset[ControlFile]): """HEC-HMS Control file asset.""" + ext = r".*\.control$" __roles__ = ["hms-control", MediaType.TEXT] __description__ = "Defines time control information for HEC-HMS simulations." __file_class__ = ControlFile @@ -117,6 +125,7 @@ def extra_fields(self): class MetAsset(GenericAsset[MetFile]): """HEC-HMS Meteorological file asset.""" + ext = r".*\.met$" __roles__ = ["hms-met", MediaType.TEXT] __description__ = "Contains meteorological data such as precipitation and temperature." __file_class__ = MetFile @@ -132,6 +141,7 @@ def extra_fields(self): class DSSAsset(GenericAsset): """DSS asset.""" + ext = r".*\.dss$" __roles__ = ["hec-dss", "application/octet-stream"] __description__ = "HEC-DSS file." @@ -143,6 +153,7 @@ def extra_fields(self): class SqliteAsset(GenericAsset[SqliteDB]): """HEC-HMS SQLite database asset.""" + ext = r".*\.sqlite$" __roles__ = ["hms-sqlite", "application/x-sqlite3"] __description__ = "Stores spatial data for HEC-HMS basin files." __file_class__ = SqliteDB @@ -155,6 +166,7 @@ def extra_fields(self): class GageAsset(GenericAsset[GageFile]): """Gage asset.""" + ext = r".*\.gage$" __roles__ = ["hms-gage", MediaType.TEXT] __description__ = "Contains data for HEC-HMS gages." __file_class__ = GageFile @@ -169,6 +181,7 @@ def extra_fields(self): class GridAsset(GenericAsset[GridFile]): """Grid asset.""" + ext = r".*\.grid$" __roles__ = ["hms-grid", MediaType.TEXT] __description__ = "Contains data for HEC-HMS grid files." __file_class__ = GridFile @@ -185,6 +198,7 @@ def extra_fields(self): class LogAsset(GenericAsset): """Log asset.""" + ext = r".*\.log$" __roles__ = ["hms-log", "results", MediaType.TEXT] __description__ = "Contains log data for HEC-HMS simulations." @@ -196,6 +210,7 @@ def extra_fields(self): class OutAsset(GenericAsset): """Out asset.""" + ext = r".*\.out$" __roles__ = ["hms-out", "results", MediaType.TEXT] __description__ = "Contains output data for HEC-HMS simulations." @@ -207,6 +222,7 @@ def extra_fields(self): class PdataAsset(GenericAsset[PairedDataFile]): """Pdata asset.""" + ext = r".*\.pdata$" __roles__ = ["hms-pdata", MediaType.TEXT] __description__ = "Contains paired data for HEC-HMS simulations." __file_class__ = PairedDataFile @@ -219,6 +235,7 @@ def extra_fields(self): class TerrainAsset(GenericAsset[TerrainFile]): """Terrain asset.""" + ext = r".*\.terrain$" __roles__ = ["hms-terrain", MediaType.GEOTIFF] __description__ = "Contains terrain data for HEC-HMS simulations." __file_class__ = TerrainFile @@ -230,22 +247,45 @@ def extra_fields(self): } -HMS_EXTENSION_MAPPING = { - ".hms": ProjectAsset, - ".basin": {"event": EventBasinAsset, "model": ModelBasinAsset}, - ".control": ControlAsset, - ".met": MetAsset, - ".sqlite": SqliteAsset, - ".gage": GageAsset, - ".run": RunAsset, - ".grid": GridAsset, - ".log": LogAsset, - ".out": OutAsset, - ".pdata": PdataAsset, - ".terrain": TerrainAsset, - ".dss": DSSAsset, - ".geojson": GeojsonAsset, - ".tiff": TiffAsset, - ".tif": TiffAsset, - ".png": ThumbnailAsset, -} +# HMS_EXTENSION_MAPPING = { +# ".hms": ProjectAsset, +# ".basin": {"event": EventBasinAsset, "model": ModelBasinAsset}, +# ".control": ControlAsset, +# ".met": MetAsset, +# ".sqlite": SqliteAsset, +# ".gage": GageAsset, +# ".run": RunAsset, +# ".grid": GridAsset, +# ".log": LogAsset, +# ".out": OutAsset, +# ".pdata": PdataAsset, +# ".terrain": TerrainAsset, +# ".dss": DSSAsset, +# ".geojson": GeojsonAsset, +# ".tiff": TiffAsset, +# ".tif": TiffAsset, +# ".png": ThumbnailAsset, +# } + +HMS_ASSET_CLASSES = [ + ProjectAsset, + EventBasinAsset, + ModelBasinAsset, + ControlAsset, + MetAsset, + SqliteAsset, + GageAsset, + RunAsset, + GridAsset, + LogAsset, + OutAsset, + PdataAsset, + TerrainAsset, + DSSAsset, + GeojsonAsset, + TiffAsset, + TiffAsset, + ThumbnailAsset, +] + +HMS_EXTENSION_MAPPING = {re.compile(cls.ext, re.IGNORECASE): cls for cls in HMS_ASSET_CLASSES} diff --git a/hecstac/hms/item.py b/hecstac/hms/item.py index 1180775..71d3d52 100644 --- a/hecstac/hms/item.py +++ b/hecstac/hms/item.py @@ -10,16 +10,23 @@ import matplotlib.pyplot as plt import numpy as np import requests -from pystac import Item +from pystac import Item, Asset from pystac.extensions.projection import ProjectionExtension from pystac.extensions.storage import StorageExtension -from shapely import to_geojson, union_all +from shapely import to_geojson, union_all, unary_union +from functools import lru_cache from hecstac.common.asset_factory import AssetFactory from hecstac.common.path_manager import LocalPathManager from hecstac.hms.assets import HMS_EXTENSION_MAPPING, ProjectAsset from hecstac.hms.parser import BasinFile, ProjectFile +from hecstac.ras.consts import ( + NULL_DATETIME, + NULL_STAC_BBOX, + NULL_STAC_GEOMETRY, +) + logger = logging.getLogger(__name__) @@ -35,85 +42,133 @@ class HMSModelItem(Item): PROJECT_UNITS = "hms:unit_system" SUMMARY = "hms:summary" - def __init__(self, hms_project_file, item_id: str, simplify_geometry: bool = True): - - self._project = None - self.assets = {} - self.links = [] - self.thumbnail_paths = [] - self.geojson_paths = [] - self.extra_fields = {} - self.stac_extensions = None - self.pm = LocalPathManager(Path(hms_project_file).parent) - self._href = self.pm.item_path(item_id) - self.hms_project_file = hms_project_file - self._simplify_geometry = simplify_geometry - - self.pf = ProjectFile(self.hms_project_file, assert_uniform_version=False) - self.factory = AssetFactory(HMS_EXTENSION_MAPPING) - - super().__init__( - Path(self.hms_project_file).stem, - self._geometry, - self._bbox, - self._datetime, - self._properties, - href=self._href, - ) + def __init__(self, *args, **kwargs): + """Add a few default properties to the base class.""" + super().__init__(*args, **kwargs) + self.simplify_geometry = True + + @classmethod + def from_prj(cls, hms_project_file, item_id: str, simplify_geometry: bool = True): + """ + Create an `HMSModelItem` from a HEC-HMS project file. + + Parameters + ---------- + hms_project_file : str + Path to the HEC-HMS project file (.hms). + item_id : str + Unique item ID for the STAC item. + simplify_geometry : bool, optional + Whether to simplify geometry. Defaults to True. - self._check_files_exists(self.pf.files + self.pf.rasters) - self.make_thumbnails(self.pf.basins) - self.write_element_geojsons(self.pf.basins[0]) - for fpath in self.thumbnail_paths + self.geojson_paths + self.pf.files + self.pf.rasters: - self.add_hms_asset(fpath) + Returns + ------- + stac : HMSModelItem + An instance of the class representing the STAC item. + """ + pm = LocalPathManager(Path(hms_project_file).parent) + href = pm.item_path(item_id) + pf = ProjectFile(hms_project_file, assert_uniform_version=False) - self._register_extensions() + # Create GeoJSON and Thumbnails + cls._check_files_exists(cls, pf.files + pf.rasters) + geojson_paths = cls.write_element_geojsons(cls, pf.basins, pm) + thumbnail_paths = cls.make_thumbnails(cls, pf.basins, pm) + + # Collect all assets + assets = {Path(i).name: Asset(i) for i in pf.files + pf.rasters + geojson_paths + thumbnail_paths} + # Create the STAC Item + stac = cls( + Path(hms_project_file).stem, + NULL_STAC_GEOMETRY, + NULL_STAC_BBOX, + NULL_DATETIME, + {"hms_project_file": hms_project_file}, + href=href, + assets=assets, + ) + stac.pm = pm + stac.simplify_geometry = simplify_geometry + + return stac def _register_extensions(self) -> None: ProjectionExtension.add_to(self) StorageExtension.add_to(self) @property - def _properties(self): + def hms_project_file(self) -> str: + """Get the path to the HEC-HMS .hms file.""" + return self._properties.get("hms_project_file") + + @property + @lru_cache + def factory(self) -> AssetFactory: + """Return AssetFactory for this item.""" + return AssetFactory(HMS_EXTENSION_MAPPING) + + @property + @lru_cache + def pf(self) -> ProjectFile: + """Get a ProjectFile instance for the HMS Model .hms file.""" + return ProjectFile(self.hms_project_file) + + @property + def properties(self) -> dict: """Properties for the HMS STAC item.""" - properties = {} + properties = self._properties properties[self.PROJECT] = f"{self.pf.name}.hms" properties[self.PROJECT_TITLE] = self.pf.name - properties[self.PROJECT_VERSION] = (self.pf.attrs["Version"],) - properties[self.PROJECT_DESCRIPTION] = (self.pf.attrs.get("Description"),) + properties[self.PROJECT_VERSION] = self.pf.attrs["Version"] + properties[self.PROJECT_DESCRIPTION] = self.pf.attrs.get("Description") - # TODO probably fine 99% of the time but we grab this info from the first basin file only + # Get data from the first basin properties[self.MODEL_UNITS] = self.pf.basins[0].attrs["Unit System"] properties[self.MODEL_GAGES] = self.pf.basins[0].gages - properties["proj:code"] = self.pf.basins[0].epsg - if self.pf.basins[0].epsg: + + if self.pf.basins[0].epsg is None: logger.warning("No EPSG code found in basin file.") + properties["proj:wkt"] = self.pf.basins[0].wkt properties[self.SUMMARY] = self.pf.file_counts + return properties + @properties.setter + def properties(self, properties: dict): + """Set properties.""" + self._properties = properties + @property - def _bbox(self) -> tuple[float, float, float, float]: - """Bounding box of the HMS STAC item.""" - if len(self.pf.basins) == 0: - return [0, 0, 0, 0] - else: - bboxes = np.array([i.bbox(4326) for i in self.pf.basins]) - bboxes = [bboxes[:, 0].min(), bboxes[:, 1].min(), bboxes[:, 2].max(), bboxes[:, 3].max()] - return [float(i) for i in bboxes] + def geometry_assets(self) -> list[BasinFile]: + """Return list of basin geometry assets.""" + return self.pf.basins + + @property + def geometry(self) -> dict: + """Return footprint of the model as a GeoJSON.""" + if not self.geometry_assets: + return NULL_STAC_GEOMETRY + + geometries = [ + b.basin_geom.simplify(0.001) if self.simplify_geometry else b.basin_geom for b in self.geometry_assets + ] + unioned_geometry = unary_union(geometries) + + return json.loads(to_geojson(unioned_geometry)) @property - def _geometry(self) -> dict | None: - """Geometry of the HMS STAC item. Union of all basins in the HMS model.""" - if self._simplify_geometry: - geometries = [b.basin_geom.simplify(0.001) for b in self.pf.basins] - else: - geometries = [b.basin_geom for b in self.pf.basins] - return json.loads(to_geojson(union_all(geometries))) + def bbox(self) -> list[float]: + """Bounding box of the HMS model.""" + if not self.geometry_assets: + return [0, 0, 0, 0] + + bboxes = np.array([b.bbox(4326) for b in self.geometry_assets]) + return [float(i) for i in [bboxes[:, 0].min(), bboxes[:, 1].min(), bboxes[:, 2].max(), bboxes[:, 3].max()]] @property - def _datetime(self) -> datetime: + def datetime(self) -> datetime: """The datetime for the HMS STAC item.""" date = datetime.strptime(self.pf.basins[0].header.attrs["Last Modified Date"], "%d %B %Y") time = datetime.strptime(self.pf.basins[0].header.attrs["Last Modified Time"], "%H:%M:%S").time() @@ -127,25 +182,30 @@ def _check_files_exists(self, files: list[str]): if not os.path.exists(file): logger.warning(f"File not found {file}") - def make_thumbnails(self, basins: list[BasinFile], overwrite: bool = False): - """Create a png for each basin. Optionally overwrite existing files.""" + def make_thumbnails(self, basins: list[BasinFile], pm: LocalPathManager, overwrite: bool = False) -> list[str]: + """Create a PNG thumbnail for each basin.""" + thumbnail_paths = [] + for bf in basins: - thumbnail_path = self.pm.derived_item_asset(f"{bf.name}.png".replace(" ", "_").replace("-", "_")) + thumbnail_path = pm.derived_item_asset(f"{bf.name}.png".replace(" ", "_").replace("-", "_")) if not overwrite and os.path.exists(thumbnail_path): logger.info(f"Thumbnail for basin `{bf.name}` already exists. Skipping creation.") else: logger.info(f"{'Overwriting' if overwrite else 'Creating'} thumbnail for basin `{bf.name}`") - fig = self.make_thumbnail(bf.hms_schematic_2_gdfs) + fig = self.make_thumbnail(gdfs=bf.hms_schematic_2_gdfs) fig.savefig(thumbnail_path) fig.clf() - self.thumbnail_paths.append(thumbnail_path) + thumbnail_paths.append(thumbnail_path) - def write_element_geojsons(self, basins: list[BasinFile], overwrite: bool = False): + return thumbnail_paths + + def write_element_geojsons(self, basins: list[BasinFile], pm: LocalPathManager, overwrite: bool = False): """Write the HMS elements (Subbasins, Juctions, Reaches, etc.) to geojson.""" - for element_type in basins.elements.element_types: + geojson_paths = [] + for element_type in basins[0].elements.element_types: logger.debug(f"Checking if geojson for {element_type} exists") - path = self.pm.derived_item_asset(f"{element_type}.geojson") + path = pm.derived_item_asset(f"{element_type}.geojson") if not overwrite and os.path.exists(path): logger.info(f"Geojson for {element_type} already exists. Skipping creation.") else: @@ -155,20 +215,16 @@ def write_element_geojsons(self, basins: list[BasinFile], overwrite: bool = Fals keep_columns = ["name", "geometry", "Last Modified Date", "Last Modified Time", "Number Subreaches"] gdf = gdf[[col for col in keep_columns if col in gdf.columns]] gdf.to_file(path) - self.geojson_paths.append(path) - - def add_hms_asset(self, fpath: str) -> None: - """Add an asset to the HMS STAC item.""" - if os.path.exists(fpath): - asset = self.factory.create_hms_asset(fpath) - if asset is not None: - self.add_asset(asset.title, asset) - if isinstance(asset, ProjectAsset): - if self._project is not None: - logger.error( - f"Only one project asset is allowed. Found {str(asset)} when {str(self._project)} was already set." - ) - self._project = asset + geojson_paths.append(path) + + return geojson_paths + + def add_asset(self, key, asset): + """Subclass asset then add.""" + subclass = self.factory.asset_from_dict(asset) + if subclass is None: + return + return super().add_asset(key, subclass) def make_thumbnail(self, gdfs: dict): """Create a png from the geodataframes (values of the dictionary). The dictionary keys are used to label the layers in the legend.""" @@ -206,3 +262,16 @@ def make_thumbnail(self, gdfs: dict): ax.set_yticks([]) fig.tight_layout() return fig + + ### Prevent external modification of dynamically generated properties ### + @geometry.setter + def geometry(self, *args, **kwargs): + pass + + @bbox.setter + def bbox(self, *args, **kwargs): + pass + + @datetime.setter + def datetime(self, *args, **kwargs): + pass From 744d979d2d0aa11b729822e61013023653ebfaca Mon Sep 17 00:00:00 2001 From: Stevenray Janke Date: Mon, 10 Feb 2025 15:40:46 -0500 Subject: [PATCH 64/71] Add ruff check --- .github/workflows/ci.yaml | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 72b3624..52178f6 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -50,14 +50,14 @@ jobs: - name: Build wheel and source distribution run: python -m build - - name: Install the built wheel - run: | - python -c "import glob; import subprocess; wheel_files = glob.glob('dist/*.whl'); subprocess.check_call(['pip', 'install', wheel_files[0]])" + - name: Install the built wheel + run: | + python -c "import glob; import subprocess; wheel_files = glob.glob('dist/*.whl'); subprocess.check_call(['pip', 'install', wheel_files[0]])" - # - name: Lint (ruff) - # run: | - # ruff check . - # ruff format --check + - name: Lint (ruff) + run: | + ruff check . + ruff format --check # - name: Run tests with coverage # run: | From 8bff8d3affaf06cb8561f86faaa77228df1aa555 Mon Sep 17 00:00:00 2001 From: Stevenray Janke Date: Mon, 10 Feb 2025 15:41:10 -0500 Subject: [PATCH 65/71] Update find_model_files to correctly return the absolute path --- hecstac/ras/utils.py | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/hecstac/ras/utils.py b/hecstac/ras/utils.py index e3c2aff..08c2a85 100644 --- a/hecstac/ras/utils.py +++ b/hecstac/ras/utils.py @@ -15,11 +15,11 @@ def find_model_files(ras_prj: str) -> list[str]: - """Find all files with same base name.""" - ras_prj = Path(ras_prj) + """Find all files with the same base name and return absolute paths.""" + ras_prj = Path(ras_prj).resolve() parent = ras_prj.parent - stem = Path(ras_prj).name.split(".")[0] - return [str(i.as_posix()) for i in parent.glob(f"{stem}*")] + stem = ras_prj.stem + return [str(i.resolve()) for i in parent.glob(f"{stem}*")] def is_ras_prj(url: str) -> bool: @@ -212,6 +212,7 @@ def __init__(self, version): self.version = tuple(int(x) for x in version.split(".")) def __call__(self, func): + """Call.""" is_compatible = lib.geos_version >= self.version is_doc_build = os.environ.get("SPHINX_DOC_BUILD") == "1" # set in docs/conf.py if is_compatible and not is_doc_build: @@ -276,7 +277,7 @@ def wrapped(*args, **kwargs): @requires_geos("3.7.0") @multithreading_enabled def reverse(geometry, **kwargs): - """Returns a copy of a Geometry with the order of coordinates reversed. + """Return a copy of a Geometry with the order of coordinates reversed. If a Geometry is a polygon with interior rings, the interior rings are also reversed. From 70408e5037a53bae55a9e7d420d9e5aaece29077 Mon Sep 17 00:00:00 2001 From: Stevenray Janke Date: Mon, 10 Feb 2025 15:41:49 -0500 Subject: [PATCH 66/71] Update thumbnail handling, comment out unused classes --- hecstac/ras/assets.py | 54 ++++++++++++++++++++----------------------- hecstac/ras/item.py | 6 ++--- 2 files changed, 28 insertions(+), 32 deletions(-) diff --git a/hecstac/ras/assets.py b/hecstac/ras/assets.py index 21a2904..8e43702 100644 --- a/hecstac/ras/assets.py +++ b/hecstac/ras/assets.py @@ -114,34 +114,35 @@ METEOROLOGY_UNITS = "ras:meteorology_units" -class PrjAsset(GenericAsset): - """A helper class to delegate .prj files into RAS project or Projection file classes.""" +# class PrjAsset(GenericAsset): +# """A helper class to delegate .prj files into RAS project or Projection file classes.""" - regex_parse_str = r".+\.prj$" +# regex_parse_str = r".+\.prj$" - def __new__(cls, *args, **kwargs): - """Delegate to Project or Projection asset.""" - if cls is PrjAsset: # Ensuring we don't instantiate Parent directly - href = kwargs.get("href") or args[0] - is_ras = is_ras_prj(href) - if is_ras: - return ProjectAsset(*args, **kwargs) - else: - return ProjectionAsset(*args, **kwargs) - return super().__new__(cls) +# def __new__(cls, *args, **kwargs): +# """Delegate to Project or Projection asset.""" +# if cls is PrjAsset: # Ensuring we don't instantiate Parent directly +# href = kwargs.get("href") or args[0] +# is_ras = is_ras_prj(href) +# if is_ras: +# return ProjectAsset(*args, **kwargs) +# else: +# return ProjectionAsset(*args, **kwargs) +# return super().__new__(cls) -class ProjectionAsset(GenericAsset): - """A geospatial projection file.""" +# class ProjectionAsset(GenericAsset): +# """A geospatial projection file.""" - __roles__ = ["projection-file", MediaType.TEXT] - __description__ = "A geospatial projection file." - __file_class__ = None +# __roles__ = ["projection-file", MediaType.TEXT] +# __description__ = "A geospatial projection file." +# __file_class__ = None class ProjectAsset(GenericAsset[ProjectFile]): """HEC-RAS Project file asset.""" + regex_parse_str = r".+\.prj$" __roles__ = ["project-file", "ras-file"] __description__ = "The HEC-RAS project file." __file_class__ = ProjectFile @@ -485,21 +486,16 @@ def _plot_bc_lines(self, ax, bc_lines: gpd.GeoDataFrame) -> list[Line2D]: def _add_thumbnail_asset(self, filepath: str) -> None: """Add the thumbnail image as an asset with a relative href.""" - if filepath.startswith("s3://"): - media_type = "image/png" - else: - if not os.path.exists(filepath): - raise FileNotFoundError(f"Thumbnail file not found: {filepath}") - media_type = "image/png" + if not filepath.startswith("s3://") and not os.path.exists(filepath): + raise FileNotFoundError(f"Thumbnail file not found: {filepath}") - return GenericAsset( + asset = GenericAsset( href=filepath, title=filepath.split("/")[-1], description="Thumbnail image for the model", - media_type=media_type, - roles=["thumbnail"], - extra_fields=None, ) + asset.roles = ["thumbnail", "image/png"] + return asset def thumbnail( self, @@ -850,7 +846,7 @@ class MiscXMLFileAsset(GenericAsset): RAS_ASSET_CLASSES = [ - PrjAsset, + ProjectAsset, PlanAsset, GeometryAsset, SteadyFlowAsset, diff --git a/hecstac/ras/item.py b/hecstac/ras/item.py index 8e4fd39..323fed9 100644 --- a/hecstac/ras/item.py +++ b/hecstac/ras/item.py @@ -78,6 +78,7 @@ def from_prj(cls, ras_project_file, item_id: str, crs: str = None, simplify_geom pm = LocalPathManager(Path(ras_project_file).parent) href = pm.item_path(item_id) + # TODO: Add option to recursively iterate through all subdirectories in a model folder. assets = {Path(i).name: Asset(i, Path(i).name) for i in find_model_files(ras_project_file)} stac = cls( @@ -92,6 +93,7 @@ def from_prj(cls, ras_project_file, item_id: str, crs: str = None, simplify_geom if crs: stac.crs = crs stac.simplify_geometry = simplify_geometry + stac.pm = pm return stac @@ -151,7 +153,6 @@ def geometry(self) -> dict: if len(self.geometry_assets) == 0: logger.error("No geometry found for RAS item.") return NULL_STAC_GEOMETRY - print(self.geometry_assets) geometries = [] for i in self.geometry_assets: @@ -245,7 +246,7 @@ def add_model_thumbnails(self, layers: list, title_prefix: str = "Model_Thumbnai for geom in self.geometry_assets: if isinstance(geom, GeometryHdfAsset) and geom.has_2d: - self.assets[f"{geom.href}_thumbnail"] = geom.thumbnail( + self.assets[f"{geom.href.rsplit('/')[-1]}_thumbnail"] = geom.thumbnail( layers=layers, title=title_prefix, thumbnail_dest=thumbnail_dest ) @@ -257,7 +258,6 @@ def add_asset(self, key, asset): if subclass is None: return if self.crs is None and isinstance(subclass, GeometryHdfAsset) and subclass.file.projection is not None: - print("setting crs") self.crs = subclass.file.projection return super().add_asset(key, subclass) From 41a192879e3ca3331c11b2c020bbef7d1e4971d0 Mon Sep 17 00:00:00 2001 From: Stevenray Janke Date: Mon, 10 Feb 2025 15:42:40 -0500 Subject: [PATCH 67/71] Minor fixes and reformatting --- hecstac/__init__.py | 2 + hecstac/common/asset_factory.py | 3 +- hecstac/common/geometry.py | 2 + hecstac/hms/assets.py | 69 +++++++++++++++------------------ hecstac/hms/item.py | 3 +- 5 files changed, 39 insertions(+), 40 deletions(-) diff --git a/hecstac/__init__.py b/hecstac/__init__.py index aa7a1b7..3b9bf65 100644 --- a/hecstac/__init__.py +++ b/hecstac/__init__.py @@ -5,4 +5,6 @@ """ from hecstac.version import __version__ + from hecstac.ras.item import RASModelItem +from hecstac.hms.item import HMSModelItem diff --git a/hecstac/common/asset_factory.py b/hecstac/common/asset_factory.py index ba93244..a08223f 100644 --- a/hecstac/common/asset_factory.py +++ b/hecstac/common/asset_factory.py @@ -57,7 +57,7 @@ def extra_fields(self, extra_fields: dict): @property def file(self) -> T: """Return class to access asset file contents.""" - return self.__file_class__(self.href) + return self.__file_class__(self.get_absolute_href()) def name_from_suffix(self, suffix: str) -> str: """Generate a name by appending a suffix to the file stem.""" @@ -112,6 +112,7 @@ def create_hms_asset(self, fpath: str, item_type: str = "model") -> Asset: return check_storage_extension(asset) def asset_from_dict(self, asset: Asset): + """Create HEC asset given a base Asset and a map of file extensions dict.""" fpath = asset.href for pattern, asset_class in self.extension_to_asset.items(): if pattern.match(fpath): diff --git a/hecstac/common/geometry.py b/hecstac/common/geometry.py index 537a543..59cf5c9 100644 --- a/hecstac/common/geometry.py +++ b/hecstac/common/geometry.py @@ -1,3 +1,5 @@ +"""Geometry utils.""" + from pyproj import CRS, Transformer from shapely import Geometry from shapely.ops import transform diff --git a/hecstac/hms/assets.py b/hecstac/hms/assets.py index c292372..2249d81 100644 --- a/hecstac/hms/assets.py +++ b/hecstac/hms/assets.py @@ -20,7 +20,7 @@ class GeojsonAsset(GenericAsset): """Geojson asset.""" - ext = r".*\.geojson$" + regex_parse_str = r".*\.geojson$" __roles__ = ["data", MediaType.GEOJSON] __description__ = "Geojson file." @@ -28,7 +28,7 @@ class GeojsonAsset(GenericAsset): class TiffAsset(GenericAsset): """Tiff Asset.""" - ext = r".*\.tiff$" + regex_parse_str = r".*\.tiff$" __roles__ = ["data", MediaType.GEOTIFF] __description__ = "Tiff file." @@ -36,7 +36,7 @@ class TiffAsset(GenericAsset): class ProjectAsset(GenericAsset[ProjectFile]): """Project asset.""" - ext = r".*\.hms$" + regex_parse_str = r".*\.hms$" __roles__ = ["hms-project", MediaType.TEXT] __description__ = "The HEC-HMS project file. Summary provied at the item level" __file_class__ = ProjectFile @@ -45,7 +45,7 @@ class ProjectAsset(GenericAsset[ProjectFile]): class ThumbnailAsset(GenericAsset): """Thumbnail asset.""" - ext = r".*\.png$" + regex_parse_str = r".*\.png$" __roles__ = ["thumbnail", MediaType.PNG] __description__ = "Thumbnail" @@ -53,13 +53,14 @@ class ThumbnailAsset(GenericAsset): class ModelBasinAsset(GenericAsset[BasinFile]): """HEC-HMS Basin file asset from authoritative model, containing geometry and other detailed data.""" - ext = r".*\.basin$" + regex_parse_str = r".*\.basin$" __roles__ = ["hms-basin", MediaType.TEXT] __description__ = "Defines the basin geometry and elements for HEC-HMS simulations." __file_class__ = BasinFile @GenericAsset.extra_fields.getter def extra_fields(self): + """Return extra fields with added dynamic keys/values.""" return { "hms:title": self.file.name, "hms:version": self.file.header.attrs["Version"], @@ -76,13 +77,14 @@ def extra_fields(self): class EventBasinAsset(GenericAsset[BasinFile]): """HEC-HMS Basin file asset from event, with limited basin info.""" - ext = r".*\.basin$" + regex_parse_str = r".*\.basin$" __roles__ = ["hms-basin", MediaType.TEXT] __description__ = "Defines the basin geometry and elements for HEC-HMS simulations." __file_class__ = BasinFile @GenericAsset.extra_fields.getter def extra_fields(self): + """Return extra fields with added dynamic keys/values.""" return { "hms:title": self.file.name, "hms:version": self.file.header.attrs["Version"], @@ -94,13 +96,14 @@ def extra_fields(self): class RunAsset(GenericAsset[RunFile]): """Run asset.""" - ext = r".*\.run$" + regex_parse_str = r".*\.run$" __file_class__ = RunFile __roles__ = ["hms-run", MediaType.TEXT] __description__ = "Contains data for HEC-HMS simulations." @GenericAsset.extra_fields.getter def extra_fields(self): + """Return extra fields with added dynamic keys/values.""" return {"hms:title": self.name} | { run.name: {f"hms:{key}".lower(): val for key, val in run.attrs.items()} for _, run in self.file.elements } @@ -109,13 +112,14 @@ def extra_fields(self): class ControlAsset(GenericAsset[ControlFile]): """HEC-HMS Control file asset.""" - ext = r".*\.control$" + regex_parse_str = r".*\.control$" __roles__ = ["hms-control", MediaType.TEXT] __description__ = "Defines time control information for HEC-HMS simulations." __file_class__ = ControlFile @GenericAsset.extra_fields.getter def extra_fields(self): + """Return extra fields with added dynamic keys/values.""" return { "hms:title": self.file.name, **{f"hms:{key}".lower(): val for key, val in self.file.attrs.items()}, @@ -125,13 +129,14 @@ def extra_fields(self): class MetAsset(GenericAsset[MetFile]): """HEC-HMS Meteorological file asset.""" - ext = r".*\.met$" + regex_parse_str = r".*\.met$" __roles__ = ["hms-met", MediaType.TEXT] __description__ = "Contains meteorological data such as precipitation and temperature." __file_class__ = MetFile @GenericAsset.extra_fields.getter def extra_fields(self): + """Return extra fields with added dynamic keys/values.""" return { "hms:title": self.file.name, **{f"hms:{key}".lower(): val for key, val in self.file.attrs.items()}, @@ -141,38 +146,41 @@ def extra_fields(self): class DSSAsset(GenericAsset): """DSS asset.""" - ext = r".*\.dss$" + regex_parse_str = r".*\.dss$" __roles__ = ["hec-dss", "application/octet-stream"] __description__ = "HEC-DSS file." @GenericAsset.extra_fields.getter def extra_fields(self): + """Return extra fields with added dynamic keys/values.""" return {"hms:title": self.name} class SqliteAsset(GenericAsset[SqliteDB]): """HEC-HMS SQLite database asset.""" - ext = r".*\.sqlite$" + regex_parse_str = r".*\.sqlite$" __roles__ = ["hms-sqlite", "application/x-sqlite3"] __description__ = "Stores spatial data for HEC-HMS basin files." __file_class__ = SqliteDB @GenericAsset.extra_fields.getter def extra_fields(self): + """Return extra fields with added dynamic keys/values.""" return {"hms:title": self.name, "hms:layers": self.file.layers} class GageAsset(GenericAsset[GageFile]): """Gage asset.""" - ext = r".*\.gage$" + regex_parse_str = r".*\.gage$" __roles__ = ["hms-gage", MediaType.TEXT] __description__ = "Contains data for HEC-HMS gages." __file_class__ = GageFile @GenericAsset.extra_fields.getter def extra_fields(self): + """Return extra fields with added dynamic keys/values.""" return {"hms:title": self.file.name, "hms:version": self.file.attrs["Version"]} | { f"hms:{gage.name}".lower(): {key: val for key, val in gage.attrs.items()} for gage in self.file.gages } @@ -181,13 +189,14 @@ def extra_fields(self): class GridAsset(GenericAsset[GridFile]): """Grid asset.""" - ext = r".*\.grid$" + regex_parse_str = r".*\.grid$" __roles__ = ["hms-grid", MediaType.TEXT] __description__ = "Contains data for HEC-HMS grid files." __file_class__ = GridFile @GenericAsset.extra_fields.getter def extra_fields(self): + """Return extra fields with added dynamic keys/values.""" return ( {"hms:title": self.file.name} | {f"hms:{key}".lower(): val for key, val in self.file.attrs.items()} @@ -198,75 +207,59 @@ def extra_fields(self): class LogAsset(GenericAsset): """Log asset.""" - ext = r".*\.log$" + regex_parse_str = r".*\.log$" __roles__ = ["hms-log", "results", MediaType.TEXT] __description__ = "Contains log data for HEC-HMS simulations." @GenericAsset.extra_fields.getter def extra_fields(self): + """Return extra fields with added dynamic keys/values.""" return {"hms:title": self.name} class OutAsset(GenericAsset): """Out asset.""" - ext = r".*\.out$" + regex_parse_str = r".*\.out$" __roles__ = ["hms-out", "results", MediaType.TEXT] __description__ = "Contains output data for HEC-HMS simulations." @GenericAsset.extra_fields.getter def extra_fields(self): + """Return extra fields with added dynamic keys/values.""" return {"hms:title": self.name} class PdataAsset(GenericAsset[PairedDataFile]): """Pdata asset.""" - ext = r".*\.pdata$" + regex_parse_str = r".*\.pdata$" __roles__ = ["hms-pdata", MediaType.TEXT] __description__ = "Contains paired data for HEC-HMS simulations." __file_class__ = PairedDataFile @GenericAsset.extra_fields.getter def extra_fields(self): + """Return extra fields with added dynamic keys/values.""" return {"hms:title": self.file.name, "hms:version": self.file.attrs["Version"]} class TerrainAsset(GenericAsset[TerrainFile]): """Terrain asset.""" - ext = r".*\.terrain$" + regex_parse_str = r".*\.terrain$" __roles__ = ["hms-terrain", MediaType.GEOTIFF] __description__ = "Contains terrain data for HEC-HMS simulations." __file_class__ = TerrainFile @GenericAsset.extra_fields.getter def extra_fields(self): + """Return extra fields with added dynamic keys/values.""" return {"hms:title": self.file.name, "hms:version": self.file.attrs["Version"]} | { f"hms:{layer['name']}".lower(): {key: val for key, val in layer.items()} for layer in self.file.layers } -# HMS_EXTENSION_MAPPING = { -# ".hms": ProjectAsset, -# ".basin": {"event": EventBasinAsset, "model": ModelBasinAsset}, -# ".control": ControlAsset, -# ".met": MetAsset, -# ".sqlite": SqliteAsset, -# ".gage": GageAsset, -# ".run": RunAsset, -# ".grid": GridAsset, -# ".log": LogAsset, -# ".out": OutAsset, -# ".pdata": PdataAsset, -# ".terrain": TerrainAsset, -# ".dss": DSSAsset, -# ".geojson": GeojsonAsset, -# ".tiff": TiffAsset, -# ".tif": TiffAsset, -# ".png": ThumbnailAsset, -# } - HMS_ASSET_CLASSES = [ ProjectAsset, EventBasinAsset, @@ -288,4 +281,4 @@ def extra_fields(self): ThumbnailAsset, ] -HMS_EXTENSION_MAPPING = {re.compile(cls.ext, re.IGNORECASE): cls for cls in HMS_ASSET_CLASSES} +HMS_EXTENSION_MAPPING = {re.compile(cls.regex_parse_str, re.IGNORECASE): cls for cls in HMS_ASSET_CLASSES} diff --git a/hecstac/hms/item.py b/hecstac/hms/item.py index 71d3d52..635fce8 100644 --- a/hecstac/hms/item.py +++ b/hecstac/hms/item.py @@ -90,6 +90,7 @@ def from_prj(cls, hms_project_file, item_id: str, simplify_geometry: bool = True stac.pm = pm stac.simplify_geometry = simplify_geometry + stac._register_extensions() return stac def _register_extensions(self) -> None: @@ -193,7 +194,7 @@ def make_thumbnails(self, basins: list[BasinFile], pm: LocalPathManager, overwri logger.info(f"Thumbnail for basin `{bf.name}` already exists. Skipping creation.") else: logger.info(f"{'Overwriting' if overwrite else 'Creating'} thumbnail for basin `{bf.name}`") - fig = self.make_thumbnail(gdfs=bf.hms_schematic_2_gdfs) + fig = self.make_thumbnail(self, gdfs=bf.hms_schematic_2_gdfs) fig.savefig(thumbnail_path) fig.clf() thumbnail_paths.append(thumbnail_path) From 2a9acc3870fb62e1aa4bbf692e4ce9651871f067 Mon Sep 17 00:00:00 2001 From: Stevenray Janke Date: Mon, 10 Feb 2025 15:44:08 -0500 Subject: [PATCH 68/71] Update for new format --- new_hms_item.py | 27 +++++++++++++++++++-------- new_ras_item.py | 25 +++++++++++++++---------- 2 files changed, 34 insertions(+), 18 deletions(-) diff --git a/new_hms_item.py b/new_hms_item.py index 322aff4..54dfedf 100644 --- a/new_hms_item.py +++ b/new_hms_item.py @@ -2,23 +2,34 @@ from pathlib import Path -from hecstac.hms.logger import initialize_logger -from hecstac.hms.item import HMSModelItem +from hecstac.common.logger import initialize_logger +from hecstac import HMSModelItem def sanitize_catalog_assets(item: HMSModelItem) -> HMSModelItem: - """Force the asset paths in the catalog relative to item root.""" - for asset in item.assets.values(): - if item.pm.model_root_dir in asset.href: - asset.href = asset.href.replace(item.pm.item_dir, ".") + """Force the asset paths in the catalog to be relative to the item root.""" + item_dir = Path(item.pm.item_dir).resolve() + + for _, asset in item.assets.items(): + asset_path = Path(asset.href).resolve() + + if asset_path.is_relative_to(item_dir): + asset.href = str(asset_path.relative_to(item_dir)) + else: + asset.href = ( + str(asset_path.relative_to(item_dir.parent)) + if item_dir.parent in asset_path.parents + else str(asset_path) + ) + return item if __name__ == "__main__": initialize_logger() - hms_project_file = "/Users/slawler/Downloads/duwamish/Duwamish_SST.hms" + hms_project_file = "duwamish/Duwamish_SST.hms" item_id = Path(hms_project_file).stem - hms_item = HMSModelItem(hms_project_file, item_id) + hms_item = HMSModelItem.from_prj(hms_project_file, item_id) hms_item = sanitize_catalog_assets(hms_item) hms_item.save_object(hms_item.pm.item_path(item_id)) diff --git a/new_ras_item.py b/new_ras_item.py index d361426..8b8e75a 100644 --- a/new_ras_item.py +++ b/new_ras_item.py @@ -4,7 +4,7 @@ from pathlib import Path from hecstac import RASModelItem -from hecstac.ras.logger import initialize_logger +from hecstac.common.logger import initialize_logger logger = logging.getLogger(__name__) @@ -12,26 +12,31 @@ def sanitize_catalog_assets(item: RASModelItem) -> RASModelItem: """Force the asset paths in the catalog to be relative to the item root.""" item_dir = Path(item.pm.item_dir).resolve() - for _, asset in item.assets.items(): + for _, asset in item.assets.items(): asset_path = Path(asset.href).resolve() if asset_path.is_relative_to(item_dir): asset.href = str(asset_path.relative_to(item_dir)) - - elif asset.href.startswith(f"{item_dir.name}/"): - asset.href = asset.href.replace(f"{item_dir.name}/", "", 1) + else: + asset.href = ( + str(asset_path.relative_to(item_dir.parent)) + if item_dir.parent in asset_path.parents + else str(asset_path) + ) return item if __name__ == "__main__": initialize_logger() - ras_project_file = "Example_Projects_6_6/2D Unsteady Flow Hydraulics/Muncie/Muncie.prj" + ras_project_file = "Muncie/Muncie.prj" item_id = Path(ras_project_file).stem + crs = None - ras_item = RASModelItem(ras_project_file, item_id, crs=None) - ras_item = sanitize_catalog_assets(ras_item) + ras_item = RASModelItem.from_prj(ras_project_file, item_id, crs=crs) # ras_item.add_model_thumbnails(["mesh_areas", "breaklines", "bc_lines"]) - ras_item.save_object(ras_item.pm.item_path(item_id)) - logger.info(f"Saved {ras_item.pm.item_path(item_id)}") + ras_item = sanitize_catalog_assets(ras_item) + + ras_item.save_object(ras_project_file) + logger.info(f"Saved {ras_project_file}") From bd0aec465b5d679fe73417e9f7b364b167214b95 Mon Sep 17 00:00:00 2001 From: Stevenray Janke Date: Mon, 10 Feb 2025 15:49:55 -0500 Subject: [PATCH 69/71] Remove unused imports --- hecstac/hms/item.py | 2 +- hecstac/hms/parser.py | 2 -- 2 files changed, 1 insertion(+), 3 deletions(-) diff --git a/hecstac/hms/item.py b/hecstac/hms/item.py index 635fce8..57266ed 100644 --- a/hecstac/hms/item.py +++ b/hecstac/hms/item.py @@ -13,7 +13,7 @@ from pystac import Item, Asset from pystac.extensions.projection import ProjectionExtension from pystac.extensions.storage import StorageExtension -from shapely import to_geojson, union_all, unary_union +from shapely import to_geojson, unary_union from functools import lru_cache from hecstac.common.asset_factory import AssetFactory diff --git a/hecstac/hms/parser.py b/hecstac/hms/parser.py index 6033479..3380993 100644 --- a/hecstac/hms/parser.py +++ b/hecstac/hms/parser.py @@ -7,7 +7,6 @@ import os from abc import ABC from collections import OrderedDict -from datetime import datetime from functools import lru_cache from pathlib import Path @@ -15,7 +14,6 @@ import geopandas as gpd import pandas as pd from pyproj import CRS -from shapely import get_point from shapely.geometry import LineString, MultiLineString, Point import hecstac.hms.utils as utils From 173874e3ffd33055a026fee5378db0f33a528cae Mon Sep 17 00:00:00 2001 From: Stevenray Janke Date: Mon, 10 Feb 2025 16:11:30 -0500 Subject: [PATCH 70/71] Remove unused import --- hecstac/hms/item.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/hecstac/hms/item.py b/hecstac/hms/item.py index 57266ed..c472b50 100644 --- a/hecstac/hms/item.py +++ b/hecstac/hms/item.py @@ -18,7 +18,7 @@ from hecstac.common.asset_factory import AssetFactory from hecstac.common.path_manager import LocalPathManager -from hecstac.hms.assets import HMS_EXTENSION_MAPPING, ProjectAsset +from hecstac.hms.assets import HMS_EXTENSION_MAPPING from hecstac.hms.parser import BasinFile, ProjectFile from hecstac.ras.consts import ( @@ -163,7 +163,7 @@ def geometry(self) -> dict: def bbox(self) -> list[float]: """Bounding box of the HMS model.""" if not self.geometry_assets: - return [0, 0, 0, 0] + return NULL_STAC_BBOX bboxes = np.array([b.bbox(4326) for b in self.geometry_assets]) return [float(i) for i in [bboxes[:, 0].min(), bboxes[:, 1].min(), bboxes[:, 2].max(), bboxes[:, 3].max()]] From 2b090e948e157115fcc42958543073a07ee1ead6 Mon Sep 17 00:00:00 2001 From: Stevenray Janke Date: Mon, 10 Feb 2025 16:12:42 -0500 Subject: [PATCH 71/71] Fix spelling --- hecstac/hms/item.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/hecstac/hms/item.py b/hecstac/hms/item.py index c472b50..a852912 100644 --- a/hecstac/hms/item.py +++ b/hecstac/hms/item.py @@ -176,7 +176,7 @@ def datetime(self) -> datetime: return datetime.combine(date, time) def _check_files_exists(self, files: list[str]): - """Ensure the files exists. If they don't rasie an error.""" + """Ensure the files exists. If they don't raise an error.""" from pathlib import Path for file in files: