Prep for v1 release

.github/workflows/ci.yaml

@@ -50,14 +50,14 @@ jobs:
     - name: Build wheel and source distribution
       run: python -m build
 
-    - name: Install the built wheel 
-      run: | 
-        python -c "import glob; import subprocess; wheel_files = glob.glob('dist/*.whl'); subprocess.check_call(['pip', 'install', wheel_files[0]])" 
+    - name: Install the built wheel
+      run: |
+        python -c "import glob; import subprocess; wheel_files = glob.glob('dist/*.whl'); subprocess.check_call(['pip', 'install', wheel_files[0]])"
 
-    # - name: Lint (ruff)
-    #   run: |
-    #     ruff check .
-    #     ruff format --check
+    - name: Lint (ruff)
+      run: |
+        ruff check .
+        ruff format --check
 
     # - name: Run tests with coverage
     #   run: |

README.md

@@ -6,5 +6,11 @@
 
 Utilities for creating STAC items from HEC models
 
-**hecstac** is an open-source Python library designed to mine metadata from HEC model simulations for use in the development of catalogs documenting probabilistic flood studies. This project automates the generation of STAC Items and Assets from HEC-HMS and HEC-RAS model files, enabling improved data and metadata management. 
+**hecstac** is an open-source Python library designed to mine metadata from HEC model simulations for use in the development of catalogs documenting probabilistic flood studies. This project automates the generation of STAC Items and Assets from HEC-HMS and HEC-RAS model files, enabling improved data and metadata management.
 
+***Testing HEC-RAS model item creation***
+
+- Download the HEC-RAS example project data from USACE and place it in your working directory. The data can be downloaded [here](https://github.com/HydrologicEngineeringCenter/hec-downloads/releases/download/1.0.33/Example_Projects_6_6.zip).
+- In 'new_ras_item.py', set the ras_project_file to the path of the 2D Muncie project file (ex. ras_project_file = "Example_Projects_6_6/2D Unsteady Flow Hydraulics/Muncie/Muncie.prj").
+- For projects that have projection information within the geometry .hdf files, the CRS info can automatically be detected. The Muncie data lacks that projection info so it must be set by extracting the projection string and setting the CRS in 'new_ras_item.py' to the projection string. The projection can be found in the Muncie/GIS_Data folder in Muncie_IA_Clip.prj.
+- Once the CRS and project file location have been set, a new item can be created with 'python -m new_ras_item' in the command line. The new item will be added inside the model directory at the same level as the project file.

docs/source/user_guide.rst

@@ -17,14 +17,14 @@ have Python already installed and setup:
 
 
 Note that it is highly recommended to create a python `virtual environment
-<https://docs.python.org/3/library/venv.html>`_ to install, test, and run hecstac. 
+<https://docs.python.org/3/library/venv.html>`_ to install, test, and run hecstac.
 
 
 
 Workflows
 ---------
 
-The following snippets provide examples for creating stac items from HEC model data. 
+The following snippets provide examples for creating stac items from HEC model data.
 
 .. code-block:: python
 
@@ -56,7 +56,7 @@ The following snippets provide examples for creating stac items from HEC model d
       ras_item.save_object(ras_item.pm.item_path(item_id))
 
 
-The following snippet provides an example of how to create stac items for an event based simulation. 
+The following snippet provides an example of how to create stac items for an event based simulation.
 
 .. code-block:: python
 
@@ -79,7 +79,7 @@ The following snippet provides an example of how to create stac items for an eve
          "<local-file-dr>/hms-model.met",
          "<local-file-dr>/Precip.dss",
       ]
-      
+
 
       # RAS Info
       ras_source_model_item_path = "/<local-file-dr>/authoritative-ras-model.json"
@@ -105,7 +105,7 @@ The following snippet provides an example of how to create stac items for an eve
          source_model_items=[
                hms_source_model_item,
                ras_source_model_item
-         ],  
+         ],
          hms_simulation_files=hms_simulation_files,
          ras_simulation_files=ras_simulation_files,
       )

hecstac/__init__.py

@@ -5,3 +5,6 @@
 """
 
 from hecstac.version import __version__
+
+from hecstac.ras.item import RASModelItem
+from hecstac.hms.item import HMSModelItem

hecstac/common/__init__.py

@@ -0,0 +1 @@
+"""Common scripts."""

hecstac/common/asset_factory.py

@@ -1,46 +1,98 @@
+"""Create instances of assets."""
+
 import logging
 from pathlib import Path
-from typing import Dict, Type
+from typing import Dict, Generic, Type, TypeVar
 
+from pyproj import CRS
 from pystac import Asset
 
 from hecstac.hms.s3_utils import check_storage_extension
 
+logger = logging.getLogger(__name__)
+
+T = TypeVar("T")  # Generic for asset file accessor classes
+
+
+class GenericAsset(Asset, Generic[T]):
+    """Provides a base structure for assets."""
+
+    regex_parse_str: str = r""
+    __roles__: list[str] = []
+    __description__: str = ""
+    __file_class__: T
+
+    def __init__(self, *args, **kwargs):
+        super().__init__(*args, **kwargs)
+        if self.description is None:
+            self.description = self.__description__
+        self._roles = []
+        self._extra_fields = {}
+        self.name = Path(self.href).name
+
+    @property
+    def roles(self) -> list[str]:
+        """Return roles with enforced values."""
+        roles = self._roles
+        for i in self.__roles__:
+            if i not in roles:
+                roles.append(i)
+        return roles
 
-class GenericAsset(Asset):
-    """Generic Asset."""
+    @roles.setter
+    def roles(self, roles: list):
+        self._roles = roles
 
-    def __init__(self, href: str, roles=None, description=None, *args, **kwargs):
-        super().__init__(href, *args, **kwargs)
-        self.href = href
-        self.name = Path(href).name
-        self.stem = Path(href).stem
-        self.roles = roles or []
-        self.description = description or ""
+    @property
+    def extra_fields(self):
+        """Return extra fields."""
+        # boilerplate here, but overwritten in subclasses
+        return self._extra_fields
+
+    @extra_fields.setter
+    def extra_fields(self, extra_fields: dict):
+        """Set user-defined extra fields."""
+        self._extra_fields = extra_fields
+
+    @property
+    def file(self) -> T:
+        """Return class to access asset file contents."""
+        return self.__file_class__(self.get_absolute_href())
 
     def name_from_suffix(self, suffix: str) -> str:
         """Generate a name by appending a suffix to the file stem."""
         return f"{self.stem}.{suffix}"
 
+    @property
+    def crs(self) -> CRS:
+        """Get the authority code for the model CRS."""
+        if self.ext.has("proj"):
+            wkt2 = self.ext.proj.wkt2
+            if wkt2 is None:
+                return
+            else:
+                return CRS(wkt2)
+
     def __repr__(self):
+        """Return string representation of the GenericAsset instance."""
         return f"<{self.__class__.__name__} name={self.name}>"
 
     def __str__(self):
+        """Return string representation of assets name."""
         return f"{self.name}"
 
 
 class AssetFactory:
     """Factory for creating HEC asset instances based on file extensions."""
 
     def __init__(self, extension_to_asset: Dict[str, Type[GenericAsset]]):
-        """
-        Initialize the AssetFactory with a mapping of file extensions to asset types and metadata.
-        """
+        """Initialize the AssetFactory with a mapping of file extensions to asset types and metadata."""
         self.extension_to_asset = extension_to_asset
 
     def create_hms_asset(self, fpath: str, item_type: str = "model") -> Asset:
         """
         Create an asset instance based on the file extension.
+
         item_type: str
 
         The type of item to create. This is used to determine the asset class.
@@ -59,11 +111,11 @@ def create_hms_asset(self, fpath: str, item_type: str = "model") -> Asset:
         asset.title = Path(fpath).name
         return check_storage_extension(asset)
 
-    def create_ras_asset(self, fpath: str):
-        logging.debug(f"Creating asset for {fpath}")
+    def asset_from_dict(self, asset: Asset):
+        """Create HEC asset given a base Asset and a map of file extensions dict."""
+        fpath = asset.href
         for pattern, asset_class in self.extension_to_asset.items():
             if pattern.match(fpath):
-                logging.debug(f"Matched {pattern} for {Path(fpath).name}: {asset_class}")
-                return asset_class(href=fpath, title=Path(fpath).name)
-
-        return GenericAsset(href=fpath, title=Path(fpath).name)
+                logger.debug(f"Matched {pattern} for {Path(fpath).name}: {asset_class}")
+                return asset_class.from_dict(asset.to_dict())
+        return asset

hecstac/common/geometry.py

@@ -0,0 +1,15 @@
+"""Geometry utils."""
+
+from pyproj import CRS, Transformer
+from shapely import Geometry
+from shapely.ops import transform
+
+
+def reproject_to_wgs84(geom: Geometry, crs: str) -> Geometry:
+    """Convert geometry CRS to EPSG:4326 for stac item geometry."""
+    pyproj_crs = CRS.from_user_input(crs)
+    wgs_crs = CRS.from_authority("EPSG", "4326")
+    if pyproj_crs != wgs_crs:
+        transformer = Transformer.from_crs(pyproj_crs, wgs_crs, True)
+        return transform(transformer.transform, geom)
+    return geom

hecstac/common/logger.py

@@ -2,12 +2,15 @@
 
 import logging
 import sys
+from re import L
 
 SUPPRESS_LOGS = ["boto3", "botocore", "geopandas", "fiona", "rasterio", "pyogrio", "xarray", "shapely", "matplotlib"]
 
 
 def initialize_logger(json_logging: bool = False, level: int = logging.INFO):
-    datefmt = "%Y-%m-%dT%H:%M:%SZ"
+    """Initialize the ras logger."""
+    logger = logging.getLogger("hecstac")
+    logger.setLevel(level)
     if json_logging:
         for module in SUPPRESS_LOGS:
             logging.getLogger(module).setLevel(logging.WARNING)
@@ -19,14 +22,14 @@ def emit(self, record):
 
         handler = FlushStreamHandler(sys.stdout)
 
-        logging.basicConfig(
-            level=level,
-            handlers=[handler],
-            format="""{"time": "%(asctime)s" , "level": "%(levelname)s", "msg": "%(message)s"}""",
-            datefmt=datefmt,
-        )
+        handler.setLevel(level)
+
+        datefmt = "%Y-%m-%dT%H:%M:%SZ"
+        fmt = """{"time": "%(asctime)s" , "level": "%(levelname)s", "msg": "%(message)s"}"""
+        formatter = logging.Formatter(fmt=fmt, datefmt=datefmt)
+        handler.setFormatter(formatter)
+
+        logger.addHandler(handler)
     else:
         for package in SUPPRESS_LOGS:
             logging.getLogger(package).setLevel(logging.ERROR)
-        logging.basicConfig(level=level, format="%(asctime)s | %(levelname)s | %(message)s", datefmt=datefmt)
-    # boto3.set_stream_logger(name="botocore.credentials", level=logging.ERROR)

hecstac/common/path_manager.py

@@ -1,29 +1,33 @@
+"""Path manager."""
+
 from pathlib import Path
 
 
 class LocalPathManager:
-    """
-    Builds consistent paths for STAC items and collections assuming a top level local catalog
-    """
+    """Builds consistent paths for STAC items and collections assuming a top level local catalog."""
 
     def __init__(self, model_root_dir: str):
         self._model_root_dir = model_root_dir
 
     @property
     def model_root_dir(self) -> str:
+        """Model root directory."""
         return str(self._model_root_dir)
 
     @property
     def model_parent_dir(self) -> str:
+        """Model parent directory."""
         return str(Path(self._model_root_dir).parent)
 
     @property
     def item_dir(self) -> str:
-        """Duplicate of model_root, added for clarity in the calling code"""
+        """Duplicate of model_root, added for clarity in the calling code."""
         return self.model_root_dir
 
     def item_path(self, item_id: str) -> str:
-        return f"{self._model_root_dir}/{item_id}.json"
+        """Item path."""
+        return str(Path(self._model_root_dir) / f"{item_id}.json")
 
     def derived_item_asset(self, filename: str) -> str:
-        return f"{self._model_root_dir}/{filename}"
+        """Derive item asset path."""
+        return str(Path(self._model_root_dir) / filename)

hecstac/common/schemas.py

@@ -1,3 +1,5 @@
+"""Schema parsing."""
+
 # TODO: update these, add imports, etc.
 # def extract_schema_definition(definition_name: str) -> dict[str, Any]:
 #     """Extract asset specific schema from ras extension schema"""

hecstac/events/__init__.py

@@ -0,0 +1 @@
+"""HEC event stac items."""

hecstac/events/ffrd.py

@@ -1,3 +1,5 @@
+"""Class for event items."""
+
 import json
 import logging
 import os
@@ -15,8 +17,12 @@
 from hecstac.hms.assets import HMS_EXTENSION_MAPPING
 from hecstac.ras.assets import RAS_EXTENSION_MAPPING
 
+logger = logging.getLogger(__name__)
+
 
 class FFRDEventItem(Item):
+    """Class for event items."""
+
     FFRD_REALIZATION = "FFRD:realization"
     FFRD_BLOCK_GROUP = "FFRD:block_group"
     FFRD_EVENT = "FFRD:event"
@@ -66,7 +72,7 @@ def _register_extensions(self) -> None:
     def _add_model_links(self) -> None:
         """Add links to the model items."""
         for item in self.source_model_items:
-            logging.info(f"Adding link from source model item: {item.id}")
+            logger.info(f"Adding link from source model item: {item.id}")
             link = Link(
                 rel="derived_from",
                 target=item,
@@ -116,15 +122,15 @@ def _bbox(self) -> list[float]:
     def add_hms_asset(self, fpath: str, item_type: str = "event") -> None:
         """Add an asset to the FFRD Event STAC item."""
         if os.path.exists(fpath):
-            logging.info(f"Adding asset: {fpath}")
+            logger.info(f"Adding asset: {fpath}")
             asset = self.hms_factory.create_hms_asset(fpath, item_type=item_type)
             if asset is not None:
                 self.add_asset(asset.title, asset)
 
     def add_ras_asset(self, fpath: str) -> None:
         """Add an asset to the FFRD Event STAC item."""
         if os.path.exists(fpath):
-            logging.info(f"Adding asset: {fpath}")
+            logger.info(f"Adding asset: {fpath}")
             asset = self.ras_factory.create_ras_asset(fpath)
             if asset is not None:
                 self.add_asset(asset.title, asset)

hecstac/hms/__init__.py

@@ -0,0 +1 @@
+"""HEC-HMS STAC Item module."""