diff --git a/.gitignore b/.gitignore index 52c5011..69ab47f 100644 --- a/.gitignore +++ b/.gitignore @@ -171,4 +171,7 @@ cython_debug/ .pypirc catalogs holding -.DS_Store \ No newline at end of file +.DS_Store + +bighorn +indian-creek \ No newline at end of file diff --git a/build-local.sh b/build-local.sh old mode 100755 new mode 100644 diff --git a/docs/make.sh b/docs/make.sh old mode 100755 new mode 100644 diff --git a/docs/source/change_log.rst b/docs/source/change_log.rst index 25255dc..bcf76eb 100644 --- a/docs/source/change_log.rst +++ b/docs/source/change_log.rst @@ -1,6 +1,21 @@ .. note:: Go to the `Releases `__ page for a list of all releases. +Release v0.1.0 +============== + +**Tag:** v0.1.0 + +**Published at:** 2025-01-31T21:13:10Z + +**Author:** github-actions[bot] + +**Release Notes:** + +Summary ^^^^^^^^ +This feature release adds routines for developing STAC catalogs using AORC data. Source code was ported from existing internal libraries and refactored for this initial release. + + Release v0.1.0rc1 ================= @@ -12,6 +27,6 @@ Release v0.1.0rc1 **Release Notes:** - +Initial code compilation from existing internal libraries, adapted for STAC catalogs. diff --git a/docs/source/tech_summary.rst b/docs/source/tech_summary.rst index 42a4831..d0d6783 100644 --- a/docs/source/tech_summary.rst +++ b/docs/source/tech_summary.rst @@ -8,14 +8,14 @@ Storm Transposition Module Data Source ----------- -The Analysis Of Record for Calibration (AORC) dataset is available on the AWS `Registry of Open Data `_, and provides the -source data for precipitation and temperature data used in this module. (Other sources may be added but are not currently available). This gridded / houlry data is available for the CONUS -beginning on 1972-02-01 and is updated regularly. +The Analysis Of Record for Calibration (AORC) dataset is available on the AWS `Registry of Open Data `_, and provides the +source data for precipitation and temperature data used in this module. (Other sources may be added but are not currently available). This gridded / hourly data is available for the CONUS +beginning on 1972-02-01 and is updated regularly. The primary inputs used for the data development workflow are as follows. - Watershed = The waterhsed which will be used for hydrologic modeling. + Watershed = The watershed which will be used for hydrologic modeling. Transposition Area = A region containing the watershed that has been developed as a hydro-meteorologically homogenous region. @@ -46,10 +46,10 @@ cells of the watershed within the transposition region. The centroid location an .. image:: ./images/2011-event.png -3. **Iterate over the period of record or desired date range**: In order to process multiple dates for a range (from start_date - end_date), there is an optional argumen `check_every_n_hours`. If set to 1, the process will sum up the storm duration for every hour from the start_date +3. **Iterate over the period of record or desired date range**: In order to process multiple dates for a range (from start_date - end_date), there is an optional argument `check_every_n_hours`. If set to 1, the process will sum up the storm duration for every hour from the start_date to the end_date. For a 72-hour event, this would require processing 350,400 datasets (every hour for the period) for 40 years of record and would represent the most precise estimate to aid in identifying the start hour for the event. To save in processing time and data, an alternate interval can be used. For example, selecting `check_every_n_hours` = 24 would result in 14,600 datasets processed for the same 40 year period. - + check_every_n_hours = 6 (This would get check the totals every 6 hours, or 4 times a day) @@ -67,15 +67,15 @@ After processing the data for every date in the requested date range, a csv is c |1979-02-01T18 | 0.02 | 0.02 | 0.02 | -92.66397464708892 |40.50038658823523| +------------------------+------------+----------+----------+---------------------+-----------------+ -4. **Top events and date declustering** With the staticics in place, user settings can be used to create a STAC collection for the watershed / transpositon region / storm duration using the following inputs. - +4. **Top events and date declustering** With the statistics in place, user settings can be used to create a STAC collection for the watershed / transpositon region / storm duration using the following inputs. + min_precip_threshold = 2 (Defaults to 1, this can be used to filter out events based on a minimum threshold) - + top_n_events = 440 (This will be the total # of events in the collection. 440 would represent the top 10 events for 44 years) To avoid double counting what is essentially the same storm because the N hour duration for several consecutive periods may result in a top storm, results of the query are iterated and added to a list, a process filters storms to be skipped if there is any temporal overlap with a storm already existing in the list (the overlap is determined using the start time and duration of the top storm). As shown -in these images, these records are considered to be a single storm, and would be declustered, wherein the day with the greater mean precipitation would be included in the top storms collection and the other +in these images, these records are considered to be a single storm, and would be declustered, wherein the day with the greater mean precipitation would be included in the top storms collection and the other would be dropped. @@ -91,7 +91,7 @@ would be dropped. 5. The following additional arguments are available. - + .. code:: bash specific_dates # Can be provided to resume processing in the event of a failure or other use cases @@ -102,23 +102,23 @@ would be dropped. -Results +Results ------- -A Storm Catalog is created containing a copy of the watershed, transposition domain, and the *valid transpositon domain* which is the space within the transposition domain wherein a -watershed can be transposed without encountering null space (i.e. part of the watershed extending outside of the trnasposition domain). +A Storm Catalog is created containing a copy of the watershed, transposition domain, and the *valid transpositon domain* which is the space within the transposition domain wherein a +watershed can be transposed without encountering null space (i.e. part of the watershed extending outside of the transposition domain). .. image:: ./images/catalog.png -STAC Collections will be added to the catalog for each storm duration requested. The collection will include relevant data including summary statistics, plots, and other assets to rpovide +STAC Collections will be added to the catalog for each storm duration requested. The collection will include relevant data including summary statistics, plots, and other assets to provide context and metadata for the data. .. image:: ./images/storm-collection.png -The collection is compised of STAC Items, which provide links to source data and derivative products. For example, a model speciric timeseries file may be required for hydrologic modeling. -These files can be created and added to the event item alongside metadata and other information. Assets may include additional data required for modeling (i.e. temperature data, also available via AORC). +The collection is composed of STAC Items, which provide links to source data and derivative products. For example, a model specific timeseries file may be required for hydrologic modeling. +These files can be created and added to the event item alongside metadata and other information. Assets may include additional data required for modeling (i.e. temperature data, also available via AORC). .. image:: ./images/storm-item.png @@ -129,5 +129,5 @@ These files can be created and added to the event item alongside metadata and ot This feature was evaluated and used in pilot projects, does not currently exist in this repository, but may be incorporated in the future. -Where possible, `NOAA Atlas-14 precipitation frequency estimates `_ may be considered to normalize the average accumulation for each storm. -.. image:: ./images/2yr03da.PNG +Where possible, `NOAA Atlas-14 precipitation frequency estimates `_ may be considered to normalize the average accumulation for each storm. +.. image:: ./images/2yr03da.png diff --git a/docs/source/user_guide.rst b/docs/source/user_guide.rst index 2406c75..c239036 100644 --- a/docs/source/user_guide.rst +++ b/docs/source/user_guide.rst @@ -2,7 +2,7 @@ Getting Started ################ -This section provides a high level overview for using stormhub for production, including starting the stormhub server and create objects. +This section provides a high level overview for using stormhub for production, including starting the stormhub server and creating objects. Installation ------------ @@ -18,20 +18,21 @@ have Python already installed and setup: Note that it is highly recommended to create a python `virtual environment `_ to install, test, and run -stormhub. +stormhub. It is also recommended to avoid use of Windows Subsystem for Linux (WSL) +as issues can arise with the parallel processing within stormhub. Starting the server ------------------- For convenience, a local file server is provided. This server is not necessary for data -production, but is useful for visualizing and exploring the data. +production, but is useful for visualizing and exploring the data. **Start the stormhub file:** .. code-block:: bash - stormhub-server + stormhub-server Local file server is useful for interacting with STAC browser for viewing the data locally. This is not required.... @@ -44,7 +45,7 @@ Local file server is useful for interacting with STAC browser for viewing the da Workflows --------- -A config file shown in below includes the information required to create a new catalog. +A config file shown below includes the information required to create a new catalog. .. code-block:: json @@ -62,7 +63,7 @@ A config file shown in below includes the information required to create a new c } -The following snippet provides an example of how to build and create a storm catalog. Requires an example watershed and transposition domain (examples availble in the `repo `_). +The following snippet provides an example of how to build and create a storm catalog. Requires an example watershed and transposition domain (examples available in the `repo `_). .. code-block:: python diff --git a/stormhub/hydro_domain.py b/stormhub/hydro_domain.py index 7ef4da0..b03f1ef 100644 --- a/stormhub/hydro_domain.py +++ b/stormhub/hydro_domain.py @@ -1,11 +1,11 @@ from datetime import datetime from typing import Any, Union - +import logging import fiona.errors import geopandas as gpd from pystac import Item from pystac.extensions.projection import ProjectionExtension -from shapely.geometry import Polygon, mapping, shape +from shapely.geometry import Polygon, mapping, shape, MultiPolygon HYDRO_DOMAIN_DESCRIPTION = "hydro_domain:description" HYDRO_DOMAIN_TYPE = "hydro_domain:type" @@ -121,7 +121,16 @@ def load_geometry(self, geometry_source: Union[str, Polygon]) -> Polygon: else: raise ValueError("geometry_source must be a file path or a Polygon object") - if len(gdf) != 1 or not isinstance(gdf.geometry.iloc[0], Polygon): + if len(gdf) != 1: + raise ValueError("The geometry must contain a single polygon") + + geometry = gdf.geometry.iloc[0] + if isinstance(geometry, MultiPolygon) and len(geometry.geoms) == 1: + logging.warning("Multipolygon type detected, attempting conversion to Polygon.") + geometry = geometry.geoms[0] + gdf.at[gdf.index[0], "geometry"] = geometry + + if not isinstance(geometry, Polygon): raise ValueError("The geometry must contain a single polygon") try: diff --git a/stormhub/met/aorc/aorc.py b/stormhub/met/aorc/aorc.py index f05790c..84572a6 100644 --- a/stormhub/met/aorc/aorc.py +++ b/stormhub/met/aorc/aorc.py @@ -267,6 +267,13 @@ def aorc_thumbnail( edgecolor="gray", ) ax.add_patch(valid_area_plt_polygon) + watershed_plt_polygon = patches.Polygon( + np.column_stack(self.watershed_geometry.exterior.coords.xy), + lw=0.7, + facecolor="none", + edgecolor="gray", + ) + ax.add_patch(watershed_plt_polygon) transposed_watershed_plt_polygon = patches.Polygon( np.column_stack(self._transposed_watershed.exterior.coords.xy), lw=1, @@ -283,7 +290,7 @@ def aorc_thumbnail( filename = f"{self.item_id}.thumbnail.png" fn = os.path.join(self.local_directory, filename) fig.savefig(fn, bbox_inches="tight") - asset = Asset(fn, media_type=MediaType.PNG, roles=["thumbnail"]) + asset = Asset(filename, media_type=MediaType.PNG, roles=["thumbnail"]) self.add_asset("thumbnail", asset) if return_fig: return fig diff --git a/stormhub/met/storm_catalog.py b/stormhub/met/storm_catalog.py index 517e8e6..20e451a 100644 --- a/stormhub/met/storm_catalog.py +++ b/stormhub/met/storm_catalog.py @@ -312,10 +312,10 @@ def sanitize_catalog_assets(self): """ for collection in self.get_all_collections(): for asset in collection.assets.values(): - if self.spm.collection_dir(collection.id) in asset.href: - asset.href = asset.href.replace(self.spm.collection_dir(collection.id), ".") - elif self.spm.catalog_dir in asset.href: - asset.href = asset.href.replace(self.spm.catalog_dir, "..") + if self.spm.collection_dir(collection.id).replace('\\', '/') in asset.href: + asset.href = asset.href.replace(self.spm.collection_dir(collection.id).replace('\\', '/'), ".") + elif self.spm.catalog_dir.replace('\\', '/') in asset.href: + asset.href = asset.href.replace(self.spm.catalog_dir.replace('\\', '/'), "..") for item in collection.get_all_items(): for asset in item.assets.values(): @@ -342,21 +342,21 @@ def add_hydro_domain(self, hydro_domain: Union[HydroDomain, Item]) -> str: try: title = hydro_domain.title except AttributeError: - title = hydro_domain.item_id + title = hydro_domain.id self.add_link( Link( - rel="Hydro_Domains", - target=self.spm.catalog_asset(hydro_domain.item_id).replace(self.spm.catalog_dir, "."), + rel="item", + target=self.spm.catalog_asset(hydro_domain.id).replace(self.spm.catalog_dir, "."), title=title, media_type=pystac.MediaType.GEOJSON, extra_fields={ - "Name": hydro_domain.item_id, - "Description": f"Input {hydro_domain.item_id} used to generate this catalog", + "Name": hydro_domain.id, + "Description": f"Input {hydro_domain.id} used to generate this catalog", }, ) ) - return hydro_domain.item_id + return hydro_domain.id def get_storm_collection(self, collection_id: str) -> StormCollection: """ @@ -518,6 +518,7 @@ def storm_search( catalog: StormCatalog, storm_start_date: datetime, storm_duration_hours: int, + por_rank: int= None, return_item: bool = False, scale_max: float = 12.0, collection_id: str = None, @@ -547,8 +548,10 @@ def storm_search( watershed.id, storm_duration_hours, ) - - item_id = f"{storm_start_date.strftime('%Y-%m-%dT%H')}" + if por_rank: + item_id = f"{por_rank}" + else: + item_id = f"{storm_start_date.strftime('%Y-%m-%dT%H')}" item_dir = catalog.spm.collection_item_dir(collection_id, item_id) event_item = AORCItem( @@ -582,6 +585,46 @@ def storm_search( } +def serial_processor( + func: callable, + catalog: StormCatalog, + storm_duration: int, + output_csv: str, + event_dates: list[datetime], + with_tb: bool = False, +): + """ + Run function in serial using only one processor. + + Args: + func (callable): The function to run. + catalog (StormCatalog): The storm catalog. + storm_duration (int): The duration of the storm. + output_csv (str): Path to the output CSV file. + event_dates (list[datetime]): List of event dates. + with_tb (bool): Whether to include traceback in error logs. + """ + if not os.path.exists(output_csv): + with open(output_csv, "w", encoding="utf-8") as f: + f.write("storm_date,min,mean,max,x,y\n") + + count = len(event_dates) + + with open(output_csv, "a", encoding="utf-8") as f: + for date in event_dates: + try: + r = func(catalog, date, storm_duration) + f.write(storm_search_results_to_csv_line(r)) + logging.info("%s processed (%d remaining)", r["storm_date"], count) + count -= 1 + except Exception as e: + if with_tb: + tb = traceback.format_exc() + logging.error("Error processing: %s\n%s", e, tb) + else: + logging.error("Error processing: %s", e) + + def multi_processor( func: callable, catalog: StormCatalog, @@ -646,6 +689,7 @@ def collect_event_stats( num_workers: int = None, use_threads: bool = False, with_tb: bool = False, + use_parallel_processing: bool = True ): """ Collect statistics for storm events. @@ -658,6 +702,7 @@ def collect_event_stats( num_workers (int, optional): Number of workers to use. use_threads (bool): Whether to use threads instead of processes. with_tb (bool): Whether to include traceback in error logs. + use_parallel_processing (bool): Whether to process storm stats using parallel processing. """ if not collection_id: @@ -677,17 +722,29 @@ def collect_event_stats( num_workers = 15 output_csv = os.path.join(collection_dir, "storm-stats.csv") + if use_parallel_processing: + logging.info("Using %s cpu's for collecting event stats", num_workers) + multi_processor( + func=storm_search, + catalog=catalog, + storm_duration=storm_duration, + output_csv=output_csv, + event_dates=event_dates, + num_workers=num_workers, + use_threads=use_threads, + with_tb=with_tb, + ) + else: + logging.info("Processing event stats serially.") + serial_processor( + func=storm_search, + catalog=catalog, + storm_duration=storm_duration, + output_csv=output_csv, + event_dates=event_dates, + with_tb=with_tb, + ) - multi_processor( - func=storm_search, - catalog=catalog, - storm_duration=storm_duration, - output_csv=output_csv, - event_dates=event_dates, - num_workers=num_workers, - use_threads=use_threads, - with_tb=with_tb, - ) def create_items( @@ -699,10 +756,10 @@ def create_items( with_tb: bool = False, ) -> List: """ - Create items for storm events. + Create items for storm events, setting the item ID to `por_rank` instead of storm_date. Args: - event_dates (list[dict]): List of event dates. + event_dates (list[dict]): List of event metadata (includes storm_date & por_rank). catalog (StormCatalog): The storm catalog. collection_id (str, optional): The ID of the collection. storm_duration (int): The duration of the storm. @@ -726,14 +783,22 @@ def create_items( if not num_workers: num_workers = os.cpu_count() - events = [e["storm_date"] for e in event_dates] + storm_data = [(e["storm_date"], e["por_rank"]) for e in event_dates] + with ProcessPoolExecutor(max_workers=num_workers) as executor: futures = [ executor.submit( - storm_search, catalog, storm_date, storm_duration, collection_id=collection_id, return_item=True + storm_search, + catalog, + storm_date, + storm_duration, + por_rank=por_rank, + collection_id=collection_id, + return_item=True, ) - for storm_date in events + for storm_date, por_rank in storm_data ] + for future in as_completed(futures): count -= 1 try: @@ -745,13 +810,13 @@ def create_items( if with_tb: tb = traceback.format_exc() logging.error("Error processing: %s\n%s", e, tb) - continue else: logging.error("Error processing: %s", e) - continue + return event_items + def init_storm_catalog( catalog_id: str, config: dict, local_catalog_dir: str, create_valid_transposition_region: bool = False ) -> pystac.Catalog: @@ -829,7 +894,12 @@ def get_item_from_catalog_link(links: list, link_title: str, spm: StacPathManage return None if len(matched_links) > 1: raise ValueError(f"Multiple links found with title: {link_title}") - absolute_path = f'{spm.catalog_dir}/{matched_links[0].target.replace("./", "")}' + + relative_path = matched_links[0].target.replace("./", "") + absolute_path = os.path.join(spm.catalog_dir, relative_path) + + absolute_path = os.path.abspath(absolute_path) + item = pystac.read_file(absolute_path) if not isinstance(item, Item): raise ValueError(f"Expected an Item object at {absolute_path} not : {type(item)}") @@ -928,6 +998,7 @@ def new_catalog( ) storm_catalog.save() + logging.info("Catalog has been created.") return storm_catalog @@ -989,11 +1060,10 @@ def new_collection( collection_id = storm_catalog.spm.storm_collection_id(storm_duration) logging.info("Creating collection `%s` for period %s - %s", collection_id, start_date, end_date) - stats_csv = os.path.join(storm_catalog.spm.collection_dir(collection_id), "storm-stats.csv") if dates: logging.info("Collecting event stats for %d dates", len(dates)) collect_event_stats(dates, storm_catalog, collection_id, num_workers=num_workers, with_tb=with_tb) - + stats_csv = os.path.join(storm_catalog.spm.collection_dir(collection_id), "storm-stats.csv") try: logging.info("Starting storm analysis for: %s", stats_csv) analyzer = StormAnalyzer(stats_csv, min_precip_threshold, storm_duration) @@ -1019,6 +1089,7 @@ def new_collection( storm_catalog.add_collection_to_catalog(collection, override=True) storm_catalog.save_catalog() + return collection def resume_collection( diff --git a/stormhub/utils.py b/stormhub/utils.py index 0f6bdcd..2725a8e 100644 --- a/stormhub/utils.py +++ b/stormhub/utils.py @@ -3,6 +3,7 @@ import socket from datetime import datetime, timedelta from typing import List +import os from pystac import Link, Collection from shapely.geometry import mapping, shape @@ -97,11 +98,11 @@ def create_feature_collection_from_items( class StacPathManager: """ - Builds consistent paths for STAC items and collections assuming a top level local catalog + Builds consistent paths for STAC items and collections assuming a top-level local catalog. """ def __init__(self, local_catalog_dir: str): - self._catalog_dir = local_catalog_dir + self._catalog_dir = os.path.abspath(local_catalog_dir) @property def catalog_dir(self): @@ -109,31 +110,31 @@ def catalog_dir(self): @property def catalog_file(self): - return f"{self._catalog_dir}/catalog.json" + return os.path.join(self._catalog_dir, "catalog.json") def storm_collection_id(self, duration: int) -> str: return f"{duration}hr-events" def catalog_item(self, item_id: str) -> str: - return f"{self.catalog_dir}/{item_id}/{item_id}.json" + return os.path.join(self.catalog_dir, item_id, f"{item_id}.json") def catalog_asset(self, item_id: str, asset_dir: str = "hydro_domains") -> str: - return f"{self.catalog_dir}/{asset_dir}/{item_id}.json" + return os.path.join(self.catalog_dir, asset_dir, f"{item_id}.json") def collection_file(self, collection_id: str) -> str: - return f"{self.catalog_dir}/{collection_id}/collection.json" + return os.path.join(self.catalog_dir, collection_id, "collection.json") def collection_dir(self, collection_id: str) -> str: - return f"{self.catalog_dir}/{collection_id}" + return os.path.join(self.catalog_dir, collection_id) def collection_asset(self, collection_id: str, filename: str) -> str: - return f"{self.catalog_dir}/{collection_id}/{filename}" + return os.path.join(self.catalog_dir, collection_id, filename) def collection_item_dir(self, collection_id: str, item_id: str) -> str: - return f"{self.catalog_dir}/{collection_id}/{item_id}" + return os.path.join(self.catalog_dir, collection_id, item_id) def collection_item(self, collection_id: str, item_id: str) -> str: - return f"{self.catalog_dir}/{collection_id}/{item_id}/{item_id}.json" + return os.path.join(self.catalog_dir, collection_id, item_id, f"{item_id}.json") def collection_item_asset(self, collection_id: str, item_id: str, filename: str) -> str: - return f"{self.catalog_dir}/{collection_id}/{item_id}/{filename}" + return os.path.join(self.catalog_dir, collection_id, item_id, filename) diff --git a/workflows/new.py b/workflows/new.py index 3b955a8..ee00df3 100644 --- a/workflows/new.py +++ b/workflows/new.py @@ -1,5 +1,39 @@ from stormhub.logger import initialize_logger from stormhub.met.storm_catalog import new_catalog, new_collection +import json +import logging +import shutil +import pystac + +def save_config(config_file="params-config.json", **kwargs): + """Automatically saves configuration parameters to a JSON file.""" + with open(config_file, "w") as file: + json.dump(kwargs, file, indent=4) + +def add_config_to_collection(storm_collection, config_filename="params-config.json"): + """Add config file to collection assets.""" + collection_path = storm_collection.self_href + collection_dir = os.path.dirname(collection_path) + os.makedirs(collection_dir, exist_ok=True) + + config_dest_path = os.path.join(collection_dir, config_filename) + shutil.copy(config_filename, config_dest_path) + + collection = pystac.Collection.from_file(collection_path) + + collection.add_asset( + "params-config", + pystac.Asset( + href=config_filename, + media_type=pystac.MediaType.JSON, + description= "Contains the configuration parameters used to generate the bighorn storm items.", + roles=["metadata"], + title="Configuration Parameters", + ), + ) + collection.save_object() + + if __name__ == "__main__": initialize_logger() @@ -25,6 +59,17 @@ # Collection Args storm_duration_hours = 48 min_precip_threshold = 2.5 + + save_config( + start_date=start_date, + end_date=end_date, + top_n_events=top_n_events, + storm_duration_hours=storm_duration_hours, + min_precip_threshold=min_precip_threshold, + root_dir=root_dir, + catalog_id=catalog_id, + local_directory=local_directory, + ) storm_collection = new_collection( storm_catalog, start_date, @@ -32,5 +77,7 @@ storm_duration_hours, min_precip_threshold, top_n_events, - check_every_n_hours=6, + check_every_n_hours=24, ) + # Add config file as a STAC collection asset + add_config_to_collection(storm_collection) diff --git a/workflows/resume.py b/workflows/resume.py index 9e3a23c..753bb14 100644 --- a/workflows/resume.py +++ b/workflows/resume.py @@ -1,30 +1,37 @@ import os - +import json from stormhub.logger import initialize_logger from stormhub.met.storm_catalog import resume_collection from stormhub.utils import StacPathManager + +def load_config(config_path): + """Load configuration from a JSON file.""" + with open(config_path, "r") as file: + return json.load(file) + + if __name__ == "__main__": initialize_logger() - # Catalog Args - root_dir = "" - config_file = f"{root_dir}/duwamish/config.json" - catalog_id = "duwamish" - local_directory = f"{root_dir}" + config_path = "params-config.json" + config = load_config(config_path) + # Extract config values + start_date = config["start_date"] + end_date = config["end_date"] + top_n_events = config["top_n_events"] + storm_duration_hours = config["storm_duration_hours"] + min_precip_threshold = config["min_precip_threshold"] + root_dir = config["root_dir"] + catalog_id = config["catalog_id"] + local_directory = config["local_directory"] + + # Set up storm catalog file spm = StacPathManager(os.path.join(local_directory, catalog_id)) storm_catalog_file = spm.catalog_file - # All Collection Args - start_date = "1979-02-01" - end_date = "2024-12-31" - top_n_events = 440 - - # Collection 1 Args - storm_duration_hours = 96 - min_precip_threshold = 4 - + # Resume collection with loaded parameters storm_collection = resume_collection( catalog=storm_catalog_file, start_date=start_date, @@ -32,7 +39,7 @@ storm_duration=storm_duration_hours, min_precip_threshold=min_precip_threshold, top_n_events=top_n_events, - check_every_n_hours=6, + check_every_n_hours=24, with_tb=False, create_items=True, )