generated from fastai/nbdev_template
-
-
Notifications
You must be signed in to change notification settings - Fork 31
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Add GOES Data Download Manager Script (#240)
* added new file of download manager for goes * added new file of download manager for goes * added new file of download manager for goes * added check for existing file * Determine time increment based on product/domain * made changes in eumesat.py file * added new file download_manager.py * made changes in app.py * minor changes * minor fix in eumetsat.py file * changed name of downloadmanager * changed name of downloadmanager * changed name of downloadmanager * changed name of downloadmanager * changed name of downloadmanager * changed name of downloadmanager * fixed test_utils.py file
- Loading branch information
Showing
10 changed files
with
244 additions
and
56 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,90 @@ | ||
| """Satip Download Manager | ||
| This module provides a unified interface for downloading EUMETSAT and GOES | ||
| satellite data via the `DownloadManager` class. Users specify the provider | ||
| ('EUMETSAT' or 'GOES'), and the manager delegates tasks to dedicated | ||
| sub-modules for retrieval, storage, and logging. | ||
| Key functionalities: | ||
| * Download data for a specified time range. | ||
| * Handle user authentication (for EUMETSAT data). | ||
| * Manage data retrieval, storage, and logging for both providers. | ||
| """ | ||
|
|
||
| import warnings | ||
|
|
||
| import structlog | ||
|
|
||
| from satip.eumetsat import EUMETSATDownloadManager | ||
| from satip.goes_download_manager import GOESDownloadManager | ||
|
|
||
| log = structlog.stdlib.get_logger() | ||
|
|
||
| # Suppress FutureWarning related to 'H' argument | ||
| warnings.filterwarnings('ignore', category=FutureWarning) | ||
| # constants for different data sources | ||
| EUMETSAT_PROVIDER = "EUMETSAT" | ||
| GOES_PROVIDER = "GOES" | ||
|
|
||
|
|
||
|
|
||
| class DownloadManager: | ||
| """ | ||
| Main download manager class to handle both EUMETSAT | ||
| and GOES data downloading based on the provider. | ||
| Example usage: | ||
| if __name__ == "__main__": | ||
| provider = "GOES" | ||
| user_key = "your_user_key" | ||
| user_secret = "your_user_secret" | ||
| data_dir = "path to data directory" | ||
| log_directory = "path to log directory" | ||
| start_time = datetime.datetime(2024, 3, 1, 0, 0) | ||
| end_time = datetime.datetime(2024, 3, 1, 6, 0) | ||
| if data_dir is not None: | ||
| manager = DownloadManager(provider, None, None, data_dir, log_directory) | ||
| manager.download_data(start_time, end_time) | ||
| else: | ||
| print("Error: 'data_dir' is not properly set.") | ||
| """ | ||
|
|
||
| def __init__(self, provider, user_key=None, | ||
| user_secret=None, data_dir=None, | ||
| log_directory=None): | ||
| """ | ||
| Initialize the DownloadManager. | ||
| Args: | ||
| provider (str): Provider name ('EUMETSAT' or 'GOES'). | ||
| user_key (str): User key for accessing data (for EUMETSAT). | ||
| user_secret (str): User secret for accessing data (for EUMETSAT). | ||
| data_dir (str): Directory to save downloaded data. | ||
| log_directory (str): Directory to save logs. | ||
| """ | ||
| self.provider = provider | ||
|
|
||
| if self.provider == "EUMETSAT": | ||
| self.download_manager = EUMETSATDownloadManager(user_key, user_secret, | ||
| data_dir, log_directory) | ||
| elif self.provider == "GOES": | ||
| self.download_manager = GOESDownloadManager(data_dir, log_directory) | ||
| else: | ||
| raise ValueError("Invalid provider. Supported providers are 'EUMETSAT' and 'GOES'.") | ||
|
|
||
| def download_data(self, start_time, end_time): | ||
| """ | ||
| Download data for the specified time range. | ||
| Args: | ||
| start_time (datetime): Start of the download period. | ||
| end_time (datetime): End of the download period. | ||
| """ | ||
| if self.provider == "GOES": | ||
| self.download_manager.download_goes_data(start_time, end_time) |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,96 @@ | ||
| """ | ||
| Script for downloading GOES data. | ||
| """ | ||
|
|
||
| import datetime | ||
| import logging | ||
| import os | ||
|
|
||
| from goes2go import GOES | ||
|
|
||
|
|
||
| class GOESDownloadManager: | ||
| """ | ||
| Manager class for downloading GOES data. | ||
| """ | ||
| def __init__(self, data_dir, log_directory=None): | ||
| """ | ||
| Initialize the GOESDownloadManager. | ||
| Args: | ||
| data_dir (str): Directory to save downloaded GOES data. | ||
| log_directory (str, optional): Directory to save logs. | ||
| If None, logging is printed to STDOUT. | ||
| """ | ||
| self.data_dir = data_dir | ||
| self.ensure_directory_exists(self.data_dir) | ||
|
|
||
| if log_directory: | ||
| self.ensure_directory_exists(log_directory) | ||
| logging.basicConfig( | ||
| filename=os.path.join(log_directory, 'goes_download.log'), | ||
| level=logging.INFO) | ||
| else: | ||
| logging.basicConfig(level=logging.INFO) | ||
|
|
||
| logging.info(f"GOESDownloadManager initialized. Data will be saved to: {data_dir}") | ||
|
|
||
| @staticmethod | ||
| def ensure_directory_exists(directory): | ||
| """Ensures the specified directory exists, creating it if necessary.""" | ||
| if not os.path.exists(directory): | ||
| try: | ||
| os.makedirs(directory) | ||
| logging.info(f"Created directory: {directory}") | ||
| except Exception as e: | ||
| logging.error(f"Error creating directory {directory}: {e}") | ||
| raise | ||
| def download_goes_data(self, start_time, end_time, product='ABI-L1b-RadC', | ||
| domain='F', satellite=16): | ||
| """ | ||
| Download GOES data for a specified time range and product. | ||
| Args: | ||
| start_time (datetime): Start of the download period. | ||
| end_time (datetime): End of the download period. | ||
| product (str): GOES product identifier. Default is 'ABI-L1b-RadC'. | ||
| domain (str): Domain for the product. Default is 'F' (Full Disk). | ||
| satellite (int): GOES satellite number. Default is 16. | ||
| """ | ||
| G = GOES(satellite=satellite, product=product, domain=domain) | ||
| current_time = start_time | ||
|
|
||
| # Determine time increment based on product/domain | ||
| time_increment = 1 # Default time increment (minutes) | ||
| if product == 'ABI-L1b-RadC' and domain == 'F': | ||
| time_increment = 10 | ||
|
|
||
| while current_time <= end_time: | ||
| try: | ||
| # Download the data | ||
| ds = G.nearesttime(current_time) | ||
|
|
||
| # Get acquisition time from the dataset | ||
| acquisition_time = ds.time.data.item() | ||
|
|
||
| # Format the acquisition time for filename | ||
| date_string = acquisition_time.strftime("%Y-%m-%d_%H-%M-%S") | ||
| filename = f"goes_data_{date_string}.nc" | ||
| filepath = os.path.join(self.data_dir, filename) | ||
|
|
||
| # Check if data for current acquisition time already exists | ||
| if os.path.exists(filepath): | ||
| logging.info(f"Data for {date_string} already exists. Skipping.") | ||
| current_time += datetime.timedelta(minutes=time_increment) | ||
| continue | ||
|
|
||
| # Save to NetCDF | ||
| ds.to_netcdf(filepath) | ||
|
|
||
| logging.info(f"Downloaded and saved GOES data to: {filename}") | ||
| except Exception as e: | ||
| logging.error(f"Error downloading GOES data for {current_time}: {e}") | ||
|
|
||
| current_time += datetime.timedelta(minutes=time_increment) | ||
|
|
||
| logging.info("Completed GOES data download.") |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Oops, something went wrong.