Merge branch 'main' into 127_OM2_ERA5

Author: dougiesquire

.readthedocs.yml

@@ -1,5 +1,10 @@
 version: 2
 
+build:
+  os: "ubuntu-22.04"
+  tools:
+    python: "mambaforge-22.9"
+
 sphinx:
   configuration: docs/conf.py
 

src/access_nri_intake/catalog/manager.py

@@ -69,7 +69,7 @@ def build_esm(
         **kwargs,
     ):
         """
-        Build an Intake-ESM datastore
+        Build an Intake-ESM datastore and add it to the catalog
 
         Parameters
         ----------
@@ -81,98 +81,99 @@ def build_esm(
             The builder to use to build the Intake-ESM datastore
         path: str or list of str
             Path or list of paths to crawl for assets/files to add to the Intake-ESM datastore.
-        translator: :py:class:`~access_nri_intake.catalog.translators.DefaultTranslator`
+        translator: :py:class:`~access_nri_intake.catalog.translators.DefaultTranslator`, optional
             An instance of the :py:class:`~access_nri_intake.catalog.translators.DefaultTranslator` class
             for translating info in the Intake-ESM datastore into intake-dataframe-catalog column metadata.
-            Defaults to access_nri_intake.catalog.translators.DefaultTranslator.
+            Defaults to access_nri_intake.catalog.translators.DefaultTranslator
         metadata: dict, optional
             Additional info to store in the intake cat.metadata attribute. This info will be available
             to the translator and to users of the Intake-ESM datastore
         directory: str
             The directory to save the Intake-ESM datastore to. If None, use the current directory
         overwrite: bool, optional
-            Whether to overwrite any existing entries in the catalog with the same name
+            Whether to overwrite if an Intake-ESM datastore with the same name already exists
         kwargs: dict
             Additional kwargs to pass to the builder
         """
 
         metadata = metadata or {}
+        directory = directory or ""
 
         json_file = os.path.abspath(f"{os.path.join(directory, name)}.json")
         if os.path.isfile(json_file):
             if not overwrite:
                 raise CatalogManagerError(
                     f"An Intake-ESM datastore already exists for {name}. To overwrite, "
-                    "pass `overwrite=True` to CatalogBuilder.build"
+                    "pass `overwrite=True` to CatalogBuilder.build_esm"
                 )
 
         builder = builder(path, **kwargs).build()
         builder.save(name=name, description=description, directory=directory)
 
         self.source, self.source_metadata = _open_and_translate(
             json_file,
+            "esm_datastore",
             name,
             description,
             metadata,
             translator,
             columns_with_iterables=list(builder.columns_with_iterables),
         )
 
-        return self
+        self._add()
 
     def load(
         self,
         name,
         description,
         path,
-        translator,
+        driver="esm_datastore",
+        translator=DefaultTranslator,
         metadata=None,
         **kwargs,
     ):
         """
-        Load an existing intake catalog and add it to the catalog
+        Load an existing data source using Intake and add it to the catalog
 
         Parameters
         ----------
         name: str
-            The name of the catalog
+            The name of the data source
         description: str
-            Description of the contents of the catalog
+            Description of the contents of the data source
         path: str
-            The path to the intake-esm catalog JSON file
-        translator: :py:class:`~access_nri_catalog.metacat.translators.DefaultTranslator`
+            The path to the Intake data source
+        driver: str
+            The name of the Intake driver to use to open the data source
+        translator: :py:class:`~access_nri_catalog.metacat.translators.DefaultTranslator`, optional
             An instance of the :py:class:`~access_nri_catalog.metacat.translators.DefaultTranslator` class for
-            translating intake-esm column metadata into intake-dataframe-catalog column metadata
+            translating data source metadata into intake-dataframe-catalog column metadata. Defaults to
+            access_nri_intake.catalog.translators.DefaultTranslator
         metadata: dict, optional
-            Additional info to store in the intake cat.metadata attribute. This info will be available to
-            the translator and to users of the catalog
+            Additional info to store in the intake metadata attribute for this data source. This info will be
+            available to the translator and to users of the catalog
         kwargs: dict, optional
-            Additional kwargs to pass to :py:class:`~intake.open_esm_datastore`
+            Additional kwargs to pass to :py:class:`~intake.open_<driver>`
         """
 
         if isinstance(path, list):
             if len(path) != 1:
-                raise ValueError(
-                    "Only a single JSON file can be passed to CatalogManager.load_esm. Received {len(path)}"
+                raise CatalogManagerError(
+                    f"Only a single data source can be passed to CatalogManager.load. Received {len(path)}"
                 )
             path = path[0]
 
         metadata = metadata or {}
 
         self.source, self.source_metadata = _open_and_translate(
-            path, name, description, metadata, translator, **kwargs
+            path, driver, name, description, metadata, translator, **kwargs
         )
 
-        return self
+        self._add()
 
-    def add(self, **kwargs):
+    def _add(self):
         """
         Add a source to the catalog
-
-        Parameters
-        ----------
-        kwargs: dict, optional
-            Additional keyword arguments passed to :py:func:`~pandas.DataFrame.to_csv`.
         """
 
         if self.source is None:
@@ -183,7 +184,7 @@ def add(self, **kwargs):
         # Overwrite the catalog name with the name_column entry in metadata
         name = self.source_metadata[NAME_COLUMN].unique()
         if len(name) != 1:
-            raise ValueError(
+            raise CatalogManagerError(
                 f"Metadata column '{NAME_COLUMN}' must be the same for all rows in source_metadata "
                 "since this corresponds to the source name"
             )
@@ -199,19 +200,31 @@ def add(self, **kwargs):
             self.dfcat.add(self.source, row.to_dict(), overwrite=overwrite)
             overwrite = False
 
+    def save(self, **kwargs):
+        """
+        Save the catalog
+
+        Parameters
+        ----------
+        kwargs: dict, optional
+            Additional keyword arguments passed to :py:func:`~pandas.DataFrame.to_csv`.
+        """
         self.dfcat.save(**kwargs)
 
 
-def _open_and_translate(json_file, name, description, metadata, translator, **kwargs):
+def _open_and_translate(
+    file, driver, name, description, metadata, translator, **kwargs
+):
     """
-    Open an esm-datastore, assign name, description and metadata attrs and
+    Open an Intake data source, assign name, description and metadata attrs and
     translate using the provided translator
     """
-    cat = intake.open_esm_datastore(json_file, **kwargs)
-    cat.name = name
-    cat.description = description
-    cat.metadata = metadata
+    open_ = getattr(intake, f"open_{driver}")
+    source = open_(file, **kwargs)
+    source.name = name
+    source.description = description
+    source.metadata = metadata
 
-    metadata = translator(cat, CORE_COLUMNS).translate(TRANSLATOR_GROUPBY_COLUMNS)
+    metadata = translator(source, CORE_COLUMNS).translate(TRANSLATOR_GROUPBY_COLUMNS)
 
-    return cat, metadata
+    return source, metadata

src/access_nri_intake/cli.py

@@ -188,13 +188,13 @@ def _get_project(path):
     storage_flags = "+".join(sorted([f"gdata/{proj}" for proj in project]))
 
     # Build the catalog
+    cm = CatalogManager(path=metacatalog_path)
     for method, args in parsed_sources:
-        man = CatalogManager(path=metacatalog_path)
         logger.info(f"Adding '{args['name']}' to metacatalog '{metacatalog_path}'")
-        getattr(man, method)(**args).add()
+        getattr(cm, method)(**args)
 
     # Write catalog yaml file
-    cat = man.dfcat
+    cat = cm.dfcat
     cat.name = "access_nri"
     cat.description = "ACCESS-NRI intake catalog"
     yaml_dict = yaml.safe_load(cat.yaml())
@@ -211,6 +211,9 @@ def _get_project(path):
         "version": {"description": "Catalog version", "type": "str", "default": version}
     }
 
+    # Save the catalog
+    cm.save()
+
     _here = os.path.abspath(os.path.dirname(__file__))
     if update:
         with open(os.path.join(_here, "data", "catalog.yaml"), "w") as fobj:

tests/test_manager.py

@@ -25,7 +25,7 @@ def test_CatalogManager_init(tmp_path):
     assert hasattr(cat, "dfcat")
 
     with pytest.raises(CatalogManagerError) as excinfo:
-        cat.add()
+        cat._add()
     assert "first load or build the source" in str(excinfo.value)
 
 
@@ -55,16 +55,17 @@ def test_CatalogManager_build_esm(tmp_path, test_data, builder, basedir, kwargs)
         directory=str(tmp_path),
         **kwargs,
     )
-    cat.build_esm(**args).add()
+    cat.build_esm(**args)
 
     # Try to rebuild without setting overwrite
     with pytest.raises(CatalogManagerError) as excinfo:
         cat.build_esm(**args)
     assert "An Intake-ESM datastore already exists" in str(excinfo.value)
 
     # Overwrite
-    cat.build_esm(**args, overwrite=True).add()
+    cat.build_esm(**args, overwrite=True)
 
+    cat.save()
     cat = CatalogManager(path)
     assert cat.mode == "a"
 
@@ -89,7 +90,8 @@ def test_CatalogManager_load(tmp_path, test_data, translator, datastore, metadat
         translator=translator,
         metadata=metadata,
     )
-    cat.load(**args).add()
+    cat.load(**args)
+    cat.save()
 
     cat = CatalogManager(path)
     assert cat.mode == "a"
@@ -110,9 +112,9 @@ def test_CatalogManager_load_error(tmp_path, test_data):
     cat.load(**args, path=[path])
 
     # Test fails when len > 1
-    with pytest.raises(ValueError) as excinfo:
+    with pytest.raises(CatalogManagerError) as excinfo:
         cat.load(**args, path=[path, path])
-    assert "Only a single JSON file" in str(excinfo.value)
+    assert "Only a single data source" in str(excinfo.value)
 
 
 def test_CatalogManager_all(tmp_path, test_data):
@@ -129,7 +131,10 @@ def test_CatalogManager_all(tmp_path, test_data):
     )
     cat.load(
         **load_args,
-    ).add()
+    )
+    assert len(cat.dfcat) == 1
+    cat.save()
+    assert len(CatalogManager(path).dfcat) == 1
 
     # Build source
     cat.build_esm(
@@ -143,12 +148,15 @@ def test_CatalogManager_all(tmp_path, test_data):
             ]
         ),
         directory=str(tmp_path),
-    ).add()
-
+    )
+    # Still only one entry on disk
     assert len(cat.dfcat) == 2
+    assert len(CatalogManager(path).dfcat) == 1
 
     # Check that entry with same name overwrites correctly
     cat.load(
         **load_args,
-    ).add()
+    )
     assert len(cat.dfcat) == 2
+    cat.save()
+    assert len(CatalogManager(path).dfcat) == 2