Merge branch 'main' into feature/2733-validate-node-values

RELEASE.md

@@ -8,6 +8,7 @@
 
 ## Bug fixes and other changes
 * Updated `kedro pipeline create` and `kedro pipeline delete` to read the base environment from the project settings.
+* Updated CLI command `kedro catalog resolve` to read credentials properly.
 
 ## Breaking changes to the API
 * Methods `_is_project` and `_find_kedro_project` have been moved to `kedro.utils`. We recommend not using private methods in your code, but if you do, please update your code to use the new location.

kedro/framework/cli/catalog.py

@@ -233,7 +233,10 @@ def resolve_patterns(metadata: ProjectMetadata, env: str) -> None:
     context = session.load_context()
 
     catalog_config = context.config_loader["catalog"]
-    data_catalog = DataCatalog.from_config(catalog_config)
+    credentials_config = context.config_loader.get("credentials", None)
+    data_catalog = DataCatalog.from_config(
+        catalog=catalog_config, credentials=credentials_config
+    )
 
     explicit_datasets = {
         ds_name: ds_config

tests/config/test_omegaconf_config.py

@@ -403,9 +403,8 @@ def test_empty_catalog_file(self, tmp_path):
         )["catalog"]
         assert catalog == {}
 
-    @pytest.mark.xfail(reason="Logic failing")
-    def test_overlapping_patterns(self, tmp_path, mocker):
-        """Check that same configuration file is not loaded more than once."""
+    def test_overlapping_patterns(self, tmp_path):
+        """Check that configuration is loaded correctly from overlapping patterns."""
         _write_yaml(
             tmp_path / _BASE_ENV / "catalog0.yml",
             {"env": _BASE_ENV, "common": "common"},
@@ -424,7 +423,6 @@ def test_overlapping_patterns(self, tmp_path, mocker):
             ]
         }
 
-        load_spy = mocker.spy(OmegaConf, "load")
         catalog = OmegaConfigLoader(
             conf_source=str(tmp_path),
             base_env=_BASE_ENV,
@@ -439,9 +437,31 @@ def test_overlapping_patterns(self, tmp_path, mocker):
         }
         assert catalog == expected_catalog
 
-        # Assert any specific config file was only loaded once
-        expected_path = (tmp_path / "dev" / "user1" / "catalog2.yml").resolve()
-        load_spy.assert_called_once_with(expected_path)
+    def test_overlapping_patterns_in_same_env(self, tmp_path, mocker):
+        """Check that configuration files that match several patterns are only loaded once in each env."""
+        _write_yaml(tmp_path / _BASE_ENV / "user1" / "catalog.yml", {"user1_c2": True})
+
+        catalog_patterns = {
+            "catalog": [
+                "catalog*",
+                "user1/catalog*",
+                "*/catalog2*",
+            ]
+        }
+
+        load_spy = mocker.spy(OmegaConf, "load")
+        catalog = OmegaConfigLoader(
+            conf_source=str(tmp_path),
+            base_env=_BASE_ENV,
+            config_patterns=catalog_patterns,
+        )["catalog"]
+        expected_catalog = {
+            "user1_c2": True,
+        }
+        assert catalog == expected_catalog
+
+        # Assert load is only called once
+        load_spy.assert_called_once()
 
     def test_yaml_parser_error(self, tmp_path):
         conf_path = tmp_path / _BASE_ENV

tests/framework/cli/test_catalog.py

@@ -28,12 +28,18 @@ def mock_pipelines(mocker):
     return mocker.patch("kedro.framework.cli.catalog.pipelines", dummy_pipelines)
 
 
+@pytest.fixture()
+def fake_credentials_config(tmp_path):
+    return {"db_connection": {"con": "foo"}}
+
+
 @pytest.fixture
 def fake_catalog_config():
     config = {
         "parquet_{factory_pattern}": {
             "type": "pandas.ParquetDataset",
             "filepath": "data/01_raw/{factory_pattern}.pq",
+            "credentials": "db_connection",
         },
         "csv_{factory_pattern}": {
             "type": "pandas.CSVDataset",
@@ -50,6 +56,7 @@ def fake_catalog_config_resolved():
         "parquet_example": {
             "type": "pandas.ParquetDataset",
             "filepath": "data/01_raw/example.pq",
+            "credentials": {"con": "foo"},
         },
         "csv_example": {
             "type": "pandas.CSVDataset",
@@ -271,13 +278,16 @@ def test_list_factory_generated_datasets(
         mocker,
         mock_pipelines,
         fake_catalog_config,
+        fake_credentials_config,
     ):
         """Test that datasets generated from factory patterns in the catalog
         are resolved correctly under the correct dataset classes.
         """
         yaml_dump_mock = mocker.patch("yaml.dump", return_value="Result YAML")
         mocked_context = fake_load_context.return_value
-        mocked_context.catalog = DataCatalog.from_config(fake_catalog_config)
+        mocked_context.catalog = DataCatalog.from_config(
+            catalog=fake_catalog_config, credentials=fake_credentials_config
+        )
         mocker.patch.object(
             mock_pipelines[PIPELINE_NAME],
             "datasets",
@@ -526,11 +536,17 @@ def test_catalog_resolve(
         mock_pipelines,
         fake_catalog_config,
         fake_catalog_config_resolved,
+        fake_credentials_config,
     ):
         """Test that datasets factories are correctly resolved to the explicit datasets in the pipeline."""
         mocked_context = fake_load_context.return_value
-        mocked_context.config_loader = {"catalog": fake_catalog_config}
-        mocked_context.catalog = DataCatalog.from_config(fake_catalog_config)
+        mocked_context.config_loader = {
+            "catalog": fake_catalog_config,
+            "credentials": fake_credentials_config,
+        }
+        mocked_context.catalog = DataCatalog.from_config(
+            catalog=fake_catalog_config, credentials=fake_credentials_config
+        )
         placeholder_ds = mocked_context.catalog._dataset_patterns.keys()
         pipeline_datasets = {"csv_example", "parquet_example", "explicit_dataset"}