diff --git a/dev-requirements.txt b/dev-requirements.txt index ae5300865..a1d03eae8 100644 --- a/dev-requirements.txt +++ b/dev-requirements.txt @@ -5,7 +5,7 @@ dbt-tests-adapter~=1.7.7 flake8~=7.0 Flake8-pyproject~=1.2 isort~=5.13 -moto~=4.2.13 +moto~=5.0.1 pre-commit~=3.5 pyparsing~=3.1.1 pytest~=8.0 diff --git a/tests/unit/test_adapter.py b/tests/unit/test_adapter.py index 1c8c14225..ed62c936d 100644 --- a/tests/unit/test_adapter.py +++ b/tests/unit/test_adapter.py @@ -7,9 +7,7 @@ import boto3 import botocore import pytest - -# from botocore.client.BaseClient import _make_api_call -from moto import mock_athena, mock_glue, mock_s3, mock_sts +from moto import mock_aws from moto.core import DEFAULT_ACCOUNT_ID from dbt.adapters.athena import AthenaAdapter @@ -436,10 +434,7 @@ def test_generate_s3_location( relation, s3_data_dir, s3_data_naming, s3_tmp_table_dir, external_location, is_temporary_table ) - @mock_glue - @mock_s3 - @mock_athena - @mock_sts + @mock_aws def test_get_table_location(self, dbt_debug_caplog, mock_aws_service): table_name = "test_table" self.adapter.acquire_connection("dummy") @@ -453,10 +448,7 @@ def test_get_table_location(self, dbt_debug_caplog, mock_aws_service): ) assert self.adapter.get_glue_table_location(relation) == "s3://test-dbt-athena/tables/test_table" - @mock_glue - @mock_s3 - @mock_athena - @mock_sts + @mock_aws def test_get_table_location_raise_s3_location_exception(self, dbt_debug_caplog, mock_aws_service): table_name = "test_table" self.adapter.acquire_connection("dummy") @@ -475,10 +467,7 @@ def test_get_table_location_raise_s3_location_exception(self, dbt_debug_caplog, "location, but no location returned by Glue." ) - @mock_glue - @mock_s3 - @mock_athena - @mock_sts + @mock_aws def test_get_table_location_for_view(self, dbt_debug_caplog, mock_aws_service): view_name = "view" self.adapter.acquire_connection("dummy") @@ -490,10 +479,7 @@ def test_get_table_location_for_view(self, dbt_debug_caplog, mock_aws_service): ) assert self.adapter.get_glue_table_location(relation) is None - @mock_glue - @mock_s3 - @mock_athena - @mock_sts + @mock_aws def test_get_table_location_with_failure(self, dbt_debug_caplog, mock_aws_service): table_name = "test_table" self.adapter.acquire_connection("dummy") @@ -507,10 +493,7 @@ def test_get_table_location_with_failure(self, dbt_debug_caplog, mock_aws_servic assert self.adapter.get_glue_table_location(relation) is None assert f"Table {relation.render()} does not exists - Ignoring" in dbt_debug_caplog.getvalue() - @mock_glue - @mock_s3 - @mock_athena - @mock_sts + @mock_aws def test_clean_up_partitions_will_work(self, dbt_debug_caplog, mock_aws_service): table_name = "table" mock_aws_service.create_data_catalog() @@ -541,9 +524,7 @@ def test_clean_up_partitions_will_work(self, dbt_debug_caplog, mock_aws_service) keys = [obj["Key"] for obj in s3.list_objects_v2(Bucket=BUCKET)["Contents"]] assert set(keys) == {"tables/table/dt=2022-01-03/data1.parquet", "tables/table/dt=2022-01-03/data2.parquet"} - @mock_glue - @mock_athena - @mock_sts + @mock_aws def test_clean_up_table_table_does_not_exist(self, dbt_debug_caplog, mock_aws_service): mock_aws_service.create_data_catalog() mock_aws_service.create_database() @@ -559,9 +540,7 @@ def test_clean_up_table_table_does_not_exist(self, dbt_debug_caplog, mock_aws_se 'Table "awsdatacatalog"."test_dbt_athena"."table" does not exists - Ignoring' in dbt_debug_caplog.getvalue() ) - @mock_glue - @mock_athena - @mock_sts + @mock_aws def test_clean_up_table_view(self, dbt_debug_caplog, mock_aws_service): mock_aws_service.create_data_catalog() mock_aws_service.create_database() @@ -576,10 +555,7 @@ def test_clean_up_table_view(self, dbt_debug_caplog, mock_aws_service): result = self.adapter.clean_up_table(relation) assert result is None - @mock_glue - @mock_s3 - @mock_athena - @mock_sts + @mock_aws def test_clean_up_table_delete_table(self, dbt_debug_caplog, mock_aws_service): mock_aws_service.create_data_catalog() mock_aws_service.create_database() @@ -613,9 +589,7 @@ def test_clean_up_table_delete_table(self, dbt_debug_caplog, mock_aws_service): def test_quote_seed_column(self, column, quote_config, quote_character, expected): assert self.adapter.quote_seed_column(column, quote_config, quote_character) == expected - @mock_glue - @mock_athena - @mock_sts + @mock_aws def test__get_one_catalog(self, mock_aws_service): mock_aws_service.create_data_catalog() mock_aws_service.create_database("foo") @@ -665,15 +639,13 @@ def test__get_one_catalog(self, mock_aws_service): for row in actual.rows.values(): assert row.values() in expected_rows - @mock_glue - @mock_athena - @mock_sts + @mock_aws def test__get_one_catalog_by_relations(self, mock_aws_service): mock_aws_service.create_data_catalog() mock_aws_service.create_database("foo") mock_aws_service.create_database("quux") mock_aws_service.create_table(database_name="foo", table_name="bar") - # we create another relations + # we create another relation mock_aws_service.create_table(table_name="bar", database_name="quux") mock_information_schema = mock.MagicMock() @@ -710,8 +682,7 @@ def test__get_one_catalog_by_relations(self, mock_aws_service): assert actual.column_names == expected_column_names assert actual.rows == expected_rows - @mock_glue - @mock_athena + @mock_aws def test__get_one_catalog_shared_catalog(self, mock_aws_service): mock_aws_service.create_data_catalog(catalog_name=SHARED_DATA_CATALOG_NAME, catalog_id=SHARED_DATA_CATALOG_NAME) mock_aws_service.create_database("foo", catalog_id=SHARED_DATA_CATALOG_NAME) @@ -751,7 +722,7 @@ def test__get_one_catalog_shared_catalog(self, mock_aws_service): for row in actual.rows.values(): assert row.values() in expected_rows - @mock_athena + @mock_aws def test__get_one_catalog_federated_query_catalog(self, mock_aws_service): mock_aws_service.create_data_catalog( catalog_name=FEDERATED_QUERY_CATALOG_NAME, catalog_type=AthenaCatalogType.LAMBDA @@ -762,7 +733,7 @@ def test__get_one_catalog_federated_query_catalog(self, mock_aws_service): # Original botocore _make_api_call function orig = botocore.client.BaseClient._make_api_call - # Mocking this as list_table_metadata and creating non glue tables is not supported by moto. + # Mocking this as list_table_metadata and creating non-glue tables is not supported by moto. # Followed this guide: http://docs.getmoto.org/en/latest/docs/services/patching_other_services.html def mock_athena_list_table_metadata(self, operation_name, kwarg): if operation_name == "ListTableMetadata": @@ -854,8 +825,7 @@ def test__get_catalog_schemas(self): assert set(relations.keys()) == {"foo"} assert list(relations.values()) == [{"bar"}] - @mock_athena - @mock_sts + @mock_aws def test__get_data_catalog(self, mock_aws_service): mock_aws_service.create_data_catalog() self.adapter.acquire_connection("dummy") @@ -885,9 +855,7 @@ def _test_list_relations_without_caching(self, schema_relation): assert view.type == "view" assert view.detailed_table_type == "" - @mock_athena - @mock_glue - @mock_sts + @mock_aws def test_list_relations_without_caching_with_awsdatacatalog(self, mock_aws_service): mock_aws_service.create_data_catalog() mock_aws_service.create_database() @@ -903,8 +871,7 @@ def test_list_relations_without_caching_with_awsdatacatalog(self, mock_aws_servi ) self._test_list_relations_without_caching(schema_relation) - @mock_athena - @mock_glue + @mock_aws def test_list_relations_without_caching_with_other_glue_data_catalog(self, mock_aws_service): data_catalog_name = "other_data_catalog" mock_aws_service.create_data_catalog(data_catalog_name) @@ -921,7 +888,7 @@ def test_list_relations_without_caching_with_other_glue_data_catalog(self, mock_ ) self._test_list_relations_without_caching(schema_relation) - @mock_athena + @mock_aws @patch("dbt.adapters.athena.impl.SQLAdapter.list_relations_without_caching", return_value=[]) def test_list_relations_without_caching_with_non_glue_data_catalog( self, parent_list_relations_without_caching, mock_aws_service @@ -947,10 +914,7 @@ def test_list_relations_without_caching_with_non_glue_data_catalog( def test_parse_s3_path(self, s3_path, expected): assert self.adapter._parse_s3_path(s3_path) == expected - @mock_athena - @mock_glue - @mock_s3 - @mock_sts + @mock_aws def test_swap_table_with_partitions(self, mock_aws_service): mock_aws_service.create_data_catalog() mock_aws_service.create_database() @@ -974,10 +938,7 @@ def test_swap_table_with_partitions(self, mock_aws_service): self.adapter.swap_table(source_relation, target_relation) assert self.adapter.get_glue_table_location(target_relation) == f"s3://{BUCKET}/tables/{source_table}" - @mock_athena - @mock_glue - @mock_s3 - @mock_sts + @mock_aws def test_swap_table_without_partitions(self, mock_aws_service): mock_aws_service.create_data_catalog() mock_aws_service.create_database() @@ -999,10 +960,7 @@ def test_swap_table_without_partitions(self, mock_aws_service): self.adapter.swap_table(source_relation, target_relation) assert self.adapter.get_glue_table_location(target_relation) == f"s3://{BUCKET}/tables/{source_table}" - @mock_athena - @mock_glue - @mock_s3 - @mock_sts + @mock_aws def test_swap_table_with_partitions_to_one_without(self, mock_aws_service): mock_aws_service.create_data_catalog() mock_aws_service.create_database() @@ -1037,10 +995,7 @@ def test_swap_table_with_partitions_to_one_without(self, mock_aws_service): assert self.adapter.get_glue_table_location(target_relation) == f"s3://{BUCKET}/tables/{source_table}" assert len(target_table_partitions) == 0 - @mock_athena - @mock_glue - @mock_s3 - @mock_sts + @mock_aws def test_swap_table_with_no_partitions_to_one_with(self, mock_aws_service): mock_aws_service.create_data_catalog() mock_aws_service.create_database() @@ -1073,8 +1028,7 @@ def test_swap_table_with_no_partitions_to_one_with(self, mock_aws_service): assert self.adapter.get_glue_table_location(target_relation) == f"s3://{BUCKET}/tables/{source_table}" assert len(target_table_partitions_after) == 26 - @mock_athena - @mock_glue + @mock_aws def test__get_glue_table_versions_to_expire(self, mock_aws_service, dbt_debug_caplog): mock_aws_service.create_data_catalog() mock_aws_service.create_database() @@ -1097,10 +1051,7 @@ def test__get_glue_table_versions_to_expire(self, mock_aws_service, dbt_debug_ca assert len(versions_to_expire) == 3 assert [v["VersionId"] for v in versions_to_expire] == ["3", "2", "1"] - @mock_athena - @mock_glue - @mock_s3 - @mock_sts + @mock_aws def test_expire_glue_table_versions(self, mock_aws_service): mock_aws_service.create_data_catalog() mock_aws_service.create_database() @@ -1124,7 +1075,7 @@ def test_expire_glue_table_versions(self, mock_aws_service): # TODO moto issue https://github.com/getmoto/moto/issues/5952 # assert len(result) == 3 - @mock_s3 + @mock_aws def test_upload_seed_to_s3(self, mock_aws_service): seed_table = agate.Table.from_object(seed_data) self.adapter.acquire_connection("dummy") @@ -1156,7 +1107,7 @@ def test_upload_seed_to_s3(self, mock_aws_service): assert len(objects) == 1 assert objects[0].get("Key").endswith(".csv") - @mock_s3 + @mock_aws def test_upload_seed_to_s3_external_location(self, mock_aws_service): seed_table = agate.Table.from_object(seed_data) self.adapter.acquire_connection("dummy") @@ -1188,31 +1139,28 @@ def test_upload_seed_to_s3_external_location(self, mock_aws_service): assert len(objects) == 1 assert objects[0].get("Key").endswith(".csv") - @mock_athena + @mock_aws def test_get_work_group_output_location(self, mock_aws_service): self.adapter.acquire_connection("dummy") mock_aws_service.create_work_group_with_output_location_enforced(ATHENA_WORKGROUP) work_group_location_enforced = self.adapter.is_work_group_output_location_enforced() assert work_group_location_enforced - @mock_athena + @mock_aws def test_get_work_group_output_location_no_location(self, mock_aws_service): self.adapter.acquire_connection("dummy") mock_aws_service.create_work_group_no_output_location(ATHENA_WORKGROUP) work_group_location_enforced = self.adapter.is_work_group_output_location_enforced() assert not work_group_location_enforced - @mock_athena + @mock_aws def test_get_work_group_output_location_not_enforced(self, mock_aws_service): self.adapter.acquire_connection("dummy") mock_aws_service.create_work_group_with_output_location_not_enforced(ATHENA_WORKGROUP) work_group_location_enforced = self.adapter.is_work_group_output_location_enforced() assert not work_group_location_enforced - @mock_athena - @mock_glue - @mock_s3 - @mock_sts + @mock_aws def test_persist_docs_to_glue_no_comment(self, mock_aws_service): mock_aws_service.create_data_catalog() mock_aws_service.create_database() @@ -1251,10 +1199,7 @@ def test_persist_docs_to_glue_no_comment(self, mock_aws_service): assert not table["Parameters"].get("comment") assert all(not col.get("Comment") for col in table["StorageDescriptor"]["Columns"]) - @mock_athena - @mock_glue - @mock_s3 - @mock_sts + @mock_aws def test_persist_docs_to_glue_comment(self, mock_aws_service): mock_aws_service.create_data_catalog() mock_aws_service.create_database() @@ -1294,8 +1239,7 @@ def test_persist_docs_to_glue_comment(self, mock_aws_service): col_id = [col for col in table["StorageDescriptor"]["Columns"] if col["Name"] == "id"][0] assert col_id["Comment"] == "A column with str, 123, &^% \" and ' and an other paragraph." - @mock_athena - @mock_glue + @mock_aws def test_list_schemas(self, mock_aws_service): mock_aws_service.create_data_catalog() mock_aws_service.create_database(name="foo") @@ -1305,9 +1249,7 @@ def test_list_schemas(self, mock_aws_service): res = self.adapter.list_schemas("") assert sorted(res) == ["bar", "foo", "quux"] - @mock_athena - @mock_glue - @mock_sts + @mock_aws def test_get_columns_in_relation(self, mock_aws_service): mock_aws_service.create_data_catalog() mock_aws_service.create_database() @@ -1326,9 +1268,7 @@ def test_get_columns_in_relation(self, mock_aws_service): AthenaColumn(column="dt", dtype="date", table_type=TableType.TABLE), ] - @mock_athena - @mock_glue - @mock_sts + @mock_aws def test_get_columns_in_relation_not_found_table(self, mock_aws_service): mock_aws_service.create_data_catalog() mock_aws_service.create_database() @@ -1342,9 +1282,7 @@ def test_get_columns_in_relation_not_found_table(self, mock_aws_service): ) assert columns == [] - @mock_athena - @mock_glue - @mock_sts + @mock_aws def test_delete_from_glue_catalog(self, mock_aws_service): mock_aws_service.create_data_catalog() mock_aws_service.create_database() @@ -1356,9 +1294,7 @@ def test_delete_from_glue_catalog(self, mock_aws_service): tables_list = glue.get_tables(DatabaseName=DATABASE_NAME).get("TableList") assert tables_list == [] - @mock_athena - @mock_glue - @mock_sts + @mock_aws def test_delete_from_glue_catalog_not_found_table(self, dbt_debug_caplog, mock_aws_service): mock_aws_service.create_data_catalog() mock_aws_service.create_database() @@ -1372,10 +1308,7 @@ def test_delete_from_glue_catalog_not_found_table(self, dbt_debug_caplog, mock_a error_msg = f"Table {relation.render()} does not exist and will not be deleted, ignoring" assert error_msg in dbt_debug_caplog.getvalue() - @mock_glue - @mock_s3 - @mock_athena - @mock_sts + @mock_aws def test__get_relation_type_table(self, dbt_debug_caplog, mock_aws_service): mock_aws_service.create_data_catalog() mock_aws_service.create_database() @@ -1387,10 +1320,7 @@ def test__get_relation_type_table(self, dbt_debug_caplog, mock_aws_service): table_type = self.adapter.get_glue_table_type(relation) assert table_type == TableType.TABLE - @mock_glue - @mock_s3 - @mock_athena - @mock_sts + @mock_aws def test__get_relation_type_with_no_type(self, dbt_debug_caplog, mock_aws_service): mock_aws_service.create_data_catalog() mock_aws_service.create_database() @@ -1402,10 +1332,7 @@ def test__get_relation_type_with_no_type(self, dbt_debug_caplog, mock_aws_servic with pytest.raises(ValueError): self.adapter.get_glue_table_type(relation) - @mock_glue - @mock_s3 - @mock_athena - @mock_sts + @mock_aws def test__get_relation_type_view(self, dbt_debug_caplog, mock_aws_service): mock_aws_service.create_data_catalog() mock_aws_service.create_database() @@ -1417,10 +1344,7 @@ def test__get_relation_type_view(self, dbt_debug_caplog, mock_aws_service): table_type = self.adapter.get_glue_table_type(relation) assert table_type == TableType.VIEW - @mock_glue - @mock_s3 - @mock_athena - @mock_sts + @mock_aws def test__get_relation_type_iceberg(self, dbt_debug_caplog, mock_aws_service): mock_aws_service.create_data_catalog() mock_aws_service.create_database()