From b5fb78f6f2c43327ad60b2fcb87e71acc28ac638 Mon Sep 17 00:00:00 2001 From: gjskibinski Date: Sun, 9 Feb 2025 10:10:38 +0100 Subject: [PATCH 1/6] code fix --- dbt-bigquery/src/dbt/adapters/bigquery/impl.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/dbt-bigquery/src/dbt/adapters/bigquery/impl.py b/dbt-bigquery/src/dbt/adapters/bigquery/impl.py index 51c457129..83ac7b725 100644 --- a/dbt-bigquery/src/dbt/adapters/bigquery/impl.py +++ b/dbt-bigquery/src/dbt/adapters/bigquery/impl.py @@ -144,6 +144,7 @@ class BigQueryAdapter(BaseAdapter): def __init__(self, config, mp_context: SpawnContext) -> None: super().__init__(config, mp_context) self.connections: BigQueryConnectionManager = self.connections + self.job_execution_timeout_seconds = self.config.credentials.job_execution_timeout_seconds or 60 * 60 * 12 ### # Implementations of abstract methods @@ -774,7 +775,7 @@ def get_table_options( opts["kms_key_name"] = f"'{config.get('kms_key_name')}'" if temporary: - opts["expiration_timestamp"] = "TIMESTAMP_ADD(CURRENT_TIMESTAMP(), INTERVAL 12 hour)" + opts["expiration_timestamp"] = f"TIMESTAMP_ADD(CURRENT_TIMESTAMP(), INTERVAL {self.job_execution_timeout_seconds} second)" else: # It doesn't apply the `require_partition_filter` option for a temporary table # so that we avoid the error by not specifying a partition with a temporary table From 02cf72b7c9202544adbfced64192c80c7f6ec4ff Mon Sep 17 00:00:00 2001 From: Georgie Date: Sun, 9 Feb 2025 12:48:13 +0100 Subject: [PATCH 2/6] changie? + release? --- dbt-adapters/.changes/1.14.0.md | 2 -- dbt-adapters/CHANGELOG.md | 1 - .../.changes/unreleased/Features-20250209-123750.yaml | 6 ++++++ dbt-bigquery/src/dbt/adapters/bigquery/impl.py | 8 ++++++-- dbt-tests-adapter/.changes/1.11.0.md | 2 -- dbt-tests-adapter/CHANGELOG.md | 3 --- 6 files changed, 12 insertions(+), 10 deletions(-) create mode 100644 dbt-bigquery/.changes/unreleased/Features-20250209-123750.yaml diff --git a/dbt-adapters/.changes/1.14.0.md b/dbt-adapters/.changes/1.14.0.md index be861a060..bd4a6257c 100644 --- a/dbt-adapters/.changes/1.14.0.md +++ b/dbt-adapters/.changes/1.14.0.md @@ -1,3 +1 @@ ## dbt-adapters 1.14.0 - February 07, 2025 - - diff --git a/dbt-adapters/CHANGELOG.md b/dbt-adapters/CHANGELOG.md index 9c858cfc5..e93f1dcab 100644 --- a/dbt-adapters/CHANGELOG.md +++ b/dbt-adapters/CHANGELOG.md @@ -281,4 +281,3 @@ and is generated by [Changie](https://github.com/miniscruff/changie) ### Security * Pin `black>=24.3` in `pyproject.toml` ([#140](https://github.com/dbt-labs/dbt-adapters/issues/140)) - diff --git a/dbt-bigquery/.changes/unreleased/Features-20250209-123750.yaml b/dbt-bigquery/.changes/unreleased/Features-20250209-123750.yaml new file mode 100644 index 000000000..c2f39e34c --- /dev/null +++ b/dbt-bigquery/.changes/unreleased/Features-20250209-123750.yaml @@ -0,0 +1,6 @@ +kind: Features +body: Make expiration of temp table dynamic, based on model timeout. +time: 2025-02-09T12:37:50.314467072+01:00 +custom: + Author: gjskibinski + Issue: "163" diff --git a/dbt-bigquery/src/dbt/adapters/bigquery/impl.py b/dbt-bigquery/src/dbt/adapters/bigquery/impl.py index 83ac7b725..d6b38f0cb 100644 --- a/dbt-bigquery/src/dbt/adapters/bigquery/impl.py +++ b/dbt-bigquery/src/dbt/adapters/bigquery/impl.py @@ -144,7 +144,9 @@ class BigQueryAdapter(BaseAdapter): def __init__(self, config, mp_context: SpawnContext) -> None: super().__init__(config, mp_context) self.connections: BigQueryConnectionManager = self.connections - self.job_execution_timeout_seconds = self.config.credentials.job_execution_timeout_seconds or 60 * 60 * 12 + self.job_execution_timeout_seconds = ( + self.config.credentials.job_execution_timeout_seconds or 60 * 60 * 12 + ) ### # Implementations of abstract methods @@ -775,7 +777,9 @@ def get_table_options( opts["kms_key_name"] = f"'{config.get('kms_key_name')}'" if temporary: - opts["expiration_timestamp"] = f"TIMESTAMP_ADD(CURRENT_TIMESTAMP(), INTERVAL {self.job_execution_timeout_seconds} second)" + opts["expiration_timestamp"] = ( + f"TIMESTAMP_ADD(CURRENT_TIMESTAMP(), INTERVAL {self.job_execution_timeout_seconds} second)" + ) else: # It doesn't apply the `require_partition_filter` option for a temporary table # so that we avoid the error by not specifying a partition with a temporary table diff --git a/dbt-tests-adapter/.changes/1.11.0.md b/dbt-tests-adapter/.changes/1.11.0.md index ca1ee7e9e..ca28b7453 100644 --- a/dbt-tests-adapter/.changes/1.11.0.md +++ b/dbt-tests-adapter/.changes/1.11.0.md @@ -1,3 +1 @@ ## dbt-tests-adapter 1.11.0 - February 07, 2025 - - diff --git a/dbt-tests-adapter/CHANGELOG.md b/dbt-tests-adapter/CHANGELOG.md index 89935545c..cdc09e32a 100644 --- a/dbt-tests-adapter/CHANGELOG.md +++ b/dbt-tests-adapter/CHANGELOG.md @@ -11,6 +11,3 @@ and is generated by [Changie](https://github.com/miniscruff/changie) - For details on how to document a change, see the [contributing guide](/CONTRIBUTING.md#changelog-entry) ## dbt-tests-adapter 1.11.0 - February 07, 2025 - - - From 09d2963e8e8e62787e5263c94851c4de88bb1945 Mon Sep 17 00:00:00 2001 From: gjskibinski <54750503+gjskibinski@users.noreply.github.com> Date: Sun, 9 Feb 2025 14:32:45 +0100 Subject: [PATCH 3/6] Update Features-20250209-123750.yaml --- dbt-bigquery/.changes/unreleased/Features-20250209-123750.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dbt-bigquery/.changes/unreleased/Features-20250209-123750.yaml b/dbt-bigquery/.changes/unreleased/Features-20250209-123750.yaml index c2f39e34c..386857699 100644 --- a/dbt-bigquery/.changes/unreleased/Features-20250209-123750.yaml +++ b/dbt-bigquery/.changes/unreleased/Features-20250209-123750.yaml @@ -3,4 +3,4 @@ body: Make expiration of temp table dynamic, based on model timeout. time: 2025-02-09T12:37:50.314467072+01:00 custom: Author: gjskibinski - Issue: "163" + Issue: "805" From cf982a01d0d8d67849fd6bde708571e41cabf95a Mon Sep 17 00:00:00 2001 From: Georgie Date: Sun, 9 Feb 2025 14:38:06 +0100 Subject: [PATCH 4/6] fix athena changelog... --- dbt-athena/.changes/1.9.1.md | 2 -- dbt-athena/CHANGELOG.md | 1 - 2 files changed, 3 deletions(-) diff --git a/dbt-athena/.changes/1.9.1.md b/dbt-athena/.changes/1.9.1.md index 75f0d91c8..71a385742 100644 --- a/dbt-athena/.changes/1.9.1.md +++ b/dbt-athena/.changes/1.9.1.md @@ -1,3 +1 @@ ## dbt-athena 1.9.1 - February 07, 2025 - - diff --git a/dbt-athena/CHANGELOG.md b/dbt-athena/CHANGELOG.md index dbe6d2a71..b9fb4d3ef 100644 --- a/dbt-athena/CHANGELOG.md +++ b/dbt-athena/CHANGELOG.md @@ -19,4 +19,3 @@ and is generated by [Changie](https://github.com/miniscruff/changie) For information on prior major and minor releases, see their changelogs: - [1.8](https://github.com/dbt-labs/dbt-athena/blob/main/CHANGELOG.md) - From dfd2d9ab196758ab286c86b631757d2f22ad0a35 Mon Sep 17 00:00:00 2001 From: gjskibinski <54750503+gjskibinski@users.noreply.github.com> Date: Sun, 9 Feb 2025 15:49:28 +0100 Subject: [PATCH 5/6] Update test_bigquery_adapter.py --- dbt-bigquery/tests/unit/test_bigquery_adapter.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/dbt-bigquery/tests/unit/test_bigquery_adapter.py b/dbt-bigquery/tests/unit/test_bigquery_adapter.py index e57db9a62..3bc74e934 100644 --- a/dbt-bigquery/tests/unit/test_bigquery_adapter.py +++ b/dbt-bigquery/tests/unit/test_bigquery_adapter.py @@ -775,14 +775,14 @@ def test_hours_to_expiration(self): actual = adapter.get_table_options(mock_config, node={}, temporary=False) self.assertEqual(expected, actual) - def test_hours_to_expiration_temporary(self): + def test_seconds_to_expiration_temporary(self): adapter = self.get_adapter("oauth") mock_config = create_autospec(RuntimeConfigObject) config = {"hours_to_expiration": 4} mock_config.get.side_effect = lambda name: config.get(name) expected = { - "expiration_timestamp": ("TIMESTAMP_ADD(CURRENT_TIMESTAMP(), INTERVAL 12 hour)"), + "expiration_timestamp": ("TIMESTAMP_ADD(CURRENT_TIMESTAMP(), INTERVAL 43200 second)"), } actual = adapter.get_table_options(mock_config, node={}, temporary=True) self.assertEqual(expected, actual) From d4f8c8a5b2d2d01803e6086de8ff320cbfff75d4 Mon Sep 17 00:00:00 2001 From: gjskibinski <54750503+gjskibinski@users.noreply.github.com> Date: Mon, 10 Feb 2025 09:55:32 +0100 Subject: [PATCH 6/6] Update impl.py --- dbt-bigquery/src/dbt/adapters/bigquery/impl.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dbt-bigquery/src/dbt/adapters/bigquery/impl.py b/dbt-bigquery/src/dbt/adapters/bigquery/impl.py index d6b38f0cb..073faa5a4 100644 --- a/dbt-bigquery/src/dbt/adapters/bigquery/impl.py +++ b/dbt-bigquery/src/dbt/adapters/bigquery/impl.py @@ -144,7 +144,7 @@ class BigQueryAdapter(BaseAdapter): def __init__(self, config, mp_context: SpawnContext) -> None: super().__init__(config, mp_context) self.connections: BigQueryConnectionManager = self.connections - self.job_execution_timeout_seconds = ( + self.job_execution_timeout_seconds: int = int( self.config.credentials.job_execution_timeout_seconds or 60 * 60 * 12 )