From b4b2e42d0ffdf7318bfff8ecd5fc4027361c9c90 Mon Sep 17 00:00:00 2001 From: Gerda Shank Date: Mon, 10 Feb 2025 12:33:13 -0500 Subject: [PATCH 1/5] Add invocation_started_at --- .changes/unreleased/Breaking Changes-20250210-123306.yaml | 6 ++++++ core/dbt/artifacts/schemas/base.py | 3 ++- dev-requirements.txt | 2 +- 3 files changed, 9 insertions(+), 2 deletions(-) create mode 100644 .changes/unreleased/Breaking Changes-20250210-123306.yaml diff --git a/.changes/unreleased/Breaking Changes-20250210-123306.yaml b/.changes/unreleased/Breaking Changes-20250210-123306.yaml new file mode 100644 index 00000000000..99c1c314e85 --- /dev/null +++ b/.changes/unreleased/Breaking Changes-20250210-123306.yaml @@ -0,0 +1,6 @@ +kind: Breaking Changes +body: Add invocations_started_at field to artifact metadata +time: 2025-02-10T12:33:06.722803-05:00 +custom: + Author: gshank + Issue: "11272" diff --git a/core/dbt/artifacts/schemas/base.py b/core/dbt/artifacts/schemas/base.py index 6cca031603e..5cf933d43de 100644 --- a/core/dbt/artifacts/schemas/base.py +++ b/core/dbt/artifacts/schemas/base.py @@ -12,7 +12,7 @@ from dbt_common.dataclass_schema import dbtClassMixin from dbt_common.events.functions import get_metadata_vars from dbt_common.exceptions import DbtInternalError, DbtRuntimeError -from dbt_common.invocation import get_invocation_id +from dbt_common.invocation import get_invocation_id, get_invocation_started_at BASE_SCHEMAS_URL = "https://schemas.getdbt.com/" SCHEMA_PATH = "dbt/{name}/v{version}.json" @@ -57,6 +57,7 @@ class BaseArtifactMetadata(dbtClassMixin): dbt_version: str = __version__ generated_at: datetime = dataclasses.field(default_factory=datetime.utcnow) invocation_id: Optional[str] = dataclasses.field(default_factory=get_invocation_id) + invocation_started_at: datetime = dataclasses.field(default_factory=get_invocation_started_at) env: Dict[str, str] = dataclasses.field(default_factory=get_metadata_vars) def __post_serialize__(self, dct: Dict, context: Optional[Dict] = None): diff --git a/dev-requirements.txt b/dev-requirements.txt index ea34d0d42e6..63014493011 100644 --- a/dev-requirements.txt +++ b/dev-requirements.txt @@ -1,6 +1,6 @@ git+https://github.com/dbt-labs/dbt-adapters.git@main#subdirectory=dbt-adapters git+https://github.com/dbt-labs/dbt-adapters.git@main#subdirectory=dbt-tests-adapter -git+https://github.com/dbt-labs/dbt-common.git@main +git+https://github.com/dbt-labs/dbt-common.git@invocation_started_at git+https://github.com/dbt-labs/dbt-adapters.git@main#subdirectory=dbt-postgres # black must match what's in .pre-commit-config.yaml to be sure local env matches CI black==24.3.0 From 35f3746678fe1fa28170fdc16daa3ae21e45edb9 Mon Sep 17 00:00:00 2001 From: Gerda Shank Date: Mon, 10 Feb 2025 16:31:21 -0500 Subject: [PATCH 2/5] Update schemas --- core/dbt/artifacts/schemas/base.py | 4 +++- schemas/dbt/catalog/v1.json | 10 ++++++++++ schemas/dbt/manifest/v12.json | 28 ++++++++++++++++++++++++++++ schemas/dbt/run-results/v6.json | 10 ++++++++++ schemas/dbt/sources/v3.json | 10 ++++++++++ 5 files changed, 61 insertions(+), 1 deletion(-) diff --git a/core/dbt/artifacts/schemas/base.py b/core/dbt/artifacts/schemas/base.py index 5cf933d43de..5b4ae200769 100644 --- a/core/dbt/artifacts/schemas/base.py +++ b/core/dbt/artifacts/schemas/base.py @@ -57,7 +57,9 @@ class BaseArtifactMetadata(dbtClassMixin): dbt_version: str = __version__ generated_at: datetime = dataclasses.field(default_factory=datetime.utcnow) invocation_id: Optional[str] = dataclasses.field(default_factory=get_invocation_id) - invocation_started_at: datetime = dataclasses.field(default_factory=get_invocation_started_at) + invocation_started_at: Optional[datetime] = dataclasses.field( + default_factory=get_invocation_started_at + ) env: Dict[str, str] = dataclasses.field(default_factory=get_metadata_vars) def __post_serialize__(self, dct: Dict, context: Optional[Dict] = None): diff --git a/schemas/dbt/catalog/v1.json b/schemas/dbt/catalog/v1.json index ef54c19d714..94ea81b0043 100644 --- a/schemas/dbt/catalog/v1.json +++ b/schemas/dbt/catalog/v1.json @@ -27,6 +27,16 @@ } ] }, + "invocation_started_at": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ] + }, "env": { "type": "object", "additionalProperties": { diff --git a/schemas/dbt/manifest/v12.json b/schemas/dbt/manifest/v12.json index c19549cc0f4..c729dd4f3db 100644 --- a/schemas/dbt/manifest/v12.json +++ b/schemas/dbt/manifest/v12.json @@ -28,6 +28,16 @@ } ] }, + "invocation_started_at": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ] + }, "env": { "type": "object", "additionalProperties": { @@ -8546,6 +8556,12 @@ { "type": "string" }, + { + "type": "array", + "items": { + "type": "string" + } + }, { "type": "null" } @@ -9903,6 +9919,12 @@ { "type": "string" }, + { + "type": "array", + "items": { + "type": "string" + } + }, { "type": "null" } @@ -18184,6 +18206,12 @@ { "type": "string" }, + { + "type": "array", + "items": { + "type": "string" + } + }, { "type": "null" } diff --git a/schemas/dbt/run-results/v6.json b/schemas/dbt/run-results/v6.json index 32f0ec46748..d90afc1183a 100644 --- a/schemas/dbt/run-results/v6.json +++ b/schemas/dbt/run-results/v6.json @@ -27,6 +27,16 @@ } ] }, + "invocation_started_at": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ] + }, "env": { "type": "object", "additionalProperties": { diff --git a/schemas/dbt/sources/v3.json b/schemas/dbt/sources/v3.json index c07cb574df6..fc4345fd4a7 100644 --- a/schemas/dbt/sources/v3.json +++ b/schemas/dbt/sources/v3.json @@ -27,6 +27,16 @@ } ] }, + "invocation_started_at": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ] + }, "env": { "type": "object", "additionalProperties": { From 11c372754a09a1935697c0317ec924e903743a75 Mon Sep 17 00:00:00 2001 From: Gerda Shank Date: Mon, 10 Feb 2025 17:24:17 -0500 Subject: [PATCH 3/5] Fix unit tests --- tests/unit/contracts/graph/test_manifest.py | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/tests/unit/contracts/graph/test_manifest.py b/tests/unit/contracts/graph/test_manifest.py index 526f46c7884..314c83cfe13 100644 --- a/tests/unit/contracts/graph/test_manifest.py +++ b/tests/unit/contracts/graph/test_manifest.py @@ -382,6 +382,7 @@ def test_no_nodes(self, mock_user): ) invocation_id = dbt_common.invocation._INVOCATION_ID + invocation_started_at = dbt_common.invocation._INVOCATION_STARTED_AT mock_user.id = "cfc9500f-dc7f-4c83-9ea7-2c581c1b38cf" set_from_args(Namespace(SEND_ANONYMOUS_USAGE_STATS=False), None) self.assertEqual( @@ -403,6 +404,7 @@ def test_no_nodes(self, mock_user): "dbt_version": dbt.version.__version__, "env": {ENV_KEY_NAME: "value"}, "invocation_id": invocation_id, + "invocation_started_at": str(invocation_started_at).replace(" ", "T") + "Z", "send_anonymous_usage_stats": False, "user_id": "cfc9500f-dc7f-4c83-9ea7-2c581c1b38cf", }, @@ -530,11 +532,13 @@ def test_build_flat_graph(self): def test_no_nodes_with_metadata(self, mock_user): mock_user.id = "cfc9500f-dc7f-4c83-9ea7-2c581c1b38cf" dbt_common.invocation._INVOCATION_ID = "01234567-0123-0123-0123-0123456789ab" + dbt_common.invocation._INVOCATION_STARTED_AT = datetime.utcnow() set_from_args(Namespace(SEND_ANONYMOUS_USAGE_STATS=False), None) metadata = ManifestMetadata( project_id="098f6bcd4621d373cade4e832627b4f6", adapter_type="postgres", generated_at=datetime.utcnow(), + invocation_started_at=dbt_common.invocation._INVOCATION_STARTED_AT, user_id="cfc9500f-dc7f-4c83-9ea7-2c581c1b38cf", send_anonymous_usage_stats=False, ) @@ -552,6 +556,9 @@ def test_no_nodes_with_metadata(self, mock_user): saved_queries={}, ) + invocation_started_at = ( + str(dbt_common.invocation._INVOCATION_STARTED_AT).replace(" ", "T") + "Z" + ) self.assertEqual( manifest.writable_manifest().to_dict(omit_none=True), { @@ -575,6 +582,7 @@ def test_no_nodes_with_metadata(self, mock_user): "send_anonymous_usage_stats": False, "adapter_type": "postgres", "invocation_id": "01234567-0123-0123-0123-0123456789ab", + "invocation_started_at": invocation_started_at, "env": {ENV_KEY_NAME: "value"}, }, "disabled": {}, @@ -883,8 +891,11 @@ def tearDown(self): def test_no_nodes(self, mock_user): mock_user.id = "cfc9500f-dc7f-4c83-9ea7-2c581c1b38cf" set_from_args(Namespace(SEND_ANONYMOUS_USAGE_STATS=False), None) + invocation_started_at = datetime.utcnow() metadata = ManifestMetadata( - generated_at=datetime.utcnow(), invocation_id="01234567-0123-0123-0123-0123456789ab" + generated_at=datetime.utcnow(), + invocation_id="01234567-0123-0123-0123-0123456789ab", + invocation_started_at=invocation_started_at, ) manifest = Manifest( nodes={}, @@ -917,6 +928,7 @@ def test_no_nodes(self, mock_user): "dbt_schema_version": "https://schemas.getdbt.com/dbt/manifest/v12.json", "dbt_version": dbt.version.__version__, "invocation_id": "01234567-0123-0123-0123-0123456789ab", + "invocation_started_at": str(invocation_started_at).replace(" ", "T") + "Z", "env": {ENV_KEY_NAME: "value"}, "send_anonymous_usage_stats": False, "user_id": "cfc9500f-dc7f-4c83-9ea7-2c581c1b38cf", From 61790ebe8faa7f487bea1e0c4fa2a68481afd817 Mon Sep 17 00:00:00 2001 From: Gerda Shank Date: Thu, 13 Feb 2025 16:24:04 -0500 Subject: [PATCH 4/5] Use get_invocation_started_at in microbatch builder --- core/dbt/config/runtime.py | 6 +----- core/dbt/task/run.py | 3 ++- 2 files changed, 3 insertions(+), 6 deletions(-) diff --git a/core/dbt/config/runtime.py b/core/dbt/config/runtime.py index d2c0183e1cb..e1c24cf5f0c 100644 --- a/core/dbt/config/runtime.py +++ b/core/dbt/config/runtime.py @@ -1,8 +1,7 @@ import itertools import os from copy import deepcopy -from dataclasses import dataclass, field -from datetime import datetime +from dataclasses import dataclass from pathlib import Path from typing import ( Any, @@ -16,8 +15,6 @@ Type, ) -import pytz - from dbt import tracking from dbt.adapters.contracts.connection import ( AdapterRequiredConfig, @@ -101,7 +98,6 @@ class RuntimeConfig(Project, Profile, AdapterRequiredConfig): profile_name: str cli_vars: Dict[str, Any] dependencies: Optional[Mapping[str, "RuntimeConfig"]] = None - invoked_at: datetime = field(default_factory=lambda: datetime.now(pytz.UTC)) def __post_init__(self): self.validate() diff --git a/core/dbt/task/run.py b/core/dbt/task/run.py index ac9fe761df6..0f5f65694d9 100644 --- a/core/dbt/task/run.py +++ b/core/dbt/task/run.py @@ -55,6 +55,7 @@ from dbt_common.events.functions import fire_event, get_invocation_id from dbt_common.events.types import Formatting from dbt_common.exceptions import DbtValidationError +from dbt_common.invocation import get_invocation_started_at @functools.total_ordering @@ -574,7 +575,7 @@ def _execute_microbatch_materialization( is_incremental=self._is_incremental(model), event_time_start=event_time_start, event_time_end=event_time_end, - default_end_time=self.config.invoked_at, + default_end_time=get_invocation_started_at(), ) if self.batch_idx is None: From a5825e1d081311171a536ef90ab5474344f67a7b Mon Sep 17 00:00:00 2001 From: Gerda Shank Date: Fri, 14 Feb 2025 10:57:25 -0500 Subject: [PATCH 5/5] Bump dbt-common version, remove branch from dev-requirements.txt --- core/setup.py | 2 +- dev-requirements.txt | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/core/setup.py b/core/setup.py index 7a3df25ec61..e357aa30097 100644 --- a/core/setup.py +++ b/core/setup.py @@ -71,7 +71,7 @@ "dbt-extractor>=0.5.0,<=0.6", "dbt-semantic-interfaces>=0.8.3,<0.9", # Minor versions for these are expected to be backwards-compatible - "dbt-common>=1.13.0,<2.0", + "dbt-common>=1.15.0,<2.0", "dbt-adapters>=1.13.0,<2.0", # ---- # Expect compatibility with all new versions of these packages, so lower bounds only. diff --git a/dev-requirements.txt b/dev-requirements.txt index 63014493011..ea34d0d42e6 100644 --- a/dev-requirements.txt +++ b/dev-requirements.txt @@ -1,6 +1,6 @@ git+https://github.com/dbt-labs/dbt-adapters.git@main#subdirectory=dbt-adapters git+https://github.com/dbt-labs/dbt-adapters.git@main#subdirectory=dbt-tests-adapter -git+https://github.com/dbt-labs/dbt-common.git@invocation_started_at +git+https://github.com/dbt-labs/dbt-common.git@main git+https://github.com/dbt-labs/dbt-adapters.git@main#subdirectory=dbt-postgres # black must match what's in .pre-commit-config.yaml to be sure local env matches CI black==24.3.0