From c18defab05a975c483716454225ca32eb69e07b4 Mon Sep 17 00:00:00 2001 From: Pulkit Gaur Date: Fri, 15 Nov 2024 17:27:06 +0530 Subject: [PATCH 01/13] project governance llm checks --- src/datapilot/clients/altimate/client.py | 13 ++++++++++ src/datapilot/clients/altimate/utils.py | 21 ++++++++++++++++ src/datapilot/core/platforms/dbt/cli/cli.py | 27 ++++++++++++++++++++- 3 files changed, 60 insertions(+), 1 deletion(-) diff --git a/src/datapilot/clients/altimate/client.py b/src/datapilot/clients/altimate/client.py index 497e5ed..118d0e5 100644 --- a/src/datapilot/clients/altimate/client.py +++ b/src/datapilot/clients/altimate/client.py @@ -91,3 +91,16 @@ def validate_upload_to_integration(self): def start_dbt_ingestion(self, params=None): endpoint = "/dbt/v1/start_dbt_ingestion" return self.post(endpoint, data=params) + + def get_project_governance_llm_checks(self, params=None): + endpoint = "/project_governance/checks" + return self.get(endpoint, params=params) + + def run_project_governance_llm_checks(self, manifest, catalog, check_names): + endpoint = "/project_governance/run_checks" + data = { + "manifest": manifest, + "catalog": catalog, + "check_names": check_names, + } + return self.post(endpoint, data=data) diff --git a/src/datapilot/clients/altimate/utils.py b/src/datapilot/clients/altimate/utils.py index 14baf1b..cc4b4fa 100644 --- a/src/datapilot/clients/altimate/utils.py +++ b/src/datapilot/clients/altimate/utils.py @@ -103,3 +103,24 @@ def start_dbt_ingestion(api_token, tenant, dbt_core_integration_id, dbt_core_int "ok": False, "message": "Error starting dbt ingestion worker. ", } + + +def get_project_governance_llm_checks( + api_token, + tenant, + backend_url, +): + api_client = APIClient(api_token=api_token, base_url=backend_url, tenant=tenant) + return api_client.get_project_governance_llm_checks() + + +def run_project_governance_llm_checks( + api_token, + tenant, + backend_url, + manifest, + catalog, + check_names, +): + api_client = APIClient(api_token=api_token, base_url=backend_url, tenant=tenant) + return api_client.run_project_governance_llm_checks(manifest, catalog, check_names) diff --git a/src/datapilot/core/platforms/dbt/cli/cli.py b/src/datapilot/core/platforms/dbt/cli/cli.py index 0cf3f1a..7921d01 100644 --- a/src/datapilot/core/platforms/dbt/cli/cli.py +++ b/src/datapilot/core/platforms/dbt/cli/cli.py @@ -3,7 +3,9 @@ import click from datapilot.clients.altimate.utils import check_token_and_instance +from datapilot.clients.altimate.utils import get_project_governance_llm_checks from datapilot.clients.altimate.utils import onboard_file +from datapilot.clients.altimate.utils import run_project_governance_llm_checks from datapilot.clients.altimate.utils import start_dbt_ingestion from datapilot.clients.altimate.utils import validate_credentials from datapilot.clients.altimate.utils import validate_permissions @@ -28,6 +30,8 @@ def dbt(): @dbt.command("project-health") +@click.option("--token", prompt="API Token", help="Your API token for authentication.") +@click.option("--instance-name", prompt="Instance Name", help="Your tenant ID.") @click.option( "--manifest-path", required=True, @@ -49,7 +53,10 @@ def dbt(): default=None, help="Selective model testing. Specify one or more models to run tests on.", ) -def project_health(manifest_path, catalog_path, config_path=None, select=None): +@click.option("--backend-url", required=False, help="Altimate's Backend URL", default="https://api.myaltimate.com") +def project_health( + token, instance_name, manifest_path, catalog_path, config_path=None, select=None, backend_url="https://api.myaltimate.com" +): """ Validate the DBT project's configuration and structure. :param manifest_path: Path to the DBT manifest file. @@ -62,6 +69,11 @@ def project_health(manifest_path, catalog_path, config_path=None, select=None): selected_models = select.split(" ") manifest = load_manifest(manifest_path) catalog = load_catalog(catalog_path) if catalog_path else None + + llm_checks = get_project_governance_llm_checks(token, instance_name, backend_url) + check_names = [check["name"] for check in llm_checks if check["alias"] not in config.get("disabled_insights", [])] + llm_check_results = run_project_governance_llm_checks(token, instance_name, backend_url, manifest, catalog, check_names) + insight_generator = DBTInsightGenerator(manifest=manifest, catalog=catalog, config=config, selected_models=selected_models) reports = insight_generator.run() @@ -85,6 +97,19 @@ def project_health(manifest_path, catalog_path, config_path=None, select=None): click.echo("--" * 50) click.echo(tabulate_data(project_report, headers="keys")) + if llm_check_results: + click.echo("--" * 50) + click.echo("Project Governance LLM Insights") + click.echo("--" * 50) + for check in llm_check_results["results"]: + click.echo(f"Check: {check['name']}") + for answer in check["answer"]: + click.echo(f"Rule: {answer['Rule']}") + click.echo(f"Location: {answer['Location']}") + click.echo(f"Issue: {answer['Issue']}") + click.echo(f"Fix: {answer['Fix']}") + click.echo("\n") + @dbt.command("onboard") @click.option("--token", prompt="API Token", help="Your API token for authentication.") From 54ba98347517a0b45fdc3248a224e264d9cf02c9 Mon Sep 17 00:00:00 2001 From: Pulkit Gaur Date: Mon, 18 Nov 2024 13:02:53 +0530 Subject: [PATCH 02/13] changes --- src/datapilot/core/platforms/dbt/cli/cli.py | 26 +++++++++++-------- src/datapilot/core/platforms/dbt/constants.py | 2 ++ src/datapilot/core/platforms/dbt/executor.py | 23 ++++++++++++++++ 3 files changed, 40 insertions(+), 11 deletions(-) diff --git a/src/datapilot/core/platforms/dbt/cli/cli.py b/src/datapilot/core/platforms/dbt/cli/cli.py index 7921d01..3eceb7a 100644 --- a/src/datapilot/core/platforms/dbt/cli/cli.py +++ b/src/datapilot/core/platforms/dbt/cli/cli.py @@ -3,13 +3,12 @@ import click from datapilot.clients.altimate.utils import check_token_and_instance -from datapilot.clients.altimate.utils import get_project_governance_llm_checks from datapilot.clients.altimate.utils import onboard_file -from datapilot.clients.altimate.utils import run_project_governance_llm_checks from datapilot.clients.altimate.utils import start_dbt_ingestion from datapilot.clients.altimate.utils import validate_credentials from datapilot.clients.altimate.utils import validate_permissions from datapilot.config.config import load_config +from datapilot.core.platforms.dbt.constants import LLM from datapilot.core.platforms.dbt.constants import MODEL from datapilot.core.platforms.dbt.constants import PROJECT from datapilot.core.platforms.dbt.executor import DBTInsightGenerator @@ -30,8 +29,8 @@ def dbt(): @dbt.command("project-health") -@click.option("--token", prompt="API Token", help="Your API token for authentication.") -@click.option("--instance-name", prompt="Instance Name", help="Your tenant ID.") +@click.option("--token", required=False, prompt="API Token", help="Your API token for authentication.") +@click.option("--instance-name", required=False, prompt="Instance Name", help="Your tenant ID.") @click.option( "--manifest-path", required=True, @@ -70,15 +69,20 @@ def project_health( manifest = load_manifest(manifest_path) catalog = load_catalog(catalog_path) if catalog_path else None - llm_checks = get_project_governance_llm_checks(token, instance_name, backend_url) - check_names = [check["name"] for check in llm_checks if check["alias"] not in config.get("disabled_insights", [])] - llm_check_results = run_project_governance_llm_checks(token, instance_name, backend_url, manifest, catalog, check_names) - - insight_generator = DBTInsightGenerator(manifest=manifest, catalog=catalog, config=config, selected_models=selected_models) + insight_generator = DBTInsightGenerator( + manifest=manifest, + catalog=catalog, + config=config, + selected_models=selected_models, + token=token, + instance_name=instance_name, + backend_url=backend_url, + ) reports = insight_generator.run() package_insights = reports[PROJECT] model_insights = reports[MODEL] + llm_insights = reports[LLM] model_report = generate_model_insights_table(model_insights) if len(model_report) > 0: click.echo("--" * 50) @@ -97,11 +101,11 @@ def project_health( click.echo("--" * 50) click.echo(tabulate_data(project_report, headers="keys")) - if llm_check_results: + if len(llm_insights): click.echo("--" * 50) click.echo("Project Governance LLM Insights") click.echo("--" * 50) - for check in llm_check_results["results"]: + for check in llm_insights: click.echo(f"Check: {check['name']}") for answer in check["answer"]: click.echo(f"Rule: {answer['Rule']}") diff --git a/src/datapilot/core/platforms/dbt/constants.py b/src/datapilot/core/platforms/dbt/constants.py index 5357b66..cbf3560 100644 --- a/src/datapilot/core/platforms/dbt/constants.py +++ b/src/datapilot/core/platforms/dbt/constants.py @@ -4,6 +4,8 @@ MODEL = "model" SOURCE = "source" +LLM = "llm" + PROJECT = "project" SQL = "sql" diff --git a/src/datapilot/core/platforms/dbt/executor.py b/src/datapilot/core/platforms/dbt/executor.py index b698962..410bbbe 100644 --- a/src/datapilot/core/platforms/dbt/executor.py +++ b/src/datapilot/core/platforms/dbt/executor.py @@ -5,6 +5,9 @@ from typing import List from typing import Optional +from datapilot.clients.altimate.utils import get_project_governance_llm_checks +from datapilot.clients.altimate.utils import run_project_governance_llm_checks +from datapilot.core.platforms.dbt.constants import LLM from datapilot.core.platforms.dbt.constants import MODEL from datapilot.core.platforms.dbt.constants import PROJECT from datapilot.core.platforms.dbt.exceptions import AltimateCLIArgumentError @@ -29,11 +32,17 @@ def __init__( target: str = "dev", selected_models: Optional[str] = None, selected_model_ids: Optional[List[str]] = None, + token: Optional[str] = None, + instance_name: Optional[str] = None, + backend_url: Optional[str] = None, ): self.run_results_path = run_results_path self.target = target self.env = env self.config = config or {} + self.token = token + self.instance_name = instance_name + self.backend_url = backend_url self.manifest_wrapper = DBTFactory.get_manifest_wrapper(manifest) self.manifest_present = True @@ -85,10 +94,19 @@ def _check_if_skipped(self, insight): return True return False + def run_llm_checks(self): + llm_checks = get_project_governance_llm_checks(self.token, self.instance_name, self.backend_url) + check_names = [check["name"] for check in llm_checks if check["alias"] not in self.config.get("disabled_insights", [])] + llm_check_results = run_project_governance_llm_checks( + self.token, self.instance_name, self.backend_url, self.manifest, self.catalog, check_names + ) + return llm_check_results + def run(self): reports = { MODEL: {}, PROJECT: [], + LLM: [], } for insight_class in INSIGHTS: # TODO: Skip insight based on config @@ -154,4 +172,9 @@ def run(self): else: self.logger.info(color_text(f"Skipping insight {insight_class.NAME} as {message}", YELLOW)) + if self.token and self.instance_name and self.backend_url: + llm_check_results = self.run_llm_checks() + if llm_check_results: + reports[LLM].extend(llm_check_results["results"]) + return reports From 26380e3a546786ce2076e3b8dea5d1832c3a6887 Mon Sep 17 00:00:00 2001 From: Pulkit Gaur Date: Mon, 18 Nov 2024 18:24:37 +0530 Subject: [PATCH 03/13] changes --- src/datapilot/clients/altimate/client.py | 2 +- src/datapilot/core/platforms/dbt/executor.py | 4 +++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/src/datapilot/clients/altimate/client.py b/src/datapilot/clients/altimate/client.py index 118d0e5..8abd088 100644 --- a/src/datapilot/clients/altimate/client.py +++ b/src/datapilot/clients/altimate/client.py @@ -97,7 +97,7 @@ def get_project_governance_llm_checks(self, params=None): return self.get(endpoint, params=params) def run_project_governance_llm_checks(self, manifest, catalog, check_names): - endpoint = "/project_governance/run_checks" + endpoint = "/project_governance/check/run" data = { "manifest": manifest, "catalog": catalog, diff --git a/src/datapilot/core/platforms/dbt/executor.py b/src/datapilot/core/platforms/dbt/executor.py index 410bbbe..cc4b3a4 100644 --- a/src/datapilot/core/platforms/dbt/executor.py +++ b/src/datapilot/core/platforms/dbt/executor.py @@ -43,6 +43,8 @@ def __init__( self.token = token self.instance_name = instance_name self.backend_url = backend_url + self.manifest = manifest + self.catalog = catalog self.manifest_wrapper = DBTFactory.get_manifest_wrapper(manifest) self.manifest_present = True @@ -98,7 +100,7 @@ def run_llm_checks(self): llm_checks = get_project_governance_llm_checks(self.token, self.instance_name, self.backend_url) check_names = [check["name"] for check in llm_checks if check["alias"] not in self.config.get("disabled_insights", [])] llm_check_results = run_project_governance_llm_checks( - self.token, self.instance_name, self.backend_url, self.manifest, self.catalog, check_names + self.token, self.instance_name, self.backend_url, self.manifest.json(), self.catalog.json(), check_names ) return llm_check_results From ae7423c1548aa672137bce4a5149c9bdf05939cf Mon Sep 17 00:00:00 2001 From: rakendd Date: Wed, 20 Nov 2024 00:34:07 +0530 Subject: [PATCH 04/13] updated fields --- src/datapilot/core/platforms/dbt/cli/cli.py | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/src/datapilot/core/platforms/dbt/cli/cli.py b/src/datapilot/core/platforms/dbt/cli/cli.py index 3eceb7a..133331a 100644 --- a/src/datapilot/core/platforms/dbt/cli/cli.py +++ b/src/datapilot/core/platforms/dbt/cli/cli.py @@ -108,10 +108,9 @@ def project_health( for check in llm_insights: click.echo(f"Check: {check['name']}") for answer in check["answer"]: - click.echo(f"Rule: {answer['Rule']}") - click.echo(f"Location: {answer['Location']}") - click.echo(f"Issue: {answer['Issue']}") - click.echo(f"Fix: {answer['Fix']}") + click.echo(f"Path: {answer['path']}") + click.echo(f"Reason to Flag: {answer['reason_to_flag']}") + click.echo(f"Recommendation: {answer['recommendation']}") click.echo("\n") From bf82ad61c33e0cac57587b5665541b922e8b290b Mon Sep 17 00:00:00 2001 From: Pulkit Gaur Date: Wed, 20 Nov 2024 14:12:43 +0530 Subject: [PATCH 05/13] add llm insights to datapilot --- src/datapilot/core/platforms/dbt/cli/cli.py | 14 -------- src/datapilot/core/platforms/dbt/executor.py | 38 ++++++++++++++++++-- 2 files changed, 35 insertions(+), 17 deletions(-) diff --git a/src/datapilot/core/platforms/dbt/cli/cli.py b/src/datapilot/core/platforms/dbt/cli/cli.py index 133331a..4fb866e 100644 --- a/src/datapilot/core/platforms/dbt/cli/cli.py +++ b/src/datapilot/core/platforms/dbt/cli/cli.py @@ -8,7 +8,6 @@ from datapilot.clients.altimate.utils import validate_credentials from datapilot.clients.altimate.utils import validate_permissions from datapilot.config.config import load_config -from datapilot.core.platforms.dbt.constants import LLM from datapilot.core.platforms.dbt.constants import MODEL from datapilot.core.platforms.dbt.constants import PROJECT from datapilot.core.platforms.dbt.executor import DBTInsightGenerator @@ -82,7 +81,6 @@ def project_health( package_insights = reports[PROJECT] model_insights = reports[MODEL] - llm_insights = reports[LLM] model_report = generate_model_insights_table(model_insights) if len(model_report) > 0: click.echo("--" * 50) @@ -101,18 +99,6 @@ def project_health( click.echo("--" * 50) click.echo(tabulate_data(project_report, headers="keys")) - if len(llm_insights): - click.echo("--" * 50) - click.echo("Project Governance LLM Insights") - click.echo("--" * 50) - for check in llm_insights: - click.echo(f"Check: {check['name']}") - for answer in check["answer"]: - click.echo(f"Path: {answer['path']}") - click.echo(f"Reason to Flag: {answer['reason_to_flag']}") - click.echo(f"Recommendation: {answer['recommendation']}") - click.echo("\n") - @dbt.command("onboard") @click.option("--token", prompt="API Token", help="Your API token for authentication.") diff --git a/src/datapilot/core/platforms/dbt/executor.py b/src/datapilot/core/platforms/dbt/executor.py index cc4b3a4..46dd2e5 100644 --- a/src/datapilot/core/platforms/dbt/executor.py +++ b/src/datapilot/core/platforms/dbt/executor.py @@ -13,6 +13,8 @@ from datapilot.core.platforms.dbt.exceptions import AltimateCLIArgumentError from datapilot.core.platforms.dbt.factory import DBTFactory from datapilot.core.platforms.dbt.insights import INSIGHTS +from datapilot.core.platforms.dbt.insights.schema import DBTInsightResult +from datapilot.core.platforms.dbt.insights.schema import DBTModelInsightResponse from datapilot.core.platforms.dbt.schemas.manifest import Catalog from datapilot.core.platforms.dbt.schemas.manifest import Manifest from datapilot.core.platforms.dbt.utils import get_models @@ -108,7 +110,6 @@ def run(self): reports = { MODEL: {}, PROJECT: [], - LLM: [], } for insight_class in INSIGHTS: # TODO: Skip insight based on config @@ -176,7 +177,38 @@ def run(self): if self.token and self.instance_name and self.backend_url: llm_check_results = self.run_llm_checks() - if llm_check_results: - reports[LLM].extend(llm_check_results["results"]) + llm_reports = llm_check_results["results"] + llm_insights = {} + for report in llm_reports: + for answer in report["answer"]: + location = answer["unique_id"] + if location not in llm_insights: + llm_insights[location] = [] + metadata = answer.get("metadata", {}) + metadata["source"] = LLM + metadata["llm_id"] = report["id"] + metadata["catagory"] = report["type"] + llm_insights[location].append( + DBTModelInsightResponse( + insight=DBTInsightResult( + type="Custom", + name=report["name"], + message=answer["message"], + reason_to_flag=answer["reason_to_flag"], + recommendation=answer["recommendation"], + metadata=metadata, + ), + severity=answer["severity"], + path=answer["path"], + original_file_path=answer["original_file_path"], + package_name=answer["package_name"], + unique_id=answer["unique_id"], + ) + ) + for key, value in llm_insights.items(): + if key in reports[MODEL]: + reports[MODEL][key].extend(value) + else: + reports[MODEL][key] = value return reports From 94d55634552d2e3ce1354b0aae81e2ad47bc2a0d Mon Sep 17 00:00:00 2001 From: Pulkit Gaur Date: Wed, 20 Nov 2024 14:42:58 +0530 Subject: [PATCH 06/13] category --- src/datapilot/core/platforms/dbt/executor.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/datapilot/core/platforms/dbt/executor.py b/src/datapilot/core/platforms/dbt/executor.py index 46dd2e5..1ffd982 100644 --- a/src/datapilot/core/platforms/dbt/executor.py +++ b/src/datapilot/core/platforms/dbt/executor.py @@ -187,7 +187,7 @@ def run(self): metadata = answer.get("metadata", {}) metadata["source"] = LLM metadata["llm_id"] = report["id"] - metadata["catagory"] = report["type"] + metadata["category"] = report["type"] llm_insights[location].append( DBTModelInsightResponse( insight=DBTInsightResult( From 71dce56d4c1bdf3119db541616113731fb198e9a Mon Sep 17 00:00:00 2001 From: Pulkit Gaur Date: Wed, 20 Nov 2024 14:47:23 +0530 Subject: [PATCH 07/13] teammate_check_id --- src/datapilot/core/platforms/dbt/executor.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/datapilot/core/platforms/dbt/executor.py b/src/datapilot/core/platforms/dbt/executor.py index 1ffd982..a11558f 100644 --- a/src/datapilot/core/platforms/dbt/executor.py +++ b/src/datapilot/core/platforms/dbt/executor.py @@ -186,7 +186,7 @@ def run(self): llm_insights[location] = [] metadata = answer.get("metadata", {}) metadata["source"] = LLM - metadata["llm_id"] = report["id"] + metadata["teammate_check_id"] = report["id"] metadata["category"] = report["type"] llm_insights[location].append( DBTModelInsightResponse( From c636504c543e811237e4ec988905803d38ef5319 Mon Sep 17 00:00:00 2001 From: Pulkit Gaur Date: Wed, 20 Nov 2024 15:45:09 +0530 Subject: [PATCH 08/13] no check condition --- src/datapilot/core/platforms/dbt/executor.py | 22 ++++++++++++++------ 1 file changed, 16 insertions(+), 6 deletions(-) diff --git a/src/datapilot/core/platforms/dbt/executor.py b/src/datapilot/core/platforms/dbt/executor.py index a11558f..2d766ce 100644 --- a/src/datapilot/core/platforms/dbt/executor.py +++ b/src/datapilot/core/platforms/dbt/executor.py @@ -101,8 +101,16 @@ def _check_if_skipped(self, insight): def run_llm_checks(self): llm_checks = get_project_governance_llm_checks(self.token, self.instance_name, self.backend_url) check_names = [check["name"] for check in llm_checks if check["alias"] not in self.config.get("disabled_insights", [])] + if len(check_names) == 0: + return {"results": []} + llm_check_results = run_project_governance_llm_checks( - self.token, self.instance_name, self.backend_url, self.manifest.json(), self.catalog.json(), check_names + self.token, + self.instance_name, + self.backend_url, + self.manifest.json() if self.manifest else "", + self.catalog.json() if self.catalog else "", + check_names, ) return llm_check_results @@ -205,10 +213,12 @@ def run(self): unique_id=answer["unique_id"], ) ) - for key, value in llm_insights.items(): - if key in reports[MODEL]: - reports[MODEL][key].extend(value) - else: - reports[MODEL][key] = value + + if llm_insights: + for key, value in llm_insights.items(): + if key in reports[MODEL]: + reports[MODEL][key].extend(value) + else: + reports[MODEL][key] = value return reports From 603be2590f8da56d1bec139e378d08df9dbe5ceb Mon Sep 17 00:00:00 2001 From: Pulkit Gaur Date: Wed, 20 Nov 2024 15:56:57 +0530 Subject: [PATCH 09/13] change --- src/datapilot/core/platforms/dbt/cli/cli.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/datapilot/core/platforms/dbt/cli/cli.py b/src/datapilot/core/platforms/dbt/cli/cli.py index 4fb866e..24558dc 100644 --- a/src/datapilot/core/platforms/dbt/cli/cli.py +++ b/src/datapilot/core/platforms/dbt/cli/cli.py @@ -28,8 +28,8 @@ def dbt(): @dbt.command("project-health") -@click.option("--token", required=False, prompt="API Token", help="Your API token for authentication.") -@click.option("--instance-name", required=False, prompt="Instance Name", help="Your tenant ID.") +@click.option("--token", required=False, help="Your API token for authentication.") +@click.option("--instance-name", required=False, help="Your tenant ID.") @click.option( "--manifest-path", required=True, From de31c42249e6fdbb91d45c403a891a22e4314b09 Mon Sep 17 00:00:00 2001 From: Pulkit Gaur Date: Fri, 29 Nov 2024 20:31:33 +0530 Subject: [PATCH 10/13] handle null package_name --- src/datapilot/core/platforms/dbt/executor.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/datapilot/core/platforms/dbt/executor.py b/src/datapilot/core/platforms/dbt/executor.py index 2d766ce..56841a6 100644 --- a/src/datapilot/core/platforms/dbt/executor.py +++ b/src/datapilot/core/platforms/dbt/executor.py @@ -209,7 +209,7 @@ def run(self): severity=answer["severity"], path=answer["path"], original_file_path=answer["original_file_path"], - package_name=answer["package_name"], + package_name=answer["package_name"] if answer.get("package_name") else "", unique_id=answer["unique_id"], ) ) From 0cff308b4a95c74f509cd7cacc4ee6d2fe4899e2 Mon Sep 17 00:00:00 2001 From: rakendd Date: Mon, 2 Dec 2024 08:38:09 +0530 Subject: [PATCH 11/13] handling none values --- src/datapilot/core/platforms/dbt/executor.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/datapilot/core/platforms/dbt/executor.py b/src/datapilot/core/platforms/dbt/executor.py index 56841a6..3dd9cc5 100644 --- a/src/datapilot/core/platforms/dbt/executor.py +++ b/src/datapilot/core/platforms/dbt/executor.py @@ -207,9 +207,9 @@ def run(self): metadata=metadata, ), severity=answer["severity"], - path=answer["path"], - original_file_path=answer["original_file_path"], - package_name=answer["package_name"] if answer.get("package_name") else "", + path=answer["path"] if answer.get("path") else "", + original_file_path=answer["original_file_path"] if answer.get("original_file_path") else "", + package_name=answer.get["package_name"] if answer.get("package_name") else "", unique_id=answer["unique_id"], ) ) From dfd43ea3bdb75111e4bb8666489e3e81cc0200a6 Mon Sep 17 00:00:00 2001 From: saravmajestic Date: Wed, 11 Dec 2024 16:33:39 +0530 Subject: [PATCH 12/13] fix: null condition --- src/datapilot/core/platforms/dbt/executor.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/datapilot/core/platforms/dbt/executor.py b/src/datapilot/core/platforms/dbt/executor.py index 3dd9cc5..f665b33 100644 --- a/src/datapilot/core/platforms/dbt/executor.py +++ b/src/datapilot/core/platforms/dbt/executor.py @@ -185,7 +185,7 @@ def run(self): if self.token and self.instance_name and self.backend_url: llm_check_results = self.run_llm_checks() - llm_reports = llm_check_results["results"] + llm_reports = llm_check_results.get("results", []) llm_insights = {} for report in llm_reports: for answer in report["answer"]: From a499fee798ecec698b983fffef8c75638c312961 Mon Sep 17 00:00:00 2001 From: Pulkit Gaur Date: Wed, 18 Dec 2024 09:18:25 +0530 Subject: [PATCH 13/13] fix --- src/datapilot/core/platforms/dbt/executor.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/datapilot/core/platforms/dbt/executor.py b/src/datapilot/core/platforms/dbt/executor.py index f665b33..6b98aef 100644 --- a/src/datapilot/core/platforms/dbt/executor.py +++ b/src/datapilot/core/platforms/dbt/executor.py @@ -185,7 +185,7 @@ def run(self): if self.token and self.instance_name and self.backend_url: llm_check_results = self.run_llm_checks() - llm_reports = llm_check_results.get("results", []) + llm_reports = llm_check_results.get("results", []) llm_insights = {} for report in llm_reports: for answer in report["answer"]: @@ -209,7 +209,7 @@ def run(self): severity=answer["severity"], path=answer["path"] if answer.get("path") else "", original_file_path=answer["original_file_path"] if answer.get("original_file_path") else "", - package_name=answer.get["package_name"] if answer.get("package_name") else "", + package_name=answer["package_name"] if answer.get("package_name") else "", unique_id=answer["unique_id"], ) )