From ecd160dddcbfb8102583607545f133a317742f75 Mon Sep 17 00:00:00 2001 From: Pedro Silva Date: Mon, 20 Jan 2025 13:44:11 +0000 Subject: [PATCH 1/3] fix(slack source): Add retries to pulling information from DataHub --- .../src/datahub/ingestion/source/slack/slack.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/metadata-ingestion/src/datahub/ingestion/source/slack/slack.py b/metadata-ingestion/src/datahub/ingestion/source/slack/slack.py index 00b783a89774cc..26ca525536c22e 100644 --- a/metadata-ingestion/src/datahub/ingestion/source/slack/slack.py +++ b/metadata-ingestion/src/datahub/ingestion/source/slack/slack.py @@ -2,6 +2,8 @@ import textwrap from dataclasses import dataclass from typing import Iterable, List, Optional, Tuple +from tenacity import retry, wait_exponential +from tenacity.before_sleep import before_sleep_log from pydantic import Field, SecretStr from slack_sdk import WebClient @@ -294,6 +296,10 @@ def populate_slack_id_from_email(self, user_obj: CorpUser) -> None: return raise e + @retry( + wait=wait_exponential(multiplier=2, min=4, max=60), + before_sleep=before_sleep_log(logger, logging.ERROR, True) + ) def get_user_to_be_updated(self) -> Iterable[CorpUser]: graphql_query = textwrap.dedent( """ From 1bb367e1b56bbf689714c663613c370996f021be Mon Sep 17 00:00:00 2001 From: Pedro Silva Date: Tue, 21 Jan 2025 15:02:44 +0000 Subject: [PATCH 2/3] fix lint --- .../src/datahub/ingestion/source/slack/slack.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/metadata-ingestion/src/datahub/ingestion/source/slack/slack.py b/metadata-ingestion/src/datahub/ingestion/source/slack/slack.py index 26ca525536c22e..6d147e27405818 100644 --- a/metadata-ingestion/src/datahub/ingestion/source/slack/slack.py +++ b/metadata-ingestion/src/datahub/ingestion/source/slack/slack.py @@ -2,11 +2,11 @@ import textwrap from dataclasses import dataclass from typing import Iterable, List, Optional, Tuple -from tenacity import retry, wait_exponential -from tenacity.before_sleep import before_sleep_log from pydantic import Field, SecretStr from slack_sdk import WebClient +from tenacity import retry, wait_exponential +from tenacity.before_sleep import before_sleep_log import datahub.emitter.mce_builder as builder from datahub.configuration.common import ConfigModel @@ -298,7 +298,7 @@ def populate_slack_id_from_email(self, user_obj: CorpUser) -> None: @retry( wait=wait_exponential(multiplier=2, min=4, max=60), - before_sleep=before_sleep_log(logger, logging.ERROR, True) + before_sleep=before_sleep_log(logger, logging.ERROR, True), ) def get_user_to_be_updated(self) -> Iterable[CorpUser]: graphql_query = textwrap.dedent( From 3db75509595019fd79a674bcb0de0ebfcaf5062b Mon Sep 17 00:00:00 2001 From: Pedro Silva Date: Wed, 22 Jan 2025 15:57:18 +0000 Subject: [PATCH 3/3] Add tenacity as a slack module dependency --- metadata-ingestion/setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/metadata-ingestion/setup.py b/metadata-ingestion/setup.py index d5dbb98d3cb17b..df46bffd9feb24 100644 --- a/metadata-ingestion/setup.py +++ b/metadata-ingestion/setup.py @@ -312,7 +312,7 @@ powerbi_report_server = {"requests", "requests_ntlm"} -slack = {"slack-sdk==3.18.1"} +slack = {"slack-sdk==3.18.1", "tenacity>=8.0.1",} databricks = { # 0.1.11 appears to have authentication issues with azure databricks