From 8db8d8481959463851412ed12689944c28bbe05e Mon Sep 17 00:00:00 2001 From: Ivan Trusov Date: Sat, 10 Jun 2023 21:05:06 +0200 Subject: [PATCH] =?UTF-8?q?=E2=AC=86=EF=B8=8F=20update=20security-related?= =?UTF-8?q?=20dependencies=20and=20apply=20`isort`=20(#798)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * apply isort and fix deps * update requests dep * apply formatting * fix pydantic version * fix versions * move to 11.3 LTS in the deployment file * add changelog --- CHANGELOG.md | 10 +++++++ dbx/__init__.py | 2 +- dbx/api/adjuster/adjuster.py | 2 +- dbx/api/adjuster/mixins/existing_cluster.py | 2 +- dbx/api/adjuster/mixins/instance_profile.py | 2 +- dbx/api/auth.py | 17 ++++++------ dbx/api/build.py | 2 +- dbx/api/client_provider.py | 4 +-- dbx/api/config_reader.py | 2 +- dbx/api/context.py | 2 +- dbx/api/deployment.py | 2 +- dbx/api/destroyer.py | 2 +- dbx/api/execute.py | 6 ++--- dbx/api/launch/functions.py | 2 +- dbx/api/launch/pipeline_models.py | 2 +- dbx/api/launch/runners/asset_based.py | 2 +- dbx/api/launch/runners/pipeline.py | 2 +- dbx/api/launch/runners/standard.py | 4 +-- dbx/api/launch/tracer.py | 4 +-- dbx/api/output_provider.py | 2 +- dbx/api/services/_base.py | 2 +- dbx/api/services/pipelines.py | 2 +- dbx/api/storage/io.py | 2 +- dbx/commands/configure.py | 4 +-- dbx/commands/deploy.py | 6 +---- dbx/commands/destroy.py | 6 ++--- dbx/commands/execute.py | 12 ++++----- dbx/commands/launch.py | 20 +++++--------- dbx/commands/sync/functions.py | 2 +- dbx/commands/sync/sync.py | 26 +++++++++---------- dbx/custom.py | 2 +- dbx/models/build.py | 4 +-- dbx/models/cli/destroyer.py | 4 +-- dbx/models/cli/execute.py | 4 +-- dbx/models/deployment.py | 4 +-- dbx/models/files/project.py | 2 +- dbx/models/validators.py | 2 +- dbx/models/workflow/common/access_control.py | 2 +- .../common/job_email_notifications.py | 2 +- dbx/models/workflow/common/libraries.py | 2 +- dbx/models/workflow/common/parameters.py | 3 +-- dbx/models/workflow/common/pipeline.py | 2 +- dbx/models/workflow/common/task.py | 2 +- dbx/models/workflow/v2dot0/workflow.py | 6 ++--- dbx/models/workflow/v2dot1/_parameters.py | 6 ++--- dbx/models/workflow/v2dot1/job_cluster.py | 2 +- .../workflow/v2dot1/job_task_settings.py | 4 +-- dbx/models/workflow/v2dot1/parameters.py | 6 ++--- dbx/models/workflow/v2dot1/task.py | 6 +---- dbx/models/workflow/v2dot1/workflow.py | 2 +- dbx/options.py | 2 +- dbx/sync/__init__.py | 1 + dbx/sync/event_handler.py | 1 + .../components/conf/deployment.yml | 2 +- .../render/hooks/post_gen_project.py | 2 +- .../tests/integration/e2e_test.py | 1 + .../tests/unit/conftest.py | 6 ++--- .../tests/unit/sample_test.py | 10 ++++--- .../{{cookiecutter.project_slug}}/common.py | 7 ++--- .../tasks/sample_etl_task.py | 4 +-- .../tasks/sample_ml_task.py | 15 ++++++----- dbx/utils/common.py | 2 +- dbx/utils/file_uploader.py | 2 +- dbx/utils/json.py | 2 +- setup.py | 10 +++---- tests/unit/api/adjuster/test_complex.py | 2 +- .../api/adjuster/test_existing_cluster.py | 2 +- tests/unit/api/adjuster/test_instance_pool.py | 3 ++- .../api/adjuster/test_instance_profile.py | 2 +- tests/unit/api/adjuster/test_pipeline.py | 3 ++- tests/unit/api/adjuster/test_policy.py | 2 +- .../api/adjuster/test_service_principals.py | 3 ++- tests/unit/api/launch/test_runners.py | 2 +- tests/unit/api/launch/test_tracer.py | 4 +-- tests/unit/api/storage/test_mlflow_storage.py | 2 +- tests/unit/api/test_client_provider.py | 2 +- tests/unit/api/test_destroyer.py | 4 +-- tests/unit/api/test_execution_controller.py | 2 +- tests/unit/api/test_jobs_service.py | 2 +- tests/unit/commands/test_deploy.py | 5 +--- .../test_deploy_jinja_variables_file.py | 5 +--- tests/unit/commands/test_destroy.py | 2 +- tests/unit/commands/test_execute.py | 4 +-- tests/unit/commands/test_launch.py | 2 +- tests/unit/models/test_deployment.py | 2 +- tests/unit/models/test_destroyer.py | 2 +- tests/unit/models/test_new_cluster.py | 2 +- tests/unit/models/test_parameters.py | 6 ++--- tests/unit/models/test_task.py | 2 +- tests/unit/sync/clients/conftest.py | 3 +-- tests/unit/sync/clients/test_dbfs_client.py | 4 +-- tests/unit/sync/clients/test_repos_client.py | 3 +-- .../unit/sync/sync/test_unmatched_deletes.py | 2 +- tests/unit/sync/test_commands.py | 7 ++--- tests/unit/sync/test_config.py | 2 +- tests/unit/sync/test_event_handler.py | 1 + tests/unit/sync/test_main_loop.py | 1 + tests/unit/sync/test_path_matcher.py | 1 + tests/unit/sync/test_snapshot.py | 1 + tests/unit/sync/utils.py | 1 - tests/unit/utils/test_common.py | 6 +---- 101 files changed, 191 insertions(+), 197 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 981cd682..d494bf16 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -16,6 +16,16 @@ This may lead to instability when using dbx API methods directly. ### Fixed +- Several dependency issues related to both security and library versions + +### Changed + +- Switched to 11.3 LTS in the main template + +## [0.8.15] - 2023-06-02 + +### Fixed + - Allow provisioning of `source` property for `spark_python_task` ## [0.8.14] - 2023-05-11 diff --git a/dbx/__init__.py b/dbx/__init__.py index 846679f4..8aa65d9c 100644 --- a/dbx/__init__.py +++ b/dbx/__init__.py @@ -1 +1 @@ -__version__ = "0.8.15" +__version__ = "0.8.16" diff --git a/dbx/api/adjuster/adjuster.py b/dbx/api/adjuster/adjuster.py index cfcf38c3..4ae4301c 100644 --- a/dbx/api/adjuster/adjuster.py +++ b/dbx/api/adjuster/adjuster.py @@ -1,4 +1,4 @@ -from typing import Any, Optional, Union, List +from typing import Any, List, Optional, Union from databricks_cli.sdk import ApiClient from pydantic import BaseModel diff --git a/dbx/api/adjuster/mixins/existing_cluster.py b/dbx/api/adjuster/mixins/existing_cluster.py index 58301603..474140aa 100644 --- a/dbx/api/adjuster/mixins/existing_cluster.py +++ b/dbx/api/adjuster/mixins/existing_cluster.py @@ -1,5 +1,5 @@ import functools -from typing import Optional, List, Any +from typing import Any, List, Optional from databricks_cli.sdk import ClusterService diff --git a/dbx/api/adjuster/mixins/instance_profile.py b/dbx/api/adjuster/mixins/instance_profile.py index 2c683d94..7c68744f 100644 --- a/dbx/api/adjuster/mixins/instance_profile.py +++ b/dbx/api/adjuster/mixins/instance_profile.py @@ -1,5 +1,5 @@ import functools -from typing import Optional, Any, List +from typing import Any, List, Optional from dbx.api.adjuster.mixins.base import ApiClientMixin, ElementSetterMixin from dbx.models.workflow.common.flexible import FlexibleModel diff --git a/dbx/api/auth.py b/dbx/api/auth.py index dab3fcdd..7986a3ae 100644 --- a/dbx/api/auth.py +++ b/dbx/api/auth.py @@ -3,23 +3,22 @@ from typing import List, Optional, Protocol, Union from databricks_cli.configure.provider import ( + DEFAULT_SECTION, + HOST, + INSECURE, + JOBS_API_VERSION, + PASSWORD, + REFRESH_TOKEN, + TOKEN, + USERNAME, DatabricksConfig, DatabricksConfigProvider, _fetch_from_fs, _get_option_if_exists, - HOST, - USERNAME, - PASSWORD, - TOKEN, - REFRESH_TOKEN, - INSECURE, - JOBS_API_VERSION, - DEFAULT_SECTION, ) from dbx.utils import dbx_echo - AZURE_SERVICE_PRINCIPAL_TOKEN = "azure_service_principal_token" WORKSPACE_ID = "workspace_id" ORG_ID = "org_id" diff --git a/dbx/api/build.py b/dbx/api/build.py index ccdfbef6..2ffa09b1 100644 --- a/dbx/api/build.py +++ b/dbx/api/build.py @@ -2,7 +2,7 @@ import subprocess import sys from pathlib import Path -from typing import Union, List, Optional +from typing import List, Optional, Union from dbx.utils import dbx_echo diff --git a/dbx/api/client_provider.py b/dbx/api/client_provider.py index 856377fa..0925eed3 100644 --- a/dbx/api/client_provider.py +++ b/dbx/api/client_provider.py @@ -1,9 +1,9 @@ import copy -from typing import Dict, Any, Optional +from typing import Any, Dict, Optional import requests from databricks_cli.sdk import ApiClient -from tenacity import retry, wait_exponential, stop_after_attempt +from tenacity import retry, stop_after_attempt, wait_exponential from dbx.api.auth import AuthConfigProvider diff --git a/dbx/api/config_reader.py b/dbx/api/config_reader.py index 674dc108..344740a8 100644 --- a/dbx/api/config_reader.py +++ b/dbx/api/config_reader.py @@ -2,7 +2,7 @@ import os from abc import ABC, abstractmethod from pathlib import Path -from typing import List, Optional, Dict, Any +from typing import Any, Dict, List, Optional import jinja2 import yaml diff --git a/dbx/api/context.py b/dbx/api/context.py index 9497d216..8de24460 100644 --- a/dbx/api/context.py +++ b/dbx/api/context.py @@ -1,7 +1,7 @@ import time from base64 import b64encode from pathlib import Path -from typing import Optional, List, Any +from typing import Any, List, Optional import typer from databricks_cli.sdk import ApiClient diff --git a/dbx/api/deployment.py b/dbx/api/deployment.py index c3689c14..922e1de9 100644 --- a/dbx/api/deployment.py +++ b/dbx/api/deployment.py @@ -4,7 +4,7 @@ from dbx.api.services.jobs import NamedJobsService from dbx.api.services.permissions import PermissionsService from dbx.api.services.pipelines import NamedPipelinesService -from dbx.models.deployment import WorkflowList, AnyWorkflow +from dbx.models.deployment import AnyWorkflow, WorkflowList from dbx.models.workflow.common.workflow_types import WorkflowType from dbx.utils import dbx_echo diff --git a/dbx/api/destroyer.py b/dbx/api/destroyer.py index b2912302..c6caedc9 100644 --- a/dbx/api/destroyer.py +++ b/dbx/api/destroyer.py @@ -14,7 +14,7 @@ from dbx.api.services.jobs import NamedJobsService from dbx.api.services.pipelines import NamedPipelinesService -from dbx.models.cli.destroyer import DestroyerConfig, DeletionMode +from dbx.models.cli.destroyer import DeletionMode, DestroyerConfig from dbx.models.deployment import AnyWorkflow from dbx.models.files.project import EnvironmentInfo from dbx.models.workflow.common.workflow_types import WorkflowType diff --git a/dbx/api/execute.py b/dbx/api/execute.py index 76bda1de..43d86392 100644 --- a/dbx/api/execute.py +++ b/dbx/api/execute.py @@ -1,17 +1,17 @@ from pathlib import Path -from typing import Optional, List, Union, Dict +from typing import Dict, List, Optional, Union import mlflow from rich.console import Console -from dbx.api.adjuster.adjuster import Adjuster, AdditionalLibrariesProvider +from dbx.api.adjuster.adjuster import AdditionalLibrariesProvider, Adjuster from dbx.api.context import RichExecutionContextClient from dbx.models.workflow.common.libraries import Library from dbx.models.workflow.common.task_type import TaskType from dbx.models.workflow.v2dot1.task import PythonWheelTask from dbx.types import ExecuteTask from dbx.utils import dbx_echo -from dbx.utils.file_uploader import MlflowFileUploader, ContextBasedUploader +from dbx.utils.file_uploader import ContextBasedUploader, MlflowFileUploader class ExecutionController: diff --git a/dbx/api/launch/functions.py b/dbx/api/launch/functions.py index b038e03e..bab2e74e 100644 --- a/dbx/api/launch/functions.py +++ b/dbx/api/launch/functions.py @@ -1,5 +1,5 @@ import time -from typing import Dict, Any, List +from typing import Any, Dict, List import mlflow from databricks_cli.sdk import ApiClient, JobsService diff --git a/dbx/api/launch/pipeline_models.py b/dbx/api/launch/pipeline_models.py index 3208fe1e..64217882 100644 --- a/dbx/api/launch/pipeline_models.py +++ b/dbx/api/launch/pipeline_models.py @@ -1,5 +1,5 @@ from enum import Enum -from typing import Optional, List +from typing import List, Optional from pydantic import BaseModel diff --git a/dbx/api/launch/runners/asset_based.py b/dbx/api/launch/runners/asset_based.py index a42afe31..4e6290d3 100644 --- a/dbx/api/launch/runners/asset_based.py +++ b/dbx/api/launch/runners/asset_based.py @@ -1,6 +1,6 @@ import inspect import json -from typing import Optional, Union, Tuple, Dict, Any +from typing import Any, Dict, Optional, Tuple, Union from databricks_cli.sdk import ApiClient, JobsService diff --git a/dbx/api/launch/runners/pipeline.py b/dbx/api/launch/runners/pipeline.py index 19a0e743..afb58f92 100644 --- a/dbx/api/launch/runners/pipeline.py +++ b/dbx/api/launch/runners/pipeline.py @@ -1,7 +1,7 @@ import json import time from functools import partial -from typing import Optional, List, Tuple +from typing import List, Optional, Tuple from databricks_cli.sdk import ApiClient from pydantic import BaseModel diff --git a/dbx/api/launch/runners/standard.py b/dbx/api/launch/runners/standard.py index d60ccb49..08ef1616 100644 --- a/dbx/api/launch/runners/standard.py +++ b/dbx/api/launch/runners/standard.py @@ -1,9 +1,9 @@ import json -from typing import Optional, Union, Tuple +from typing import Optional, Tuple, Union from databricks_cli.sdk import ApiClient, JobsService -from dbx.api.launch.functions import wait_run, cancel_run +from dbx.api.launch.functions import cancel_run, wait_run from dbx.api.launch.runners.base import RunData from dbx.api.services.jobs import NamedJobsService from dbx.models.cli.options import ExistingRunsOption diff --git a/dbx/api/launch/tracer.py b/dbx/api/launch/tracer.py index dd9a3d82..d1c2ed46 100644 --- a/dbx/api/launch/tracer.py +++ b/dbx/api/launch/tracer.py @@ -3,9 +3,9 @@ from databricks_cli.sdk import ApiClient from rich.console import Console -from dbx.api.launch.functions import trace_run, cancel_run +from dbx.api.launch.functions import cancel_run, trace_run from dbx.api.launch.pipeline_models import PipelineUpdateState, PipelineUpdateStatus -from dbx.api.launch.runners.base import RunData, PipelineUpdateResponse +from dbx.api.launch.runners.base import PipelineUpdateResponse, RunData from dbx.utils import dbx_echo diff --git a/dbx/api/output_provider.py b/dbx/api/output_provider.py index c0be7979..cde421bf 100644 --- a/dbx/api/output_provider.py +++ b/dbx/api/output_provider.py @@ -1,4 +1,4 @@ -from typing import Dict, Any, List +from typing import Any, Dict, List from databricks_cli.sdk import JobsService diff --git a/dbx/api/services/_base.py b/dbx/api/services/_base.py index 8dcf370a..654c5bb1 100644 --- a/dbx/api/services/_base.py +++ b/dbx/api/services/_base.py @@ -1,5 +1,5 @@ from abc import abstractmethod -from typing import Optional, Any +from typing import Any, Optional from dbx.api.adjuster.mixins.base import ApiClientMixin from dbx.models.deployment import AnyWorkflow diff --git a/dbx/api/services/pipelines.py b/dbx/api/services/pipelines.py index a2603ea8..9af1415c 100644 --- a/dbx/api/services/pipelines.py +++ b/dbx/api/services/pipelines.py @@ -1,4 +1,4 @@ -from typing import Optional, List +from typing import List, Optional from requests import HTTPError from rich.markup import escape diff --git a/dbx/api/storage/io.py b/dbx/api/storage/io.py index c3760304..999b56de 100644 --- a/dbx/api/storage/io.py +++ b/dbx/api/storage/io.py @@ -2,7 +2,7 @@ import shutil import tempfile from pathlib import Path -from typing import Dict, Any +from typing import Any, Dict import mlflow from mlflow.tracking import MlflowClient diff --git a/dbx/commands/configure.py b/dbx/commands/configure.py index 24428e00..a1d96cbc 100644 --- a/dbx/commands/configure.py +++ b/dbx/commands/configure.py @@ -1,9 +1,9 @@ import typer -from dbx.api.configure import ProjectConfigurationManager, EnvironmentInfo +from dbx.api.configure import EnvironmentInfo, ProjectConfigurationManager from dbx.models.files.project import MlflowStorageProperties, StorageType from dbx.options import ENVIRONMENT_OPTION, PROFILE_OPTION -from dbx.utils import dbx_echo, current_folder_name +from dbx.utils import current_folder_name, dbx_echo def configure( diff --git a/dbx/commands/deploy.py b/dbx/commands/deploy.py index 8af01133..f2a3434b 100644 --- a/dbx/commands/deploy.py +++ b/dbx/commands/deploy.py @@ -26,11 +26,7 @@ WORKFLOW_ARGUMENT, ) from dbx.utils import dbx_echo -from dbx.utils.common import ( - get_current_branch_name, - parse_multiple, - prepare_environment, -) +from dbx.utils.common import get_current_branch_name, parse_multiple, prepare_environment from dbx.utils.file_uploader import MlflowFileUploader diff --git a/dbx/commands/destroy.py b/dbx/commands/destroy.py index 52f6ea19..07d8728a 100644 --- a/dbx/commands/destroy.py +++ b/dbx/commands/destroy.py @@ -9,13 +9,13 @@ from dbx.api.config_reader import ConfigReader from dbx.api.destroyer import Destroyer -from dbx.models.cli.destroyer import DestroyerConfig, DeletionMode +from dbx.models.cli.destroyer import DeletionMode, DestroyerConfig from dbx.options import ( - HEADERS_OPTION, - WORKFLOW_ARGUMENT, DEPLOYMENT_FILE_OPTION, ENVIRONMENT_OPTION, + HEADERS_OPTION, JINJA_VARIABLES_FILE_OPTION, + WORKFLOW_ARGUMENT, ) from dbx.utils import dbx_echo from dbx.utils.common import parse_multiple, prepare_environment diff --git a/dbx/commands/execute.py b/dbx/commands/execute.py index 4298d2a9..15747e77 100644 --- a/dbx/commands/execute.py +++ b/dbx/commands/execute.py @@ -4,7 +4,7 @@ import typer from dbx.api.cluster import ClusterController -from dbx.api.config_reader import ConfigReader, BuildProperties +from dbx.api.config_reader import BuildProperties, ConfigReader from dbx.api.configure import ProjectConfigurationManager from dbx.api.context import RichExecutionContextClient from dbx.api.dependency.core_package import CorePackageManager @@ -12,16 +12,16 @@ from dbx.models.cli.execute import ExecuteParametersPayload from dbx.models.workflow.common.workflow_types import WorkflowType from dbx.options import ( + DEBUG_OPTION, DEPLOYMENT_FILE_OPTION, ENVIRONMENT_OPTION, + EXECUTE_PARAMETERS_OPTION, HEADERS_OPTION, - REQUIREMENTS_FILE_OPTION, - NO_REBUILD_OPTION, - NO_PACKAGE_OPTION, JINJA_VARIABLES_FILE_OPTION, - DEBUG_OPTION, + NO_PACKAGE_OPTION, + NO_REBUILD_OPTION, + REQUIREMENTS_FILE_OPTION, WORKFLOW_ARGUMENT, - EXECUTE_PARAMETERS_OPTION, ) from dbx.types import ExecuteTask from dbx.utils import dbx_echo diff --git a/dbx/commands/launch.py b/dbx/commands/launch.py index 74c03c82..e5f5057c 100644 --- a/dbx/commands/launch.py +++ b/dbx/commands/launch.py @@ -1,37 +1,31 @@ -from typing import List, Dict, Any -from typing import Optional +from typing import Any, Dict, List, Optional import mlflow import typer from databricks_cli.jobs.api import JobsService from rich.markup import escape -from dbx.api.launch.contexts import EmptyContext, AssetBasedLaunchContext +from dbx.api.launch.contexts import AssetBasedLaunchContext, EmptyContext from dbx.api.launch.functions import find_deployment_run from dbx.api.launch.pipeline_models import PipelineUpdateState from dbx.api.launch.runners.asset_based import AssetBasedLauncher from dbx.api.launch.runners.base import RunData from dbx.api.launch.runners.pipeline import PipelineLauncher from dbx.api.launch.runners.standard import StandardLauncher -from dbx.api.launch.tracer import RunTracer, PipelineTracer +from dbx.api.launch.tracer import PipelineTracer, RunTracer from dbx.api.output_provider import OutputProvider from dbx.models.cli.options import ExistingRunsOption, IncludeOutputOption from dbx.options import ( + BRANCH_NAME_OPTION, + DEBUG_OPTION, ENVIRONMENT_OPTION, HEADERS_OPTION, + LAUNCH_PARAMETERS_OPTION, TAGS_OPTION, - BRANCH_NAME_OPTION, - DEBUG_OPTION, WORKFLOW_ARGUMENT, - LAUNCH_PARAMETERS_OPTION, ) from dbx.utils import dbx_echo -from dbx.utils.common import ( - generate_filter_string, - prepare_environment, - parse_multiple, - get_current_branch_name, -) +from dbx.utils.common import generate_filter_string, get_current_branch_name, parse_multiple, prepare_environment def launch( diff --git a/dbx/commands/sync/functions.py b/dbx/commands/sync/functions.py index a4e93786..f800a957 100644 --- a/dbx/commands/sync/functions.py +++ b/dbx/commands/sync/functions.py @@ -7,7 +7,7 @@ from databricks_cli.configure.provider import DatabricksConfig from dbx.constants import DBX_SYNC_DEFAULT_IGNORES -from dbx.sync import DeleteUnmatchedOption, PathMatcher, BaseClient, RemoteSyncer +from dbx.sync import BaseClient, DeleteUnmatchedOption, PathMatcher, RemoteSyncer from dbx.sync.clients import get_user from dbx.sync.event_handler import file_watcher from dbx.utils import dbx_echo diff --git a/dbx/commands/sync/sync.py b/dbx/commands/sync/sync.py index c5c9d3e0..9d06b969 100644 --- a/dbx/commands/sync/sync.py +++ b/dbx/commands/sync/sync.py @@ -10,26 +10,26 @@ from dbx.api.configure import ProjectConfigurationManager from dbx.commands.sync.functions import ( create_path_matcher, - main_loop, - handle_source, - get_user_name, get_source_base_name, + get_user_name, + handle_source, + main_loop, ) from dbx.commands.sync.options import ( - SOURCE_OPTION, - FULL_SYNC_OPTION, DRY_RUN_OPTION, - INCLUDE_DIRS_OPTION, - FORCE_INCLUDE_DIRS_OPTION, EXCLUDE_DIRS_OPTION, - WATCH_OPTION, - POLLING_INTERVAL_OPTION, - INCLUDE_PATTERNS_OPTION, - FORCE_INCLUDE_PATTERNS_OPTION, EXCLUDE_PATTERNS_OPTION, - USE_GITIGNORE_OPTION, - UNMATCHED_BEHAVIOUR_OPTION, + FORCE_INCLUDE_DIRS_OPTION, + FORCE_INCLUDE_PATTERNS_OPTION, + FULL_SYNC_OPTION, + INCLUDE_DIRS_OPTION, + INCLUDE_PATTERNS_OPTION, + POLLING_INTERVAL_OPTION, + SOURCE_OPTION, SYNC_ENVIRONMENT_OPTION, + UNMATCHED_BEHAVIOUR_OPTION, + USE_GITIGNORE_OPTION, + WATCH_OPTION, ) from dbx.options import PROFILE_OPTION from dbx.sync import DeleteUnmatchedOption diff --git a/dbx/custom.py b/dbx/custom.py index 4d662ee0..23db371f 100644 --- a/dbx/custom.py +++ b/dbx/custom.py @@ -1,5 +1,5 @@ import inspect -from typing import Union, Iterable +from typing import Iterable, Union import click from rich.console import group diff --git a/dbx/models/build.py b/dbx/models/build.py index afb2cb16..4b14bbf2 100644 --- a/dbx/models/build.py +++ b/dbx/models/build.py @@ -1,12 +1,12 @@ from __future__ import annotations from enum import Enum -from typing import Optional, List +from typing import List, Optional from pydantic import BaseModel from rich.console import Console -from dbx.api.build import execute_shell_command, cleanup_dist +from dbx.api.build import cleanup_dist, execute_shell_command from dbx.utils import dbx_echo diff --git a/dbx/models/cli/destroyer.py b/dbx/models/cli/destroyer.py index fdcfe457..6da7f030 100644 --- a/dbx/models/cli/destroyer.py +++ b/dbx/models/cli/destroyer.py @@ -1,9 +1,9 @@ from enum import Enum -from typing import Optional, List +from typing import List, Optional from pydantic import BaseModel -from dbx.models.deployment import EnvironmentDeploymentInfo, AnyWorkflow +from dbx.models.deployment import AnyWorkflow, EnvironmentDeploymentInfo class DeletionMode(str, Enum): diff --git a/dbx/models/cli/execute.py b/dbx/models/cli/execute.py index 0b48acd8..3c77c701 100644 --- a/dbx/models/cli/execute.py +++ b/dbx/models/cli/execute.py @@ -4,8 +4,8 @@ from pydantic import root_validator -from dbx.models.validators import mutually_exclusive, at_least_one_of -from dbx.models.workflow.common.parameters import ParametersMixin, NamedParametersMixin +from dbx.models.validators import at_least_one_of, mutually_exclusive +from dbx.models.workflow.common.parameters import NamedParametersMixin, ParametersMixin class ExecuteParametersPayload(ParametersMixin, NamedParametersMixin): diff --git a/dbx/models/deployment.py b/dbx/models/deployment.py index 75861e5f..f0664989 100644 --- a/dbx/models/deployment.py +++ b/dbx/models/deployment.py @@ -1,9 +1,9 @@ from __future__ import annotations import collections -from typing import Optional, Dict, Any, List, Union +from typing import Any, Dict, List, Optional, Union -from pydantic import BaseModel, validator, Field +from pydantic import BaseModel, Field, validator from rich.markup import escape from typing_extensions import Annotated diff --git a/dbx/models/files/project.py b/dbx/models/files/project.py index 296e85ab..26649f4a 100644 --- a/dbx/models/files/project.py +++ b/dbx/models/files/project.py @@ -1,7 +1,7 @@ from __future__ import annotations from enum import Enum -from typing import Dict, Union, Optional +from typing import Dict, Optional, Union from pydantic import BaseModel diff --git a/dbx/models/validators.py b/dbx/models/validators.py index 045ee252..2b66b37c 100644 --- a/dbx/models/validators.py +++ b/dbx/models/validators.py @@ -1,4 +1,4 @@ -from typing import Dict, Any, List +from typing import Any, Dict, List def at_least_one_by_suffix(suffix: str, values: Dict[str, Any]): diff --git a/dbx/models/workflow/common/access_control.py b/dbx/models/workflow/common/access_control.py index 74e4b390..c4abc873 100644 --- a/dbx/models/workflow/common/access_control.py +++ b/dbx/models/workflow/common/access_control.py @@ -1,5 +1,5 @@ from enum import Enum -from typing import Optional, List, Dict, Any +from typing import Any, Dict, List, Optional from pydantic import root_validator, validator diff --git a/dbx/models/workflow/common/job_email_notifications.py b/dbx/models/workflow/common/job_email_notifications.py index 0ac32c12..99db6d10 100644 --- a/dbx/models/workflow/common/job_email_notifications.py +++ b/dbx/models/workflow/common/job_email_notifications.py @@ -1,4 +1,4 @@ -from typing import Optional, List +from typing import List, Optional from dbx.models.workflow.common.flexible import FlexibleModel diff --git a/dbx/models/workflow/common/libraries.py b/dbx/models/workflow/common/libraries.py index c36a96c0..a78bb368 100644 --- a/dbx/models/workflow/common/libraries.py +++ b/dbx/models/workflow/common/libraries.py @@ -1,4 +1,4 @@ -from typing import Optional, List +from typing import List, Optional from pydantic import root_validator diff --git a/dbx/models/workflow/common/parameters.py b/dbx/models/workflow/common/parameters.py index 9a731937..8219be88 100644 --- a/dbx/models/workflow/common/parameters.py +++ b/dbx/models/workflow/common/parameters.py @@ -1,7 +1,6 @@ from __future__ import annotations -from typing import Dict, List, Any -from typing import Optional +from typing import Any, Dict, List, Optional from pydantic import BaseModel diff --git a/dbx/models/workflow/common/pipeline.py b/dbx/models/workflow/common/pipeline.py index abf6b093..048498fb 100644 --- a/dbx/models/workflow/common/pipeline.py +++ b/dbx/models/workflow/common/pipeline.py @@ -1,4 +1,4 @@ -from typing import Optional, Dict, List, Any, Literal +from typing import Any, Dict, List, Literal, Optional from pydantic import validator diff --git a/dbx/models/workflow/common/task.py b/dbx/models/workflow/common/task.py index 8faa30f6..ffadbbff 100644 --- a/dbx/models/workflow/common/task.py +++ b/dbx/models/workflow/common/task.py @@ -2,7 +2,7 @@ from pathlib import Path from typing import Optional -from pydantic import validator, root_validator +from pydantic import root_validator, validator from dbx.constants import TASKS_SUPPORTED_IN_EXECUTE from dbx.models.cli.execute import ExecuteParametersPayload diff --git a/dbx/models/workflow/v2dot0/workflow.py b/dbx/models/workflow/v2dot0/workflow.py index 5d7b5358..e15adb60 100644 --- a/dbx/models/workflow/v2dot0/workflow.py +++ b/dbx/models/workflow/v2dot0/workflow.py @@ -1,4 +1,4 @@ -from typing import Optional, List, Union, Literal +from typing import List, Literal, Optional, Union from pydantic import root_validator, validator @@ -6,12 +6,12 @@ from dbx.models.workflow.common.deployment_config import DbxDeploymentConfig from dbx.models.workflow.common.libraries import Library from dbx.models.workflow.common.new_cluster import NewCluster -from dbx.models.workflow.common.task import SparkPythonTask, SparkJarTask, SparkSubmitTask +from dbx.models.workflow.common.task import SparkJarTask, SparkPythonTask, SparkSubmitTask from dbx.models.workflow.common.task_type import TaskType from dbx.models.workflow.common.workflow import WorkflowBase from dbx.models.workflow.common.workflow_types import WorkflowType from dbx.models.workflow.v2dot0.parameters import AssetBasedRunPayload -from dbx.models.workflow.v2dot0.task import TaskMixin, NotebookTask +from dbx.models.workflow.v2dot0.task import NotebookTask, TaskMixin ALLOWED_TASK_TYPES = Union[SparkPythonTask, NotebookTask, SparkJarTask, SparkSubmitTask] diff --git a/dbx/models/workflow/v2dot1/_parameters.py b/dbx/models/workflow/v2dot1/_parameters.py index c2dc31b3..7dab2324 100644 --- a/dbx/models/workflow/v2dot1/_parameters.py +++ b/dbx/models/workflow/v2dot1/_parameters.py @@ -3,12 +3,12 @@ from typing import Optional, Union from dbx.models.workflow.common.parameters import ( + BaseParametersMixin, + NamedParametersMixin, ParametersMixin, ParamPair, - StringArray, - BaseParametersMixin, PipelineTaskParametersPayload, - NamedParametersMixin, + StringArray, ) diff --git a/dbx/models/workflow/v2dot1/job_cluster.py b/dbx/models/workflow/v2dot1/job_cluster.py index 4c506034..3da8deb4 100644 --- a/dbx/models/workflow/v2dot1/job_cluster.py +++ b/dbx/models/workflow/v2dot1/job_cluster.py @@ -1,5 +1,5 @@ import collections -from typing import Dict, Any, List, Optional +from typing import Any, Dict, List, Optional from pydantic import root_validator diff --git a/dbx/models/workflow/v2dot1/job_task_settings.py b/dbx/models/workflow/v2dot1/job_task_settings.py index 4084e666..75907c34 100644 --- a/dbx/models/workflow/v2dot1/job_task_settings.py +++ b/dbx/models/workflow/v2dot1/job_task_settings.py @@ -1,10 +1,10 @@ -from typing import Optional, List +from typing import List, Optional from dbx.models.workflow.common.deployment_config import DbxDeploymentConfig from dbx.models.workflow.common.flexible import FlexibleModel +from dbx.models.workflow.common.job_email_notifications import JobEmailNotifications from dbx.models.workflow.common.libraries import Library from dbx.models.workflow.common.new_cluster import NewCluster -from dbx.models.workflow.common.job_email_notifications import JobEmailNotifications from dbx.models.workflow.v2dot1.task import TaskMixin diff --git a/dbx/models/workflow/v2dot1/parameters.py b/dbx/models/workflow/v2dot1/parameters.py index 1bc23e79..e4f91f48 100644 --- a/dbx/models/workflow/v2dot1/parameters.py +++ b/dbx/models/workflow/v2dot1/parameters.py @@ -1,16 +1,16 @@ from __future__ import annotations import json -from typing import Optional, List +from typing import List, Optional from pydantic import BaseModel, validator from dbx.models.validators import check_dbt_commands from dbx.models.workflow.common.parameters import ( ParamPair, - StringArray, - StandardBasePayload, PipelineTaskParametersPayload, + StandardBasePayload, + StringArray, ) from dbx.models.workflow.v2dot1._parameters import PayloadElement diff --git a/dbx/models/workflow/v2dot1/task.py b/dbx/models/workflow/v2dot1/task.py index 9f405374..e063bb49 100644 --- a/dbx/models/workflow/v2dot1/task.py +++ b/dbx/models/workflow/v2dot1/task.py @@ -4,11 +4,7 @@ from pydantic import BaseModel, root_validator, validator from pydantic.fields import Field -from dbx.models.validators import ( - at_least_one_of, - check_dbt_commands, - mutually_exclusive, -) +from dbx.models.validators import at_least_one_of, check_dbt_commands, mutually_exclusive from dbx.models.workflow.common.flexible import FlexibleModel from dbx.models.workflow.common.task import ( BaseNotebookTask, diff --git a/dbx/models/workflow/v2dot1/workflow.py b/dbx/models/workflow/v2dot1/workflow.py index 8e1535cf..8d6c7ec6 100644 --- a/dbx/models/workflow/v2dot1/workflow.py +++ b/dbx/models/workflow/v2dot1/workflow.py @@ -1,5 +1,5 @@ import collections -from typing import Optional, List, Dict, Any, Literal +from typing import Any, Dict, List, Literal, Optional from pydantic import root_validator, validator diff --git a/dbx/options.py b/dbx/options.py index 5465d951..a0938988 100644 --- a/dbx/options.py +++ b/dbx/options.py @@ -4,9 +4,9 @@ from dbx.callbacks import ( debug_callback, deployment_file_callback, - verify_jinja_variables_file, execute_parameters_callback, launch_parameters_callback, + verify_jinja_variables_file, ) ENVIRONMENT_OPTION = typer.Option("default", "--environment", "-e", help="Environment name.") diff --git a/dbx/sync/__init__.py b/dbx/sync/__init__.py index 284fddc4..75801a99 100644 --- a/dbx/sync/__init__.py +++ b/dbx/sync/__init__.py @@ -14,6 +14,7 @@ from watchdog.utils.dirsnapshot import DirectorySnapshot, EmptyDirectorySnapshot from dbx.utils import dbx_echo + from .clients import BaseClient from .constants import DBX_SYNC_DIR from .path_matcher import PathMatcher, filtered_listdir diff --git a/dbx/sync/event_handler.py b/dbx/sync/event_handler.py index 61a4bd99..d458414d 100644 --- a/dbx/sync/event_handler.py +++ b/dbx/sync/event_handler.py @@ -9,6 +9,7 @@ from watchdog.observers.polling import PollingObserverVFS from dbx.utils import dbx_echo + from .path_matcher import PathMatcher, filtered_listdir diff --git a/dbx/templates/projects/python_basic/components/conf/deployment.yml b/dbx/templates/projects/python_basic/components/conf/deployment.yml index 7e36ee2e..76c79454 100644 --- a/dbx/templates/projects/python_basic/components/conf/deployment.yml +++ b/dbx/templates/projects/python_basic/components/conf/deployment.yml @@ -2,7 +2,7 @@ # Please read YAML documentation for details on how to use substitutions and anchors. custom: basic-cluster-props: &basic-cluster-props - spark_version: "10.4.x-cpu-ml-scala2.12" + spark_version: "11.3.x-cpu-ml-scala2.12" basic-static-cluster: &basic-static-cluster new_cluster: diff --git a/dbx/templates/projects/python_basic/render/hooks/post_gen_project.py b/dbx/templates/projects/python_basic/render/hooks/post_gen_project.py index 0a7b54ae..449a4351 100644 --- a/dbx/templates/projects/python_basic/render/hooks/post_gen_project.py +++ b/dbx/templates/projects/python_basic/render/hooks/post_gen_project.py @@ -1,5 +1,5 @@ import pathlib -from typing import Callable, Dict, Optional, Any +from typing import Any, Callable, Dict, Optional from jinja2 import Environment, FileSystemLoader diff --git a/dbx/templates/projects/python_basic/render/{{cookiecutter.project_name}}/tests/integration/e2e_test.py b/dbx/templates/projects/python_basic/render/{{cookiecutter.project_name}}/tests/integration/e2e_test.py index c681676e..53d1cdd4 100644 --- a/dbx/templates/projects/python_basic/render/{{cookiecutter.project_name}}/tests/integration/e2e_test.py +++ b/dbx/templates/projects/python_basic/render/{{cookiecutter.project_name}}/tests/integration/e2e_test.py @@ -1,5 +1,6 @@ from {{cookiecutter.project_slug}}.tasks.sample_etl_task import SampleETLTask + def test_etl(): common_config = {"database": "default", "table": "sklearn_housing"} test_etl_config = {"output": common_config} diff --git a/dbx/templates/projects/python_basic/render/{{cookiecutter.project_name}}/tests/unit/conftest.py b/dbx/templates/projects/python_basic/render/{{cookiecutter.project_name}}/tests/unit/conftest.py index e4303649..7b13c52e 100644 --- a/dbx/templates/projects/python_basic/render/{{cookiecutter.project_name}}/tests/unit/conftest.py +++ b/dbx/templates/projects/python_basic/render/{{cookiecutter.project_name}}/tests/unit/conftest.py @@ -2,10 +2,12 @@ This conftest.py contains handy components that prepare SparkSession and other relevant objects. """ +import logging import os -from pathlib import Path import shutil import tempfile +from dataclasses import dataclass +from pathlib import Path from typing import Iterator from unittest.mock import patch @@ -13,8 +15,6 @@ import pytest from delta import configure_spark_with_delta_pip from pyspark.sql import SparkSession -import logging -from dataclasses import dataclass @dataclass diff --git a/dbx/templates/projects/python_basic/render/{{cookiecutter.project_name}}/tests/unit/sample_test.py b/dbx/templates/projects/python_basic/render/{{cookiecutter.project_name}}/tests/unit/sample_test.py index 4d6b02b8..5c7fc45f 100644 --- a/dbx/templates/projects/python_basic/render/{{cookiecutter.project_name}}/tests/unit/sample_test.py +++ b/dbx/templates/projects/python_basic/render/{{cookiecutter.project_name}}/tests/unit/sample_test.py @@ -1,9 +1,11 @@ -from {{cookiecutter.project_slug}}.tasks.sample_etl_task import SampleETLTask -from {{cookiecutter.project_slug}}.tasks.sample_ml_task import SampleMLTask -from pyspark.sql import SparkSession +import logging from pathlib import Path + import mlflow -import logging +from pyspark.sql import SparkSession +from {{cookiecutter.project_slug}}.tasks.sample_etl_task import SampleETLTask +from {{cookiecutter.project_slug}}.tasks.sample_ml_task import SampleMLTask + def test_jobs(spark: SparkSession, tmp_path: Path): logging.info("Testing the ETL job") diff --git a/dbx/templates/projects/python_basic/render/{{cookiecutter.project_name}}/{{cookiecutter.project_slug}}/common.py b/dbx/templates/projects/python_basic/render/{{cookiecutter.project_name}}/{{cookiecutter.project_slug}}/common.py index 96e8b234..51036a6e 100644 --- a/dbx/templates/projects/python_basic/render/{{cookiecutter.project_name}}/{{cookiecutter.project_slug}}/common.py +++ b/dbx/templates/projects/python_basic/render/{{cookiecutter.project_name}}/{{cookiecutter.project_slug}}/common.py @@ -1,10 +1,11 @@ +import pathlib +import sys from abc import ABC, abstractmethod from argparse import ArgumentParser -from typing import Dict, Any +from typing import Any, Dict + import yaml -import pathlib from pyspark.sql import SparkSession -import sys def get_dbutils( diff --git a/dbx/templates/projects/python_basic/render/{{cookiecutter.project_name}}/{{cookiecutter.project_slug}}/tasks/sample_etl_task.py b/dbx/templates/projects/python_basic/render/{{cookiecutter.project_name}}/{{cookiecutter.project_slug}}/tasks/sample_etl_task.py index 0c33b928..b914fe58 100644 --- a/dbx/templates/projects/python_basic/render/{{cookiecutter.project_name}}/{{cookiecutter.project_slug}}/tasks/sample_etl_task.py +++ b/dbx/templates/projects/python_basic/render/{{cookiecutter.project_name}}/{{cookiecutter.project_slug}}/tasks/sample_etl_task.py @@ -1,6 +1,6 @@ -from {{cookiecutter.project_slug}}.common import Task -from sklearn.datasets import fetch_california_housing import pandas as pd +from sklearn.datasets import fetch_california_housing +from {{cookiecutter.project_slug}}.common import Task class SampleETLTask(Task): diff --git a/dbx/templates/projects/python_basic/render/{{cookiecutter.project_name}}/{{cookiecutter.project_slug}}/tasks/sample_ml_task.py b/dbx/templates/projects/python_basic/render/{{cookiecutter.project_name}}/{{cookiecutter.project_slug}}/tasks/sample_ml_task.py index 26000ebf..d079da4c 100644 --- a/dbx/templates/projects/python_basic/render/{{cookiecutter.project_name}}/{{cookiecutter.project_slug}}/tasks/sample_ml_task.py +++ b/dbx/templates/projects/python_basic/render/{{cookiecutter.project_name}}/{{cookiecutter.project_slug}}/tasks/sample_ml_task.py @@ -1,12 +1,13 @@ -from {{cookiecutter.project_slug}}.common import Task -from sklearn.pipeline import Pipeline -from sklearn.preprocessing import StandardScaler +import mlflow +import mlflow.sklearn +import pandas as pd from sklearn.ensemble import RandomForestRegressor -from sklearn.model_selection import train_test_split from sklearn.metrics import r2_score -import pandas as pd -import mlflow.sklearn -import mlflow +from sklearn.model_selection import train_test_split +from sklearn.pipeline import Pipeline +from sklearn.preprocessing import StandardScaler +from {{cookiecutter.project_slug}}.common import Task + class SampleMLTask(Task): TARGET_COLUMN: str = "MedHouseVal" diff --git a/dbx/utils/common.py b/dbx/utils/common.py index e0f3e0a2..f92f6627 100644 --- a/dbx/utils/common.py +++ b/dbx/utils/common.py @@ -6,7 +6,7 @@ from dbx.api.auth import ProfileEnvConfigProvider from dbx.api.client_provider import DatabricksClientProvider -from dbx.api.configure import ProjectConfigurationManager, EnvironmentInfo +from dbx.api.configure import EnvironmentInfo, ProjectConfigurationManager from dbx.api.storage.mlflow_based import MlflowStorageConfigurationManager from dbx.utils import dbx_echo diff --git a/dbx/utils/file_uploader.py b/dbx/utils/file_uploader.py index 35878e00..1046b908 100644 --- a/dbx/utils/file_uploader.py +++ b/dbx/utils/file_uploader.py @@ -4,7 +4,7 @@ from typing import Optional, Tuple import mlflow -from tenacity import retry, wait_exponential, stop_after_attempt +from tenacity import retry, stop_after_attempt, wait_exponential from dbx.api.context import RichExecutionContextClient from dbx.utils import dbx_echo diff --git a/dbx/utils/json.py b/dbx/utils/json.py index 567f2382..3aa9dd11 100644 --- a/dbx/utils/json.py +++ b/dbx/utils/json.py @@ -1,6 +1,6 @@ import json from pathlib import Path -from typing import Dict, Any +from typing import Any, Dict JsonContent = Dict[Any, Any] diff --git a/setup.py b/setup.py index bbf8b4ec..e3199ac2 100644 --- a/setup.py +++ b/setup.py @@ -9,8 +9,8 @@ INSTALL_REQUIRES = [ # to use Databricks and MLflow APIs - "requests>=2.24.0, <2.30.0", - "mlflow-skinny>=1.28.0,<3.0.0", + "requests>=2.30.1,<3.0.0", + "mlflow-skinny>=2.0.0,<3.0.0", "databricks-cli>=0.17,<0.18", "tenacity>=8.2.2,<=9.0.0", # CLI interface @@ -18,13 +18,13 @@ "rich==12.6.0", "typer[all]==0.7.0", # for templates creation - "cookiecutter>=1.7.2, <3.0.0", + "cookiecutter>2.1.0, <3.0.0", # file formats and models "pyyaml>=6.0", - "pydantic>=1.9.1,<=2.0.0", + "pydantic>=1.10.8,<2.0.0", "Jinja2>=2.11.2", # misc - enforced to avoid issues with dependent libraries - "cryptography>=3.3.1,<41.0.0", + "cryptography>=41.0.0,<42.0.0", # required by dbx sync "aiohttp>=3.8.2", "pathspec>=0.9.0", diff --git a/tests/unit/api/adjuster/test_complex.py b/tests/unit/api/adjuster/test_complex.py index 44ab508c..ec705691 100644 --- a/tests/unit/api/adjuster/test_complex.py +++ b/tests/unit/api/adjuster/test_complex.py @@ -5,7 +5,7 @@ from databricks_cli.sdk import InstancePoolService from pytest_mock import MockerFixture -from dbx.api.adjuster.adjuster import Adjuster, AdditionalLibrariesProvider +from dbx.api.adjuster.adjuster import AdditionalLibrariesProvider, Adjuster from dbx.api.services.pipelines import NamedPipelinesService from dbx.models.deployment import DeploymentConfig from dbx.models.workflow.common.libraries import Library diff --git a/tests/unit/api/adjuster/test_existing_cluster.py b/tests/unit/api/adjuster/test_existing_cluster.py index a3b2ea88..7c574715 100644 --- a/tests/unit/api/adjuster/test_existing_cluster.py +++ b/tests/unit/api/adjuster/test_existing_cluster.py @@ -6,7 +6,7 @@ from databricks_cli.sdk import ClusterService from pytest_mock import MockerFixture -from dbx.api.adjuster.adjuster import Adjuster, AdditionalLibrariesProvider +from dbx.api.adjuster.adjuster import AdditionalLibrariesProvider, Adjuster from dbx.models.deployment import AnyWorkflow from dbx.models.workflow.v2dot0.workflow import Workflow as V2dot0Workflow from dbx.models.workflow.v2dot1.workflow import Workflow as V2dot1Workflow diff --git a/tests/unit/api/adjuster/test_instance_pool.py b/tests/unit/api/adjuster/test_instance_pool.py index 01abb60c..431c0543 100644 --- a/tests/unit/api/adjuster/test_instance_pool.py +++ b/tests/unit/api/adjuster/test_instance_pool.py @@ -4,7 +4,8 @@ from databricks_cli.sdk import InstancePoolService from pytest_mock import MockerFixture -from dbx.api.adjuster.adjuster import Adjuster, AdditionalLibrariesProvider +from dbx.api.adjuster.adjuster import AdditionalLibrariesProvider, Adjuster + from .test_instance_profile import convert_to_workflow TEST_PAYLOADS = { diff --git a/tests/unit/api/adjuster/test_instance_profile.py b/tests/unit/api/adjuster/test_instance_profile.py index 4ab6f426..a2853641 100644 --- a/tests/unit/api/adjuster/test_instance_profile.py +++ b/tests/unit/api/adjuster/test_instance_profile.py @@ -4,7 +4,7 @@ import pytest import yaml -from dbx.api.adjuster.adjuster import Adjuster, AdditionalLibrariesProvider +from dbx.api.adjuster.adjuster import AdditionalLibrariesProvider, Adjuster from dbx.models.workflow.v2dot1.workflow import Workflow TEST_PAYLOADS = { diff --git a/tests/unit/api/adjuster/test_pipeline.py b/tests/unit/api/adjuster/test_pipeline.py index 3212048a..600a9e96 100644 --- a/tests/unit/api/adjuster/test_pipeline.py +++ b/tests/unit/api/adjuster/test_pipeline.py @@ -3,7 +3,8 @@ import pytest from pytest_mock import MockerFixture -from dbx.api.adjuster.adjuster import Adjuster, AdditionalLibrariesProvider +from dbx.api.adjuster.adjuster import AdditionalLibrariesProvider, Adjuster + from .test_instance_profile import convert_to_workflow TEST_PAYLOADS = { diff --git a/tests/unit/api/adjuster/test_policy.py b/tests/unit/api/adjuster/test_policy.py index 4631e026..627fc288 100644 --- a/tests/unit/api/adjuster/test_policy.py +++ b/tests/unit/api/adjuster/test_policy.py @@ -5,7 +5,7 @@ from databricks_cli.sdk import PolicyService from pytest_mock import MockerFixture -from dbx.api.adjuster.adjuster import Adjuster, AdditionalLibrariesProvider +from dbx.api.adjuster.adjuster import AdditionalLibrariesProvider, Adjuster from dbx.api.adjuster.policy import PolicyAdjuster from dbx.models.deployment import DeploymentConfig from dbx.models.workflow.common.libraries import Library diff --git a/tests/unit/api/adjuster/test_service_principals.py b/tests/unit/api/adjuster/test_service_principals.py index ee8c05eb..6d9eac2b 100644 --- a/tests/unit/api/adjuster/test_service_principals.py +++ b/tests/unit/api/adjuster/test_service_principals.py @@ -2,7 +2,8 @@ import pytest -from dbx.api.adjuster.adjuster import Adjuster, AdditionalLibrariesProvider +from dbx.api.adjuster.adjuster import AdditionalLibrariesProvider, Adjuster + from .test_instance_profile import convert_to_workflow TEST_PAYLOADS = { diff --git a/tests/unit/api/launch/test_runners.py b/tests/unit/api/launch/test_runners.py index f8620a4e..7efba3d5 100644 --- a/tests/unit/api/launch/test_runners.py +++ b/tests/unit/api/launch/test_runners.py @@ -1,7 +1,7 @@ from unittest.mock import MagicMock import pytest -from databricks_cli.sdk import JobsService, ApiClient +from databricks_cli.sdk import ApiClient, JobsService from pytest_mock import MockerFixture from dbx.api.launch.runners.standard import StandardLauncher diff --git a/tests/unit/api/launch/test_tracer.py b/tests/unit/api/launch/test_tracer.py index 106b3221..7af742a8 100644 --- a/tests/unit/api/launch/test_tracer.py +++ b/tests/unit/api/launch/test_tracer.py @@ -4,8 +4,8 @@ from pytest_mock import MockerFixture from dbx.api.launch.pipeline_models import PipelineUpdateState, UpdateStatus -from dbx.api.launch.runners.base import RunData, PipelineUpdateResponse -from dbx.api.launch.tracer import RunTracer, PipelineTracer +from dbx.api.launch.runners.base import PipelineUpdateResponse, RunData +from dbx.api.launch.tracer import PipelineTracer, RunTracer def test_tracer_with_interruption(mocker: MockerFixture): diff --git a/tests/unit/api/storage/test_mlflow_storage.py b/tests/unit/api/storage/test_mlflow_storage.py index 60b463b5..ac74ba2c 100644 --- a/tests/unit/api/storage/test_mlflow_storage.py +++ b/tests/unit/api/storage/test_mlflow_storage.py @@ -2,7 +2,7 @@ import pytest from mlflow.exceptions import RestException -from mlflow.protos.databricks_pb2 import ErrorCode, INVALID_PARAMETER_VALUE, INTERNAL_ERROR +from mlflow.protos.databricks_pb2 import INTERNAL_ERROR, INVALID_PARAMETER_VALUE, ErrorCode from pytest_mock import MockerFixture from dbx.api.storage.mlflow_based import MlflowStorageConfigurationManager diff --git a/tests/unit/api/test_client_provider.py b/tests/unit/api/test_client_provider.py index 273dcbc6..1ffc4dd0 100644 --- a/tests/unit/api/test_client_provider.py +++ b/tests/unit/api/test_client_provider.py @@ -1,7 +1,7 @@ import pytest +from databricks_cli.sdk import ApiClient from pytest_mock import MockFixture -from databricks_cli.sdk import ApiClient from dbx.api.auth import AuthConfigProvider from dbx.api.client_provider import DatabricksClientProvider diff --git a/tests/unit/api/test_destroyer.py b/tests/unit/api/test_destroyer.py index cbc64f2a..740c973d 100644 --- a/tests/unit/api/test_destroyer.py +++ b/tests/unit/api/test_destroyer.py @@ -6,9 +6,9 @@ import pytest from pytest_mock import MockerFixture -from dbx.api.destroyer import Destroyer, WorkflowEraser, AssetEraser +from dbx.api.destroyer import AssetEraser, Destroyer, WorkflowEraser from dbx.api.services.jobs import NamedJobsService -from dbx.models.cli.destroyer import DestroyerConfig, DeletionMode +from dbx.models.cli.destroyer import DeletionMode, DestroyerConfig from dbx.models.deployment import EnvironmentDeploymentInfo from dbx.models.workflow.v2dot1.workflow import Workflow from tests.unit.conftest import invoke_cli_runner diff --git a/tests/unit/api/test_execution_controller.py b/tests/unit/api/test_execution_controller.py index 74d79ce8..fc261169 100644 --- a/tests/unit/api/test_execution_controller.py +++ b/tests/unit/api/test_execution_controller.py @@ -2,7 +2,7 @@ from pytest_mock import MockerFixture -from dbx.api.context import RichExecutionContextClient, LowLevelExecutionContextClient +from dbx.api.context import LowLevelExecutionContextClient, RichExecutionContextClient def test_execution_controller(mocker: MockerFixture, temp_project): diff --git a/tests/unit/api/test_jobs_service.py b/tests/unit/api/test_jobs_service.py index a7e2e69a..13bf4936 100644 --- a/tests/unit/api/test_jobs_service.py +++ b/tests/unit/api/test_jobs_service.py @@ -3,7 +3,7 @@ import pytest from pytest_mock import MockerFixture -from dbx.api.services.jobs import NamedJobsService, JobListing, ListJobsResponse +from dbx.api.services.jobs import JobListing, ListJobsResponse, NamedJobsService def test_duplicated_jobs(mocker: MockerFixture): diff --git a/tests/unit/commands/test_deploy.py b/tests/unit/commands/test_deploy.py index dd6ab2e9..64bb9bad 100644 --- a/tests/unit/commands/test_deploy.py +++ b/tests/unit/commands/test_deploy.py @@ -14,10 +14,7 @@ from dbx.commands import deploy from dbx.models.files.project import MlflowStorageProperties from dbx.utils.json import JsonUtils -from tests.unit.conftest import ( - get_path_with_relation_to_current_file, - invoke_cli_runner, -) +from tests.unit.conftest import get_path_with_relation_to_current_file, invoke_cli_runner def test_deploy_smoke_default(temp_project: Path, mlflow_file_uploader, mock_storage_io, mock_api_v2_client): diff --git a/tests/unit/commands/test_deploy_jinja_variables_file.py b/tests/unit/commands/test_deploy_jinja_variables_file.py index 34331582..c19970b9 100644 --- a/tests/unit/commands/test_deploy_jinja_variables_file.py +++ b/tests/unit/commands/test_deploy_jinja_variables_file.py @@ -3,10 +3,7 @@ from dbx.api.config_reader import ConfigReader, Jinja2ConfigReader from dbx.api.configure import ProjectConfigurationManager -from tests.unit.conftest import ( - get_path_with_relation_to_current_file, - invoke_cli_runner, -) +from tests.unit.conftest import get_path_with_relation_to_current_file, invoke_cli_runner def test_incorrect_file_name(temp_project: Path, mlflow_file_uploader, mock_storage_io, mock_api_v2_client): diff --git a/tests/unit/commands/test_destroy.py b/tests/unit/commands/test_destroy.py index 42a28197..a3d76bd3 100644 --- a/tests/unit/commands/test_destroy.py +++ b/tests/unit/commands/test_destroy.py @@ -9,7 +9,7 @@ from dbx.api.config_reader import ConfigReader from dbx.api.destroyer import Destroyer from dbx.commands.destroy import ask_for_confirmation, parse_multiple -from dbx.models.cli.destroyer import DestroyerConfig, DeletionMode +from dbx.models.cli.destroyer import DeletionMode, DestroyerConfig from tests.unit.conftest import invoke_cli_runner diff --git a/tests/unit/commands/test_execute.py b/tests/unit/commands/test_execute.py index 051e0cb2..cf4e190d 100644 --- a/tests/unit/commands/test_execute.py +++ b/tests/unit/commands/test_execute.py @@ -1,9 +1,9 @@ from pathlib import Path -from unittest.mock import patch, Mock, MagicMock +from unittest.mock import MagicMock, Mock, patch import pytest -from pytest_mock import MockerFixture from databricks_cli.sdk import ApiClient, ClusterService +from pytest_mock import MockerFixture from dbx.api.cluster import ClusterController from dbx.api.context import LocalContextManager diff --git a/tests/unit/commands/test_launch.py b/tests/unit/commands/test_launch.py index f17a25b5..7803777b 100644 --- a/tests/unit/commands/test_launch.py +++ b/tests/unit/commands/test_launch.py @@ -12,7 +12,7 @@ from dbx.api.launch.pipeline_models import PipelineUpdateState from dbx.api.launch.runners.base import PipelineUpdateResponse from dbx.api.launch.runners.pipeline import PipelineLauncher -from dbx.api.launch.tracer import RunTracer, PipelineTracer +from dbx.api.launch.tracer import PipelineTracer, RunTracer from dbx.api.services.jobs import JobListing, ListJobsResponse from dbx.api.services.pipelines import NamedPipelinesService from dbx.api.storage.io import StorageIO diff --git a/tests/unit/models/test_deployment.py b/tests/unit/models/test_deployment.py index fa26798d..9cc9fbf7 100644 --- a/tests/unit/models/test_deployment.py +++ b/tests/unit/models/test_deployment.py @@ -2,7 +2,7 @@ import yaml from dbx.api.config_reader import ConfigReader -from dbx.models.deployment import DeploymentConfig, EnvironmentDeploymentInfo, WorkflowListMixin, Deployment +from dbx.models.deployment import Deployment, DeploymentConfig, EnvironmentDeploymentInfo, WorkflowListMixin from dbx.models.workflow.common.pipeline import Pipeline from dbx.models.workflow.v2dot0.workflow import Workflow as V2dot0Workflow from dbx.models.workflow.v2dot1.workflow import Workflow as V2dot1Workflow diff --git a/tests/unit/models/test_destroyer.py b/tests/unit/models/test_destroyer.py index a3487e2b..26aa8741 100644 --- a/tests/unit/models/test_destroyer.py +++ b/tests/unit/models/test_destroyer.py @@ -1,7 +1,7 @@ from pathlib import Path from dbx.api.config_reader import ConfigReader -from dbx.models.cli.destroyer import DestroyerConfig, DeletionMode +from dbx.models.cli.destroyer import DeletionMode, DestroyerConfig def test_destroy_model(temp_project): diff --git a/tests/unit/models/test_new_cluster.py b/tests/unit/models/test_new_cluster.py index 57e43be5..75a89f9d 100644 --- a/tests/unit/models/test_new_cluster.py +++ b/tests/unit/models/test_new_cluster.py @@ -1,6 +1,6 @@ import pytest -from dbx.models.workflow.common.new_cluster import NewCluster, AutoScale +from dbx.models.workflow.common.new_cluster import AutoScale, NewCluster def test_legacy_msg(capsys): diff --git a/tests/unit/models/test_parameters.py b/tests/unit/models/test_parameters.py index 2252feac..0697865c 100644 --- a/tests/unit/models/test_parameters.py +++ b/tests/unit/models/test_parameters.py @@ -1,10 +1,10 @@ import pytest -from dbx.models.workflow.v2dot0.parameters import StandardRunPayload as V2dot0StandardRunPayload +from dbx.models.cli.execute import ExecuteParametersPayload from dbx.models.workflow.v2dot0.parameters import AssetBasedRunPayload as V2dot0AssetBasedRunPayload -from dbx.models.workflow.v2dot1.parameters import StandardRunPayload as V2dot1StandardRunPayload +from dbx.models.workflow.v2dot0.parameters import StandardRunPayload as V2dot0StandardRunPayload from dbx.models.workflow.v2dot1.parameters import AssetBasedRunPayload as V2dot1AssetBasedRunPayload -from dbx.models.cli.execute import ExecuteParametersPayload +from dbx.models.workflow.v2dot1.parameters import StandardRunPayload as V2dot1StandardRunPayload def test_empty_execute(): diff --git a/tests/unit/models/test_task.py b/tests/unit/models/test_task.py index 9b213f71..1b28a1f5 100644 --- a/tests/unit/models/test_task.py +++ b/tests/unit/models/test_task.py @@ -4,7 +4,7 @@ from pydantic import ValidationError from dbx.models.cli.execute import ExecuteParametersPayload -from dbx.models.workflow.common.task import SparkPythonTask, SparkJarTask, SparkSubmitTask, BaseTaskMixin +from dbx.models.workflow.common.task import BaseTaskMixin, SparkJarTask, SparkPythonTask, SparkSubmitTask from dbx.models.workflow.common.task_type import TaskType from dbx.models.workflow.v2dot1.task import SqlTask diff --git a/tests/unit/sync/clients/conftest.py b/tests/unit/sync/clients/conftest.py index 1baedd76..6b4ec916 100644 --- a/tests/unit/sync/clients/conftest.py +++ b/tests/unit/sync/clients/conftest.py @@ -2,8 +2,7 @@ import pytest -from tests.unit.sync.utils import mocked_props -from tests.unit.sync.utils import temporary_directory +from tests.unit.sync.utils import mocked_props, temporary_directory @pytest.fixture diff --git a/tests/unit/sync/clients/test_dbfs_client.py b/tests/unit/sync/clients/test_dbfs_client.py index e983390f..8b222a3a 100644 --- a/tests/unit/sync/clients/test_dbfs_client.py +++ b/tests/unit/sync/clients/test_dbfs_client.py @@ -1,14 +1,12 @@ import asyncio import base64 import textwrap -from tests.unit.sync.utils import create_async_with_result from unittest.mock import AsyncMock, MagicMock, PropertyMock, call import pytest from dbx.sync.clients import ClientError, DBFSClient -from tests.unit.sync.utils import create_async_with_result -from tests.unit.sync.utils import mocked_props, is_dbfs_user_agent +from tests.unit.sync.utils import create_async_with_result, is_dbfs_user_agent, mocked_props @pytest.fixture diff --git a/tests/unit/sync/clients/test_repos_client.py b/tests/unit/sync/clients/test_repos_client.py index ed2c033e..44b766a8 100644 --- a/tests/unit/sync/clients/test_repos_client.py +++ b/tests/unit/sync/clients/test_repos_client.py @@ -4,8 +4,7 @@ import pytest from dbx.sync.clients import ClientError, ReposClient -from tests.unit.sync.utils import create_async_with_result -from tests.unit.sync.utils import mocked_props, is_repos_user_agent +from tests.unit.sync.utils import create_async_with_result, is_repos_user_agent, mocked_props @pytest.fixture diff --git a/tests/unit/sync/sync/test_unmatched_deletes.py b/tests/unit/sync/sync/test_unmatched_deletes.py index 913440fa..4ca9a31d 100644 --- a/tests/unit/sync/sync/test_unmatched_deletes.py +++ b/tests/unit/sync/sync/test_unmatched_deletes.py @@ -5,7 +5,7 @@ import pytest from dbx.commands.sync.functions import create_path_matcher -from dbx.sync import RemoteSyncer, DeleteUnmatchedOption +from dbx.sync import DeleteUnmatchedOption, RemoteSyncer from tests.unit.sync.utils import temporary_directory diff --git a/tests/unit/sync/test_commands.py b/tests/unit/sync/test_commands.py index f95c4e2a..96503239 100644 --- a/tests/unit/sync/test_commands.py +++ b/tests/unit/sync/test_commands.py @@ -1,19 +1,20 @@ import asyncio import os -from unittest.mock import patch, call, MagicMock, AsyncMock +from unittest.mock import AsyncMock, MagicMock, call, patch import click import pytest from databricks_cli.configure.provider import ProfileConfigProvider +from dbx.commands.sync.functions import get_source_base_name, get_user_name from dbx.commands.sync.sync import repo_exists -from dbx.commands.sync.functions import get_user_name, get_source_base_name from dbx.constants import DBX_SYNC_DEFAULT_IGNORES from dbx.sync import DeleteUnmatchedOption from dbx.sync.clients import DBFSClient, ReposClient, WorkspaceClient from tests.unit.sync.utils import mocked_props + from .conftest import invoke_cli_runner -from .utils import temporary_directory, pushd +from .utils import pushd, temporary_directory def get_config(): diff --git a/tests/unit/sync/test_config.py b/tests/unit/sync/test_config.py index 803cbc22..c089dfd9 100644 --- a/tests/unit/sync/test_config.py +++ b/tests/unit/sync/test_config.py @@ -4,7 +4,7 @@ import pytest from databricks_cli.utils import InvalidConfigurationError -from dbx.sync.config import has_valid_token, get_databricks_config +from dbx.sync.config import get_databricks_config, has_valid_token from tests.unit.sync.utils import mocked_props diff --git a/tests/unit/sync/test_event_handler.py b/tests/unit/sync/test_event_handler.py index e3e4f651..9681be1c 100644 --- a/tests/unit/sync/test_event_handler.py +++ b/tests/unit/sync/test_event_handler.py @@ -9,6 +9,7 @@ from dbx.sync.event_handler import CollectingEventHandler, file_watcher from dbx.sync.path_matcher import PathMatcher + from .utils import temporary_directory diff --git a/tests/unit/sync/test_main_loop.py b/tests/unit/sync/test_main_loop.py index af23c992..9f6ac59c 100644 --- a/tests/unit/sync/test_main_loop.py +++ b/tests/unit/sync/test_main_loop.py @@ -1,6 +1,7 @@ from unittest.mock import MagicMock, patch from dbx.commands.sync.functions import main_loop + from .utils import temporary_directory diff --git a/tests/unit/sync/test_path_matcher.py b/tests/unit/sync/test_path_matcher.py index e29be6d1..a55fa0f7 100644 --- a/tests/unit/sync/test_path_matcher.py +++ b/tests/unit/sync/test_path_matcher.py @@ -5,6 +5,7 @@ from dbx.commands.sync.functions import create_path_matcher from dbx.sync.path_matcher import PathMatcher, filtered_listdir + from .utils import temporary_directory diff --git a/tests/unit/sync/test_snapshot.py b/tests/unit/sync/test_snapshot.py index 774e1d00..280f45d2 100644 --- a/tests/unit/sync/test_snapshot.py +++ b/tests/unit/sync/test_snapshot.py @@ -6,6 +6,7 @@ from dbx.sync import get_relative_path from dbx.sync.snapshot import compute_snapshot_diff + from .utils import temporary_directory diff --git a/tests/unit/sync/utils.py b/tests/unit/sync/utils.py index e50a9db0..b9f39faf 100644 --- a/tests/unit/sync/utils.py +++ b/tests/unit/sync/utils.py @@ -2,7 +2,6 @@ import re from contextlib import contextmanager from tempfile import TemporaryDirectory - from unittest.mock import AsyncMock, MagicMock, PropertyMock diff --git a/tests/unit/utils/test_common.py b/tests/unit/utils/test_common.py index af62664e..0d5bffe6 100644 --- a/tests/unit/utils/test_common.py +++ b/tests/unit/utils/test_common.py @@ -9,11 +9,7 @@ from dbx.api.config_reader import ConfigReader from dbx.models.build import BuildConfiguration -from dbx.utils.common import ( - generate_filter_string, - get_current_branch_name, - get_environment_data, -) +from dbx.utils.common import generate_filter_string, get_current_branch_name, get_environment_data from tests.unit.conftest import get_path_with_relation_to_current_file json_file_01 = get_path_with_relation_to_current_file("../deployment-configs/01-json-test.json")