Skip to content

Commit 262dd76

Browse files
anshbansalanshbansal
authored
dev: remove black in favor of ruff for formatting (#12378)
1 parent 654728e commit 262dd76

File tree

208 files changed

+1366
-1326
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

208 files changed

+1366
-1326
lines changed

‎metadata-ingestion-modules/airflow-plugin/build.gradle

+6-4
Original file line numberDiff line numberDiff line change
@@ -73,16 +73,15 @@ task lint(type: Exec, dependsOn: installDev) {
7373
commandLine 'bash', '-c',
7474
"find ${venv_name}/lib -path *airflow/_vendor/connexion/spec.py -exec sed -i.bak -e '169,169s/ # type: List\\[str\\]//g' {} \\; && " +
7575
"source ${venv_name}/bin/activate && set -x && " +
76-
"black --check --diff src/ tests/ && " +
7776
"ruff check src/ tests/ && " +
77+
"ruff format --check src/ tests/ && " +
7878
"mypy --show-traceback --show-error-codes src/ tests/"
7979
}
8080
task lintFix(type: Exec, dependsOn: installDev) {
8181
commandLine 'bash', '-c',
8282
"source ${venv_name}/bin/activate && set -x && " +
83-
"black src/ tests/ && " +
84-
"ruff check --fix src/ tests/"
85-
"mypy src/ tests/ "
83+
"ruff check --fix src/ tests/ && " +
84+
"ruff format src/ tests/ "
8685
}
8786

8887
// HACK: Some of the Airflow constraint files conflict with packages that we install (e.g. black).
@@ -119,5 +118,8 @@ clean {
119118
delete venv_name
120119
delete 'build'
121120
delete 'dist'
121+
delete '.ruff_cache'
122+
delete '.mypy_cache'
123+
delete '.pytest_cache'
122124
}
123125
clean.dependsOn cleanPythonCache

‎metadata-ingestion-modules/airflow-plugin/pyproject.toml

+24-24
Original file line numberDiff line numberDiff line change
@@ -2,13 +2,21 @@
22
build-backend = "setuptools.build_meta"
33
requires = ["setuptools>=54.0.0", "wheel", "pip>=21.0.0"]
44

5-
[tool.black]
6-
extend-exclude = '''
7-
# A regex preceded with ^/ will apply only to files and directories
8-
# in the root of the project.
9-
^/tmp
10-
'''
11-
include = '\.pyi?$'
5+
[tool.ruff]
6+
line-length = 88
7+
target-version = "py38"
8+
exclude = [
9+
".git",
10+
"venv",
11+
".tox",
12+
"__pycache__",
13+
]
14+
15+
[tool.ruff.format]
16+
quote-style = "double"
17+
indent-style = "space"
18+
skip-magic-trailing-comma = false
19+
line-ending = "auto"
1220

1321
[tool.ruff.lint.isort]
1422
combine-as-imports = true
@@ -28,31 +36,23 @@ required-imports = []
2836
classes = ["typing"]
2937

3038
[tool.ruff.lint]
31-
select = [
32-
"B",
33-
"C90",
34-
"E",
35-
"F",
36-
"I", # For isort
37-
"TID",
39+
extend-select = [
40+
"B", # flake8-bugbear
41+
"C90", # mccabe complexity
42+
"E", # pycodestyle errors
43+
"F", # pyflakes
44+
"G010", # logging.warn -> logging.warning
45+
"I", # isort
46+
"TID", # flake8-tidy-imports
3847
]
3948
ignore = [
40-
# Ignore line length violations (handled by Black)
41-
"E501",
42-
# Ignore whitespace before ':' (matches Black)
43-
"E203",
44-
"E203",
45-
# Allow usages of functools.lru_cache
46-
"B019",
47-
# Allow function call in argument defaults
48-
"B008",
49+
"E501", # Line length violations (handled by formatter)
4950
]
5051

5152
[tool.ruff.lint.mccabe]
5253
max-complexity = 15
5354

5455
[tool.ruff.lint.flake8-tidy-imports]
55-
# Disallow all relative imports.
5656
ban-relative-imports = "all"
5757

5858
[tool.ruff.lint.per-file-ignores]

‎metadata-ingestion-modules/airflow-plugin/setup.py

-1
Original file line numberDiff line numberDiff line change
@@ -71,7 +71,6 @@ def get_long_description():
7171
dev_requirements = {
7272
*base_requirements,
7373
*mypy_stubs,
74-
"black==22.12.0",
7574
"coverage>=5.1",
7675
"ruff==0.9.2",
7776
"mypy==1.10.1",

‎metadata-ingestion-modules/airflow-plugin/src/datahub_airflow_plugin/_extractors.py

+3-3
Original file line numberDiff line numberDiff line change
@@ -63,9 +63,9 @@ def __init__(self):
6363

6464
self.task_to_extractor.extractors["AthenaOperator"] = AthenaOperatorExtractor
6565

66-
self.task_to_extractor.extractors[
67-
"BigQueryInsertJobOperator"
68-
] = BigQueryInsertJobOperatorExtractor
66+
self.task_to_extractor.extractors["BigQueryInsertJobOperator"] = (
67+
BigQueryInsertJobOperatorExtractor
68+
)
6969

7070
self._graph: Optional["DataHubGraph"] = None
7171

‎metadata-ingestion-modules/airflow-plugin/src/datahub_airflow_plugin/datahub_listener.py

+3-3
Original file line numberDiff line numberDiff line change
@@ -286,9 +286,9 @@ def _extract_lineage(
286286
if sql_parsing_result:
287287
if error := sql_parsing_result.debug_info.error:
288288
logger.info(f"SQL parsing error: {error}", exc_info=error)
289-
datajob.properties[
290-
"datahub_sql_parser_error"
291-
] = f"{type(error).__name__}: {error}"
289+
datajob.properties["datahub_sql_parser_error"] = (
290+
f"{type(error).__name__}: {error}"
291+
)
292292
if not sql_parsing_result.debug_info.table_error:
293293
input_urns.extend(sql_parsing_result.in_tables)
294294
output_urns.extend(sql_parsing_result.out_tables)

‎metadata-ingestion-modules/airflow-plugin/src/datahub_airflow_plugin/datahub_plugin_v22.py

+3-5
Original file line numberDiff line numberDiff line change
@@ -44,11 +44,9 @@ def get_task_inlets_advanced(task: BaseOperator, context: Any) -> Iterable[Any]:
4444

4545
if task_inlets and isinstance(task_inlets, list):
4646
inlets = []
47-
task_ids = (
48-
{o for o in task_inlets if isinstance(o, str)}
49-
.union(op.task_id for op in task_inlets if isinstance(op, BaseOperator))
50-
.intersection(task.get_flat_relative_ids(upstream=True))
51-
)
47+
task_ids = {o for o in task_inlets if isinstance(o, str)}.union(
48+
op.task_id for op in task_inlets if isinstance(op, BaseOperator)
49+
).intersection(task.get_flat_relative_ids(upstream=True))
5250

5351
from airflow.lineage import AUTO
5452
from cattr import structure

‎metadata-ingestion-modules/airflow-plugin/src/datahub_airflow_plugin/example_dags/lineage_emission_dag.py

+1
Original file line numberDiff line numberDiff line change
@@ -2,6 +2,7 @@
22
33
This example demonstrates how to emit lineage to DataHub within an Airflow DAG.
44
"""
5+
56
from datetime import timedelta
67

78
from airflow import DAG

‎metadata-ingestion-modules/airflow-plugin/tests/integration/test_plugin.py

+15-7
Original file line numberDiff line numberDiff line change
@@ -273,13 +273,21 @@ def _run_airflow(
273273
subprocess.check_call(
274274
[
275275
# fmt: off
276-
"airflow", "users", "create",
277-
"--username", "airflow",
278-
"--password", "airflow",
279-
"--firstname", "admin",
280-
"--lastname", "admin",
281-
"--role", "Admin",
282-
"--email", "[email protected]",
276+
"airflow",
277+
"users",
278+
"create",
279+
"--username",
280+
"airflow",
281+
"--password",
282+
"airflow",
283+
"--firstname",
284+
"admin",
285+
"--lastname",
286+
"admin",
287+
"--role",
288+
"Admin",
289+
"--email",
290+
"[email protected]",
283291
# fmt: on
284292
],
285293
env=environment,

‎metadata-ingestion-modules/airflow-plugin/tests/unit/test_airflow.py

+5-4
Original file line numberDiff line numberDiff line change
@@ -242,9 +242,7 @@ def test_lineage_backend(mock_emit, inlets, outlets, capture_executions):
242242
},
243243
), mock.patch("airflow.models.BaseOperator.xcom_pull"), mock.patch(
244244
"airflow.models.BaseOperator.xcom_push"
245-
), patch_airflow_connection(
246-
datahub_rest_connection_config
247-
):
245+
), patch_airflow_connection(datahub_rest_connection_config):
248246
func = mock.Mock()
249247
func.__name__ = "foo"
250248

@@ -275,7 +273,10 @@ def test_lineage_backend(mock_emit, inlets, outlets, capture_executions):
275273
if AIRFLOW_VERSION < packaging.version.parse("2.2.0"):
276274
ti = TaskInstance(task=op2, execution_date=DEFAULT_DATE)
277275
# Ignoring type here because DagRun state is just a sring at Airflow 1
278-
dag_run = DagRun(state="success", run_id=f"scheduled_{DEFAULT_DATE.isoformat()}") # type: ignore
276+
dag_run = DagRun(
277+
state="success", # type: ignore[arg-type]
278+
run_id=f"scheduled_{DEFAULT_DATE.isoformat()}",
279+
)
279280
else:
280281
from airflow.utils.state import DagRunState
281282

‎metadata-ingestion-modules/dagster-plugin/build.gradle

+7-5
Original file line numberDiff line numberDiff line change
@@ -54,16 +54,15 @@ task installDev(type: Exec, dependsOn: [install]) {
5454
task lint(type: Exec, dependsOn: installDev) {
5555
commandLine 'bash', '-c',
5656
"source ${venv_name}/bin/activate && set -x && " +
57-
"black --check --diff src/ tests/ examples/ && " +
58-
"ruff check src/ tests/ && " +
57+
"ruff check src/ tests/ examples/ && " +
58+
"ruff format --check src/ tests/ && " +
5959
"mypy --show-traceback --show-error-codes src/ tests/ examples/"
6060
}
6161
task lintFix(type: Exec, dependsOn: installDev) {
6262
commandLine 'bash', '-x', '-c',
6363
"source ${venv_name}/bin/activate && " +
64-
"black src/ tests/ examples/ && " +
65-
"ruff check --fix src/ tests/"
66-
"mypy src/ tests/ examples/"
64+
"ruff check --fix src/ tests/ examples/ && " +
65+
"ruff format src/ tests/ examples/ "
6766
}
6867

6968
task installDevTest(type: Exec, dependsOn: [installDev]) {
@@ -105,5 +104,8 @@ clean {
105104
delete venv_name
106105
delete 'build'
107106
delete 'dist'
107+
delete '.ruff_cache'
108+
delete '.mypy_cache'
109+
delete '.pytest_cache'
108110
}
109111
clean.dependsOn cleanPythonCache

‎metadata-ingestion-modules/dagster-plugin/examples/advanced_ops_jobs.py

+1-1
Original file line numberDiff line numberDiff line change
@@ -9,9 +9,9 @@
99
job,
1010
op,
1111
)
12+
1213
from datahub.ingestion.graph.client import DatahubClientConfig, DataHubGraph
1314
from datahub.utilities.urns.dataset_urn import DatasetUrn
14-
1515
from datahub_dagster_plugin.client.dagster_generator import (
1616
DagsterGenerator,
1717
DatasetLineage,

‎metadata-ingestion-modules/dagster-plugin/examples/assets_job.py

+1-1
Original file line numberDiff line numberDiff line change
@@ -7,9 +7,9 @@
77
define_asset_job,
88
multi_asset,
99
)
10+
1011
from datahub.ingestion.graph.config import DatahubClientConfig
1112
from datahub.utilities.urns.dataset_urn import DatasetUrn
12-
1313
from datahub_dagster_plugin.sensors.datahub_sensors import (
1414
DatahubDagsterSourceConfig,
1515
make_datahub_sensor,

‎metadata-ingestion-modules/dagster-plugin/examples/basic_setup.py

+1-1
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,6 @@
11
from dagster import Definitions
2-
from datahub.ingestion.graph.client import DatahubClientConfig
32

3+
from datahub.ingestion.graph.client import DatahubClientConfig
44
from datahub_dagster_plugin.sensors.datahub_sensors import (
55
DatahubDagsterSourceConfig,
66
make_datahub_sensor,

‎metadata-ingestion-modules/dagster-plugin/examples/ops_job.py

+1-1
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,7 @@
11
from dagster import Definitions, In, Out, PythonObjectDagsterType, job, op
2+
23
from datahub.ingestion.graph.config import DatahubClientConfig
34
from datahub.utilities.urns.dataset_urn import DatasetUrn
4-
55
from datahub_dagster_plugin.sensors.datahub_sensors import (
66
DatahubDagsterSourceConfig,
77
make_datahub_sensor,

‎metadata-ingestion-modules/dagster-plugin/pyproject.toml

+24-24
Original file line numberDiff line numberDiff line change
@@ -2,13 +2,21 @@
22
build-backend = "setuptools.build_meta"
33
requires = ["setuptools>=54.0.0", "wheel", "pip>=21.0.0"]
44

5-
[tool.black]
6-
extend-exclude = '''
7-
# A regex preceded with ^/ will apply only to files and directories
8-
# in the root of the project.
9-
^/tmp
10-
'''
11-
include = '\.pyi?$'
5+
[tool.ruff]
6+
line-length = 88
7+
target-version = "py38"
8+
exclude = [
9+
".git",
10+
"venv",
11+
".tox",
12+
"__pycache__",
13+
]
14+
15+
[tool.ruff.format]
16+
quote-style = "double"
17+
indent-style = "space"
18+
skip-magic-trailing-comma = false
19+
line-ending = "auto"
1220

1321
[tool.ruff.lint.isort]
1422
combine-as-imports = true
@@ -28,31 +36,23 @@ required-imports = []
2836
classes = ["typing"]
2937

3038
[tool.ruff.lint]
31-
select = [
32-
"B",
33-
"C90",
34-
"E",
35-
"F",
36-
"I", # For isort
37-
"TID",
39+
extend-select = [
40+
"B", # flake8-bugbear
41+
"C90", # mccabe complexity
42+
"E", # pycodestyle errors
43+
"F", # pyflakes
44+
"G010", # logging.warn -> logging.warning
45+
"I", # isort
46+
"TID", # flake8-tidy-imports
3847
]
3948
ignore = [
40-
# Ignore line length violations (handled by Black)
41-
"E501",
42-
# Ignore whitespace before ':' (matches Black)
43-
"E203",
44-
"E203",
45-
# Allow usages of functools.lru_cache
46-
"B019",
47-
# Allow function call in argument defaults
48-
"B008",
49+
"E501", # Line length violations (handled by formatter)
4950
]
5051

5152
[tool.ruff.lint.mccabe]
5253
max-complexity = 15
5354

5455
[tool.ruff.lint.flake8-tidy-imports]
55-
# Disallow all relative imports.
5656
ban-relative-imports = "all"
5757

5858
[tool.ruff.lint.per-file-ignores]

‎metadata-ingestion-modules/dagster-plugin/setup.py

-1
Original file line numberDiff line numberDiff line change
@@ -51,7 +51,6 @@ def get_long_description():
5151
"dagster-aws >= 0.11.0",
5252
"dagster-snowflake >= 0.11.0",
5353
"dagster-snowflake-pandas >= 0.11.0",
54-
"black==22.12.0",
5554
"coverage>=5.1",
5655
"ruff==0.9.2",
5756
"mypy>=1.4.0",

‎metadata-ingestion-modules/dagster-plugin/src/datahub_dagster_plugin/client/dagster_generator.py

+4-4
Original file line numberDiff line numberDiff line change
@@ -507,15 +507,15 @@ def generate_datajob(
507507
job_property_bag: Dict[str, str] = {}
508508
if input_datasets:
509509
self.logger.info(
510-
f"Input datasets for {op_def_snap.name} are { list(input_datasets.get(op_def_snap.name, []))}"
510+
f"Input datasets for {op_def_snap.name} are {list(input_datasets.get(op_def_snap.name, []))}"
511511
)
512512
inlets.update(input_datasets.get(op_def_snap.name, []))
513513

514514
datajob.inlets = list(inlets)
515515

516516
if output_datasets:
517517
self.logger.info(
518-
f"Output datasets for {op_def_snap.name} are { list(output_datasets.get(op_def_snap.name, []))}"
518+
f"Output datasets for {op_def_snap.name} are {list(output_datasets.get(op_def_snap.name, []))}"
519519
)
520520
datajob.outlets = list(output_datasets.get(op_def_snap.name, []))
521521

@@ -606,7 +606,7 @@ def emit_job_run(
606606
if run.status not in status_result_map:
607607
raise Exception(
608608
f"Job run status should be either complete, failed or cancelled and it was "
609-
f"{run.status }"
609+
f"{run.status}"
610610
)
611611

612612
if run_stats.start_time is not None:
@@ -673,7 +673,7 @@ def emit_op_run(
673673
if run_step_stats.status not in status_result_map:
674674
raise Exception(
675675
f"Step run status should be either complete, failed or cancelled and it was "
676-
f"{run_step_stats.status }"
676+
f"{run_step_stats.status}"
677677
)
678678

679679
if run_step_stats.start_time is not None:

‎metadata-ingestion-modules/dagster-plugin/src/datahub_dagster_plugin/sensors/datahub_sensors.py

-1
Original file line numberDiff line numberDiff line change
@@ -262,7 +262,6 @@ def get_dagster_environment(
262262
and context.dagster_run.job_code_origin.repository_origin
263263
and context.dagster_run.job_code_origin.repository_origin.code_pointer
264264
):
265-
266265
code_pointer = (
267266
context.dagster_run.job_code_origin.repository_origin.code_pointer
268267
)

0 commit comments

Comments
 (0)