From 9c338eca025982f5810aee6629091d75c399144e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?St=C3=A9phane=20Brunner?= Date: Thu, 30 Jan 2025 14:25:28 +0100 Subject: [PATCH] Ignore the blended associated code --- prospector/postfilter.py | 14 +++- prospector/run.py | 13 +++- prospector/suppression.py | 65 +++++++++++++++---- prospector/tools/base.py | 7 ++ prospector/tools/mypy/__init__.py | 9 +++ prospector/tools/pylint/__init__.py | 8 +++ prospector/tools/ruff/__init__.py | 9 +++ tests/suppression/test_blender_suppression.py | 60 +++++++++++++++++ .../test_blender_suppressions/test.py | 3 + 9 files changed, 169 insertions(+), 19 deletions(-) create mode 100644 tests/suppression/test_blender_suppression.py create mode 100644 tests/suppression/testdata/test_blender_suppressions/test.py diff --git a/prospector/postfilter.py b/prospector/postfilter.py index 35ce4272..c4c36961 100644 --- a/prospector/postfilter.py +++ b/prospector/postfilter.py @@ -1,10 +1,18 @@ from pathlib import Path +from typing import Optional from prospector.message import Message from prospector.suppression import get_suppressions +from prospector.tools.base import ToolBase -def filter_messages(filepaths: list[Path], messages: list[Message]) -> list[Message]: +def filter_messages( + filepaths: list[Path], + messages: list[Message], + tools: Optional[dict[str, ToolBase]] = None, + blending: bool = False, + blend_combos: Optional[list[list[tuple[str, str]]]] = None, +) -> list[Message]: """ This method post-processes all messages output by all tools, in order to filter out any based on the overall output. @@ -23,7 +31,9 @@ def filter_messages(filepaths: list[Path], messages: list[Message]) -> list[Mess This method uses the information about suppressed messages from pylint to squash the unwanted redundant error from pyflakes and frosted. """ - paths_to_ignore, lines_to_ignore, messages_to_ignore = get_suppressions(filepaths, messages) + paths_to_ignore, lines_to_ignore, messages_to_ignore = get_suppressions( + filepaths, messages, tools, blending, blend_combos + ) filtered = [] for message in messages: diff --git a/prospector/run.py b/prospector/run.py index 31b0f385..b2339788 100644 --- a/prospector/run.py +++ b/prospector/run.py @@ -16,6 +16,7 @@ from prospector.formatters import FORMATTERS, Formatter from prospector.message import Location, Message from prospector.tools import DEPRECATED_TOOL_NAMES +from prospector.tools.base import ToolBase from prospector.tools.utils import CaptureOutput @@ -25,7 +26,9 @@ def __init__(self, config: ProspectorConfig) -> None: self.summary: Optional[dict[str, Any]] = None self.messages = config.messages - def process_messages(self, found_files: FileFinder, messages: list[Message]) -> list[Message]: + def process_messages( + self, found_files: FileFinder, messages: list[Message], tools: dict[str, tools.ToolBase] + ) -> list[Message]: if self.config.blending: messages = blender.blend(messages) @@ -37,7 +40,7 @@ def process_messages(self, found_files: FileFinder, messages: list[Message]) -> updated.append(msg) messages = updated - return postfilter.filter_messages(found_files.python_modules, messages) + return postfilter.filter_messages(found_files.python_modules, messages, tools, self.config.blending) def execute(self) -> None: deprecated_names = self.config.replace_deprecated_tool_names() @@ -70,6 +73,8 @@ def execute(self) -> None: messages.append(message) warnings.warn(msg, category=DeprecationWarning, stacklevel=0) + running_tools: dict[str, ToolBase] = {} + # Run the tools for tool in self.config.get_tools(found_files): for name, cls in tools.TOOLS.items(): @@ -79,6 +84,8 @@ def execute(self) -> None: else: toolname = "Unknown" + running_tools[toolname] = tool + try: # Tools can output to stdout/stderr in unexpected places, for example, # pydocstyle emits warnings about __all__ and as pyroma exec's the setup.py @@ -116,7 +123,7 @@ def execute(self) -> None: ) messages.append(message) - messages = self.process_messages(found_files, messages) + messages = self.process_messages(found_files, messages, running_tools) summary["message_count"] = len(messages) summary["completed"] = datetime.now() diff --git a/prospector/suppression.py b/prospector/suppression.py index c958d3cf..94d51068 100644 --- a/prospector/suppression.py +++ b/prospector/suppression.py @@ -27,8 +27,10 @@ from typing import Optional from prospector import encoding +from prospector.blender import BLEND_COMBOS from prospector.exceptions import FatalProspectorException from prospector.message import Message +from prospector.tools.base import ToolBase _FLAKE8_IGNORE_FILE = re.compile(r"flake8[:=]\s*noqa", re.IGNORECASE) _PEP8_IGNORE_LINE = re.compile(r"#\s*noqa(\s*#.*)?$", re.IGNORECASE) @@ -51,6 +53,17 @@ def __init__( def __str__(self) -> str: return self.code if self.source is None else f"{self.source}.{self.code}" + def __repr__(self) -> str: + return f"<{type(self).__name__} {self}>" + + def __eq__(self, value: object) -> bool: + if not isinstance(value, Ignore): + return False + return self.code == value.code and self.source == value.source + + def __hash__(self) -> int: + return hash((self.source, self.code)) + def get_noqa_suppressions(file_contents: list[str]) -> tuple[bool, set[int], dict[int, set[Ignore]]]: """ @@ -81,15 +94,6 @@ def get_noqa_suppressions(file_contents: list[str]) -> tuple[bool, set[int], dic return ignore_whole_file, ignore_lines, messages_to_ignore -_PYLINT_EQUIVALENTS = { - # TODO: blending has this info already? - "unused-import": ( - ("pyflakes", "FL0001"), - ("frosted", "E101"), - ) -} - - def _parse_pylint_informational( messages: list[Message], ) -> tuple[set[Optional[Path]], dict[Optional[Path], dict[int, list[str]]]]: @@ -113,17 +117,43 @@ def _parse_pylint_informational( return ignore_files, ignore_messages +def _process_tool_ignores( + tools_ignore: dict[Path, dict[int, set[Ignore]]], + blend_combos_dict: dict[Ignore, set[Ignore]], + messages_to_ignore: dict[Optional[Path], dict[int, set[Ignore]]], +) -> None: + for path, lines_ignore in tools_ignore.items(): + for line, ignores in lines_ignore.items(): + for ignore in ignores: + if ignore in blend_combos_dict: + messages_to_ignore[path][line].update(blend_combos_dict[ignore]) + + def get_suppressions( - filepaths: list[Path], messages: list[Message] + filepaths: list[Path], + messages: list[Message], + tools: Optional[dict[str, ToolBase]] = None, + blending: bool = False, + blend_combos: Optional[list[list[tuple[str, str]]]] = None, ) -> tuple[set[Optional[Path]], dict[Path, set[int]], dict[Optional[Path], dict[int, set[Ignore]]]]: """ Given every message which was emitted by the tools, and the list of files to inspect, create a list of files to ignore, and a map of filepath -> line-number -> codes to ignore """ + tools = tools or {} + blend_combos = blend_combos or BLEND_COMBOS + blend_combos_dict: dict[Ignore, set[Ignore]] = {} + if blending: + for combo in blend_combos: + ignore_combos = {Ignore(tool, code) for tool, code in combo} + for ignore in ignore_combos: + blend_combos_dict[ignore] = ignore_combos + paths_to_ignore: set[Optional[Path]] = set() lines_to_ignore: dict[Path, set[int]] = defaultdict(set) messages_to_ignore: dict[Optional[Path], dict[int, set[Ignore]]] = defaultdict(lambda: defaultdict(set)) + tools_ignore: dict[Path, dict[int, set[Ignore]]] = defaultdict(lambda: defaultdict(set)) # First deal with 'noqa' style messages for filepath in filepaths: @@ -141,6 +171,17 @@ def get_suppressions( for line, codes_ignore in file_messages_to_ignore.items(): messages_to_ignore[filepath][line] |= codes_ignore + if blending: + for line_number, line_content in enumerate(file_contents): + for tool_name, tool in tools.items(): + tool_ignores = tool.get_ignored_codes(line_content) + for tool_ignore in tool_ignores: + tools_ignore[filepath][line_number + 1].add(Ignore(tool_name, tool_ignore)) + + # Ignore the blending messages + if blending: + _process_tool_ignores(tools_ignore, blend_combos_dict, messages_to_ignore) + # Now figure out which messages were suppressed by pylint pylint_ignore_files, pylint_ignore_messages = _parse_pylint_informational(messages) paths_to_ignore |= pylint_ignore_files @@ -149,9 +190,5 @@ def get_suppressions( for code in codes: ignore = Ignore("pylint", code) messages_to_ignore[pylint_filepath][line_number].add(ignore) - if code in _PYLINT_EQUIVALENTS: - for ignore_source, ignore_code in _PYLINT_EQUIVALENTS[code]: - ignore = Ignore(ignore_source, ignore_code) - messages_to_ignore[pylint_filepath][line_number].add(ignore) return paths_to_ignore, lines_to_ignore, messages_to_ignore diff --git a/prospector/tools/base.py b/prospector/tools/base.py index 16b17f8a..9ff30da5 100644 --- a/prospector/tools/base.py +++ b/prospector/tools/base.py @@ -40,3 +40,10 @@ def run(self, found_files: FileFinder) -> list[Message]: standard prospector Message and Location objects. """ raise NotImplementedError + + def get_ignored_codes(self, line: str) -> list[str]: + """ + Return a list of error codes that the tool will ignore from a line of code. + """ + del line # unused + return [] diff --git a/prospector/tools/mypy/__init__.py b/prospector/tools/mypy/__init__.py index 45275e45..19884cfc 100644 --- a/prospector/tools/mypy/__init__.py +++ b/prospector/tools/mypy/__init__.py @@ -1,3 +1,4 @@ +import re from multiprocessing import Process, Queue from typing import TYPE_CHECKING, Any, Callable, Optional @@ -14,6 +15,8 @@ if TYPE_CHECKING: from prospector.config import ProspectorConfig +_IGNORE_RE = re.compile(r"#\s*type:\s*ignore\[([^#]*[^# ])\](\s*#.*)?$", re.IGNORECASE) + def format_message(message: str) -> Message: character: Optional[int] @@ -105,3 +108,9 @@ def run(self, found_files: FileFinder) -> list[Message]: report, _ = result[0], result[1:] # noqa return [format_message(message) for message in report.splitlines()] + + def get_ignored_codes(self, line: str) -> list[str]: + match = _IGNORE_RE.search(line) + if match: + return match.group(1).split(",") + return [] diff --git a/prospector/tools/pylint/__init__.py b/prospector/tools/pylint/__init__.py index 6af3122d..3fca5397 100644 --- a/prospector/tools/pylint/__init__.py +++ b/prospector/tools/pylint/__init__.py @@ -21,6 +21,8 @@ _UNUSED_WILDCARD_IMPORT_RE = re.compile(r"^Unused import(\(s\))? (.*) from wildcard import") +_IGNORE_RE = re.compile(r"#\s*pylint:\s*disable=([^#]*[^#\s])(?:\s*#.*)?$", re.IGNORECASE) + def _is_in_dir(subpath: Path, path: Path) -> bool: return subpath.parent == path @@ -266,3 +268,9 @@ def run(self, found_files: FileFinder) -> list[Message]: messages = self._collector.get_messages() return self.combine(messages) + + def get_ignored_codes(self, line: str) -> list[str]: + match = _IGNORE_RE.search(line) + if match: + return match.group(1).split(",") + return [] diff --git a/prospector/tools/ruff/__init__.py b/prospector/tools/ruff/__init__.py index 3add2ea8..556549c9 100644 --- a/prospector/tools/ruff/__init__.py +++ b/prospector/tools/ruff/__init__.py @@ -1,4 +1,5 @@ import json +import re import subprocess # nosec from typing import TYPE_CHECKING, Any @@ -11,6 +12,8 @@ if TYPE_CHECKING: from prospector.config import ProspectorConfig +_IGNORE_RE = re.compile(r"#\s*noqa:([^#]*[^# ])(?:\s*#.*)?$", re.IGNORECASE) + class RuffTool(ToolBase): def configure(self, prospector_config: "ProspectorConfig", _: Any) -> None: @@ -84,3 +87,9 @@ def run(self, found_files: FileFinder) -> list[Message]: ) ) return messages + + def get_ignored_codes(self, line: str) -> list[str]: + match = _IGNORE_RE.search(line) + if match: + return match.group(1).split(",") + return [] diff --git a/tests/suppression/test_blender_suppression.py b/tests/suppression/test_blender_suppression.py new file mode 100644 index 00000000..9927fee0 --- /dev/null +++ b/tests/suppression/test_blender_suppression.py @@ -0,0 +1,60 @@ +import unittest +from pathlib import Path + +from prospector.message import Location, Message +from prospector.suppression import Ignore, get_suppressions +from prospector.tools.mypy import MypyTool +from prospector.tools.pylint import PylintTool + + +class BlenderSuppressionsTest(unittest.TestCase): + def test_blender_suppressions_pylint(self): + path = Path(__file__).parent / "testdata" / "test_blender_suppressions" / "test.py" + messages = [ + Message("pylint", "n2", Location(path, None, None, 2, 0), "message1"), + Message("other", "o2", Location(path, None, None, 2, 0), "message1"), + ] + tools = {"pylint": PylintTool()} + blend_combos = [[("pylint", "n2"), ("other", "o2")]] + + _, _, messages_to_ignore = get_suppressions([path], messages, tools, blending=False, blend_combos=blend_combos) + assert messages_to_ignore == {path: {1: {Ignore(None, "n1")}}} + + _, _, messages_to_ignore = get_suppressions([path], messages, tools, blending=True, blend_combos=blend_combos) + assert path in messages_to_ignore + assert 2 in messages_to_ignore[path] + assert messages_to_ignore[path][2] == {Ignore("pylint", "n2"), Ignore("other", "o2")} + + def test_blender_suppressions_mypy(self): + path = Path(__file__).parent / "testdata" / "test_blender_suppressions" / "test.py" + messages = [ + Message("mypy", "n3", Location(path, None, None, 3, 0), "message1"), + Message("other", "o3", Location(path, None, None, 3, 0), "message1"), + ] + tools = {"mypy": MypyTool()} + blend_combos = [[("mypy", "n3"), ("other", "o3")]] + + _, _, messages_to_ignore = get_suppressions([path], messages, tools, blending=False, blend_combos=blend_combos) + assert messages_to_ignore == {path: {1: {Ignore(None, "n1")}}} + + _, _, messages_to_ignore = get_suppressions([path], messages, tools, blending=True, blend_combos=blend_combos) + assert path in messages_to_ignore + assert 3 in messages_to_ignore[path] + assert messages_to_ignore[path][3] == {Ignore("mypy", "n3"), Ignore("other", "o3")} + + def test_blender_suppressions_ruff(self): + path = Path(__file__).parent / "testdata" / "test_blender_suppressions" / "test.py" + messages = [ + Message("ruff", "n1", Location(path, None, None, 1, 0), "message1"), + Message("other", "o1", Location(path, None, None, 1, 0), "message1"), + ] + tools = {"ruff": MypyTool()} + blend_combos = [[("ruff", "n1"), ("other", "o1")]] + + _, _, messages_to_ignore = get_suppressions([path], messages, tools, blending=False, blend_combos=blend_combos) + assert messages_to_ignore == {path: {1: {Ignore(None, "n1")}}} + + _, _, messages_to_ignore = get_suppressions([path], messages, tools, blending=True, blend_combos=blend_combos) + assert path in messages_to_ignore + assert 4 in messages_to_ignore[path] + assert messages_to_ignore[path][4] == {Ignore("ruff", "n1"), Ignore("other", "o1"), Ignore(None, "n1")} diff --git a/tests/suppression/testdata/test_blender_suppressions/test.py b/tests/suppression/testdata/test_blender_suppressions/test.py new file mode 100644 index 00000000..79ef3361 --- /dev/null +++ b/tests/suppression/testdata/test_blender_suppressions/test.py @@ -0,0 +1,3 @@ +import test # noqa: n1 +import test # pylint: disable=n2 +import test # type: ignore[n3]