Add an option to exclude everything in .gitignore (#18696)

Fixes #12505

This is (somewhat surprisingly) one of the most upvoted issues, and
looks like a simple thing to add. I essentially do what other tools do,
but optimize for how we work with sources discovery (to avoid
performance issues). I am making this opt-in for now, we can change this
later if needed.

Author: ilevkivskyi

docs/source/command_line.rst

@@ -81,6 +81,10 @@ for full details, see :ref:`running-mypy`.
     never recursively discover files with extensions other than ``.py`` or
     ``.pyi``.
 
+.. option:: --exclude-gitignore
+
+    This flag will add everything that matches ``.gitignore`` file(s) to :option:`--exclude`.
+
 
 Optional arguments
 ******************

docs/source/config_file.rst

@@ -288,6 +288,14 @@ section of the command line docs.
 
        See :ref:`using-a-pyproject-toml`.
 
+.. confval:: exclude_gitignore
+
+    :type: boolean
+    :default: False
+
+    This flag will add everything that matches ``.gitignore`` file(s) to :confval:`exclude`.
+    This option may only be set in the global section (``[mypy]``).
+
 .. confval:: namespace_packages
 
     :type: boolean

mypy-requirements.txt

@@ -2,4 +2,5 @@
 # and the pins in setup.py
 typing_extensions>=4.6.0
 mypy_extensions>=1.0.0
+pathspec>=0.9.0
 tomli>=1.1.0; python_version<'3.11'

mypy/find_sources.py

@@ -8,7 +8,13 @@
 from typing import Final
 
 from mypy.fscache import FileSystemCache
-from mypy.modulefinder import PYTHON_EXTENSIONS, BuildSource, matches_exclude, mypy_path
+from mypy.modulefinder import (
+    PYTHON_EXTENSIONS,
+    BuildSource,
+    matches_exclude,
+    matches_gitignore,
+    mypy_path,
+)
 from mypy.options import Options
 
 PY_EXTENSIONS: Final = tuple(PYTHON_EXTENSIONS)
@@ -94,6 +100,7 @@ def __init__(self, fscache: FileSystemCache, options: Options) -> None:
         self.explicit_package_bases = get_explicit_package_bases(options)
         self.namespace_packages = options.namespace_packages
         self.exclude = options.exclude
+        self.exclude_gitignore = options.exclude_gitignore
         self.verbosity = options.verbosity
 
     def is_explicit_package_base(self, path: str) -> bool:
@@ -113,6 +120,10 @@ def find_sources_in_dir(self, path: str) -> list[BuildSource]:
 
             if matches_exclude(subpath, self.exclude, self.fscache, self.verbosity >= 2):
                 continue
+            if self.exclude_gitignore and matches_gitignore(
+                subpath, self.fscache, self.verbosity >= 2
+            ):
+                continue
 
             if self.fscache.isdir(subpath):
                 sub_sources = self.find_sources_in_dir(subpath)

mypy/main.py

@@ -1252,6 +1252,15 @@ def add_invertible_flag(
             "May be specified more than once, eg. --exclude a --exclude b"
         ),
     )
+    add_invertible_flag(
+        "--exclude-gitignore",
+        default=False,
+        help=(
+            "Use .gitignore file(s) to exclude files from checking "
+            "(in addition to any explicit --exclude if present)"
+        ),
+        group=code_group,
+    )
     code_group.add_argument(
         "-m",
         "--module",

mypy/modulefinder.py

@@ -16,6 +16,9 @@
 from typing import Final, Optional, Union
 from typing_extensions import TypeAlias as _TypeAlias
 
+from pathspec import PathSpec
+from pathspec.patterns.gitwildmatch import GitWildMatchPatternError
+
 from mypy import pyinfo
 from mypy.errors import CompileError
 from mypy.fscache import FileSystemCache
@@ -625,6 +628,12 @@ def find_modules_recursive(self, module: str) -> list[BuildSource]:
                 subpath, self.options.exclude, self.fscache, self.options.verbosity >= 2
             ):
                 continue
+            if (
+                self.options
+                and self.options.exclude_gitignore
+                and matches_gitignore(subpath, self.fscache, self.options.verbosity >= 2)
+            ):
+                continue
 
             if self.fscache.isdir(subpath):
                 # Only recurse into packages
@@ -664,6 +673,42 @@ def matches_exclude(
     return False
 
 
+def matches_gitignore(subpath: str, fscache: FileSystemCache, verbose: bool) -> bool:
+    dir, _ = os.path.split(subpath)
+    for gi_path, gi_spec in find_gitignores(dir):
+        relative_path = os.path.relpath(subpath, gi_path)
+        if fscache.isdir(relative_path):
+            relative_path = relative_path + "/"
+        if gi_spec.match_file(relative_path):
+            if verbose:
+                print(
+                    f"TRACE: Excluding {relative_path} (matches .gitignore) in {gi_path}",
+                    file=sys.stderr,
+                )
+            return True
+    return False
+
+
+@functools.lru_cache
+def find_gitignores(dir: str) -> list[tuple[str, PathSpec]]:
+    parent_dir = os.path.dirname(dir)
+    if parent_dir == dir:
+        parent_gitignores = []
+    else:
+        parent_gitignores = find_gitignores(parent_dir)
+
+    gitignore = os.path.join(dir, ".gitignore")
+    if os.path.isfile(gitignore):
+        with open(gitignore) as f:
+            lines = f.readlines()
+        try:
+            return parent_gitignores + [(dir, PathSpec.from_lines("gitwildmatch", lines))]
+        except GitWildMatchPatternError:
+            print(f"error: could not parse {gitignore}", file=sys.stderr)
+            return parent_gitignores
+    return parent_gitignores
+
+
 def is_init_file(path: str) -> bool:
     return os.path.basename(path) in ("__init__.py", "__init__.pyi")
 

mypy/options.py

@@ -136,6 +136,7 @@ def __init__(self) -> None:
         self.explicit_package_bases = False
         # File names, directory names or subpaths to avoid checking
         self.exclude: list[str] = []
+        self.exclude_gitignore: bool = False
 
         # disallow_any options
         self.disallow_any_generics = False

pyproject.toml

@@ -7,6 +7,7 @@ requires = [
     # the following is from mypy-requirements.txt/setup.py
     "typing_extensions>=4.6.0",
     "mypy_extensions>=1.0.0",
+    "pathspec>=0.9.0",
     "tomli>=1.1.0; python_version<'3.11'",
     # the following is from build-requirements.txt
     "types-psutil",
@@ -49,6 +50,7 @@ dependencies = [
   # When changing this, also update build-system.requires and mypy-requirements.txt
   "typing_extensions>=4.6.0",
   "mypy_extensions>=1.0.0",
+  "pathspec>=0.9.0",
   "tomli>=1.1.0; python_version<'3.11'",
 ]
 dynamic = ["version"]

test-data/unit/cmdline.test

@@ -1135,6 +1135,21 @@ b/bpkg.py:1: error: "int" not callable
 [out]
 c/cpkg.py:1: error: "int" not callable
 
+[case testCmdlineExcludeGitignore]
+# cmd: mypy --exclude-gitignore .
+[file .gitignore]
+abc
+[file abc/apkg.py]
+1()
+[file b/.gitignore]
+bpkg.*
+[file b/bpkg.py]
+1()
+[file c/cpkg.py]
+1()
+[out]
+c/cpkg.py:1: error: "int" not callable
+
 [case testCmdlineCfgExclude]
 # cmd: mypy .
 [file mypy.ini]

test-requirements.txt

@@ -1,5 +1,5 @@
 #
-# This file is autogenerated by pip-compile with Python 3.11
+# This file is autogenerated by pip-compile with Python 3.12
 # by the following command:
 #
 #    pip-compile --allow-unsafe --output-file=test-requirements.txt --strip-extras test-requirements.in
@@ -30,6 +30,8 @@ nodeenv==1.9.1
     # via pre-commit
 packaging==24.2
     # via pytest
+pathspec==0.12.1
+    # via -r mypy-requirements.txt
 platformdirs==4.3.6
     # via virtualenv
 pluggy==1.5.0