feat: Support linting in out-of-source directories

Author: gremat

CONTRIBUTORS.txt

@@ -339,6 +339,7 @@ contributors:
 - kdestin <[email protected]>
 - jaydesl <[email protected]>
 - jab <[email protected]>
+- gremat <[email protected]>
 - gracejiang16 <[email protected]>
 - glmdgrielson <[email protected]>
 - glegoux <[email protected]>

doc/user_guide/usage/run.rst

@@ -39,7 +39,7 @@ you can give it a file name if it's possible to guess a module name from the fil
 path using the python path. Some examples:
 
 ``pylint mymodule.py`` should always work since the current working
-directory is automatically added on top of the python path
+directory is automatically added on top of the python path.
 
 ``pylint directory/mymodule.py`` will work if: ``directory`` is a python
 package (i.e. has an ``__init__.py`` file), an implicit namespace package
@@ -52,6 +52,19 @@ If the analyzed sources use implicit namespace packages (PEP 420), the source ro
 be specified using the ``--source-roots`` option. Otherwise, the package names are
 detected incorrectly, since implicit namespace packages don't contain an ``__init__.py``.
 
+In out-of-source directories
+----------------------------
+
+If you are analyzing a file that is not located under the main source directory of your
+project but needs to import modules from there, for instance and most prominantly a test
+file in ``tests/``, you can use ``--pythonpath`` to add the main source directory to the
+python path.
+For example, if your project features a directory layout with a dedicated source
+directory ``src/`` and a test directory ``tests/`` at the top level, you can use
+``--pythonpath=src`` (or the appropriate configuration setting) to successfully lint
+your tests.
+
+
 Globbing support
 ----------------
 

doc/whatsnew/fragments/9507.feature

@@ -0,0 +1,5 @@
+Support linting in out-of-source directories with new main.pythonpath argument that adds relative or absolute paths to sys.path.
+
+Refs #9507
+Refs #7357
+Refs #5644

pylint/lint/__init__.py

@@ -27,7 +27,11 @@
     report_total_messages_stats,
 )
 from pylint.lint.run import Run
-from pylint.lint.utils import _augment_sys_path, augmented_sys_path
+from pylint.lint.utils import (
+    _augment_sys_path,
+    augmented_sys_path,
+    realpath_transformer,
+)
 
 __all__ = [
     "check_parallel",
@@ -39,6 +43,7 @@
     "ArgumentPreprocessingError",
     "_augment_sys_path",
     "augmented_sys_path",
+    "realpath_transformer",
     "discover_package_path",
     "save_results",
     "load_results",

pylint/lint/base_options.py

@@ -377,6 +377,16 @@ def _make_linter_options(linter: PyLinter) -> Options:
                 ),
             },
         ),
+        (
+            "pythonpath",
+            {
+                "type": "glob_paths_csv",
+                "metavar": "<path>[,<path>...]",
+                "default": (),
+                "help": "Add paths to sys.path. Supports globbing patterns. Paths are absolute "
+                "or relative to the current working directory.",
+            },
+        ),
         (
             "ignored-modules",
             {

pylint/lint/parallel.py

@@ -36,12 +36,12 @@
 
 
 def _worker_initialize(
-    linter: bytes, extra_packages_paths: Sequence[str] | None = None
+    linter: bytes, extra_sys_paths: Sequence[str] | None = None
 ) -> None:
     """Function called to initialize a worker for a Process within a concurrent Pool.
 
     :param linter: A linter-class (PyLinter) instance pickled with dill
-    :param extra_packages_paths: Extra entries to be added to `sys.path`
+    :param extra_sys_paths: Extra entries to be added to `sys.path`
     """
     global _worker_linter  # pylint: disable=global-statement
     _worker_linter = dill.loads(linter)
@@ -57,8 +57,8 @@ def _worker_initialize(
     _worker_linter.load_plugin_modules(_worker_linter._dynamic_plugins, force=True)
     _worker_linter.load_plugin_configuration()
 
-    if extra_packages_paths:
-        _augment_sys_path(extra_packages_paths)
+    if extra_sys_paths:
+        _augment_sys_path(extra_sys_paths)
 
 
 def _worker_check_single_file(
@@ -125,7 +125,7 @@ def check_parallel(
     linter: PyLinter,
     jobs: int,
     files: Iterable[FileItem],
-    extra_packages_paths: Sequence[str] | None = None,
+    extra_sys_paths: Sequence[str] | None = None,
 ) -> None:
     """Use the given linter to lint the files with given amount of workers (jobs).
 
@@ -136,7 +136,7 @@ def check_parallel(
     # is identical to the linter object here. This is required so that
     # a custom PyLinter object can be used.
     initializer = functools.partial(
-        _worker_initialize, extra_packages_paths=extra_packages_paths
+        _worker_initialize, extra_sys_paths=extra_sys_paths
     )
     with ProcessPoolExecutor(
         max_workers=jobs, initializer=initializer, initargs=(dill.dumps(linter),)

pylint/lint/pylinter.py

@@ -50,6 +50,7 @@
 from pylint.lint.utils import (
     _is_relative_to,
     augmented_sys_path,
+    realpath_transformer,
     get_fatal_error_message,
     prepare_crash_report,
 )
@@ -74,7 +75,8 @@
 class GetAstProtocol(Protocol):
     def __call__(
         self, filepath: str, modname: str, data: str | None = None
-    ) -> nodes.Module: ...
+    ) -> nodes.Module:
+        ...
 
 
 def _read_stdin() -> str:
@@ -308,9 +310,9 @@ def __init__(
         """Dictionary of possible but non-initialized reporters."""
 
         # Attributes for checkers and plugins
-        self._checkers: defaultdict[str, list[checkers.BaseChecker]] = (
-            collections.defaultdict(list)
-        )
+        self._checkers: defaultdict[
+            str, list[checkers.BaseChecker]
+        ] = collections.defaultdict(list)
         """Dictionary of registered and initialized checkers."""
         self._dynamic_plugins: dict[str, ModuleType | ModuleNotFoundError | bool] = {}
         """Set of loaded plugin names."""
@@ -671,6 +673,8 @@ def check(self, files_or_modules: Sequence[str]) -> None:
                 for file_or_module in files_or_modules
             }
         )
+        # Prefer package paths detected per module over user-defined PYTHONPATH additions
+        extra_sys_paths = extra_packages_paths + realpath_transformer(self.config.pythonpath)
 
         # TODO: Move the parallel invocation into step 3 of the checking process
         if not self.config.from_stdin and self.config.jobs > 1:
@@ -679,13 +683,13 @@ def check(self, files_or_modules: Sequence[str]) -> None:
                 self,
                 self.config.jobs,
                 self._iterate_file_descrs(files_or_modules),
-                extra_packages_paths,
+                extra_sys_paths,
             )
             sys.path = original_sys_path
             return
 
         # 1) Get all FileItems
-        with augmented_sys_path(extra_packages_paths):
+        with augmented_sys_path(extra_sys_paths):
             if self.config.from_stdin:
                 fileitems = self._get_file_descr_from_stdin(files_or_modules[0])
                 data: str | None = _read_stdin()
@@ -694,7 +698,7 @@ def check(self, files_or_modules: Sequence[str]) -> None:
                 data = None
 
         # The contextmanager also opens all checkers and sets up the PyLinter class
-        with augmented_sys_path(extra_packages_paths):
+        with augmented_sys_path(extra_sys_paths):
             with self._astroid_module_checker() as check_astroid_module:
                 # 2) Get the AST for each FileItem
                 ast_per_fileitem = self._get_asts(fileitems, data)

pylint/lint/utils.py

@@ -112,6 +112,11 @@ def get_fatal_error_message(filepath: str, issue_template_path: Path) -> str:
     )
 
 
+def realpath_transformer(paths: Sequence[str]) -> list[str]:
+    """Transforms paths to real paths while expanding user vars."""
+    return [str(Path(path).resolve().expanduser()) for path in paths]
+
+
 def _augment_sys_path(additional_paths: Sequence[str]) -> list[str]:
     original = list(sys.path)
     changes = []

pylint/pyreverse/main.py

@@ -14,7 +14,7 @@
 from pylint.config.arguments_manager import _ArgumentsManager
 from pylint.config.arguments_provider import _ArgumentsProvider
 from pylint.lint import discover_package_path
-from pylint.lint.utils import augmented_sys_path
+from pylint.lint.utils import augmented_sys_path, realpath_transformer
 from pylint.pyreverse import writer
 from pylint.pyreverse.diadefslib import DiadefsHandler
 from pylint.pyreverse.inspector import Linker, project_from_files
@@ -304,7 +304,9 @@ def run(self, args: list[str]) -> int:
         extra_packages_paths = list(
             {discover_package_path(arg, self.config.source_roots) for arg in args}
         )
-        with augmented_sys_path(extra_packages_paths):
+        # Prefer package paths detected per module over global PYTHONPATH additons
+        extra_sys_paths = extra_packages_paths + realpath_transformer(self.config.pythonpath)
+        with augmented_sys_path(extra_sys_paths):
             project = project_from_files(
                 args,
                 project_name=self.config.project,

tests/lint/unittest_lint.py

@@ -1250,6 +1250,31 @@ def test_import_sibling_module_from_namespace(initialized_linter: PyLinter) -> N
     assert not linter.stats.by_msg
 
 
+def test_import_external_module_with_relative_pythonpath_config(initialized_linter: PyLinter) -> None:
+    """Given a module that imports an external module, ensure that the external module
+    is found when the path to the external module is configured in `main.pythonpath`.
+
+    Note: The setup is similar to `test_import_sibling_module_from_namespace` but the
+    manual sys.path setup is replaced with a `main.pythonpath` configuration.
+    """
+    linter = initialized_linter
+    with tempdir() as tmpdir:
+        create_files(["namespace_main/module.py", "namespace_ext/ext_module.py"])
+        main_path = Path("namespace_main/module.py")
+        with open(main_path, "w", encoding="utf-8") as f:
+            f.write(
+                """\"\"\"This module imports ext_module.\"\"\"
+import ext_module
+print(ext_module)
+"""
+            )
+
+        os.chdir(tmpdir)
+        linter.config.pythonpath = ["namespace_ext"]
+        linter.check(["namespace_main/module.py"])
+    assert not linter.stats.by_msg
+
+
 def test_lint_namespace_package_under_dir(initialized_linter: PyLinter) -> None:
     """Regression test for https://github.com/pylint-dev/pylint/issues/1667."""
     linter = initialized_linter

tests/test_check_parallel.py

@@ -187,7 +187,7 @@ def test_worker_initialize_with_package_paths(self) -> None:
         linter = PyLinter(reporter=Reporter())
         with augmented_sys_path([]):
             worker_initialize(
-                linter=dill.dumps(linter), extra_packages_paths=["fake-path"]
+                linter=dill.dumps(linter), extra_sys_paths=["fake-path"]
             )
             assert "fake-path" in sys.path
 
@@ -629,7 +629,7 @@ def test_no_deadlock_due_to_initializer_error(self) -> None:
                 files=iter(single_file_container),
                 # This will trigger an exception in the initializer for the parallel jobs
                 # because arguments has to be an Iterable.
-                extra_packages_paths=1,  # type: ignore[arg-type]
+                extra_sys_paths=1,  # type: ignore[arg-type]
             )
 
     @pytest.mark.needs_two_cores