diff --git a/Lib/test/support/__init__.py b/Lib/test/support/__init__.py index c51a1f26f29a6b..281900b1ab035a 100644 --- a/Lib/test/support/__init__.py +++ b/Lib/test/support/__init__.py @@ -1827,6 +1827,7 @@ def missing_compiler_executable(cmd_names=[]): """ # TODO (PEP 632): alternate check without using distutils + # Remove with Lib/distutils/tests from distutils import ccompiler, sysconfig, spawn, errors compiler = ccompiler.new_compiler() sysconfig.customize_compiler(compiler) diff --git a/Lib/test/support/import_helper.py b/Lib/test/support/import_helper.py index 5201dc84cf6df4..ae01d4b1857858 100644 --- a/Lib/test/support/import_helper.py +++ b/Lib/test/support/import_helper.py @@ -1,14 +1,20 @@ +import tempfile import contextlib +import ensurepip import _imp import importlib import importlib.util +import importlib.resources import os +import site import shutil import sys import unittest import warnings +import zipfile from .os_helper import unlink +from . import requires_zlib @contextlib.contextmanager @@ -246,3 +252,40 @@ def modules_cleanup(oldmodules): # do currently). Implicitly imported *real* modules should be left alone # (see issue 10556). sys.modules.update(oldmodules) + + +@requires_zlib() +@contextlib.contextmanager +def inject_setuptools(): + sud = "SETUPTOOLS_USE_DISTUTILS" + package = ensurepip._get_packages()["setuptools"] + if package.wheel_name: + # Use bundled wheel package + ensurepip_res = importlib.resources.files("ensurepip") + wheel_path = ensurepip_res / "_bundled" / package.wheel_name + else: + wheel_path = package.wheel_path + orig_path = sys.path[:] + if sud in os.environ: + orig_sud = os.environ[sud] + else: + orig_sud = None + os.environ[sud] = "local" + tmpdir = tempfile.mkdtemp() + try: + zf = zipfile.ZipFile(wheel_path) + zf.extractall(tmpdir) + site.addsitedir(tmpdir) + import setuptools._distutils + sys.modules["distutils"] = setuptools._distutils + yield + finally: + sys.path[:] = orig_path + if orig_sud is not None: + os.environ[sud] = orig_sud + else: + os.environ.pop(sud) + shutil.rmtree(tmpdir) + for name in list(sys.modules): + if name.startswith(("setuptools", "distutils", "pkg_resources")): + forget(name) diff --git a/Lib/test/test_peg_generator/test_c_parser.py b/Lib/test/test_peg_generator/test_c_parser.py index 1b3fcbb92f8292..d38d61d4209efa 100644 --- a/Lib/test/test_peg_generator/test_c_parser.py +++ b/Lib/test/test_peg_generator/test_c_parser.py @@ -8,6 +8,7 @@ from test import test_tools from test import support +from test.support import import_helper from test.support import os_helper from test.support.script_helper import assert_python_ok @@ -88,10 +89,11 @@ def setUpClass(cls): # runtime overhead of spawning compiler processes. cls.library_dir = tempfile.mkdtemp(dir=cls.tmp_base) cls.addClassCleanup(shutil.rmtree, cls.library_dir) + cls.enterClassContext(import_helper.inject_setuptools()) def setUp(self): self._backup_config_vars = dict(sysconfig._CONFIG_VARS) - cmd = support.missing_compiler_executable() + cmd = self.missing_compiler_executable() if cmd is not None: self.skipTest("The %r command is not found" % cmd) self.old_cwd = os.getcwd() @@ -104,6 +106,38 @@ def tearDown(self): sysconfig._CONFIG_VARS.clear() sysconfig._CONFIG_VARS.update(self._backup_config_vars) + def missing_compiler_executable(self, cmd_names=()): + """Check if the compiler components used to build the interpreter exist. + + Check for the existence of the compiler executables whose names are listed + in 'cmd_names' or all the compiler executables when 'cmd_names' is empty + and return the first missing executable or None when none is found + missing. + + """ + # TODO (PEP 632): alternate check without using distutils + # uses distutils from setuptools + from distutils import ccompiler, sysconfig, spawn, errors + compiler = ccompiler.new_compiler() + sysconfig.customize_compiler(compiler) + if compiler.compiler_type == "msvc": + # MSVC has no executables, so check whether initialization succeeds + try: + compiler.initialize() + except errors.DistutilsPlatformError: + return "msvc" + for name in compiler.executables: + if cmd_names and name not in cmd_names: + continue + cmd = getattr(compiler, name) + if cmd_names: + assert cmd is not None, \ + "the '%s' executable is not configured" % name + elif not cmd: + continue + if spawn.find_executable(cmd[0]) is None: + return cmd[0] + def build_extension(self, grammar_source): grammar = parse_string(grammar_source, GrammarParser) # Because setUp() already changes the current directory to the diff --git a/Tools/peg_generator/pegen/build.py b/Tools/peg_generator/pegen/build.py index 5805ff63717440..30ff719a1c5af7 100644 --- a/Tools/peg_generator/pegen/build.py +++ b/Tools/peg_generator/pegen/build.py @@ -1,4 +1,5 @@ import itertools +import os import pathlib import sys import sysconfig @@ -19,6 +20,40 @@ TokenDefinitions = Tuple[Dict[int, str], Dict[str, int], Set[str]] +def fixup_build_ext(cmd) -> None: + """Function needed to make build_ext tests pass. + When Python was built with --enable-shared on Unix, -L. is not enough to + find libpython.so, because regrtest runs in a tempdir, not in the + source directory where the .so lives. + When Python was built with in debug mode on Windows, build_ext commands + need their debug attribute set, and it is not done automatically for + some reason. + This function handles both of these things. Example use: + cmd = build_ext(dist) + support.fixup_build_ext(cmd) + cmd.ensure_finalized() + Unlike most other Unix platforms, Mac OS X embeds absolute paths + to shared libraries into executables, so the fixup is not needed there. + + Copied from distutils.tests.support. + """ + if os.name == 'nt': + cmd.debug = sys.executable.endswith('_d.exe') + elif sysconfig.get_config_var('Py_ENABLE_SHARED'): + # To further add to the shared builds fun on Unix, we can't just add + # library_dirs to the Extension() instance because that doesn't get + # plumbed through to the final compiler command. + runshared = sysconfig.get_config_var('RUNSHARED') + if runshared is None: + cmd.library_dirs = ['.'] + else: + if sys.platform == 'darwin': + cmd.library_dirs = [] + else: + name, equals, value = runshared.partition('=') + cmd.library_dirs = [d for d in value.split(os.pathsep) if d] + + def get_extra_flags(compiler_flags: str, compiler_py_flags_nodist: str) -> List[str]: flags = sysconfig.get_config_var(compiler_flags) py_flags_nodist = sysconfig.get_config_var(compiler_py_flags_nodist) @@ -51,7 +86,6 @@ def compile_c_extension( """ import distutils.log from distutils.core import Distribution, Extension - from distutils.tests.support import fixup_build_ext # type: ignore from distutils.ccompiler import new_compiler from distutils.dep_util import newer_group