From b12ddb7eba1bbd042c3fe1d6ca6e89bc2b409ced Mon Sep 17 00:00:00 2001 From: Christian Heimes Date: Mon, 25 Jul 2022 22:22:13 +0200 Subject: [PATCH 1/2] gh-85454: Extract and inject distutils from setuptools whl Extract the wheel from ensurepip's bundle and inject setuptools and distutils into sys.path. --- Lib/test/support/__init__.py | 34 +--------------- Lib/test/support/import_helper.py | 43 ++++++++++++++++++++ Lib/test/test_peg_generator/test_c_parser.py | 36 +++++++++++++++- Tools/peg_generator/pegen/build.py | 36 +++++++++++++++- 4 files changed, 114 insertions(+), 35 deletions(-) diff --git a/Lib/test/support/__init__.py b/Lib/test/support/__init__.py index c51a1f26f29a6b..c79c480f318dd1 100644 --- a/Lib/test/support/__init__.py +++ b/Lib/test/support/__init__.py @@ -56,7 +56,7 @@ # miscellaneous "run_with_locale", "swap_item", "findfile", "infinite_recursion", "swap_attr", "Matcher", "set_memlimit", "SuppressCrashReport", "sortdict", - "run_with_tz", "PGO", "missing_compiler_executable", + "run_with_tz", "PGO", "ALWAYS_EQ", "NEVER_EQ", "LARGEST", "SMALLEST", "LOOPBACK_TIMEOUT", "INTERNET_TIMEOUT", "SHORT_TIMEOUT", "LONG_TIMEOUT", "Py_DEBUG", @@ -1817,38 +1817,6 @@ def __del__(self): test.assertTrue(done) -def missing_compiler_executable(cmd_names=[]): - """Check if the compiler components used to build the interpreter exist. - - Check for the existence of the compiler executables whose names are listed - in 'cmd_names' or all the compiler executables when 'cmd_names' is empty - and return the first missing executable or None when none is found - missing. - - """ - # TODO (PEP 632): alternate check without using distutils - from distutils import ccompiler, sysconfig, spawn, errors - compiler = ccompiler.new_compiler() - sysconfig.customize_compiler(compiler) - if compiler.compiler_type == "msvc": - # MSVC has no executables, so check whether initialization succeeds - try: - compiler.initialize() - except errors.DistutilsPlatformError: - return "msvc" - for name in compiler.executables: - if cmd_names and name not in cmd_names: - continue - cmd = getattr(compiler, name) - if cmd_names: - assert cmd is not None, \ - "the '%s' executable is not configured" % name - elif not cmd: - continue - if spawn.find_executable(cmd[0]) is None: - return cmd[0] - - _is_android_emulator = None def setswitchinterval(interval): # Setting a very low gil interval on the Android emulator causes python diff --git a/Lib/test/support/import_helper.py b/Lib/test/support/import_helper.py index 5201dc84cf6df4..ae01d4b1857858 100644 --- a/Lib/test/support/import_helper.py +++ b/Lib/test/support/import_helper.py @@ -1,14 +1,20 @@ +import tempfile import contextlib +import ensurepip import _imp import importlib import importlib.util +import importlib.resources import os +import site import shutil import sys import unittest import warnings +import zipfile from .os_helper import unlink +from . import requires_zlib @contextlib.contextmanager @@ -246,3 +252,40 @@ def modules_cleanup(oldmodules): # do currently). Implicitly imported *real* modules should be left alone # (see issue 10556). sys.modules.update(oldmodules) + + +@requires_zlib() +@contextlib.contextmanager +def inject_setuptools(): + sud = "SETUPTOOLS_USE_DISTUTILS" + package = ensurepip._get_packages()["setuptools"] + if package.wheel_name: + # Use bundled wheel package + ensurepip_res = importlib.resources.files("ensurepip") + wheel_path = ensurepip_res / "_bundled" / package.wheel_name + else: + wheel_path = package.wheel_path + orig_path = sys.path[:] + if sud in os.environ: + orig_sud = os.environ[sud] + else: + orig_sud = None + os.environ[sud] = "local" + tmpdir = tempfile.mkdtemp() + try: + zf = zipfile.ZipFile(wheel_path) + zf.extractall(tmpdir) + site.addsitedir(tmpdir) + import setuptools._distutils + sys.modules["distutils"] = setuptools._distutils + yield + finally: + sys.path[:] = orig_path + if orig_sud is not None: + os.environ[sud] = orig_sud + else: + os.environ.pop(sud) + shutil.rmtree(tmpdir) + for name in list(sys.modules): + if name.startswith(("setuptools", "distutils", "pkg_resources")): + forget(name) diff --git a/Lib/test/test_peg_generator/test_c_parser.py b/Lib/test/test_peg_generator/test_c_parser.py index 1b3fcbb92f8292..d38d61d4209efa 100644 --- a/Lib/test/test_peg_generator/test_c_parser.py +++ b/Lib/test/test_peg_generator/test_c_parser.py @@ -8,6 +8,7 @@ from test import test_tools from test import support +from test.support import import_helper from test.support import os_helper from test.support.script_helper import assert_python_ok @@ -88,10 +89,11 @@ def setUpClass(cls): # runtime overhead of spawning compiler processes. cls.library_dir = tempfile.mkdtemp(dir=cls.tmp_base) cls.addClassCleanup(shutil.rmtree, cls.library_dir) + cls.enterClassContext(import_helper.inject_setuptools()) def setUp(self): self._backup_config_vars = dict(sysconfig._CONFIG_VARS) - cmd = support.missing_compiler_executable() + cmd = self.missing_compiler_executable() if cmd is not None: self.skipTest("The %r command is not found" % cmd) self.old_cwd = os.getcwd() @@ -104,6 +106,38 @@ def tearDown(self): sysconfig._CONFIG_VARS.clear() sysconfig._CONFIG_VARS.update(self._backup_config_vars) + def missing_compiler_executable(self, cmd_names=()): + """Check if the compiler components used to build the interpreter exist. + + Check for the existence of the compiler executables whose names are listed + in 'cmd_names' or all the compiler executables when 'cmd_names' is empty + and return the first missing executable or None when none is found + missing. + + """ + # TODO (PEP 632): alternate check without using distutils + # uses distutils from setuptools + from distutils import ccompiler, sysconfig, spawn, errors + compiler = ccompiler.new_compiler() + sysconfig.customize_compiler(compiler) + if compiler.compiler_type == "msvc": + # MSVC has no executables, so check whether initialization succeeds + try: + compiler.initialize() + except errors.DistutilsPlatformError: + return "msvc" + for name in compiler.executables: + if cmd_names and name not in cmd_names: + continue + cmd = getattr(compiler, name) + if cmd_names: + assert cmd is not None, \ + "the '%s' executable is not configured" % name + elif not cmd: + continue + if spawn.find_executable(cmd[0]) is None: + return cmd[0] + def build_extension(self, grammar_source): grammar = parse_string(grammar_source, GrammarParser) # Because setUp() already changes the current directory to the diff --git a/Tools/peg_generator/pegen/build.py b/Tools/peg_generator/pegen/build.py index 5805ff63717440..30ff719a1c5af7 100644 --- a/Tools/peg_generator/pegen/build.py +++ b/Tools/peg_generator/pegen/build.py @@ -1,4 +1,5 @@ import itertools +import os import pathlib import sys import sysconfig @@ -19,6 +20,40 @@ TokenDefinitions = Tuple[Dict[int, str], Dict[str, int], Set[str]] +def fixup_build_ext(cmd) -> None: + """Function needed to make build_ext tests pass. + When Python was built with --enable-shared on Unix, -L. is not enough to + find libpython.so, because regrtest runs in a tempdir, not in the + source directory where the .so lives. + When Python was built with in debug mode on Windows, build_ext commands + need their debug attribute set, and it is not done automatically for + some reason. + This function handles both of these things. Example use: + cmd = build_ext(dist) + support.fixup_build_ext(cmd) + cmd.ensure_finalized() + Unlike most other Unix platforms, Mac OS X embeds absolute paths + to shared libraries into executables, so the fixup is not needed there. + + Copied from distutils.tests.support. + """ + if os.name == 'nt': + cmd.debug = sys.executable.endswith('_d.exe') + elif sysconfig.get_config_var('Py_ENABLE_SHARED'): + # To further add to the shared builds fun on Unix, we can't just add + # library_dirs to the Extension() instance because that doesn't get + # plumbed through to the final compiler command. + runshared = sysconfig.get_config_var('RUNSHARED') + if runshared is None: + cmd.library_dirs = ['.'] + else: + if sys.platform == 'darwin': + cmd.library_dirs = [] + else: + name, equals, value = runshared.partition('=') + cmd.library_dirs = [d for d in value.split(os.pathsep) if d] + + def get_extra_flags(compiler_flags: str, compiler_py_flags_nodist: str) -> List[str]: flags = sysconfig.get_config_var(compiler_flags) py_flags_nodist = sysconfig.get_config_var(compiler_py_flags_nodist) @@ -51,7 +86,6 @@ def compile_c_extension( """ import distutils.log from distutils.core import Distribution, Extension - from distutils.tests.support import fixup_build_ext # type: ignore from distutils.ccompiler import new_compiler from distutils.dep_util import newer_group From 9a7222012f31386bc8fbf2a5fa742ef3abe1bbf6 Mon Sep 17 00:00:00 2001 From: Christian Heimes Date: Tue, 26 Jul 2022 07:27:47 +0200 Subject: [PATCH 2/2] distutils tests need missing_compiler_executable --- Lib/test/support/__init__.py | 35 ++++++++++++++++++++++++++++++++++- 1 file changed, 34 insertions(+), 1 deletion(-) diff --git a/Lib/test/support/__init__.py b/Lib/test/support/__init__.py index c79c480f318dd1..281900b1ab035a 100644 --- a/Lib/test/support/__init__.py +++ b/Lib/test/support/__init__.py @@ -56,7 +56,7 @@ # miscellaneous "run_with_locale", "swap_item", "findfile", "infinite_recursion", "swap_attr", "Matcher", "set_memlimit", "SuppressCrashReport", "sortdict", - "run_with_tz", "PGO", + "run_with_tz", "PGO", "missing_compiler_executable", "ALWAYS_EQ", "NEVER_EQ", "LARGEST", "SMALLEST", "LOOPBACK_TIMEOUT", "INTERNET_TIMEOUT", "SHORT_TIMEOUT", "LONG_TIMEOUT", "Py_DEBUG", @@ -1817,6 +1817,39 @@ def __del__(self): test.assertTrue(done) +def missing_compiler_executable(cmd_names=[]): + """Check if the compiler components used to build the interpreter exist. + + Check for the existence of the compiler executables whose names are listed + in 'cmd_names' or all the compiler executables when 'cmd_names' is empty + and return the first missing executable or None when none is found + missing. + + """ + # TODO (PEP 632): alternate check without using distutils + # Remove with Lib/distutils/tests + from distutils import ccompiler, sysconfig, spawn, errors + compiler = ccompiler.new_compiler() + sysconfig.customize_compiler(compiler) + if compiler.compiler_type == "msvc": + # MSVC has no executables, so check whether initialization succeeds + try: + compiler.initialize() + except errors.DistutilsPlatformError: + return "msvc" + for name in compiler.executables: + if cmd_names and name not in cmd_names: + continue + cmd = getattr(compiler, name) + if cmd_names: + assert cmd is not None, \ + "the '%s' executable is not configured" % name + elif not cmd: + continue + if spawn.find_executable(cmd[0]) is None: + return cmd[0] + + _is_android_emulator = None def setswitchinterval(interval): # Setting a very low gil interval on the Android emulator causes python