From 6daba07ad6477fb98b93c8db0cf6ad7608f878bb Mon Sep 17 00:00:00 2001 From: Henry Schreiner Date: Fri, 2 Jun 2023 17:21:14 -0400 Subject: [PATCH 01/12] fix: custom cross compiling environment Signed-off-by: Henry Schreiner --- src/scikit_build_core/builder/builder.py | 10 ++- .../builder/cross_compile.py | 80 +++++++++++++++++++ .../resources/_cross_compile.py | 7 ++ 3 files changed, 93 insertions(+), 4 deletions(-) create mode 100644 src/scikit_build_core/builder/cross_compile.py create mode 100644 src/scikit_build_core/resources/_cross_compile.py diff --git a/src/scikit_build_core/builder/builder.py b/src/scikit_build_core/builder/builder.py index 395466a7..3c5d97a0 100644 --- a/src/scikit_build_core/builder/builder.py +++ b/src/scikit_build_core/builder/builder.py @@ -15,6 +15,7 @@ from ..cmake import CMaker from ..resources import find_python from ..settings.skbuild_model import ScikitBuildSettings +from .cross_compile import auto_cross_compile_env from .generator import set_environment_for_gen from .sysconfig import ( get_platform, @@ -183,10 +184,11 @@ def configure( # Add the pre-defined or passed CMake defines cmake_defines.update(self.settings.cmake.define) - self.config.configure( - defines=cmake_defines, - cmake_args=[*self.get_cmake_args(), *configure_args], - ) + with auto_cross_compile_env(self.config.env): + self.config.configure( + defines=cmake_defines, + cmake_args=[*self.get_cmake_args(), *configure_args], + ) def build(self, build_args: list[str]) -> None: self.config.build(build_args=build_args, verbose=self.settings.cmake.verbose) diff --git a/src/scikit_build_core/builder/cross_compile.py b/src/scikit_build_core/builder/cross_compile.py new file mode 100644 index 00000000..87ef3618 --- /dev/null +++ b/src/scikit_build_core/builder/cross_compile.py @@ -0,0 +1,80 @@ +from __future__ import annotations + +import contextlib +import os +import string +import sysconfig +import tempfile +from collections.abc import Generator, MutableMapping +from pathlib import Path + +from .._logging import logger +from ..resources import resources + +__all__ = ["set_cross_compile_env"] + + +def __dir__() -> list[str]: + return __all__ + + +@contextlib.contextmanager +def auto_cross_compile_env( + env: MutableMapping[str, str] +) -> Generator[None, None, None]: + if "SETUPTOOLS_EXT_SUFFIX" not in env: + yield + return + + with set_cross_compile_env(env["SETUPTOOLS_EXT_SUFFIX"], env): + yield + + +@contextlib.contextmanager +def set_cross_compile_env( + ext_suffix: str, env: MutableMapping[str, str] +) -> Generator[None, None, None]: + """ + Generate python file and set environment variables to cross-compile Python + extensions. Do not call if _PYTHON_SYSCONFIGDATA_NAME is already set. + """ + + if "_PYTHON_SYSCONFIGDATA_NAME" in env: + logger.debug( + "Not setting up cross compiling explicitly due to _PYTHON_SYSCONFIGDATA_NAME already set." + ) + yield + return + + if getattr(sysconfig, "_get_sysconfigdata_name", ""): + logger.warning( + "Cross-compiling is not supported due to sysconfig._get_sysconfigdata_name missing." + ) + yield + return + + with tempfile.TemporaryDirectory() as tmpdir: + tmp_dir = Path(tmpdir).resolve() + cross_compile_file = tmp_dir / f"_cross_compile_{ext_suffix}.py" + input_txt = resources.read_text("_cross_compile.py") + output_text = string.Template(input_txt).substitute( + host_name=sysconfig._get_sysconfigdata_name(), # type: ignore[attr-defined] + SOABI=ext_suffix.rsplit(maxsplit=1)[0], + EXT_SUFFIX=ext_suffix, + ) + cross_compile_file.write_text(output_text) + current_path = env.get("PYTHONPATH", "") + env["PYTHONPATH"] = ( + os.pathsep.join([current_path, str(tmp_dir)]) + if current_path + else str(tmp_dir) + ) + env["_PYTHON_SYSCONFIGDATA_NAME"] = f"_cross_compile_{ext_suffix}.py" + try: + yield + finally: + del env["_PYTHON_SYSCONFIGDATA_NAME"] + if current_path: + env["PYTHONPATH"] = current_path + else: + del env["PYTHONPATH"] diff --git a/src/scikit_build_core/resources/_cross_compile.py b/src/scikit_build_core/resources/_cross_compile.py new file mode 100644 index 00000000..5d116466 --- /dev/null +++ b/src/scikit_build_core/resources/_cross_compile.py @@ -0,0 +1,7 @@ +# Cross compiling activation template + +host_name = "${host_name}" + +build_time_vars = __import__("${host_name}").build_time_vars.copy() +build_time_vars["SOABI"] = "${SOABI}" +build_time_vars["EXT_SUFFIX"] = "${EXT_SUFFIX}" From 9b3c3bcaba449b3213f9e5de20f7b8a7408b5605 Mon Sep 17 00:00:00 2001 From: Henry Schreiner Date: Fri, 2 Jun 2023 17:38:16 -0400 Subject: [PATCH 02/12] fix: more logging Signed-off-by: Henry Schreiner --- src/scikit_build_core/builder/cross_compile.py | 10 ++++++++-- tests/test_module_dir.py | 12 ++++++++++-- 2 files changed, 18 insertions(+), 4 deletions(-) diff --git a/src/scikit_build_core/builder/cross_compile.py b/src/scikit_build_core/builder/cross_compile.py index 87ef3618..de720a05 100644 --- a/src/scikit_build_core/builder/cross_compile.py +++ b/src/scikit_build_core/builder/cross_compile.py @@ -46,7 +46,8 @@ def set_cross_compile_env( yield return - if getattr(sysconfig, "_get_sysconfigdata_name", ""): + sysconf_name = getattr(sysconfig, "_get_sysconfigdata_name", "") + if not sysconf_name: logger.warning( "Cross-compiling is not supported due to sysconfig._get_sysconfigdata_name missing." ) @@ -58,7 +59,7 @@ def set_cross_compile_env( cross_compile_file = tmp_dir / f"_cross_compile_{ext_suffix}.py" input_txt = resources.read_text("_cross_compile.py") output_text = string.Template(input_txt).substitute( - host_name=sysconfig._get_sysconfigdata_name(), # type: ignore[attr-defined] + host_name=sysconf_name, SOABI=ext_suffix.rsplit(maxsplit=1)[0], EXT_SUFFIX=ext_suffix, ) @@ -70,6 +71,11 @@ def set_cross_compile_env( else str(tmp_dir) ) env["_PYTHON_SYSCONFIGDATA_NAME"] = f"_cross_compile_{ext_suffix}.py" + logger.info(f"Cross-compiling is enabled to {ext_suffix!r}.") + logger.debug( + f"Setting _PYTHON_SYSCONFIGDATA_NAME to {env['_PYTHON_SYSCONFIGDATA_NAME']!r}." + ) + logger.debug(f"Setting PYTHONPATH to {env['PYTHONPATH']!r}.") try: yield finally: diff --git a/tests/test_module_dir.py b/tests/test_module_dir.py index 39d3c65a..f9a7013b 100644 --- a/tests/test_module_dir.py +++ b/tests/test_module_dir.py @@ -29,7 +29,11 @@ def on_all_modules( def test_all_modules_filter_all(): all_modules = on_all_modules("scikit_build_core", pkg=False) - all_modules = (n for n in all_modules if not n.split(".")[-1].startswith("__")) + all_modules = ( + n + for n in all_modules + if not n.split(".")[-1].startswith("__") and "resources" not in n + ) for name in all_modules: module = importlib.import_module(name) @@ -45,7 +49,11 @@ def test_all_modules_filter_all(): def test_all_modules_has_all(): all_modules = on_all_modules("scikit_build_core", pkg=True) - all_modules = (n for n in all_modules if not n.split(".")[-1].startswith("_")) + all_modules = ( + n + for n in all_modules + if not n.split(".")[-1].startswith("_") and "resources" not in n + ) for name in all_modules: module = importlib.import_module(name) From db3ac569f75e774bb1c3757aac2b283cd819e95b Mon Sep 17 00:00:00 2001 From: Henry Schreiner Date: Fri, 2 Jun 2023 18:49:47 -0400 Subject: [PATCH 03/12] fix: path correction Signed-off-by: Henry Schreiner --- src/scikit_build_core/builder/cross_compile.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/scikit_build_core/builder/cross_compile.py b/src/scikit_build_core/builder/cross_compile.py index de720a05..674c781d 100644 --- a/src/scikit_build_core/builder/cross_compile.py +++ b/src/scikit_build_core/builder/cross_compile.py @@ -57,7 +57,7 @@ def set_cross_compile_env( with tempfile.TemporaryDirectory() as tmpdir: tmp_dir = Path(tmpdir).resolve() cross_compile_file = tmp_dir / f"_cross_compile_{ext_suffix}.py" - input_txt = resources.read_text("_cross_compile.py") + input_txt = resources.joinpath("_cross_compile.py").read_text(encoding="utf-8") output_text = string.Template(input_txt).substitute( host_name=sysconf_name, SOABI=ext_suffix.rsplit(maxsplit=1)[0], From 4383a28e039a65b7ccda583760a43abc7f8b3498 Mon Sep 17 00:00:00 2001 From: Henry Schreiner Date: Fri, 2 Jun 2023 19:15:33 -0400 Subject: [PATCH 04/12] fix: corrected file name Signed-off-by: Henry Schreiner --- docs/api/scikit_build_core.builder.rst | 8 ++++++++ src/scikit_build_core/builder/cross_compile.py | 6 ++++-- 2 files changed, 12 insertions(+), 2 deletions(-) diff --git a/docs/api/scikit_build_core.builder.rst b/docs/api/scikit_build_core.builder.rst index 79ef03b8..8db6bb42 100644 --- a/docs/api/scikit_build_core.builder.rst +++ b/docs/api/scikit_build_core.builder.rst @@ -17,6 +17,14 @@ scikit\_build\_core.builder.builder module :undoc-members: :show-inheritance: +scikit\_build\_core.builder.cross\_compile module +------------------------------------------------- + +.. automodule:: scikit_build_core.builder.cross_compile + :members: + :undoc-members: + :show-inheritance: + scikit\_build\_core.builder.generator module -------------------------------------------- diff --git a/src/scikit_build_core/builder/cross_compile.py b/src/scikit_build_core/builder/cross_compile.py index 674c781d..11381119 100644 --- a/src/scikit_build_core/builder/cross_compile.py +++ b/src/scikit_build_core/builder/cross_compile.py @@ -56,7 +56,9 @@ def set_cross_compile_env( with tempfile.TemporaryDirectory() as tmpdir: tmp_dir = Path(tmpdir).resolve() - cross_compile_file = tmp_dir / f"_cross_compile_{ext_suffix}.py" + cross_compile_file = ( + tmp_dir / f"_cross_compile_{ext_suffix.replace('.', '_')}.py" + ) input_txt = resources.joinpath("_cross_compile.py").read_text(encoding="utf-8") output_text = string.Template(input_txt).substitute( host_name=sysconf_name, @@ -70,7 +72,7 @@ def set_cross_compile_env( if current_path else str(tmp_dir) ) - env["_PYTHON_SYSCONFIGDATA_NAME"] = f"_cross_compile_{ext_suffix}.py" + env["_PYTHON_SYSCONFIGDATA_NAME"] = cross_compile_file.stem logger.info(f"Cross-compiling is enabled to {ext_suffix!r}.") logger.debug( f"Setting _PYTHON_SYSCONFIGDATA_NAME to {env['_PYTHON_SYSCONFIGDATA_NAME']!r}." From e6249eba8e2537dae541deb6ff738941008309de Mon Sep 17 00:00:00 2001 From: Henry Schreiner Date: Fri, 9 Jun 2023 18:23:43 -0400 Subject: [PATCH 05/12] fix: get correct SOABI when cross-compiling Signed-off-by: Henry Schreiner --- src/scikit_build_core/builder/cross_compile.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/scikit_build_core/builder/cross_compile.py b/src/scikit_build_core/builder/cross_compile.py index 11381119..093f4db0 100644 --- a/src/scikit_build_core/builder/cross_compile.py +++ b/src/scikit_build_core/builder/cross_compile.py @@ -32,7 +32,8 @@ def auto_cross_compile_env( @contextlib.contextmanager def set_cross_compile_env( - ext_suffix: str, env: MutableMapping[str, str] + ext_suffix: str, + env: MutableMapping[str, str], ) -> Generator[None, None, None]: """ Generate python file and set environment variables to cross-compile Python From 1ab8e44006c53b5c2262da2d19261bfbc6d87881 Mon Sep 17 00:00:00 2001 From: Henry Schreiner Date: Fri, 9 Jun 2023 23:58:17 -0400 Subject: [PATCH 06/12] fix: more logging --- src/scikit_build_core/builder/cross_compile.py | 8 +++++--- src/scikit_build_core/resources/_cross_compile.py | 2 +- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/src/scikit_build_core/builder/cross_compile.py b/src/scikit_build_core/builder/cross_compile.py index 093f4db0..674fea4c 100644 --- a/src/scikit_build_core/builder/cross_compile.py +++ b/src/scikit_build_core/builder/cross_compile.py @@ -74,11 +74,13 @@ def set_cross_compile_env( else str(tmp_dir) ) env["_PYTHON_SYSCONFIGDATA_NAME"] = cross_compile_file.stem - logger.info(f"Cross-compiling is enabled to {ext_suffix!r}.") + logger.info("Cross-compiling is enabled to {!r}.", ext_suffix) logger.debug( - f"Setting _PYTHON_SYSCONFIGDATA_NAME to {env['_PYTHON_SYSCONFIGDATA_NAME']!r}." + "Setting _PYTHON_SYSCONFIGDATA_NAME to {!r}.", + env["_PYTHON_SYSCONFIGDATA_NAME"], ) - logger.debug(f"Setting PYTHONPATH to {env['PYTHONPATH']!r}.") + logger.debug("Setting PYTHONPATH to {!r}.", env["PYTHONPATH"]) + logger.debug("Cross compile output file contents: {}", output_text) try: yield finally: diff --git a/src/scikit_build_core/resources/_cross_compile.py b/src/scikit_build_core/resources/_cross_compile.py index 5d116466..2d9b4c72 100644 --- a/src/scikit_build_core/resources/_cross_compile.py +++ b/src/scikit_build_core/resources/_cross_compile.py @@ -2,6 +2,6 @@ host_name = "${host_name}" -build_time_vars = __import__("${host_name}").build_time_vars.copy() +build_time_vars = __import__("${host_name}").build_time_vars build_time_vars["SOABI"] = "${SOABI}" build_time_vars["EXT_SUFFIX"] = "${EXT_SUFFIX}" From b7a73cf99a2f674d6c7392b76e6792c41b69becc Mon Sep 17 00:00:00 2001 From: Henry Schreiner Date: Sat, 10 Jun 2023 00:29:46 -0400 Subject: [PATCH 07/12] fix: avoid passing a function Signed-off-by: Henry Schreiner --- src/scikit_build_core/builder/cross_compile.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/scikit_build_core/builder/cross_compile.py b/src/scikit_build_core/builder/cross_compile.py index 674fea4c..78d178d8 100644 --- a/src/scikit_build_core/builder/cross_compile.py +++ b/src/scikit_build_core/builder/cross_compile.py @@ -47,7 +47,7 @@ def set_cross_compile_env( yield return - sysconf_name = getattr(sysconfig, "_get_sysconfigdata_name", "") + sysconf_name = getattr(sysconfig, "_get_sysconfigdata_name", lambda: None)() if not sysconf_name: logger.warning( "Cross-compiling is not supported due to sysconfig._get_sysconfigdata_name missing." From f8a78ead5193cfcc36bdd6aabd74bbffdcc84aca Mon Sep 17 00:00:00 2001 From: Henry Schreiner Date: Sat, 10 Jun 2023 10:07:19 -0400 Subject: [PATCH 08/12] fix: render out full build_time_vars Signed-off-by: Henry Schreiner --- .../builder/cross_compile.py | 20 ++++--------------- .../resources/_cross_compile.py | 7 ------- 2 files changed, 4 insertions(+), 23 deletions(-) delete mode 100644 src/scikit_build_core/resources/_cross_compile.py diff --git a/src/scikit_build_core/builder/cross_compile.py b/src/scikit_build_core/builder/cross_compile.py index 78d178d8..0577f296 100644 --- a/src/scikit_build_core/builder/cross_compile.py +++ b/src/scikit_build_core/builder/cross_compile.py @@ -2,14 +2,12 @@ import contextlib import os -import string import sysconfig import tempfile from collections.abc import Generator, MutableMapping from pathlib import Path from .._logging import logger -from ..resources import resources __all__ = ["set_cross_compile_env"] @@ -47,25 +45,15 @@ def set_cross_compile_env( yield return - sysconf_name = getattr(sysconfig, "_get_sysconfigdata_name", lambda: None)() - if not sysconf_name: - logger.warning( - "Cross-compiling is not supported due to sysconfig._get_sysconfigdata_name missing." - ) - yield - return - with tempfile.TemporaryDirectory() as tmpdir: tmp_dir = Path(tmpdir).resolve() cross_compile_file = ( tmp_dir / f"_cross_compile_{ext_suffix.replace('.', '_')}.py" ) - input_txt = resources.joinpath("_cross_compile.py").read_text(encoding="utf-8") - output_text = string.Template(input_txt).substitute( - host_name=sysconf_name, - SOABI=ext_suffix.rsplit(maxsplit=1)[0], - EXT_SUFFIX=ext_suffix, - ) + build_time_vars = sysconfig.get_config_vars() + build_time_vars["EXT_SUFFIX"] = ext_suffix + build_time_vars["SOABI"] = ext_suffix.rsplit(maxsplit=1)[0] + output_text = f"build_time_vars = {build_time_vars!r}\n" cross_compile_file.write_text(output_text) current_path = env.get("PYTHONPATH", "") env["PYTHONPATH"] = ( diff --git a/src/scikit_build_core/resources/_cross_compile.py b/src/scikit_build_core/resources/_cross_compile.py deleted file mode 100644 index 2d9b4c72..00000000 --- a/src/scikit_build_core/resources/_cross_compile.py +++ /dev/null @@ -1,7 +0,0 @@ -# Cross compiling activation template - -host_name = "${host_name}" - -build_time_vars = __import__("${host_name}").build_time_vars -build_time_vars["SOABI"] = "${SOABI}" -build_time_vars["EXT_SUFFIX"] = "${EXT_SUFFIX}" From 4672a63c7d1071a243307ba12df1d4f512bdc590 Mon Sep 17 00:00:00 2001 From: Henry Schreiner Date: Sat, 10 Jun 2023 11:16:53 -0400 Subject: [PATCH 09/12] tests: add a cross-compile test Signed-off-by: Henry Schreiner --- .../builder/cross_compile.py | 2 +- tests/test_cross_compile.py | 36 +++++++++++++++++++ 2 files changed, 37 insertions(+), 1 deletion(-) create mode 100644 tests/test_cross_compile.py diff --git a/src/scikit_build_core/builder/cross_compile.py b/src/scikit_build_core/builder/cross_compile.py index 0577f296..f1f8a1ba 100644 --- a/src/scikit_build_core/builder/cross_compile.py +++ b/src/scikit_build_core/builder/cross_compile.py @@ -9,7 +9,7 @@ from .._logging import logger -__all__ = ["set_cross_compile_env"] +__all__ = ["set_cross_compile_env", "auto_cross_compile_env"] def __dir__() -> list[str]: diff --git a/tests/test_cross_compile.py b/tests/test_cross_compile.py new file mode 100644 index 00000000..1d43b95b --- /dev/null +++ b/tests/test_cross_compile.py @@ -0,0 +1,36 @@ +from __future__ import annotations + +import os +import subprocess +import sys +import sysconfig + +import pytest + +from scikit_build_core.builder.cross_compile import set_cross_compile_env + + +@pytest.mark.skipif( + sysconfig.get_config_var("SOABI") != "cp311-win_amd64", + reason="Only tests 'cp311-win_amd64', got {sysconfig.get_config_var('SOABI')!r}", +) +def test_environment(): + env = os.environ.copy() + cmd = [ + sys.executable, + "-c", + "import sysconfig; print(sysconfig.get_config_var('SOABI'), sysconfig.get_config_var('EXT_SUFFIX'))", + ] + + with set_cross_compile_env(".cp311-win_arm64.pyd", env): + result = subprocess.run( + cmd, check=True, capture_output=True, text=True, env=env + ) + soabi, ext_suffix = result.stdout.strip().split() + assert soabi == "cp311-win_arm64" + assert ext_suffix == ".cp311-win_arm64.pyd" + + result = subprocess.run(cmd, check=True, capture_output=True, text=True, env=env) + soabi, ext_suffix = result.stdout.strip().split() + assert soabi == "cp311-win_amd64" + assert ext_suffix == ".cp311-win_amd64.pyd" From 76056d5ded262c937418899c1238e91b2f4b4779 Mon Sep 17 00:00:00 2001 From: Henry Schreiner Date: Sat, 10 Jun 2023 12:51:58 -0400 Subject: [PATCH 10/12] Update test_cross_compile.py --- tests/test_cross_compile.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_cross_compile.py b/tests/test_cross_compile.py index 1d43b95b..48e76693 100644 --- a/tests/test_cross_compile.py +++ b/tests/test_cross_compile.py @@ -12,7 +12,7 @@ @pytest.mark.skipif( sysconfig.get_config_var("SOABI") != "cp311-win_amd64", - reason="Only tests 'cp311-win_amd64', got {sysconfig.get_config_var('SOABI')!r}", + reason=f"Only tests 'cp311-win_amd64', got {sysconfig.get_config_var('SOABI')!r}", ) def test_environment(): env = os.environ.copy() From fb9428712e12aa301c5dcc6fea710978bc11ad4e Mon Sep 17 00:00:00 2001 From: Henry Schreiner Date: Sat, 10 Jun 2023 14:00:54 -0400 Subject: [PATCH 11/12] tests: try ext_suffix Signed-off-by: Henry Schreiner --- tests/test_cross_compile.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/tests/test_cross_compile.py b/tests/test_cross_compile.py index 48e76693..65ea9543 100644 --- a/tests/test_cross_compile.py +++ b/tests/test_cross_compile.py @@ -9,10 +9,12 @@ from scikit_build_core.builder.cross_compile import set_cross_compile_env +ext_suffix = sysconfig.get_config_var("EXT_SUFFIX") + @pytest.mark.skipif( - sysconfig.get_config_var("SOABI") != "cp311-win_amd64", - reason=f"Only tests 'cp311-win_amd64', got {sysconfig.get_config_var('SOABI')!r}", + ext_suffix != ".cp311-win_amd64.pyd", + reason=f"Only tests '.cp311-win_amd64.pyd', got {ext_suffix!r}", ) def test_environment(): env = os.environ.copy() From 6ed843ea3843b01dd9d221509f532dedefc8e002 Mon Sep 17 00:00:00 2001 From: Henry Schreiner Date: Sat, 10 Jun 2023 17:10:11 -0400 Subject: [PATCH 12/12] Update test_cross_compile.py --- tests/test_cross_compile.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/test_cross_compile.py b/tests/test_cross_compile.py index 65ea9543..8f9cfd6a 100644 --- a/tests/test_cross_compile.py +++ b/tests/test_cross_compile.py @@ -29,6 +29,7 @@ def test_environment(): cmd, check=True, capture_output=True, text=True, env=env ) soabi, ext_suffix = result.stdout.strip().split() + print(soabi, ext_suffix) assert soabi == "cp311-win_arm64" assert ext_suffix == ".cp311-win_arm64.pyd"