From 34ea5b2e0056d84a75022d16e382e2b219a69e6a Mon Sep 17 00:00:00 2001 From: Ignas Anikevicius <240938+aignas@users.noreply.github.com> Date: Sat, 30 Nov 2024 13:39:03 +0900 Subject: [PATCH] feat(pypi): support freethreaded in experimental_index_url With this we: * Fix the previous behaviour where `abi3` wheels would be selected when freethreaded builds are selected. Whilst this may work in practise sometimes, I am not sure it has been supported by reading PEP703. * Start selecting `cp313t` wheels when we scan what is available on PyPI. * Ensure that the `whl_library` repository rule handles `cp313t` wheel extraction. * Generate `cp313t` config_settings so that we can use them in `pkg_aliases`. * Generate `cp313t` references in `pkg_aliases` macro. * Add the 3.13 deps to dev_pip for testing. Also tested by manually running: ``` $ bazel cquery --//python/config_settings:python_version=3.13 --//python/config_settings:py_freethreaded=yes 'kind("py_library rule", deps(@dev_pip//markupsafe))' INFO: Analyzed target @@_main~pip~dev_pip//markupsafe:markupsafe (3 packages loaded, 4091 targets configured). INFO: Found 1 target... @@_main~pip~dev_pip_313_markupsafe_cp313_cp313t_manylinux_2_17_x86_64_c0ef13ea//:pkg (008c5a5) $bazel build --//python/config_settings:python_version=3.13 --//python/config_settings:py_freethreaded=yes @dev_pip//markupsafe ``` Fixes #2386 --- MODULE.bazel | 7 ++ python/config_settings/BUILD.bazel | 6 ++ python/private/pypi/config_settings.bzl | 101 ++++++++++++------ python/private/pypi/pkg_aliases.bzl | 4 +- python/private/pypi/whl_library.bzl | 2 +- python/private/pypi/whl_target_platforms.bzl | 4 + .../config_settings/config_settings_tests.bzl | 50 +++++++++ tests/pypi/pkg_aliases/pkg_aliases_test.bzl | 10 ++ .../whl_target_platforms/select_whl_tests.bzl | 20 ++++ 9 files changed, 171 insertions(+), 33 deletions(-) diff --git a/MODULE.bazel b/MODULE.bazel index 2ae3173094..e4b113e785 100644 --- a/MODULE.bazel +++ b/MODULE.bazel @@ -121,6 +121,13 @@ dev_pip.parse( python_version = "3.11", requirements_lock = "//docs:requirements.txt", ) +dev_pip.parse( + download_only = True, + experimental_index_url = "https://pypi.org/simple", + hub_name = "dev_pip", + python_version = "3.13.0", + requirements_lock = "//docs:requirements.txt", +) dev_pip.parse( download_only = True, experimental_index_url = "https://pypi.org/simple", diff --git a/python/config_settings/BUILD.bazel b/python/config_settings/BUILD.bazel index aa26e6e669..5455f5aef7 100644 --- a/python/config_settings/BUILD.bazel +++ b/python/config_settings/BUILD.bazel @@ -106,6 +106,12 @@ config_setting( visibility = ["//visibility:public"], ) +config_setting( + name = "is_py_non_freethreaded", + flag_values = {":py_freethreaded": FreeThreadedFlag.NO}, + visibility = ["//visibility:public"], +) + # pip.parse related flags string_flag( diff --git a/python/private/pypi/config_settings.bzl b/python/private/pypi/config_settings.bzl index 6f927f2a4c..620e50e997 100644 --- a/python/private/pypi/config_settings.bzl +++ b/python/private/pypi/config_settings.bzl @@ -20,20 +20,21 @@ that matches the target platform. We can leverage this fact to ensure that the most specialized wheels are used by default with the users being able to configure string_flag values to select the less specialized ones. -The list of specialization of the dists goes like follows: +The list of specialization of the dists goes like follows (cpxyt stands for freethreaded +environments): * sdist * py*-none-any.whl * py*-abi3-any.whl -* py*-cpxy-any.whl +* py*-cpxy-any.whl or py*-cpxyt-any.whl * cp*-none-any.whl * cp*-abi3-any.whl -* cp*-cpxy-plat.whl +* cp*-cpxy-any.whl or cp*-cpxyt-any.whl * py*-none-plat.whl * py*-abi3-plat.whl -* py*-cpxy-plat.whl +* py*-cpxy-plat.whl or py*-cpxyt-plat.whl * cp*-none-plat.whl * cp*-abi3-plat.whl -* cp*-cpxy-plat.whl +* cp*-cpxy-plat.whl or cp*-cpxyt-plat.whl Note, that here the specialization of musl vs manylinux wheels is the same in order to ensure that the matching fails if the user requests for `musl` and we don't have it or vice versa. @@ -46,19 +47,24 @@ FLAGS = struct( **{ f: str(Label("//python/config_settings:" + f)) for f in [ - "python_version", + "is_pip_whl_auto", + "is_pip_whl_no", + "is_pip_whl_only", + "is_py_freethreaded", + "is_py_non_freethreaded", "pip_whl_glibc_version", "pip_whl_muslc_version", "pip_whl_osx_arch", "pip_whl_osx_version", "py_linux_libc", - "is_pip_whl_no", - "is_pip_whl_only", - "is_pip_whl_auto", + "python_version", ] } ) +_DEFAULT = "//conditions:default" +_INCOMPATIBLE = "@platforms//:incompatible" + # Here we create extra string flags that are just to work with the select # selecting the most specialized match. We don't allow the user to change # them. @@ -170,52 +176,70 @@ def _dist_config_settings(*, suffix, plat_flag_values, **kwargs): **kwargs ) - for name, f in [ - ("py_none", _flags.whl_py2_py3), - ("py3_none", _flags.whl_py3), - ("py3_abi3", _flags.whl_py3_abi3), - ("cp3x_none", _flags.whl_pycp3x), - ("cp3x_abi3", _flags.whl_pycp3x_abi3), - ("cp3x_cp", _flags.whl_pycp3x_abicp), + used_flags = {} + + # NOTE @aignas 2024-12-01: the abi3 is not compatible with freethreaded + # builds as per PEP703 (https://peps.python.org/pep-0703/#backwards-compatibility) + # + # The discussion here also reinforces this notion: + # https://discuss.python.org/t/pep-703-making-the-global-interpreter-lock-optional-3-12-updates/26503/99 + + for name, f, abi in [ + ("py_none", _flags.whl_py2_py3, None), + ("py3_none", _flags.whl_py3, None), + ("py3_abi3", _flags.whl_py3_abi3, (FLAGS.is_py_non_freethreaded,)), + ("cp3x_none", _flags.whl_pycp3x, None), + ("cp3x_abi3", _flags.whl_pycp3x_abi3, (FLAGS.is_py_non_freethreaded,)), + # The below are not specializations of one another, they are variants + ("cp3x_cp", _flags.whl_pycp3x_abicp, (FLAGS.is_py_non_freethreaded,)), + ("cp3x_cpt", _flags.whl_pycp3x_abicp, (FLAGS.is_py_freethreaded,)), ]: - if f in flag_values: + if (f, abi) in used_flags: # This should never happen as all of the different whls should have - # unique flag values. + # unique flag values fail("BUG: the flag {} is attempted to be added twice to the list".format(f)) else: flag_values[f] = "" + used_flags[(f, abi)] = True _dist_config_setting( name = "{}_any{}".format(name, suffix), flag_values = flag_values, is_pip_whl = FLAGS.is_pip_whl_only, + abi = abi, **kwargs ) generic_flag_values = flag_values + generic_used_flags = used_flags for (suffix, flag_values) in plat_flag_values: + used_flags = {(f, None): True for f in flag_values} | generic_used_flags flag_values = flag_values | generic_flag_values - for name, f in [ - ("py_none", _flags.whl_plat), - ("py3_none", _flags.whl_plat_py3), - ("py3_abi3", _flags.whl_plat_py3_abi3), - ("cp3x_none", _flags.whl_plat_pycp3x), - ("cp3x_abi3", _flags.whl_plat_pycp3x_abi3), - ("cp3x_cp", _flags.whl_plat_pycp3x_abicp), + for name, f, abi in [ + ("py_none", _flags.whl_plat, None), + ("py3_none", _flags.whl_plat_py3, None), + ("py3_abi3", _flags.whl_plat_py3_abi3, (FLAGS.is_py_non_freethreaded,)), + ("cp3x_none", _flags.whl_plat_pycp3x, None), + ("cp3x_abi3", _flags.whl_plat_pycp3x_abi3, (FLAGS.is_py_non_freethreaded,)), + # The below are not specializations of one another, they are variants + ("cp3x_cp", _flags.whl_plat_pycp3x_abicp, (FLAGS.is_py_non_freethreaded,)), + ("cp3x_cpt", _flags.whl_plat_pycp3x_abicp, (FLAGS.is_py_freethreaded,)), ]: - if f in flag_values: + if (f, abi) in used_flags: # This should never happen as all of the different whls should have # unique flag values. fail("BUG: the flag {} is attempted to be added twice to the list".format(f)) else: flag_values[f] = "" + used_flags[(f, abi)] = True _dist_config_setting( name = "{}_{}".format(name, suffix), flag_values = flag_values, is_pip_whl = FLAGS.is_pip_whl_only, + abi = abi, **kwargs ) @@ -285,7 +309,7 @@ def _plat_flag_values(os, cpu, osx_versions, glibc_versions, muslc_versions): return ret -def _dist_config_setting(*, name, is_python, python_version, is_pip_whl = None, native = native, **kwargs): +def _dist_config_setting(*, name, is_python, python_version, is_pip_whl = None, abi = None, native = native, **kwargs): """A macro to create a target that matches is_pip_whl_auto and one more value. Args: @@ -294,6 +318,10 @@ def _dist_config_setting(*, name, is_python, python_version, is_pip_whl = None, `is_pip_whl_auto` when evaluating the config setting. is_python: The python version config_setting to match. python_version: The python version name. + abi: {type}`tuple[Label]` A collection of ABI config settings that are + compatible with the given dist config setting. For example, if only + non-freethreaded python builds are allowed, add + FLAGS.is_py_non_freethreaded here. native (struct): The struct containing alias and config_setting rules to use for creating the objects. Can be overridden for unit tests reasons. @@ -306,9 +334,9 @@ def _dist_config_setting(*, name, is_python, python_version, is_pip_whl = None, native.alias( name = "is_cp{}_{}".format(python_version, name) if python_version else "is_{}".format(name), actual = select({ - # First match by the python version - is_python: _name, - "//conditions:default": is_python, + # First match by the python version and then by ABI + is_python: _name + ("_abi" if abi else ""), + _DEFAULT: _INCOMPATIBLE, }), visibility = visibility, ) @@ -325,12 +353,23 @@ def _dist_config_setting(*, name, is_python, python_version, is_pip_whl = None, config_setting_name = _name + "_setting" native.config_setting(name = config_setting_name, **kwargs) + if abi: + native.alias( + name = _name + "_abi", + actual = select( + {k: _name for k in abi} | { + _DEFAULT: _INCOMPATIBLE, + }, + ), + visibility = visibility, + ) + # Next match by the `pip_whl` flag value and then match by the flags that # are intrinsic to the distribution. native.alias( name = _name, actual = select({ - "//conditions:default": FLAGS.is_pip_whl_auto, + _DEFAULT: _INCOMPATIBLE, FLAGS.is_pip_whl_auto: config_setting_name, is_pip_whl: config_setting_name, }), diff --git a/python/private/pypi/pkg_aliases.bzl b/python/private/pypi/pkg_aliases.bzl index 284f8e9ed0..c39c8e8696 100644 --- a/python/private/pypi/pkg_aliases.bzl +++ b/python/private/pypi/pkg_aliases.bzl @@ -298,7 +298,9 @@ def get_filename_config_settings( else: py = "py3" - if parsed.abi_tag.startswith("cp"): + if parsed.abi_tag.startswith("cp") and parsed.abi_tag.endswith("t"): + abi = "cpt" + elif parsed.abi_tag.startswith("cp"): abi = "cp" else: abi = parsed.abi_tag diff --git a/python/private/pypi/whl_library.bzl b/python/private/pypi/whl_library.bzl index 612ca2cfdf..79a58a81f2 100644 --- a/python/private/pypi/whl_library.bzl +++ b/python/private/pypi/whl_library.bzl @@ -287,7 +287,7 @@ def _whl_library_impl(rctx): p.target_platform for p in whl_target_platforms( platform_tag = parsed_whl.platform_tag, - abi_tag = parsed_whl.abi_tag, + abi_tag = parsed_whl.abi_tag.strip("tm"), ) ] diff --git a/python/private/pypi/whl_target_platforms.bzl b/python/private/pypi/whl_target_platforms.bzl index bdc44c697a..6823199bee 100644 --- a/python/private/pypi/whl_target_platforms.bzl +++ b/python/private/pypi/whl_target_platforms.bzl @@ -89,6 +89,10 @@ def select_whls(*, whls, want_platforms = [], logger = None): want_abis[abi] = None want_abis[abi + "m"] = None + # Also add freethreaded wheels if we find them since we started supporting them + _want_platforms["{}t_{}".format(abi, os_cpu)] = None + want_abis[abi + "t"] = None + want_platforms = sorted(_want_platforms) candidates = {} diff --git a/tests/pypi/config_settings/config_settings_tests.bzl b/tests/pypi/config_settings/config_settings_tests.bzl index a77fa5b66b..049556a4c6 100644 --- a/tests/pypi/config_settings/config_settings_tests.bzl +++ b/tests/pypi/config_settings/config_settings_tests.bzl @@ -39,6 +39,7 @@ _flag = struct( pip_whl_osx_arch = lambda x: (str(Label("//python/config_settings:pip_whl_osx_arch")), str(x)), py_linux_libc = lambda x: (str(Label("//python/config_settings:py_linux_libc")), str(x)), python_version = lambda x: (str(Label("//python/config_settings:python_version")), str(x)), + py_freethreaded = lambda x: (str(Label("//python/config_settings:py_freethreaded")), str(x)), ) def _analysis_test(*, name, dist, want, config_settings = [_flag.platform("linux_aarch64")]): @@ -286,6 +287,38 @@ def _test_py_none_any_versioned(name): _tests.append(_test_py_none_any_versioned) +def _test_cp_whl_is_not_prefered_over_py3_non_freethreaded(name): + _analysis_test( + name = name, + dist = { + "is_cp3.7_cp3x_abi3_any": "py3_abi3", + "is_cp3.7_cp3x_cpt_any": "cp", + "is_cp3.7_cp3x_none_any": "py3", + }, + want = "py3_abi3", + config_settings = [ + _flag.py_freethreaded("no"), + ], + ) + +_tests.append(_test_cp_whl_is_not_prefered_over_py3_non_freethreaded) + +def _test_cp_whl_is_not_prefered_over_py3_freethreaded(name): + _analysis_test( + name = name, + dist = { + "is_cp3.7_cp3x_abi3_any": "py3_abi3", + "is_cp3.7_cp3x_cp_any": "cp", + "is_cp3.7_cp3x_none_any": "py3", + }, + want = "py3", + config_settings = [ + _flag.py_freethreaded("yes"), + ], + ) + +_tests.append(_test_cp_whl_is_not_prefered_over_py3_freethreaded) + def _test_cp_cp_whl(name): _analysis_test( name = name, @@ -412,6 +445,7 @@ def _test_windows(name): name = name, dist = { "is_cp3.7_cp3x_cp_windows_x86_64": "whl", + "is_cp3.7_cp3x_cpt_windows_x86_64": "whl_freethreaded", }, want = "whl", config_settings = [ @@ -421,6 +455,22 @@ def _test_windows(name): _tests.append(_test_windows) +def _test_windows_freethreaded(name): + _analysis_test( + name = name, + dist = { + "is_cp3.7_cp3x_cp_windows_x86_64": "whl", + "is_cp3.7_cp3x_cpt_windows_x86_64": "whl_freethreaded", + }, + want = "whl_freethreaded", + config_settings = [ + _flag.platform("windows_x86_64"), + _flag.py_freethreaded("yes"), + ], + ) + +_tests.append(_test_windows_freethreaded) + def _test_osx(name): _analysis_test( name = name, diff --git a/tests/pypi/pkg_aliases/pkg_aliases_test.bzl b/tests/pypi/pkg_aliases/pkg_aliases_test.bzl index 0fa66d05eb..23a0f01db9 100644 --- a/tests/pypi/pkg_aliases/pkg_aliases_test.bzl +++ b/tests/pypi/pkg_aliases/pkg_aliases_test.bzl @@ -287,6 +287,14 @@ def _test_multiplatform_whl_aliases_filename(env): filename = "foo-0.0.1-py3-none-any.whl", version = "3.1", ): "foo-py3-0.0.1", + whl_config_setting( + filename = "foo-0.0.1-cp313-cp313-any.whl", + version = "3.1", + ): "foo-cp-0.0.1", + whl_config_setting( + filename = "foo-0.0.1-cp313-cp313t-any.whl", + version = "3.1", + ): "foo-cpt-0.0.1", whl_config_setting( filename = "foo-0.0.2-py3-none-any.whl", version = "3.1", @@ -303,6 +311,8 @@ def _test_multiplatform_whl_aliases_filename(env): osx_versions = [], ) want = { + "//_config:is_cp3.1_cp3x_cp_any": "foo-cp-0.0.1", + "//_config:is_cp3.1_cp3x_cpt_any": "foo-cpt-0.0.1", "//_config:is_cp3.1_py3_none_any": "foo-py3-0.0.1", "//_config:is_cp3.1_py3_none_any_linux_aarch64": "foo-0.0.2", "//_config:is_cp3.1_py3_none_any_linux_x86_64": "foo-0.0.2", diff --git a/tests/pypi/whl_target_platforms/select_whl_tests.bzl b/tests/pypi/whl_target_platforms/select_whl_tests.bzl index 2994bd513f..8ab24138d1 100644 --- a/tests/pypi/whl_target_platforms/select_whl_tests.bzl +++ b/tests/pypi/whl_target_platforms/select_whl_tests.bzl @@ -27,6 +27,10 @@ WHL_LIST = [ "pkg-0.0.1-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", "pkg-0.0.1-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", "pkg-0.0.1-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", + "pkg-0.0.1-cp313-cp313t-musllinux_1_1_x86_64.whl", + "pkg-0.0.1-cp313-cp313-musllinux_1_1_x86_64.whl", + "pkg-0.0.1-cp313-abi3-musllinux_1_1_x86_64.whl", + "pkg-0.0.1-cp313-none-musllinux_1_1_x86_64.whl", "pkg-0.0.1-cp311-cp311-musllinux_1_1_aarch64.whl", "pkg-0.0.1-cp311-cp311-musllinux_1_1_i686.whl", "pkg-0.0.1-cp311-cp311-musllinux_1_1_ppc64le.whl", @@ -269,6 +273,22 @@ def _test_prefer_manylinux_wheels(env): _tests.append(_test_prefer_manylinux_wheels) +def _test_freethreaded_wheels(env): + # Check we prefer platform specific wheels + got = _select_whls(whls = WHL_LIST, want_platforms = ["cp313_linux_x86_64"]) + _match( + env, + got, + "pkg-0.0.1-cp313-cp313t-musllinux_1_1_x86_64.whl", + "pkg-0.0.1-cp313-cp313-musllinux_1_1_x86_64.whl", + "pkg-0.0.1-cp313-abi3-musllinux_1_1_x86_64.whl", + "pkg-0.0.1-cp313-none-musllinux_1_1_x86_64.whl", + "pkg-0.0.1-cp39-abi3-any.whl", + "pkg-0.0.1-py3-none-any.whl", + ) + +_tests.append(_test_freethreaded_wheels) + def select_whl_test_suite(name): """Create the test suite.