refactor(pypi): implement PEP508 compliant marker evaluation (#2692)

This implements the PEP508 compliant marker evaluation in starlark and
removes the need for the Python interpreter when evaluating requirements
files passed to `pip.parse`. This makes the evaluation faster and allows
us to fix a few known issues (#2690).

In the future the intent is to move the `METADATA` parsing to pure
starlark so that the `RequiresDist` could be parsed in starlark at the
macro evaluation or analysis phases. This should make it possible to
more easily solve the design problem that more and more things need to
be passed to `whl_library` as args to have a robust dependency parsing:
* #2319 needs the full Python version to have correct cross-platform
compatible `METADATA` parsing and passing it to `Python` and back makes
  it difficult/annoying to implement.
* Parsing the `METADATA` file requires the precise list of target
  platform or the list of available packages in the `requirements.txt`.
  This means that without it we cannot trim the dependency tree in the
  `whl_library`. Doing this at macro loading phase allows us to depend
  on `.bzl` files in the `hub_repository` and more effectively pass
  information.

I can remotely see that this could become useful in `py_wheel` or an
building
wheels from sdists as the environment markers may be present in various
source
metadata as well. What is more `uv.lock` file has the env markers as
part of
the lock file information, so this might be useful there.

Work towards #2423
Work towards #260
Split from #2629

Author: aignas

Diff for: python/private/pypi/BUILD.bazel

@@ -75,7 +75,9 @@ bzl_library(
     name = "evaluate_markers_bzl",
     srcs = ["evaluate_markers.bzl"],
     deps = [
-        ":pypi_repo_utils_bzl",
+        ":pep508_env_bzl",
+        ":pep508_evaluate_bzl",
+        ":pep508_req_bzl",
     ],
 )
 
@@ -209,6 +211,37 @@ bzl_library(
     ],
 )
 
+bzl_library(
+    name = "pep508_bzl",
+    srcs = ["pep508.bzl"],
+    deps = [
+        ":pep508_env_bzl",
+        ":pep508_evaluate_bzl",
+    ],
+)
+
+bzl_library(
+    name = "pep508_env_bzl",
+    srcs = ["pep508_env.bzl"],
+)
+
+bzl_library(
+    name = "pep508_evaluate_bzl",
+    srcs = ["pep508_evaluate.bzl"],
+    deps = [
+        "//python/private:enum_bzl",
+        "//python/private:semver_bzl",
+    ],
+)
+
+bzl_library(
+    name = "pep508_req_bzl",
+    srcs = ["pep508_req.bzl"],
+    deps = [
+        "//python/private:normalize_name_bzl",
+    ],
+)
+
 bzl_library(
     name = "pip_bzl",
     srcs = ["pip.bzl"],

Diff for: python/private/pypi/evaluate_markers.bzl

@@ -14,65 +14,24 @@
 
 """A simple function that evaluates markers using a python interpreter."""
 
-load(":deps.bzl", "record_files")
-load(":pypi_repo_utils.bzl", "pypi_repo_utils")
+load(":pep508_env.bzl", "env", _platform_from_str = "platform_from_str")
+load(":pep508_evaluate.bzl", "evaluate")
+load(":pep508_req.bzl", _req = "requirement")
 
-# Used as a default value in a rule to ensure we fetch the dependencies.
-SRCS = [
-    # When the version, or any of the files in `packaging` package changes,
-    # this file will change as well.
-    record_files["pypi__packaging"],
-    Label("//python/private/pypi/requirements_parser:resolve_target_platforms.py"),
-    Label("//python/private/pypi/whl_installer:platform.py"),
-]
-
-def evaluate_markers(mrctx, *, requirements, python_interpreter, python_interpreter_target, srcs, logger = None):
+def evaluate_markers(requirements):
     """Return the list of supported platforms per requirements line.
 
     Args:
-        mrctx: repository_ctx or module_ctx.
-        requirements: list[str] of the requirement file lines to evaluate.
-        python_interpreter: str, path to the python_interpreter to use to
-            evaluate the env markers in the given requirements files. It will
-            be only called if the requirements files have env markers. This
-            should be something that is in your PATH or an absolute path.
-        python_interpreter_target: Label, same as python_interpreter, but in a
-            label format.
-        srcs: list[Label], the value of SRCS passed from the `rctx` or `mctx` to this function.
-        logger: repo_utils.logger or None, a simple struct to log diagnostic
-            messages. Defaults to None.
+        requirements: dict[str, list[str]] of the requirement file lines to evaluate.
 
     Returns:
         dict of string lists with target platforms
     """
-    if not requirements:
-        return {}
-
-    in_file = mrctx.path("requirements_with_markers.in.json")
-    out_file = mrctx.path("requirements_with_markers.out.json")
-    mrctx.file(in_file, json.encode(requirements))
-
-    pypi_repo_utils.execute_checked(
-        mrctx,
-        op = "ResolveRequirementEnvMarkers({})".format(in_file),
-        python = pypi_repo_utils.resolve_python_interpreter(
-            mrctx,
-            python_interpreter = python_interpreter,
-            python_interpreter_target = python_interpreter_target,
-        ),
-        arguments = [
-            "-m",
-            "python.private.pypi.requirements_parser.resolve_target_platforms",
-            in_file,
-            out_file,
-        ],
-        srcs = srcs,
-        environment = {
-            "PYTHONPATH": [
-                Label("@pypi__packaging//:BUILD.bazel"),
-                Label("//:BUILD.bazel"),
-            ],
-        },
-        logger = logger,
-    )
-    return json.decode(mrctx.read(out_file))
+    ret = {}
+    for req_string, platforms in requirements.items():
+        req = _req(req_string)
+        for platform in platforms:
+            if evaluate(req.marker, env = env(_platform_from_str(platform, None))):
+                ret.setdefault(req_string, []).append(platform)
+
+    return ret

Diff for: python/private/pypi/extension.bzl

@@ -22,7 +22,7 @@ load("//python/private:repo_utils.bzl", "repo_utils")
 load("//python/private:semver.bzl", "semver")
 load("//python/private:version_label.bzl", "version_label")
 load(":attrs.bzl", "use_isolated")
-load(":evaluate_markers.bzl", "evaluate_markers", EVALUATE_MARKERS_SRCS = "SRCS")
+load(":evaluate_markers.bzl", "evaluate_markers")
 load(":hub_repository.bzl", "hub_repository", "whl_config_settings_to_json")
 load(":parse_requirements.bzl", "parse_requirements")
 load(":parse_whl_name.bzl", "parse_whl_name")
@@ -167,28 +167,10 @@ def _create_whl_repos(
         ),
         extra_pip_args = pip_attr.extra_pip_args,
         get_index_urls = get_index_urls,
-        # NOTE @aignas 2024-08-02: , we will execute any interpreter that we find either
-        # in the PATH or if specified as a label. We will configure the env
-        # markers when evaluating the requirement lines based on the output
-        # from the `requirements_files_by_platform` which should have something
-        # similar to:
-        # {
-        #    "//:requirements.txt": ["cp311_linux_x86_64", ...]
-        # }
-        #
-        # We know the target python versions that we need to evaluate the
-        # markers for and thus we don't need to use multiple python interpreter
-        # instances to perform this manipulation. This function should be executed
-        # only once by the underlying code to minimize the overhead needed to
-        # spin up a Python interpreter.
-        evaluate_markers = lambda module_ctx, requirements: evaluate_markers(
-            module_ctx,
-            requirements = requirements,
-            python_interpreter = pip_attr.python_interpreter,
-            python_interpreter_target = python_interpreter_target,
-            srcs = pip_attr._evaluate_markers_srcs,
-            logger = logger,
-        ),
+        # NOTE @aignas 2025-02-24: we will use the "cp3xx_os_arch" platform labels
+        # for converting to the PEP508 environment and will evaluate them in starlark
+        # without involving the interpreter at all.
+        evaluate_markers = evaluate_markers,
         logger = logger,
     )
 
@@ -774,13 +756,6 @@ a corresponding `python.toolchain()` configured.
             doc = """\
 A dict of labels to wheel names that is typically generated by the whl_modifications.
 The labels are JSON config files describing the modifications.
-""",
-        ),
-        "_evaluate_markers_srcs": attr.label_list(
-            default = EVALUATE_MARKERS_SRCS,
-            doc = """\
-The list of labels to use as SRCS for the marker evaluation code. This ensures that the
-code will be re-evaluated when any of files in the default changes.
 """,
         ),
     }, **ATTRS)

Diff for: python/private/pypi/parse_requirements.bzl

@@ -67,10 +67,10 @@ def parse_requirements(
             of the distribution URLs from a PyPI index. Accepts ctx and
             distribution names to query.
         evaluate_markers: A function to use to evaluate the requirements.
-            Accepts the ctx and a dict where keys are requirement lines to
-            evaluate against the platforms stored as values in the input dict.
-            Returns the same dict, but with values being platforms that are
-            compatible with the requirements line.
+            Accepts a dict where keys are requirement lines to evaluate against
+            the platforms stored as values in the input dict. Returns the same
+            dict, but with values being platforms that are compatible with the
+            requirements line.
         logger: repo_utils.logger or None, a simple struct to log diagnostic messages.
 
     Returns:
@@ -93,7 +93,7 @@ def parse_requirements(
 
         The second element is extra_pip_args should be passed to `whl_library`.
     """
-    evaluate_markers = evaluate_markers or (lambda *_: {})
+    evaluate_markers = evaluate_markers or (lambda _: {})
     options = {}
     requirements = {}
     for file, plats in requirements_by_platform.items():
@@ -168,7 +168,7 @@ def parse_requirements(
     # to do, we could use Python to parse the requirement lines and infer the
     # URL of the files to download things from. This should be important for
     # VCS package references.
-    env_marker_target_platforms = evaluate_markers(ctx, reqs_with_env_markers)
+    env_marker_target_platforms = evaluate_markers(reqs_with_env_markers)
     if logger:
         logger.debug(lambda: "Evaluated env markers from:\n{}\n\nTo:\n{}".format(
             reqs_with_env_markers,

Diff for: python/private/pypi/pep508.bzl

@@ -0,0 +1,23 @@
+# Copyright 2025 The Bazel Authors. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""This module is for implementing PEP508 in starlark as FeatureFlagInfo
+"""
+
+load(":pep508_env.bzl", _env = "env")
+load(":pep508_evaluate.bzl", _evaluate = "evaluate", _to_string = "to_string")
+
+to_string = _to_string
+evaluate = _evaluate
+env = _env

Diff for: python/private/pypi/pep508_env.bzl

@@ -0,0 +1,117 @@
+# Copyright 2025 The Bazel Authors. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""This module is for implementing PEP508 environment definition.
+"""
+
+# See https://stackoverflow.com/questions/45125516/possible-values-for-uname-m
+_platform_machine_aliases = {
+    # These pairs mean the same hardware, but different values may be used
+    # on different host platforms.
+    "amd64": "x86_64",
+    "arm64": "aarch64",
+    "i386": "x86_32",
+    "i686": "x86_32",
+}
+_platform_system_values = {
+    "linux": "Linux",
+    "osx": "Darwin",
+    "windows": "Windows",
+}
+_sys_platform_values = {
+    "linux": "posix",
+    "osx": "darwin",
+    "windows": "win32",
+}
+_os_name_values = {
+    "linux": "posix",
+    "osx": "posix",
+    "windows": "nt",
+}
+
+def env(target_platform, *, extra = None):
+    """Return an env target platform
+
+    Args:
+        target_platform: {type}`str` the target platform identifier, e.g.
+            `cp33_linux_aarch64`
+        extra: {type}`str` the extra value to be added into the env.
+
+    Returns:
+        A dict that can be used as `env` in the marker evaluation.
+    """
+
+    # TODO @aignas 2025-02-13: consider moving this into config settings.
+
+    env = {"extra": extra} if extra != None else {}
+    env = env | {
+        "implementation_name": "cpython",
+        "platform_python_implementation": "CPython",
+        "platform_release": "",
+        "platform_version": "",
+    }
+    if type(target_platform) == type(""):
+        target_platform = platform_from_str(target_platform, python_version = "")
+
+    if target_platform.abi:
+        minor_version, _, micro_version = target_platform.abi[3:].partition(".")
+        micro_version = micro_version or "0"
+        env = env | {
+            "implementation_version": "3.{}.{}".format(minor_version, micro_version),
+            "python_full_version": "3.{}.{}".format(minor_version, micro_version),
+            "python_version": "3.{}".format(minor_version),
+        }
+    if target_platform.os and target_platform.arch:
+        os = target_platform.os
+        env = env | {
+            "os_name": _os_name_values.get(os, ""),
+            "platform_machine": target_platform.arch,
+            "platform_system": _platform_system_values.get(os, ""),
+            "sys_platform": _sys_platform_values.get(os, ""),
+        }
+
+    # This is split by topic
+    return env | {
+        "_aliases": {
+            "platform_machine": _platform_machine_aliases,
+        },
+    }
+
+def _platform(*, abi = None, os = None, arch = None):
+    return struct(
+        abi = abi,
+        os = os,
+        arch = arch,
+    )
+
+def platform_from_str(p, python_version):
+    """Return a platform from a string.
+
+    Args:
+        p: {type}`str` the actual string.
+        python_version: {type}`str` the python version to add to platform if needed.
+
+    Returns:
+        A struct that is returned by the `_platform` function.
+    """
+    if p.startswith("cp"):
+        abi, _, p = p.partition("_")
+    elif python_version:
+        major, _, tail = python_version.partition(".")
+        abi = "cp{}{}".format(major, tail)
+    else:
+        abi = None
+
+    os, _, arch = p.partition("_")
+    return _platform(abi = abi, os = os or None, arch = arch or None)