Skip to content

Commit 55d6836

Browse files
aignasaignas
authored
fix(pypi): fixes to the marker evaluation and utils (#2767)
These are just bugfixes to already merged code: * Fix nested bracket parsing in PEP508 marker parser. * Fix the sys_platform constants, which I noticed in #2629 but they got also pointed out in #2766. * Port some of python tests for requirement parsing and improve the implementation. Those tests will be removed in #2629. * Move the platform related code to a separate file. * Rename `pep508_req.bzl` to `pep508_requirement.bzl` to follow the convention. All of the bug fixes have added tests. Work towards #2423.
1 parent 9fb13ec commit 55d6836

File tree

7 files changed

+165
-40
lines changed

7 files changed

+165
-40
lines changed

‎python/private/pypi/BUILD.bazel

+12-3
Original file line numberDiff line numberDiff line change
@@ -77,7 +77,8 @@ bzl_library(
7777
deps = [
7878
":pep508_env_bzl",
7979
":pep508_evaluate_bzl",
80-
":pep508_req_bzl",
80+
":pep508_platform_bzl",
81+
":pep508_requirement_bzl",
8182
],
8283
)
8384

@@ -223,6 +224,9 @@ bzl_library(
223224
bzl_library(
224225
name = "pep508_env_bzl",
225226
srcs = ["pep508_env.bzl"],
227+
deps = [
228+
":pep508_platform_bzl",
229+
],
226230
)
227231

228232
bzl_library(
@@ -235,8 +239,13 @@ bzl_library(
235239
)
236240

237241
bzl_library(
238-
name = "pep508_req_bzl",
239-
srcs = ["pep508_req.bzl"],
242+
name = "pep508_platform_bzl",
243+
srcs = ["pep508_platform.bzl"],
244+
)
245+
246+
bzl_library(
247+
name = "pep508_requirement_bzl",
248+
srcs = ["pep508_requirement.bzl"],
240249
deps = [
241250
"//python/private:normalize_name_bzl",
242251
],

‎python/private/pypi/evaluate_markers.bzl

+5-4
Original file line numberDiff line numberDiff line change
@@ -14,9 +14,10 @@
1414

1515
"""A simple function that evaluates markers using a python interpreter."""
1616

17-
load(":pep508_env.bzl", "env", _platform_from_str = "platform_from_str")
17+
load(":pep508_env.bzl", "env")
1818
load(":pep508_evaluate.bzl", "evaluate")
19-
load(":pep508_req.bzl", _req = "requirement")
19+
load(":pep508_platform.bzl", "platform_from_str")
20+
load(":pep508_requirement.bzl", "requirement")
2021

2122
def evaluate_markers(requirements):
2223
"""Return the list of supported platforms per requirements line.
@@ -29,9 +30,9 @@ def evaluate_markers(requirements):
2930
"""
3031
ret = {}
3132
for req_string, platforms in requirements.items():
32-
req = _req(req_string)
33+
req = requirement(req_string)
3334
for platform in platforms:
34-
if evaluate(req.marker, env = env(_platform_from_str(platform, None))):
35+
if evaluate(req.marker, env = env(platform_from_str(platform, None))):
3536
ret.setdefault(req_string, []).append(platform)
3637

3738
return ret

‎python/private/pypi/pep508_env.bzl

+33-30
Original file line numberDiff line numberDiff line change
@@ -15,7 +15,9 @@
1515
"""This module is for implementing PEP508 environment definition.
1616
"""
1717

18-
# See https://stackoverflow.com/questions/45125516/possible-values-for-uname-m
18+
load(":pep508_platform.bzl", "platform_from_str")
19+
20+
# See https://stackoverflow.com/a/45125525
1921
_platform_machine_aliases = {
2022
# These pairs mean the same hardware, but different values may be used
2123
# on different host platforms.
@@ -24,13 +26,41 @@ _platform_machine_aliases = {
2426
"i386": "x86_32",
2527
"i686": "x86_32",
2628
}
29+
30+
# Platform system returns results from the `uname` call.
2731
_platform_system_values = {
2832
"linux": "Linux",
2933
"osx": "Darwin",
3034
"windows": "Windows",
3135
}
36+
37+
# The copy of SO [answer](https://stackoverflow.com/a/13874620) containing
38+
# all of the platforms:
39+
# ┍━━━━━━━━━━━━━━━━━━━━━┯━━━━━━━━━━━━━━━━━━━━━┑
40+
# │ System │ Value │
41+
# ┝━━━━━━━━━━━━━━━━━━━━━┿━━━━━━━━━━━━━━━━━━━━━┥
42+
# │ Linux │ linux or linux2 (*) │
43+
# │ Windows │ win32 │
44+
# │ Windows/Cygwin │ cygwin │
45+
# │ Windows/MSYS2 │ msys │
46+
# │ Mac OS X │ darwin │
47+
# │ OS/2 │ os2 │
48+
# │ OS/2 EMX │ os2emx │
49+
# │ RiscOS │ riscos │
50+
# │ AtheOS │ atheos │
51+
# │ FreeBSD 7 │ freebsd7 │
52+
# │ FreeBSD 8 │ freebsd8 │
53+
# │ FreeBSD N │ freebsdN │
54+
# │ OpenBSD 6 │ openbsd6 │
55+
# │ AIX │ aix (**) │
56+
# ┕━━━━━━━━━━━━━━━━━━━━━┷━━━━━━━━━━━━━━━━━━━━━┙
57+
#
58+
# (*) Prior to Python 3.3, the value for any Linux version is always linux2; after, it is linux.
59+
# (**) Prior Python 3.8 could also be aix5 or aix7; use sys.platform.startswith()
60+
#
61+
# We are using only the subset that we actually support.
3262
_sys_platform_values = {
33-
"linux": "posix",
63+
"linux": "linux",
3464
"osx": "darwin",
3565
"windows": "win32",
3666
}
@@ -61,6 +91,7 @@ def env(target_platform, *, extra = None):
6191
"platform_release": "",
6292
"platform_version": "",
6393
}
94+
6495
if type(target_platform) == type(""):
6596
target_platform = platform_from_str(target_platform, python_version = "")
6697

@@ -87,31 +118,3 @@ def env(target_platform, *, extra = None):
87118
"platform_machine": _platform_machine_aliases,
88119
},
89120
}
90-
91-
def _platform(*, abi = None, os = None, arch = None):
92-
return struct(
93-
abi = abi,
94-
os = os,
95-
arch = arch,
96-
)
97-
98-
def platform_from_str(p, python_version):
99-
"""Return a platform from a string.
100-
101-
Args:
102-
p: {type}`str` the actual string.
103-
python_version: {type}`str` the python version to add to platform if needed.
104-
105-
Returns:
106-
A struct that is returned by the `_platform` function.
107-
"""
108-
if p.startswith("cp"):
109-
abi, _, p = p.partition("_")
110-
elif python_version:
111-
major, _, tail = python_version.partition(".")
112-
abi = "cp{}{}".format(major, tail)
113-
else:
114-
abi = None
115-
116-
os, _, arch = p.partition("_")
117-
return _platform(abi = abi, os = os or None, arch = arch or None)

‎python/private/pypi/pep508_platform.bzl

+57
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,57 @@
1+
# Copyright 2025 The Bazel Authors. All rights reserved.
2+
#
3+
# Licensed under the Apache License, Version 2.0 (the "License");
4+
# you may not use this file except in compliance with the License.
5+
# You may obtain a copy of the License at
6+
#
7+
# http://www.apache.org/licenses/LICENSE-2.0
8+
#
9+
# Unless required by applicable law or agreed to in writing, software
10+
# distributed under the License is distributed on an "AS IS" BASIS,
11+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12+
# See the License for the specific language governing permissions and
13+
# limitations under the License.
14+
15+
"""The platform abstraction
16+
"""
17+
18+
def platform(*, abi = None, os = None, arch = None):
19+
"""platform returns a struct for the platform.
20+
21+
Args:
22+
abi: {type}`str | None` the target ABI, e.g. `"cp39"`.
23+
os: {type}`str | None` the target os, e.g. `"linux"`.
24+
arch: {type}`str | None` the target CPU, e.g. `"aarch64"`.
25+
26+
Returns:
27+
A struct.
28+
"""
29+
30+
# Note, this is used a lot as a key in dictionaries, so it cannot contain
31+
# methods.
32+
return struct(
33+
abi = abi,
34+
os = os,
35+
arch = arch,
36+
)
37+
38+
def platform_from_str(p, python_version):
39+
"""Return a platform from a string.
40+
41+
Args:
42+
p: {type}`str` the actual string.
43+
python_version: {type}`str` the python version to add to platform if needed.
44+
45+
Returns:
46+
A struct that is returned by the `_platform` function.
47+
"""
48+
if p.startswith("cp"):
49+
abi, _, p = p.partition("_")
50+
elif python_version:
51+
major, _, tail = python_version.partition(".")
52+
abi = "cp{}{}".format(major, tail)
53+
else:
54+
abi = None
55+
56+
os, _, arch = p.partition("_")
57+
return platform(abi = abi, os = os or None, arch = arch or None)

‎python/private/pypi/pep508_req.bzl renamed to ‎python/private/pypi/pep508_requirement.bzl

+6-3
Original file line numberDiff line numberDiff line change
@@ -17,7 +17,7 @@
1717

1818
load("//python/private:normalize_name.bzl", "normalize_name")
1919

20-
_STRIP = ["(", " ", ">", "=", "<", "~", "!"]
20+
_STRIP = ["(", " ", ">", "=", "<", "~", "!", "@"]
2121

2222
def requirement(spec):
2323
"""Parse a PEP508 requirement line
@@ -28,15 +28,18 @@ def requirement(spec):
2828
Returns:
2929
A struct with the information.
3030
"""
31+
spec = spec.strip()
3132
requires, _, maybe_hashes = spec.partition(";")
3233
marker, _, _ = maybe_hashes.partition("--hash")
3334
requires, _, extras_unparsed = requires.partition("[")
35+
extras_unparsed, _, _ = extras_unparsed.partition("]")
3436
for char in _STRIP:
3537
requires, _, _ = requires.partition(char)
36-
extras = extras_unparsed.strip("]").split(",")
38+
extras = extras_unparsed.replace(" ", "").split(",")
39+
name = requires.strip(" ")
3740

3841
return struct(
39-
name = normalize_name(requires.strip(" ")),
42+
name = normalize_name(name).replace("_", "-"),
4043
marker = marker.strip(" "),
4144
extras = extras,
4245
)

‎tests/pypi/pep508/BUILD.bazel

+5
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,10 @@
11
load(":evaluate_tests.bzl", "evaluate_test_suite")
2+
load(":requirement_tests.bzl", "requirement_test_suite")
23

34
evaluate_test_suite(
45
name = "evaluate_tests",
56
)
7+
8+
requirement_test_suite(
9+
name = "requirement_tests",
10+
)

‎tests/pypi/pep508/requirement_tests.bzl

+47
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,47 @@
1+
# Copyright 2025 The Bazel Authors. All rights reserved.
2+
#
3+
# Licensed under the Apache License, Version 2.0 (the "License");
4+
# you may not use this file except in compliance with the License.
5+
# You may obtain a copy of the License at
6+
#
7+
# http://www.apache.org/licenses/LICENSE-2.0
8+
#
9+
# Unless required by applicable law or agreed to in writing, software
10+
# distributed under the License is distributed on an "AS IS" BASIS,
11+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12+
# See the License for the specific language governing permissions and
13+
# limitations under the License.
14+
"""Tests for parsing the requirement specifier."""
15+
16+
load("@rules_testing//lib:test_suite.bzl", "test_suite")
17+
load("//python/private/pypi:pep508_requirement.bzl", "requirement") # buildifier: disable=bzl-visibility
18+
19+
_tests = []
20+
21+
def _test_requirement_line_parsing(env):
22+
want = {
23+
" name1[ foo ] ": ("name1", ["foo"]),
24+
"Name[foo]": ("name", ["foo"]),
25+
"name [fred,bar] @ http://foo.com ; python_version=='2.7'": ("name", ["fred", "bar"]),
26+
"name; (os_name=='a' or os_name=='b') and os_name=='c'": ("name", [""]),
27+
"name@http://foo.com": ("name", [""]),
28+
"name[ Foo123 ]": ("name", ["Foo123"]),
29+
"name[extra]@http://foo.com": ("name", ["extra"]),
30+
"name[foo]": ("name", ["foo"]),
31+
"name[quux, strange];python_version<'2.7' and platform_version=='2'": ("name", ["quux", "strange"]),
32+
"name_foo[bar]": ("name-foo", ["bar"]),
33+
}
34+
35+
got = {
36+
i: (parsed.name, parsed.extras)
37+
for i, parsed in {case: requirement(case) for case in want}.items()
38+
}
39+
env.expect.that_dict(got).contains_exactly(want)
40+
41+
_tests.append(_test_requirement_line_parsing)
42+
43+
def requirement_test_suite(name): # buildifier: disable=function-docstring
44+
test_suite(
45+
name = name,
46+
basic_tests = _tests,
47+
)

0 commit comments

Comments
 (0)