Skip to content

Commit b1dabd8

Browse files
pombredannepombredanne
committed
Update scripts
Make fetch_built_wheel work Add new strip classifiers option to fix_thirdparty Improve simple requirements parsing to get the latest versions Signed-off-by: Philippe Ombredanne <[email protected]>
1 parent a7c2efd commit b1dabd8

File tree

4 files changed

+138
-41
lines changed

4 files changed

+138
-41
lines changed

‎etc/scripts/fetch_built_wheels.py

+37-7
Original file line numberDiff line numberDiff line change
@@ -5,7 +5,7 @@
55
# ScanCode is a trademark of nexB Inc.
66
# SPDX-License-Identifier: Apache-2.0
77
# See http://www.apache.org/licenses/LICENSE-2.0 for the license text.
8-
# See https://github.com/nexB/skeleton for support or download.
8+
# See https://github.com/nexB/scancode-toolkit for support or download.
99
# See https://aboutcode.org for more information about nexB OSS projects.
1010
#
1111
import click
@@ -14,20 +14,50 @@
1414

1515

1616
@click.command()
17+
@click.option(
18+
"--remote-build-log-file",
19+
type=click.Path(readable=True),
20+
metavar="LOG-FILE",
21+
help="Path to a remote builds log file.",
22+
)
1723
@click.option(
1824
"-d",
1925
"--thirdparty-dir",
2026
type=click.Path(exists=True, readable=True, path_type=str, file_okay=False),
21-
required=True,
22-
help="Path to the thirdparty directory to check.",
27+
metavar="DIR",
28+
default=utils_thirdparty.THIRDPARTY_DIR,
29+
show_default=True,
30+
help="Path to the thirdparty directory to save built wheels.",
31+
)
32+
@click.option(
33+
"--no-wait",
34+
is_flag=True,
35+
default=False,
36+
help="Do not wait for build completion.",
37+
)
38+
@click.option(
39+
"--verbose",
40+
is_flag=True,
41+
help="Provide verbose output.",
2342
)
2443
@click.help_option("-h", "--help")
25-
def check_thirdparty_dir(thirdparty_dir):
44+
def fetch_remote_wheels(
45+
remote_build_log_file,
46+
thirdparty_dir,
47+
no_wait,
48+
verbose,
49+
):
2650
"""
27-
Check a thirdparty directory for problems.
51+
Fetch to THIRDPARTY_DIR all the wheels built in the LOG-FILE JSON lines
52+
build log file.
2853
"""
29-
utils_thirdparty.find_problems(dest_dir=thirdparty_dir)
54+
utils_thirdparty.fetch_remotely_built_wheels(
55+
remote_build_log_file=remote_build_log_file,
56+
dest_dir=thirdparty_dir,
57+
no_wait=no_wait,
58+
verbose=verbose,
59+
)
3060

3161

3262
if __name__ == "__main__":
33-
check_thirdparty_dir()
63+
fetch_remote_wheels()

‎etc/scripts/fix_thirdparty.py

+41-25
Original file line numberDiff line numberDiff line change
@@ -41,12 +41,18 @@
4141
"do not download them either). Instead create a JSON lines log file with "
4242
"one entry for each build suitable to fetch the artifacts at a later time.",
4343
)
44+
@click.option(
45+
"--strip-classifiers",
46+
is_flag=True,
47+
help="Remove danglingf classifiers",
48+
)
4449
@click.help_option("-h", "--help")
4550
def fix_thirdparty_dir(
4651
thirdparty_dir,
4752
build_wheels,
4853
build_remotely,
4954
remote_build_log_file,
55+
strip_classifiers,
5056
):
5157
"""
5258
Fix a thirdparty directory of dependent package wheels and sdist.
@@ -61,35 +67,45 @@ def fix_thirdparty_dir(
6167
Optionally build missing binary wheels for all supported OS and Python
6268
version combos locally or remotely.
6369
"""
64-
print("***FETCH*** MISSING WHEELS")
65-
package_envts_not_fetched = utils_thirdparty.fetch_missing_wheels(dest_dir=thirdparty_dir)
66-
print("***FETCH*** MISSING SOURCES")
67-
src_name_ver_not_fetched = utils_thirdparty.fetch_missing_sources(dest_dir=thirdparty_dir)
68-
69-
package_envts_not_built = []
70-
if build_wheels:
71-
print("***BUILD*** MISSING WHEELS")
72-
results = utils_thirdparty.build_missing_wheels(
73-
packages_and_envts=package_envts_not_fetched,
74-
build_remotely=build_remotely,
75-
remote_build_log_file=remote_build_log_file,
70+
if strip_classifiers:
71+
print("***ADD*** ABOUT AND LICENSES, STRIP CLASSIFIERS")
72+
utils_thirdparty.add_fetch_or_update_about_and_license_files(
7673
dest_dir=thirdparty_dir,
74+
strip_classifiers=strip_classifiers,
7775
)
78-
package_envts_not_built, _wheel_filenames_built = results
79-
80-
print("***ADD*** ABOUT AND LICENSES")
81-
utils_thirdparty.add_fetch_or_update_about_and_license_files(dest_dir=thirdparty_dir)
82-
83-
# report issues
84-
for name, version in src_name_ver_not_fetched:
85-
print(f"{name}=={version}: Failed to fetch source distribution.")
86-
87-
for package, envt in package_envts_not_built:
88-
print(
89-
f"{package.name}=={package.version}: Failed to build wheel "
90-
f"on {envt.operating_system} for Python {envt.python_version}"
76+
else:
77+
print("***FETCH*** MISSING WHEELS")
78+
package_envts_not_fetched = utils_thirdparty.fetch_missing_wheels(dest_dir=thirdparty_dir)
79+
print("***FETCH*** MISSING SOURCES")
80+
src_name_ver_not_fetched = utils_thirdparty.fetch_missing_sources(dest_dir=thirdparty_dir)
81+
82+
package_envts_not_built = []
83+
if build_wheels:
84+
print("***BUILD*** MISSING WHEELS")
85+
results = utils_thirdparty.build_missing_wheels(
86+
packages_and_envts=package_envts_not_fetched,
87+
build_remotely=build_remotely,
88+
remote_build_log_file=remote_build_log_file,
89+
dest_dir=thirdparty_dir,
90+
)
91+
package_envts_not_built, _wheel_filenames_built = results
92+
93+
print("***ADD*** ABOUT AND LICENSES")
94+
utils_thirdparty.add_fetch_or_update_about_and_license_files(
95+
dest_dir=thirdparty_dir,
96+
strip_classifiers=strip_classifiers,
9197
)
9298

99+
# report issues
100+
for name, version in src_name_ver_not_fetched:
101+
print(f"{name}=={version}: Failed to fetch source distribution.")
102+
103+
for package, envt in package_envts_not_built:
104+
print(
105+
f"{package.name}=={package.version}: Failed to build wheel "
106+
f"on {envt.operating_system} for Python {envt.python_version}"
107+
)
108+
93109
print("***FIND PROBLEMS***")
94110
utils_thirdparty.find_problems(dest_dir=thirdparty_dir)
95111

‎etc/scripts/utils_requirements.py

+51-8
Original file line numberDiff line numberDiff line change
@@ -8,11 +8,13 @@
88
# See https://github.com/nexB/skeleton for support or download.
99
# See https://aboutcode.org for more information about nexB OSS projects.
1010
#
11+
import re
1112
import subprocess
1213

1314
"""
1415
Utilities to manage requirements files and call pip.
15-
NOTE: this should use ONLY the standard library and not import anything else.
16+
NOTE: this should use ONLY the standard library and not import anything else
17+
becasue this is used for boostrapping.
1618
"""
1719

1820

@@ -27,28 +29,69 @@ def load_requirements(requirements_file="requirements.txt", force_pinned=True):
2729
return get_required_name_versions(req_lines, force_pinned)
2830

2931

30-
def get_required_name_versions(requirement_lines, force_pinned=True):
32+
def get_required_name_versions(
33+
requirement_lines,
34+
force_pinned=True,
35+
):
3136
"""
3237
Yield required (name, version) tuples given a`requirement_lines` iterable of
3338
requirement text lines. Every requirement versions must be pinned if
3439
`force_pinned` is True. Otherwise un-pinned requirements are returned with a
35-
None version
40+
None version.
41+
3642
"""
3743
for req_line in requirement_lines:
3844
req_line = req_line.strip()
3945
if not req_line or req_line.startswith("#"):
4046
continue
41-
if "==" not in req_line and force_pinned:
42-
raise Exception(f"Requirement version is not pinned: {req_line}")
47+
if force_pinned:
48+
if "==" not in req_line:
49+
raise Exception(f"Requirement version is not pinned: {req_line}")
4350
name = req_line
4451
version = None
4552
else:
46-
name, _, version = req_line.partition("==")
47-
name = name.lower().strip()
48-
version = version.lower().strip()
53+
if req_line.startswith("-"):
54+
print(f"Requirement skipped, is not supported: {req_line}")
55+
56+
if "==" in req_line:
57+
name, _, version = req_line.partition("==")
58+
version = version.lower().strip()
59+
else:
60+
# FIXME: we do not support unpinned requirements yet!
61+
name = strip_reqs(req_line)
62+
version = None
63+
64+
name = name.lower().strip()
4965
yield name, version
5066

5167

68+
def strip_reqs(line):
69+
"""
70+
Return a name given a pip reuirement text ``line` striping version and
71+
requirements.
72+
73+
For example::
74+
75+
>>> s = strip_reqs("foo <=12, >=13,!=12.6")
76+
>>> assert s == "foo"
77+
"""
78+
if "--" in line:
79+
raise Exception(f"Unsupported requirement style: {line}")
80+
81+
line = line.strip()
82+
83+
ops = "><!=~;, []"
84+
85+
def has_ops(l):
86+
return any(op in l for op in ops)
87+
88+
if not has_ops:
89+
return line
90+
91+
splitter = re.compile(r"[><!=~;, \[\]]+").split
92+
return splitter(line)[0]
93+
94+
5295
def parse_requires(requires):
5396
"""
5497
Return a list of requirement lines extracted from the `requires` text from

‎etc/scripts/utils_thirdparty.py

+9-1
Original file line numberDiff line numberDiff line change
@@ -2528,7 +2528,11 @@ def hash_requirements(dest_dir=THIRDPARTY_DIR, requirements_file="requirements.t
25282528
################################################################################
25292529

25302530

2531-
def add_fetch_or_update_about_and_license_files(dest_dir=THIRDPARTY_DIR, include_remote=True):
2531+
def add_fetch_or_update_about_and_license_files(
2532+
dest_dir=THIRDPARTY_DIR,
2533+
include_remote=True,
2534+
strip_classifiers=False,
2535+
):
25322536
"""
25332537
Given a thirdparty dir, add missing ABOUT. LICENSE and NOTICE files using
25342538
best efforts:
@@ -2560,6 +2564,10 @@ def get_other_dists(_package, _dist):
25602564
local_dist.load_about_data(dest_dir=dest_dir)
25612565
local_dist.set_checksums(dest_dir=dest_dir)
25622566

2567+
if strip_classifiers and "classifiers" in local_dist.extra_data:
2568+
local_dist.extra_data.pop("classifiers", None)
2569+
local_dist.save_about_and_notice_files(dest_dir)
2570+
25632571
# if has key data we may look to improve later, but we can move on
25642572
if local_dist.has_key_metadata():
25652573
local_dist.save_about_and_notice_files(dest_dir=dest_dir)

0 commit comments

Comments
 (0)