Improve wheel build

Allow to launch builds and then fetch built wheels later
Improve support for newer Pythons and OS versions.

Signed-off-by: Philippe Ombredanne <[email protected]>

Author: pombredanne

etc/scripts/build_wheels.py

@@ -69,6 +69,16 @@
     is_flag=True,
     help="Also include all dependent wheels.",
 )
+@click.option(
+    "--remote-build-log-file",
+    type=click.Path(writable=True),
+    default=None,
+    metavar="LOG-FILE",
+    help="Path to an optional log file where to list remote builds download URLs. "
+    "If provided, do not wait for remote builds to complete (and therefore, "
+    "do not download them either). Instead create a JSON lines log file with "
+    "one entry for each build suitable to fetch the artifacts at a later time.",
+)
 @click.option(
     "--verbose",
     is_flag=True,
@@ -83,6 +93,7 @@ def build_wheels(
     operating_system,
     with_deps,
     build_remotely,
+    remote_build_log_file,
     verbose,
 ):
     """
@@ -102,6 +113,7 @@ def build_wheels(
         build_remotely=build_remotely,
         with_deps=with_deps,
         verbose=verbose,
+        remote_build_log_file=remote_build_log_file,
     )
 
 

etc/scripts/fetch_built_wheels.py

@@ -0,0 +1,33 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+#
+# Copyright (c) nexB Inc. and others. All rights reserved.
+# ScanCode is a trademark of nexB Inc.
+# SPDX-License-Identifier: Apache-2.0
+# See http://www.apache.org/licenses/LICENSE-2.0 for the license text.
+# See https://github.com/nexB/skeleton for support or download.
+# See https://aboutcode.org for more information about nexB OSS projects.
+#
+import click
+
+import utils_thirdparty
+
+
+@click.command()
+@click.option(
+    "-d",
+    "--thirdparty-dir",
+    type=click.Path(exists=True, readable=True, path_type=str, file_okay=False),
+    required=True,
+    help="Path to the thirdparty directory to check.",
+)
+@click.help_option("-h", "--help")
+def check_thirdparty_dir(thirdparty_dir):
+    """
+    Check a thirdparty directory for problems.
+    """
+    utils_thirdparty.find_problems(dest_dir=thirdparty_dir)
+
+
+if __name__ == "__main__":
+    check_thirdparty_dir()

etc/scripts/fix_thirdparty.py

@@ -31,11 +31,22 @@
     is_flag=True,
     help="Build missing wheels remotely.",
 )
+@click.option(
+    "--remote-build-log-file",
+    type=click.Path(writable=True),
+    default=None,
+    metavar="LOG-FILE",
+    help="Path to an optional log file where to list remote builds download URLs. "
+    "If provided, do not wait for remote builds to complete (and therefore, "
+    "do not download them either). Instead create a JSON lines log file with "
+    "one entry for each build suitable to fetch the artifacts at a later time.",
+)
 @click.help_option("-h", "--help")
 def fix_thirdparty_dir(
     thirdparty_dir,
     build_wheels,
     build_remotely,
+    remote_build_log_file,
 ):
     """
     Fix a thirdparty directory of dependent package wheels and sdist.
@@ -58,11 +69,13 @@ def fix_thirdparty_dir(
     package_envts_not_built = []
     if build_wheels:
         print("***BUILD*** MISSING WHEELS")
-        package_envts_not_built, _wheel_filenames_built = utils_thirdparty.build_missing_wheels(
+        results = utils_thirdparty.build_missing_wheels(
             packages_and_envts=package_envts_not_fetched,
             build_remotely=build_remotely,
+            remote_build_log_file=remote_build_log_file,
             dest_dir=thirdparty_dir,
         )
+        package_envts_not_built, _wheel_filenames_built = results
 
     print("***ADD*** ABOUT AND LICENSES")
     utils_thirdparty.add_fetch_or_update_about_and_license_files(dest_dir=thirdparty_dir)

etc/scripts/test_utils_pip_compatibility_tags.py

@@ -47,7 +47,7 @@
     ],
 )
 def test_version_info_to_nodot(version_info, expected):
-    actual = pip_compatibility_tags.version_info_to_nodot(version_info)
+    actual = utils_pip_compatibility_tags.version_info_to_nodot(version_info)
     assert actual == expected
 
 
@@ -95,7 +95,7 @@ def test_manylinux2010_implies_manylinux1(self, manylinux2010, manylinux1):
         Specifying manylinux2010 implies manylinux1.
         """
         groups = {}
-        supported = pip_compatibility_tags.get_supported(platforms=[manylinux2010])
+        supported = utils_pip_compatibility_tags.get_supported(platforms=[manylinux2010])
         for tag in supported:
             groups.setdefault((tag.interpreter, tag.abi), []).append(tag.platform)
 
@@ -118,7 +118,7 @@ def test_manylinuxA_implies_manylinuxB(self, manylinuxA, manylinuxB):
         Specifying manylinux2014 implies manylinux2010/manylinux1.
         """
         groups = {}
-        supported = pip_compatibility_tags.get_supported(platforms=[manylinuxA])
+        supported = utils_pip_compatibility_tags.get_supported(platforms=[manylinuxA])
         for tag in supported:
             groups.setdefault((tag.interpreter, tag.abi), []).append(tag.platform)