Initial commit adding a plugin system and a PySide6 plugin

Signed-off-by: Jean-Christophe Morin <[email protected]>

docs/source/index.rst

@@ -69,5 +69,6 @@ automatically created by the `install.py <https://github.com/AcademySoftwareFoun
    command
    transition
    metadata
+   plugins
    faq
    changelog

docs/source/plugins.rst

@@ -0,0 +1,5 @@
+=======
+Plugins
+=======
+
+.. autoclass:: rez_pip.plugins.PluginSpec

src/rez_pip/cli.py

@@ -8,6 +8,7 @@
 import textwrap
 import pathlib
 import tempfile
+import itertools
 import subprocess
 
 if sys.version_info >= (3, 10):
@@ -18,13 +19,15 @@
 import rich
 import rich.text
 import rich.panel
+import rich.table
 import rez.version
 import rich.markup
 import rich.logging
 
 import rez_pip.pip
 import rez_pip.rez
 import rez_pip.data
+import rez_pip.plugins
 import rez_pip.install
 import rez_pip.download
 import rez_pip.exceptions
@@ -113,6 +116,10 @@ def _createParser() -> argparse.ArgumentParser:
         help="Print debug information that you can use when reporting an issue on GitHub.",
     )
 
+    debugGroup.add_argument(
+        "--list-plugins", action="store_true", help="List all registered plugins"
+    )
+
     parser.usage = f"""
 
   %(prog)s [options] <package(s)>
@@ -195,37 +202,55 @@ def _run(args: argparse.Namespace, pipArgs: typing.List[str], pipWorkArea: str)
             )
 
         _LOG.info(f"Resolved {len(packages)} dependencies for python {pythonVersion}")
+        packageGroups: typing.List[rez_pip.pip.PackageGroup] = list(
+            itertools.chain(*rez_pip.plugins.getHook().groupPackages(packages=packages))
+        )
+        packageGroups += [rez_pip.pip.PackageGroup([package]) for package in packages]
 
         # TODO: Should we postpone downloading to the last minute if we can?
         _LOG.info("[bold]Downloading...")
-        wheels = rez_pip.download.downloadPackages(packages, wheelsDir)
-        _LOG.info(f"[bold]Downloaded {len(wheels)} wheels")
 
-        dists: typing.Dict[importlib_metadata.Distribution, bool] = {}
+        wheelsToDownload = []
+        localWheels = []
+        for group in packageGroups:
+            for url in group.downloadUrls:
+                print(url)
+                if url.startswith("file://"):
+                    localWheels.append(url[7:])
+                else:
+                    wheelsToDownload.extend(group.packages)
+
+        downloadedWheels = rez_pip.download.downloadPackages(
+            wheelsToDownload, wheelsDir
+        )
+        _LOG.info(f"[bold]Downloaded {len(downloadedWheels)} wheels")
+
+        localWheels += downloadedWheels
 
+        # Here, we could have a mapping of <merged package>: <dists> and pass that to installWheel
         with rich.get_console().status(
             f"[bold]Installing wheels into {installedWheelsDir!r}"
         ):
-            for package, wheel in zip(packages, wheels):
-                _LOG.info(f"[bold]Installing {package.name}-{package.version} wheel")
-                dist, isPure = rez_pip.install.installWheel(
-                    package, pathlib.Path(wheel), installedWheelsDir
-                )
-
-                dists[dist] = isPure
-
-        distNames = [dist.name for dist in dists.keys()]
+            for group in packageGroups:
+                for package, wheel in zip(group.packages, group.downloadUrls):
+                    _LOG.info(f"[bold]Installing {wheel}")
+                    dist = rez_pip.install.installWheel(
+                        package,
+                        pathlib.Path(
+                            wheel[7:] if wheel.startswith("file://") else wheel
+                        ),
+                        os.path.join(installedWheelsDir, package.name),
+                    )
+                    group.dists.append(dist)
 
         with rich.get_console().status("[bold]Creating rez packages..."):
-            for dist, package in zip(dists, packages):
-                isPure = dists[dist]
+            for group in packageGroups:
+                print(list(package.name for package in group.packages))
                 rez_pip.rez.createPackage(
-                    dist,
-                    isPure,
+                    group.dists,
                     rez.version.Version(pythonVersion),
-                    distNames,
                     installedWheelsDir,
-                    wheelURL=package.download_info.url,
+                    group.downloadUrls,
                     prefix=args.prefix,
                     release=args.release,
                 )
@@ -306,10 +331,24 @@ def _debug(
     )
 
 
+def _printPlugins() -> None:
+    table = rich.table.Table("Name", "Hooks", box=None)
+    for plugin, hooks in rez_pip.plugins._getHookImplementations().items():
+        table.add_row(plugin, ", ".join(hooks))
+    rich.get_console().print(table)
+
+
 def run() -> int:
     pipWorkArea = tempfile.mkdtemp(prefix="rez-pip-target")
     args, pipArgs = _parseArgs(sys.argv[1:])
 
+    # Initialize the plugin system
+    rez_pip.plugins.getManager()
+
+    if args.list_plugins:
+        _printPlugins()
+        return 0
+
     try:
         _validateArgs(args)
 

src/rez_pip/install.py

@@ -37,9 +37,12 @@
     ScriptSection = Literal["console", "gui"]
 
 
-def isWheelPure(source: installer.sources.WheelSource) -> bool:
-    stream = source.read_dist_info("WHEEL")
-    metadata = installer.utils.parse_metadata_file(stream)
+def isWheelPure(dist: importlib_metadata.Distribution) -> bool:
+    path = next(
+        f for f in dist.files if os.fspath(f.locate()).endswith(".dist-info/WHEEL")
+    )
+    with open(path.locate()) as fd:
+        metadata = installer.utils.parse_metadata_file(fd.read())
     return typing.cast(str, metadata["Root-Is-Purelib"]) == "true"
 
 
@@ -70,7 +73,7 @@ def installWheel(
     package: rez_pip.pip.PackageInfo,
     wheelPath: pathlib.Path,
     targetPath: str,
-) -> typing.Tuple[importlib_metadata.Distribution, bool]:
+) -> importlib_metadata.Distribution:
     # TODO: Technically, target should be optional. We will always want to install in "pip install --target"
     #       mode. So right now it's a CLI option for debugging purposes.
 
@@ -81,11 +84,8 @@ def installWheel(
         script_kind=installer.utils.get_launcher_kind(),
     )
 
-    isPure = True
     _LOG.debug(f"Installing {wheelPath} into {targetPath!r}")
     with installer.sources.WheelFile.open(wheelPath) as source:
-        isPure = isWheelPure(source)
-
         installer.install(
             source=source,
             destination=destination,
@@ -118,7 +118,7 @@ def installWheel(
         if not dist.files:
             raise RuntimeError(f"{path!r} does not exist!")
 
-    return dist, isPure
+    return dist
 
 
 # TODO: Document where this code comes from.

src/rez_pip/pip.py

@@ -8,9 +8,15 @@
 import subprocess
 import dataclasses
 
+if sys.version_info >= (3, 10):
+    import importlib.metadata as importlib_metadata
+else:
+    import importlib_metadata
+
 import dataclasses_json
 
 import rez_pip.data
+import rez_pip.plugins
 import rez_pip.exceptions
 
 _LOG = logging.getLogger(__name__)
@@ -58,6 +64,21 @@ def version(self) -> str:
         return self.metadata.version
 
 
+class PackageGroup:
+    """A group of package"""
+
+    packages: typing.List[PackageInfo]
+    dists: typing.List[importlib_metadata.Distribution]
+
+    def __init__(self, packages: typing.List[PackageInfo]) -> None:
+        self.packages = packages
+        self.dists = []
+
+    @property
+    def downloadUrls(self) -> typing.List[str]:
+        return [p.download_info.url for p in self.packages]
+
+
 def getBundledPip() -> str:
     return os.path.join(os.path.dirname(rez_pip.data.__file__), "pip.pyz")
 
@@ -71,7 +92,9 @@ def getPackages(
     constraints: typing.List[str],
     extraArgs: typing.List[str],
 ) -> typing.List[PackageInfo]:
-    # python pip.pyz install -q requests --dry-run --ignore-installed --python-version 2.7 --only-binary=:all: --target /tmp/asd --report -
+    rez_pip.plugins.getHook().prePipResolve(
+        packages=packageNames, requirements=requirements
+    )
 
     _fd, tmpFile = tempfile.mkstemp(prefix="pip-install-output", text=True)
     os.close(_fd)
@@ -138,6 +161,8 @@ def getPackages(
         packageInfo = PackageInfo.from_dict(rawPackage)
         packages.append(packageInfo)
 
+    rez_pip.plugins.getHook().postPipResolve(packages=packages)
+
     return packages
 
 

src/rez_pip/plugins/PySide6.py

@@ -0,0 +1,106 @@
+"""PySide6 plugin.
+
+For PySide6, we need a merge hook. If User says "install PySide6", we need to install PySide6, PySide6-Addons and PySide6-Essentials and shiboken6.
+
+But PySide6, PySide6-Addons and PySide6-Essentials have to be merged. Additionally, shiboken6 needs to be broken down to remove PySide6 (core).
+Because shiboken6 vendors PySide6-core... See https://inspector.pypi.io/project/shiboken6/6.6.1/packages/bb/72/e54f758e49e8da0dcd9490d006c41a814b0e56898ce4ca054d60cdba97bd/shiboken6-6.6.1-cp38-abi3-manylinux_2_28_x86_64.whl/.
+
+On Windows, the PySide6/openssl folder has to be added to PATH, see https://inspector.pypi.io/project/pyside6/6.6.1/packages/ec/3d/1da1b88d74cb5318466156bac91f17ad4272c6c83a973e107ad9a9085009/PySide6-6.6.1-cp38-abi3-win_amd64.whl/PySide6/__init__.py#line.81.
+
+So it's at least a 3 steps process:
+1. Merge PySide6, PySide6-Essentials and PySide6-Addons into the same install. Unvendor shiboken.
+2. Install shiboken + cleanup. The Cleanup could be its own hook here specific to shiboken.
+"""
+import sys
+import typing
+import logging
+import collections
+
+if sys.version_info >= (3, 10):
+    import importlib.metadata as importlib_metadata
+else:
+    import importlib_metadata
+
+import packaging.utils
+import packaging.version
+import packaging.specifiers
+import packaging.requirements
+
+import rez_pip.pip
+import rez_pip.plugins
+import rez_pip.exceptions
+
+# PySide6 was initiall a single package that had shiboken as a dependency.
+# Starting from 6.3.0, the package was spit in 3, PySide6, PySide6-Essentials and
+# PySide6-Addons.
+
+
+_LOG = logging.getLogger(__name__)
+
+
+@rez_pip.plugins.hookimpl
+def prePipResolve(
+    packages: typing.List[str],
+) -> None:
+    _LOG.debug(f"prePipResolve start")
+    pyside6Seen = False
+    variantsSeens = []
+
+    for package in packages:
+        req = packaging.requirements.Requirement(package)
+        name = packaging.utils.canonicalize_name(req.name)
+
+        if name == "pyside6":
+            pyside6Seen = True
+        elif name in ["pyside6-essentials", "pyside6-addons"]:
+            variantsSeens.append(req.name)
+
+    if variantsSeens and not pyside6Seen:
+        variants = " and ".join(variantsSeens)
+        verb = "was" if len(variantsSeens) == 1 else "were"
+        raise rez_pip.exceptions.RezPipError(
+            f"{variants} {verb} requested but PySide6 was not. You must explicitly request PySide6 in addition to {variants}."
+        )
+
+
+@rez_pip.plugins.hookimpl
+def postPipResolve(packages: typing.List[rez_pip.pip.PackageInfo]) -> None:
+    """
+    This hook is implemented out of extra caution. We really don't want PySide6-Addons
+    or PySide6-Essentials to be installed without PySide6.
+
+    In this case, we cover cases where a user requests a package X and that package
+    depends on PySide6-Addons or PySide6-Essentials.
+    """
+    pyside6Seen = False
+    variantsSeens = []
+
+    for package in packages:
+        name = packaging.utils.canonicalize_name(package.name)
+        if name == "pyside6":
+            pyside6Seen = True
+        elif name in ["pyside6-essentials", "pyside6-addons"]:
+            variantsSeens.append(package.name)
+
+    if variantsSeens and not pyside6Seen:
+        variants = " and ".join(variantsSeens)
+        verb = "is" if len(variantsSeens) == 1 else "are"
+        raise rez_pip.exceptions.RezPipError(
+            f"{variants} {verb} part of the resolved packages but PySide6 was not. Dependencies and or you must explicitly request PySide6 in addition to {variants}."
+        )
+
+
+@rez_pip.plugins.hookimpl
+def groupPackages(
+    packages: typing.List[rez_pip.pip.PackageInfo],
+) -> typing.List[rez_pip.pip.PackageGroup]:
+    data = []
+    for index, package in enumerate(packages[:]):
+        if packaging.utils.canonicalize_name(package.name) in [
+            "pyside6",
+            "pyside6-addons",
+            "pyside6-essentials",
+        ]:
+            data.append(package)
+            packages.remove(package)
+    return [rez_pip.pip.PackageGroup(data)]