Add plugin system and local wheels support (#91)

Signed-off-by: Jean-Christophe Morin <[email protected]>

.coveragerc

@@ -0,0 +1,9 @@
+[run]
+branch = True
+source_pkgs=rez_pip
+
+[report]
+exclude_also =
+    def __dir__
+    if TYPE_CHECKING:
+    if typing.TYPE_CHECKING:

.gitattributes

@@ -0,0 +1,5 @@
+# Set the default behavior, in case people don't have core.autocrlf set.
+* text=auto
+
+# Denote all files that are truly binary and should not be modified.
+*.patch binary

.gitignore

@@ -57,3 +57,4 @@ tests/data/rez_repo/
 tests/data/_tmp_download/
 docs/bin/
 .idea/
+/patches

docs/source/api.rst

@@ -0,0 +1,27 @@
+===
+API
+===
+
+.. warning:: The API is only meant to be used by plugins authors.
+
+.. autoclass:: rez_pip.pip.T
+
+.. autoclass:: rez_pip.pip.DownloadInfo
+   :members:
+
+.. autoclass:: rez_pip.pip.Metadata
+   :members:
+   :undoc-members:
+
+.. autoclass:: rez_pip.pip.PackageInfo
+   :members:
+
+.. autoclass:: rez_pip.pip.PackageGroup
+   :members:
+   :show-inheritance:
+
+.. autoclass:: rez_pip.pip.DownloadedArtifact
+   :members:
+
+.. autoclass:: rez_pip.plugins.CleanupAction
+   :members:

docs/source/conf.py

@@ -4,14 +4,20 @@
 # https://www.sphinx-doc.org/en/master/usage/configuration.html
 
 import re
+import inspect
 import argparse
+import importlib
 
 import docutils.nodes
 import sphinx.transforms
+import sphinx.util.nodes
 import sphinx.application
+import sphinx.ext.autodoc
 import sphinx.util.docutils
+import docutils.statemachine
 
 import rez_pip.cli
+import rez_pip.plugins
 
 # -- Project information -----------------------------------------------------
 # https://www.sphinx-doc.org/en/master/usage/configuration.html#project-information
@@ -25,6 +31,7 @@
 
 extensions = [
     # first-party extensions
+    "sphinx.ext.todo",
     "sphinx.ext.autodoc",
     "sphinx.ext.extlinks",
     "sphinx.ext.intersphinx",
@@ -63,13 +70,26 @@
 # https://www.sphinx-doc.org/en/master/usage/extensions/intersphinx.html
 
 intersphinx_mapping = {
+    "python": ("https://docs.python.org/3", None),
     "rez": ("https://rez.readthedocs.io/en/stable/", None),
 }
 
 # Force usage of :external:
-intersphinx_disabled_reftypes = ["*"]
+# intersphinx_disabled_reftypes = ["*"]
 
 
+# -- Options for sphinx.ext.autodoc ------------------------------------------
+# https://www.sphinx-doc.org/en/master/usage/extensions/autodoc.html
+
+# autodoc_typehints = "description"
+autodoc_typehints_format = "short"
+autodoc_member_order = "bysource"
+
+# -- Options for sphinx.ext.todo --------------------------------------------
+# https://www.sphinx-doc.org/en/master/usage/extensions/todo.html
+
+todo_include_todos = True
+
 # -- Custom ------------------------------------------------------------------
 # Custom stuff
 
@@ -235,6 +255,167 @@ def run(self) -> list[docutils.nodes.Node]:
         return node.children
 
 
+class RezAutoPlugins(sphinx.util.docutils.SphinxDirective):
+    """
+    Special rez-pip-autoplugins directive. This is quite similar to "autosummary" in some ways.
+    """
+
+    required_arguments = 0
+    optional_arguments = 0
+
+    def run(self) -> list[docutils.nodes.Node]:
+        # Create the node.
+        node = docutils.nodes.section()
+        node.document = self.state.document
+
+        rst = docutils.statemachine.ViewList()
+
+        # Add rezconfig as a dependency to the current document. The document
+        # will be rebuilt if rezconfig changes.
+        self.env.note_dependency(rez_pip.plugins.__file__)
+        self.env.note_dependency(__file__)
+
+        path, lineNumber = self.get_source_info()
+
+        document = []
+        for plugin, hooks in rez_pip.plugins._getHookImplementations().items():
+            hooks = [f":func:`{hook}`" for hook in hooks]
+            document.append(f"* {plugin.split('.')[-1]}: {', '.join(hooks)}")
+
+        document = "\n".join(document)
+
+        # Add each line to the view list.
+        for index, line in enumerate(document.split("\n")):
+            # Note to future people that will look at this.
+            # "line" has to be a single line! It can't be a line like "this\nthat".
+            rst.append(line, path, lineNumber + index)
+
+        # Finally, convert the rst into the appropriate docutils/sphinx nodes.
+        sphinx.util.nodes.nested_parse_with_titles(self.state, rst, node)
+
+        # Return the generated nodes.
+        return node.children
+
+
+class RezPipAutoPluginHooks(sphinx.util.docutils.SphinxDirective):
+    """
+    Special rez-pip-autopluginhooks directive. This is quite similar to "autosummary" in some ways.
+    """
+
+    required_arguments = 1
+    optional_arguments = 0
+
+    def run(self) -> list[docutils.nodes.Node]:
+        # Create the node.
+        node = docutils.nodes.section()
+        node.document = self.state.document
+
+        rst = docutils.statemachine.ViewList()
+
+        # Add rezconfig as a dependency to the current document. The document
+        # will be rebuilt if rezconfig changes.
+        self.env.note_dependency(rez_pip.plugins.__file__)
+        self.env.note_dependency(__file__)
+
+        path, lineNumber = self.get_source_info()
+
+        fullyQualifiedClassName = self.arguments[0]
+        module, klassname = fullyQualifiedClassName.rsplit(".", 1)
+
+        mod = importlib.import_module(module)
+        klass = getattr(mod, klassname)
+
+        methods = [
+            method
+            for method in inspect.getmembers(klass, predicate=inspect.isfunction)
+            if not method[0].startswith("_")
+        ]
+
+        document = []
+        for method in sorted(methods, key=lambda x: x[1].__code__.co_firstlineno):
+            document.append(f".. autohook:: {module}.{klassname}.{method[0]}")
+
+        document = "\n".join(document)
+
+        # Add each line to the view list.
+        for index, line in enumerate(document.split("\n")):
+            # Note to future people that will look at this.
+            # "line" has to be a single line! It can't be a line like "this\nthat".
+            rst.append(line, path, lineNumber + index)
+
+        # Finally, convert the rst into the appropriate docutils/sphinx nodes.
+        sphinx.util.nodes.nested_parse_with_titles(self.state, rst, node)
+
+        # Return the generated nodes.
+        return node.children
+
+
+def autodoc_process_signature(
+    app: sphinx.application.Sphinx,
+    what: str,
+    name: str,
+    obj,
+    options: dict,
+    signature: str,
+    return_annotation,
+):
+    for name in ["Sequence", "Mapping", "MutableSequence"]:
+        signature = signature.replace(
+            f"rez_pip.compat.{name}", f"~collections.abc.{name}"
+        )
+        if return_annotation:
+            return_annotation = return_annotation.replace(
+                f"rez_pip.compat.{name}", f"~collections.abc.{name}"
+            )
+
+    signature = signature.replace(
+        "rez_pip.compat.importlib_metadata", "~importlib.metadata"
+    )
+
+    return signature, return_annotation
+
+
+class HookDocumenter(sphinx.ext.autodoc.FunctionDocumenter):
+    """
+    Custom autohook directive to document our hooks.
+    It allows us to easily document the hooks from the rez_pip.plugins.PluginSpec
+    class without exposing the class and module name.
+    """
+
+    objtype = "hook"  # auto + hook
+    directivetype = "function"  # generated reST directive
+
+    def format_signature(self, **kwargs) -> str:
+        """
+        Format the signature and remove self. We really don't want to expose
+        the class and module name or the fact that we are documenting methods.
+        """
+        sig = super().format_signature(**kwargs)
+        sig = re.sub(r"\(self(,\s)?", "(", sig)
+
+        # Also force short names for our own types
+        sig = sig.replace("rez_pip.", "~rez_pip.")
+
+        return sig
+
+    def add_directive_header(self, sig):
+        modname = self.modname
+        # Hacky, but it does the job. This should remove the module name from the directive
+        # created by autodoc.
+        self.modname = ""
+
+        data = super().add_directive_header(sig)
+
+        # We need to restore it because autodoc does lots of things with the module name.
+        self.modname = modname
+        return data
+
+
 def setup(app: sphinx.application.Sphinx):
     app.add_directive("rez-autoargparse", RezAutoArgparseDirective)
+    app.add_directive("rez-pip-autoplugins", RezAutoPlugins)
+    app.add_directive("rez-pip-autopluginhooks", RezPipAutoPluginHooks)
     app.add_transform(ReplaceGHRefs)
+
+    app.connect("autodoc-process-signature", autodoc_process_signature)
+    app.add_autodocumenter(HookDocumenter)

docs/source/faq.rst

@@ -4,8 +4,31 @@ FAQ
 
 List of commonly asked questions.
 
-Why does the rez package created by rez-pip creates a variant per platform?
-===========================================================================
+Which packages does it support?
+===============================
+
+It technically supports all packages available on PyPI that are distributed as wheels.
+Packages that only provide an sdist are not supported.
+
+We say "technically" because there are some exceptions. Some packages on PyPI rely
+on DSOs (shared libraries, i.e. ``.so``/``.DLL``/``.dylib`` files) that are not available on
+all platforms. This is normal and is supported for most packages. However, there are some
+packages that rely on methods like adding paths using :func:`os.add_dll_directory` or
+hardcoded paths.
+
+Some others rely on `path configuration (.pth) files <https://docs.python.org/3/library/site.html>`_.
+
+When a package relies on these methods, rez-pip will successfully install it, but
+the package might not function correctly (either partly or entirely).
+
+The :doc:`plugin system <plugins>` was created to handled these cases. You can use plugins
+to modify the package metatada, patch source files, add/remove files, etc.
+
+``rez-pip`` comes with some :ref:`built-in plugins <plugins:built-in plugins>` for packages that are popular
+in our communities and are known to be "broken" when installed with ``rez-pip``.
+
+Why does the rez package created by rez-pip create a variant per platform?
+==========================================================================
 
 Sometimes rez-pip creates rez packages that have variants for the platform and arch on which they were installed,
 and sometimes it even creates variants for Python versions. Bellow are the scenarios

docs/source/index.rst

@@ -22,6 +22,7 @@ Features
   create per python version variants when installing a package that has console scripts.
 * Better output logs.
 * Implemented as an out-of-tree plugin, which means faster development cycle and more frequent releases.
+* :doc:`Plugin system <plugins>` that allows for easy extensibility (experimental).
 * Maintained by the rez maintainers.
 
 Prerequisites
@@ -69,5 +70,7 @@ automatically created by the `install.py <https://github.com/AcademySoftwareFoun
    command
    transition
    metadata
+   plugins
+   api
    faq
    changelog

docs/source/plugins.rst

@@ -0,0 +1,71 @@
+=======
+Plugins
+=======
+
+.. versionadded:: 0.4.0
+
+.. warning::
+   Plugins are new and have not been tested througfully. There might be bugs, missing
+   features and rough edges.
+
+   We encourage you to try them out and report any issues you might find.
+
+   If you create your own plugins, expect them to break when you update
+   rez-pip's minor version. The plugin system is still in its early stages
+   and we will probably release breaking changes in the future.
+
+rez-pip can be extended using plugins. Plugins can be used to do various things, such as
+modifying packages (both metadata and files), etc.
+
+This page documents the hooks available to plugins and how to implement plugins.
+
+List installed plugins
+======================
+
+To list all installed plugins, use the :option:`rez-pip --list-plugins` command line argument.
+
+Register a plugin
+=================
+
+rez-pip's plugin system is based on the `pluggy <https://pluggy.readthedocs.io/en/latest/>`_ framework,
+and as such, plugins must be registered using `entry points <https://packaging.python.org/en/latest/specifications/entry-points/>`_.
+
+The entry point group is named ``rez-pip``.
+
+In a ``pyproject.toml`` file, it can be set like this:
+
+.. code-block:: toml
+   :caption: pyproject.toml
+
+   [project.entry-points."rez-pip"]
+   my_plugin = "my_plugin_module"
+
+
+Functions
+=========
+
+.. Not Using autodoc here because the decorator has a complex
+   signature to help type hinters. That signature is not needed
+   for the end user.
+.. py:decorator:: rez_pip.plugins.hookimpl
+
+   Decorator used to register a plugin hook.
+
+Hooks
+=====
+
+The list of available hooks is provided below. They are listed in the order they
+are called by rez-pip.
+
+.. rez-pip-autopluginhooks:: rez_pip.plugins.PluginSpec
+
+
+Built-in plugins
+================
+
+rez-pip comes with some built-in plugins that are enabled by default. They exists mostly
+to fix packages that are known to be "broken" if we don't fix them using plugins.
+
+This lists the plugin names and the hooks they implement.
+
+.. rez-pip-autoplugins::