minimal seemingly working approach to dynamic dependencies for legacydeps

Author: alexbarcelo

compile_protos.py

@@ -1,8 +1,9 @@
 #!/usr/bin/env python3
 
+import inspect
+from typing import Any
 from importlib import resources
 from typing import Any
-import grpc_tools.protoc
 
 try:
     from hatchling.builders.hooks.plugin.interface import BuildHookInterface
@@ -14,6 +15,10 @@
 
 
 def run_protoc():
+    # Here because during the build process, CustomBuildHook will be imported
+    # *before* knowing the dependencies of the hook itself.
+    import grpc_tools.protoc
+
     grpc_tools_proto = (resources.files("grpc_tools") / "_proto").resolve()
     grpc_tools.protoc.main(
         [
@@ -29,12 +34,38 @@ def run_protoc():
     )
 
 
+def find_config_settings_in_hatchling() -> dict[str, Any]:
+    # Terrible workaround (their words, not mine) given by @virtuald
+    # https://github.com/pypa/hatch/issues/1072#issuecomment-2448985229
+    # Hopefully this will be fixed in the future
+    for frame_info in inspect.stack():
+        frame = frame_info.frame
+        module = inspect.getmodule(frame)
+        if (
+            module
+            and module.__name__.startswith("hatchling.build")
+            and "config_settings" in frame.f_locals
+        ):
+            return frame.f_locals["config_settings"]
+
+    return {}
+
+
 class CustomBuildHook(BuildHookInterface):
     def initialize(self, version: str, build_data: dict[str, Any]) -> None:
         run_protoc()
 
     def dependencies(self):
-        return ["grpcio-tools==1.48.2"]
+        if find_config_settings_in_hatchling().get("LEGACY_DEPS", "False").lower() in (
+            "true",
+            "on",
+            "1",
+            "y",
+            "yes",
+        ):
+            return ["grpcio-tools==1.48.2"]
+        else:
+            return ["grpcio-tools==1.67.1"]
 
 
 if __name__ == "__main__":

dynamic_dependencies.py

@@ -0,0 +1,81 @@
+import inspect
+import os
+import re
+from typing import Any
+
+from hatchling.metadata.plugin.interface import MetadataHookInterface
+from packaging.requirements import Requirement
+from packaging.utils import canonicalize_name
+
+# The helper functions in this file are heavily inspired on the job layed
+# out by the hatch plugin `hatch-requirements-txt`
+
+
+COMMENT_RE = re.compile(r"(^|\s+)#.*$")
+PIP_COMMAND_RE = re.compile(r"\s+(-[A-Za-z]|--[A-Za-z]+)")
+
+
+def find_config_settings_in_hatchling() -> dict[str, Any]:
+    # Terrible workaround (their words, not mine) given by @virtuald
+    # https://github.com/pypa/hatch/issues/1072#issuecomment-2448985229
+    # Hopefully this will be fixed in the future
+    for frame_info in inspect.stack():
+        frame = frame_info.frame
+        module = inspect.getmodule(frame)
+        if (
+            module
+            and module.__name__.startswith("hatchling.build")
+            and "config_settings" in frame.f_locals
+        ):
+            return frame.f_locals["config_settings"]
+
+    return {}
+
+
+def parse_requirements(requirements: list[str]) -> tuple[list[Requirement], list[str]]:
+    comments = []
+    parsed_requirements: list[Requirement] = []
+
+    for line in requirements:
+        if line.lstrip().startswith("#"):
+            comments.append(line)
+        elif line.lstrip().startswith("-"):
+            # Likely an argument to pip from a requirements.txt file intended for pip
+            # (e.g. from pip-compile)
+            pass
+        elif line:
+            # Strip comments from end of line
+            line = COMMENT_RE.sub("", line)
+            if "-" in line:
+                line = PIP_COMMAND_RE.split(line)[0]
+            req = Requirement(line)
+            req.name = canonicalize_name(req.name)
+            parsed_requirements.append(req)
+
+    return parsed_requirements, comments
+
+
+def load_requirements(filename: str) -> list[str]:
+    if not os.path.isfile(filename):
+        raise FileNotFoundError(filename)
+    with open(filename, encoding="UTF-8") as fp:
+        contents = fp.read()
+        # Unfold lines ending with \
+        contents = re.sub(r"\\\s*\n", " ", contents)
+        parsed_requirements, _ = parse_requirements(contents.splitlines())
+    
+    return [str(r) for r in parsed_requirements]
+
+
+class DynamicDependenciesMetaDataHook(MetadataHookInterface):
+    def update(self, metadata):
+        if find_config_settings_in_hatchling().get("LEGACY_DEPS", "False").lower() in (
+            "true",
+            "on",
+            "1",
+            "y",
+            "yes",
+        ):
+            metadata["dependencies"] = load_requirements("requirements-legacydeps.txt")
+        else:
+            metadata["dependencies"] = load_requirements("requirements.txt")

pyproject.toml

@@ -26,27 +26,13 @@ classifiers = [
     "Programming Language :: Python :: 3 :: Only",
 ]
 requires-python = ">=3.9,<3.14"
-dependencies = [
-    "aiorwlock>=1.4.0",
-    "bcrypt>=4.1.1",
-    "grpcio>=1.67.1",
-    "grpcio-health-checking>=1.67.1",
-    "hiredis>=3.0.0",
-    "opentelemetry-api>=1.28.1",
-    "protobuf>=4.25.0",
-    "psutil>=6.1.0",
-    "pydantic-settings>=2.6.0",
-    "redis>=5.1.1",
-    "get-annotations;python_version<\"3.10\"",
-    "PyJWT>=2.9.0",
-    "threadpoolctl>=3.5.0",
-]
-dynamic = ["version"]
+dynamic = ["version", "dependencies"]
 
 [project.urls]
-"Documentation" = "https://dataclay.bsc.es/"
-"Code" = "https://github.com/bsc-dom/dataclay"
-"Issue tracker" = "https://github.com/bsc-dom/dataclay/issues"
+"Homepage" = "https://dataclay.bsc.es/"
+"Documentation" = "https://dataclay.readthedocs.io/"
+"Source" = "https://github.com/bsc-dom/dataclay"
+"Issues" = "https://github.com/bsc-dom/dataclay/issues"
 
 [project.optional-dependencies]
 dev = [
@@ -106,7 +92,7 @@ filterwarnings = [
 
 [tool.hatch.build.targets.wheel]
 packages = ["src/dataclay", "src/storage"]
-dependencies = ["grpcio-tools==1.48.2"]
+ignore-vcs = true
 
 [tool.hatch.build.targets.sdist]
 include = ["src/dataclay", "src/storage", "dataclay-common"]
@@ -116,4 +102,7 @@ path = "src/dataclay/__init__.py"
 
 [tool.hatch.build.hooks.custom]
 path = "compile_protos.py"
-dependencies = ["grpcio-tools==1.48.2"]
+
+[tool.hatch.metadata.hooks.custom]
+path = "dynamic_dependencies.py"
+dependencies = ["packaging"]

requirements-dev.txt

@@ -1,15 +1,3 @@
-# This requirements should be used for installing dataClay in the "legacy dependencies" flavour
-
-# You will need to install these requirements before installing dataClay. You can do it with:
-# 
-# $ pip install -r requirements-legacydeps.txt -r requirements.txt
-#
-# Tweak the previous command as you see fit, assuming requirements.txt contains
-# your own requirements. You may want to merge into a single requirements.txt.
-
-# After the requirements are in place, install dataClay without dependencies:
-# $ pip install --no-deps "dataclay=={version}"
-
 aiorwlock>=1.4.0
 bcrypt>=4.1.1
 grpcio>=1.48.2

requirements-legacydeps.txt

@@ -0,0 +1,13 @@
+aiorwlock>=1.4.0
+bcrypt>=4.1.1
+grpcio>=1.67.1
+grpcio-health-checking>=1.67.1
+hiredis>=3.0.0
+opentelemetry-api>=1.28.1
+protobuf>=4.25.0
+psutil>=6.1.0
+pydantic-settings>=2.6.0
+redis>=5.1.1
+get-annotations;python_version<"3.10"
+PyJWT>=2.9.0
+threadpoolctl>=3.5.0