Migrate to Hatch and add a _legacy dependencies_ flavour

.dockerignore

@@ -6,13 +6,17 @@
 # Now un-exclude:
 #
 !src
-!compile-protos.sh
+!compile_protos.py
+!dynamic_dependencies.py
 !dataclay-common
+!requirements.txt
 !requirements-legacydeps.txt
-!requirements-dev.txt
 !pyproject.toml
 !README.md
 !MANIFEST.in
 !tests
 !tox.ini
 .dockerignore
+
+# Reexclude protos
+src/dataclay/proto

.github/workflows/docker-publish.yml

@@ -63,14 +63,24 @@ jobs:
             type=edge,suffix=-py${{ matrix.python-version }},branch=main
             type=edge,enable=${{ matrix.python-version == '3.10' }},branch=main
 
+      # Extract metadata (tags, labels) for Docker
+      - name: Extract Docker metadata
+        id: legacy-deps-meta
+        uses: docker/metadata-action@v5
+        with:
+          images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}
+          tags: |
+            type=edge,suffix=-legacydeps-py${{ matrix.python-version }},branch=main
+            type=edge,suffix=-legacydeps,enable=${{ matrix.python-version == '3.10' }},branch=main
+
       # Build and push Docker image with Buildx (don't push on PR)
       - name: Build and push Docker image
         id: build-and-push
         uses: docker/build-push-action@v6
         with:
           context: .
           platforms: linux/amd64,linux/arm64
-          build-args: 
+          build-args: |
             PYTHON_VERSION=${{ matrix.python-version }}-bullseye
           push: ${{ github.event_name != 'pull_request' }}
           tags: ${{ steps.meta.outputs.tags }}
@@ -84,12 +94,12 @@ jobs:
         uses: docker/build-push-action@v6
         with:
           context: .
-          file: Dockerfile.legacy-deps
           platforms: linux/amd64,linux/arm64
-          build-args: 
+          build-args: |
             PYTHON_VERSION=${{ matrix.python-version }}-bullseye
+            LEGACY_DEPS=True
           push: ${{ github.event_name != 'pull_request' }}
-          tags: ${{ steps.meta.outputs.tags }}-legacydeps
-          labels: ${{ steps.meta.outputs.labels }}
+          tags: ${{ steps.legacy-deps-meta.outputs.tags }}
+          labels: ${{ steps.legacy-deps-meta.outputs.labels }}
           cache-from: type=gha
           cache-to: type=gha,mode=max

.gitignore

@@ -24,3 +24,4 @@ tests/mock/client.properties
 /.vscode
 .coverage*
 coverage.xml
+/src/dataclay/proto/

Dockerfile

@@ -5,8 +5,10 @@ ARG PYTHON_VERSION=3.10-bookworm
 # install dataclay
 FROM python:$PYTHON_VERSION
 COPY . /app
+
+ARG LEGACY_DEPS=False
 RUN python -m pip install --upgrade pip \
-  && python -m pip install /app[telemetry]
+  && python -m pip install --config-settings=LEGACY_DEPS=$LEGACY_DEPS /app[telemetry]
 
 # prepare dataclay storage dir
 RUN mkdir -p /data/storage; 

Dockerfile.dev

@@ -5,8 +5,10 @@ ARG PYTHON_VERSION=3.10-bookworm
 # install dataclay
 FROM python:$PYTHON_VERSION
 COPY . /app
+
+ARG LEGACY_DEPS=False
 RUN python -m pip install --upgrade pip \
-  && python -m pip install -e /app[telemetry,dev]
+  && python -m pip install --config-settings=LEGACY_DEPS=$LEGACY_DEPS -e /app[telemetry,dev]
 
 # prepare dataclay storage dir
 RUN mkdir -p /data/storage; 

PUBLISH.md

@@ -68,6 +68,22 @@
         docker buildx build --platform linux/amd64,linux/arm64 \
         -t ghcr.io/bsc-dom/dataclay:$VERSION-py3.13-bookworm \
         --build-arg PYTHON_VERSION=3.13-bookworm --push .
+
+        # Repeat for Python 3.9 and 3.10 with the _legacy dependency flavour_
+        # Build and push Python 3.9 bookworm
+        docker buildx build --platform linux/amd64,linux/arm64 \
+        -t ghcr.io/bsc-dom/dataclay:$VERSION-legacydeps-py3.9-bookworm \
+        --build-arg PYTHON_VERSION=3.9-bookworm \
+        --build-arg LEGACY_DEPS=True \
+        --push .
+
+        # Build and push Python 3.10 bookworm
+        docker buildx build --platform linux/amd64,linux/arm64 \
+        -t ghcr.io/bsc-dom/dataclay:$VERSION-legacydeps-py3.10-bookworm \
+        -t ghcr.io/bsc-dom/dataclay:$VERSION-legacydeps \
+        --build-arg PYTHON_VERSION=3.10-bookworm \
+        --build-arg LEGACY_DEPS=True \
+        --push .
         ```
 
 6. Publish the release distribution to PyPI:

compile_protos.py

@@ -0,0 +1,72 @@
+#!/usr/bin/env python3
+
+import inspect
+from typing import Any
+from importlib import resources
+from typing import Any
+
+try:
+    from hatchling.builders.hooks.plugin.interface import BuildHookInterface
+except ModuleNotFoundError:
+    if __name__ != "__main__":
+        # If we are not being run interactively, then that is an error
+        raise
+    BuildHookInterface = object
+
+
+def run_protoc():
+    # Here because during the build process, CustomBuildHook will be imported
+    # *before* knowing the dependencies of the hook itself.
+    import grpc_tools.protoc
+
+    grpc_tools_proto = (resources.files("grpc_tools") / "_proto").resolve()
+    grpc_tools.protoc.main(
+        [
+            "grpc_tools.protoc",
+            "--proto_path=dataclay-common",
+            "--python_out=src",
+            "--grpc_python_out=src",
+            "dataclay-common/dataclay/proto/common/common.proto",
+            "dataclay-common/dataclay/proto/backend/backend.proto",
+            "dataclay-common/dataclay/proto/metadata/metadata.proto",
+            f"-I{grpc_tools_proto}",
+        ]
+    )
+
+
+def find_config_settings_in_hatchling() -> dict[str, Any]:
+    # Terrible workaround (their words, not mine) given by @virtuald
+    # https://github.com/pypa/hatch/issues/1072#issuecomment-2448985229
+    # Hopefully this will be fixed in the future
+    for frame_info in inspect.stack():
+        frame = frame_info.frame
+        module = inspect.getmodule(frame)
+        if (
+            module
+            and module.__name__.startswith("hatchling.build")
+            and "config_settings" in frame.f_locals
+        ):
+            return frame.f_locals["config_settings"] or {}
+
+    return {}
+
+
+class CustomBuildHook(BuildHookInterface):
+    def initialize(self, version: str, build_data: dict[str, Any]) -> None:
+        run_protoc()
+
+    def dependencies(self):
+        if find_config_settings_in_hatchling().get("LEGACY_DEPS", "False").lower() in (
+            "true",
+            "on",
+            "1",
+            "y",
+            "yes",
+        ):
+            return ["grpcio-tools==1.48.2"]
+        else:
+            return ["grpcio-tools==1.67.1"]
+
+
+if __name__ == "__main__":
+    run_protoc()

dynamic_dependencies.py

@@ -0,0 +1,81 @@
+import inspect
+import os
+import re
+from typing import Any
+
+from hatchling.metadata.plugin.interface import MetadataHookInterface
+from packaging.requirements import Requirement
+from packaging.utils import canonicalize_name
+
+# The helper functions in this file are heavily inspired on the job layed
+# out by the hatch plugin `hatch-requirements-txt`
+
+
+COMMENT_RE = re.compile(r"(^|\s+)#.*$")
+PIP_COMMAND_RE = re.compile(r"\s+(-[A-Za-z]|--[A-Za-z]+)")
+
+
+def find_config_settings_in_hatchling() -> dict[str, Any]:
+    # Terrible workaround (their words, not mine) given by @virtuald
+    # https://github.com/pypa/hatch/issues/1072#issuecomment-2448985229
+    # Hopefully this will be fixed in the future
+    for frame_info in inspect.stack():
+        frame = frame_info.frame
+        module = inspect.getmodule(frame)
+        if (
+            module
+            and module.__name__.startswith("hatchling.build")
+            and "config_settings" in frame.f_locals
+        ):
+            return frame.f_locals["config_settings"] or {}
+
+    return {}
+
+
+def parse_requirements(requirements: list[str]) -> tuple[list[Requirement], list[str]]:
+    comments = []
+    parsed_requirements: list[Requirement] = []
+
+    for line in requirements:
+        if line.lstrip().startswith("#"):
+            comments.append(line)
+        elif line.lstrip().startswith("-"):
+            # Likely an argument to pip from a requirements.txt file intended for pip
+            # (e.g. from pip-compile)
+            pass
+        elif line:
+            # Strip comments from end of line
+            line = COMMENT_RE.sub("", line)
+            if "-" in line:
+                line = PIP_COMMAND_RE.split(line)[0]
+            req = Requirement(line)
+            req.name = canonicalize_name(req.name)
+            parsed_requirements.append(req)
+
+    return parsed_requirements, comments
+
+
+def load_requirements(filename: str) -> list[str]:
+    if not os.path.isfile(filename):
+        raise FileNotFoundError(filename)
+    with open(filename, encoding="UTF-8") as fp:
+        contents = fp.read()
+        # Unfold lines ending with \
+        contents = re.sub(r"\\\s*\n", " ", contents)
+        parsed_requirements, _ = parse_requirements(contents.splitlines())
+
+    return [str(r) for r in parsed_requirements]
+
+
+class DynamicDependenciesMetaDataHook(MetadataHookInterface):
+    def update(self, metadata):
+        if find_config_settings_in_hatchling().get("LEGACY_DEPS", "False").lower() in (
+            "true",
+            "on",
+            "1",
+            "y",
+            "yes",
+        ):
+            metadata["dependencies"] = load_requirements("requirements-legacydeps.txt")
+        else:
+            metadata["dependencies"] = load_requirements("requirements.txt")

noxfile.py

@@ -3,11 +3,11 @@
 # Define which Python versions to test with
 PYPROJECT = nox.project.load_toml("pyproject.toml")
 PYTHON_VERSIONS = nox.project.python_versions(PYPROJECT)
-DEFAULT_PYTHON = "3.10"  # Arbitrary decision, choose a reliable version
+DEFAULT_PYTHON = "3.10"  # Modern-ish version compatible with the legacy-deps
 
 # Default sessions (these will be executed in Github Actions)
 # Maintain a clear separation between code checking and code altering tasks (don't add format)
-nox.options.sessions = ["lint", "tests"]
+nox.options.sessions = ["lint", "tests", "legacy_deps_tests"]
 # nox.options.reuse_existing_virtualenvs = True # TODO: Check if necessary
 
 
@@ -22,21 +22,9 @@ def tests(session):
 @nox.session(python=["3.9", "3.10"], tags=["citests"])
 def legacy_deps_tests(session):
     """Run the test suite with legacy dependencies."""
-    session.install("grpcio-tools==1.48.2", "pytest", "pytest-asyncio", "pytest-docker", "pytest-cov", "-r", "requirements-legacydeps.txt")
-    session.run(
-        # See compile-protos.sh, it should be the same command
-        "python3",
-        "-m",
-        "grpc_tools.protoc",
-        "--proto_path=dataclay-common",
-        "--python_out=src",
-        "--grpc_python_out=src",
-        "dataclay-common/dataclay/proto/common/common.proto",
-        "dataclay-common/dataclay/proto/backend/backend.proto",
-        "dataclay-common/dataclay/proto/metadata/metadata.proto",
-    )
-
-    session.install(".", "--no-deps")
+    session.install("pytest", "pytest-asyncio", "pytest-docker", "pytest-cov")
+
+    session.install("--config-settings=LEGACY_DEPS=True", ".")
     session.run("pytest", "--disable-warnings", "--cov", "--cov-report=term-missing", "--build-legacy-deps", "tests/functional")