Docker optimisations for CI

.github/workflows/testing.yml

@@ -13,6 +13,10 @@ concurrency:
   group: ${{ github.head_ref || github.run_id }}
   cancel-in-progress: true
 
+permissions:
+  contents: read
+  packages: write
+
 jobs:
   lint:
     runs-on: ubuntu-latest
@@ -47,6 +51,7 @@ jobs:
         uses: docker/bake-action@v6
         with:
           load: true
+          push: false
           files: tests/integration/dockerfiles/docker-bake.hcl
           set: |
             *.cache-to=type=gha,scope=global,mode=max
@@ -83,14 +88,22 @@ jobs:
       - name: Set up Docker Buildx
         uses: docker/setup-buildx-action@v3
 
+      - name: List images
+        run: docker images
+
       - name: Load test bench containers
         uses: docker/bake-action@v6
         with:
           load: true
+          push: false
           files: tests/integration/dockerfiles/docker-bake.hcl
           set: |
+            *.cache-to=type=gha,scope=global,mode=max
             *.cache-from=type=gha,scope=global
 
+      - name: List images
+        run: docker images
+
       - name: Install dependencies
         run: |
           python -m pip install --upgrade pip

tests/integration/conftest.py

@@ -105,7 +105,7 @@ def compose_containers(
       start_period: 2s
 
   jobflow_remote_testing_slurm:
-    image: jobflow-remote-testing-slurm:latest
+    image: ghcr.io/matgenix/jobflow-remote-testing-slurm:latest
     container_name: jobflow_testing_slurm
     ports:
       - "{slurm_ssh_port}:22"
@@ -119,7 +119,7 @@ def compose_containers(
       start_period: 2s
 
   jobflow_remote_testing_sge:
-    image: jobflow-remote-testing-sge:latest
+    image: ghcr.io/matgenix/jobflow-remote-testing-sge:latest
     container_name: jobflow_testing_sge
     ports:
       - "{sge_ssh_port}:22"
@@ -133,7 +133,7 @@ def compose_containers(
       start_period: 2s
 
   jobflow_remote_testing_pbs:
-    image: jobflow-remote-testing-pbs:latest
+    image: ghcr.io/matgenix/jobflow-remote-testing-pbs:latest
     container_name: jobflow_testing_pbs
     ports:
       - "{pbs_ssh_port}:22"

tests/integration/dockerfiles/Dockerfile

@@ -4,7 +4,6 @@ ARG USERNAME=jobflow
 
 # Install jobflow-remote in its own layer with uv
 FROM --platform=linux/amd64 ubuntu:22.04 as jobflow-remote-install
-# FROM ubuntu:22.04 as jobflow-remote-install
 
 ARG USERNAME=jobflow
 ENV USERNAME=${USERNAME}
@@ -14,7 +13,7 @@ RUN apt update && \
     apt clean && \
     rm -rf /var/lib/apt/lists/*
 
-COPY --from=ghcr.io/astral-sh/uv:0.4 /uv /usr/local/bin/uv
+COPY --from=ghcr.io/astral-sh/uv:0.5 /uv /usr/local/bin/uv
 
 # As a bit of a hack, we make the path for this installation the same
 # as the eventual .venv export, to avoid needing to patch the activation
@@ -175,7 +174,6 @@ COPY ./tests/integration/dockerfiles/pbs_startup.sh /etc/startup.sh
 
 # 2) Add common options from ubuntu server
 FROM --platform=linux/amd64 ${QUEUE_SYSTEM} as jobflow
-# FROM ${QUEUE_SYSTEM} as jobflow
 
 ARG USERNAME=jobflow
 ENV USERNAME=${USERNAME}

tests/integration/dockerfiles/docker-bake.hcl

@@ -1,3 +1,16 @@
+variable "CI" {
+    // Set in GH actions to affect caching strategy
+    default = false
+}
+
+variable "IMAGE_BASE" {
+    default = "ghcr.io/matgenix/jobflow-remote-testing"
+}
+
+variable "IMAGE_TAG" {
+    default = "latest"
+}
+
 group "default" {
     targets = [
         "slurm",
@@ -11,8 +24,15 @@ target "slurm" {
     args = {
         QUEUE_SYSTEM = "slurm"
     }
+    cache-from = [
+        "type=registry,ref=${IMAGE_BASE}-slurm:${IMAGE_TAG}",
+        "type=registry,ref=${IMAGE_BASE}-slurm:cache",
+        "type=gha",
+    ]
+    // If in the CI, cache to GHA only, otherwise push to the registry cache
+    cache-to = CI ? ["type=gha,mode=max"] : ["type=registry,ref=${IMAGE_BASE}-slurm:cache,mode=max"]
     tags = [
-        "jobflow-remote-testing-slurm:latest"
+        "${IMAGE_BASE}-slurm:${IMAGE_TAG}",
     ]
     platforms = ["linux/amd64"]
 }
@@ -22,8 +42,14 @@ target "sge" {
     args = {
         QUEUE_SYSTEM = "sge"
     }
+    cache-from = [
+        "type=registry,ref=${IMAGE_BASE}-sge:${IMAGE_TAG}",
+        "type=registry,ref=${IMAGE_BASE}-sge:cache",
+        "type=gha",
+    ]
+    cache-to = CI ? ["type=gha,mode=max"] : ["type=registry,ref=${IMAGE_BASE}-sge:cache,mode=max"]
     tags = [
-        "jobflow-remote-testing-sge:latest"
+        "${IMAGE_BASE}-sge:${IMAGE_TAG}",
     ]
     platforms = ["linux/amd64"]
 }
@@ -33,8 +59,14 @@ target "pbs" {
     args = {
         QUEUE_SYSTEM = "pbs"
     }
+    cache-from = [
+        "type=registry,ref=${IMAGE_BASE}-pbs:${IMAGE_TAG}",
+        "type=registry,ref=${IMAGE_BASE}-pbs:cache",
+        "type=gha",
+    ]
+    cache-to = CI ? ["type=gha,mode=max"] : ["type=registry,ref=${IMAGE_BASE}-pbs:cache,mode=max"]
     tags = [
-        "jobflow-remote-testing-pbs:latest"
+        "${IMAGE_BASE}-pbs:${IMAGE_TAG}",
     ]
     platforms = ["linux/amd64"]
 }