Use run.sh and add tests

Author: leplatrem

.github/workflows/publish.yaml

@@ -254,3 +254,58 @@ jobs:
           webhook-type: incoming-webhook
           payload: |
             text: "⚠️  Build of ${{ env.GAR_IMAGE_NAME }}:${{ env.LATEST_TAG }} failed. Please review logs and correct issues."
+
+  git_reader_container:
+    env:
+      GAR_IMAGE_NAME: remote-settings-git-reader
+      LATEST_TAG: ""  # Set after checkout step
+    runs-on: ubuntu-latest
+    environment: build
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v5
+        with:
+          fetch-depth: 0
+      - name: Set tag version
+        run: echo "LATEST_TAG=$(git describe --tags --abbrev=4)" >> "$GITHUB_ENV"
+      - name: Extract metadata (tags, labels) for Docker
+        id: meta
+        uses: docker/metadata-action@v5
+        with:
+          images: ${{ env.GAR_LOCATION }}-docker.pkg.dev/${{ env.GCP_PROJECT_ID }}/${{ env.GAR_REPOSITORY }}/${{ env.GAR_IMAGE_NAME }}
+          tags: |
+            type=raw,value=${{ env.LATEST_TAG }}
+            type=raw,value=latest
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v3
+      - name: Authenticate on GCP
+        id: gcp_auth
+        uses: google-github-actions/auth@v3
+        with:
+          token_format: access_token
+          service_account: artifact-writer@${{ env.GCP_PROJECT_ID }}.iam.gserviceaccount.com
+          workload_identity_provider: ${{ vars.GCPV2_GITHUB_WORKLOAD_IDENTITY_PROVIDER }}
+      - name: Login to GAR
+        if: ${{ github.event_name == 'push' }}
+        uses: docker/login-action@v3
+        with:
+          registry: ${{ env.GAR_LOCATION }}-docker.pkg.dev
+          username: oauth2accesstoken
+          password: ${{ steps.gcp_auth.outputs.access_token }}
+      - name: Build and push
+        uses: docker/build-push-action@v6
+        with:
+          context: git-reader/
+          push: ${{ github.event_name == 'push' }}
+          tags: ${{ steps.meta.outputs.tags }}
+          labels: ${{ steps.meta.outputs.labels }}
+          cache-from: type=gha # Load cache from GitHub Actions
+          cache-to: type=gha,mode=max # Save cache to GitHub Actions
+      - name: Notify DEVs of build failure
+        if: failure()
+        uses: slackapi/slack-github-action@v2.1.1
+        with:
+          webhook: ${{ secrets.SLACK_WEBHOOK_URL }}
+          webhook-type: incoming-webhook
+          payload: |
+            text: "⚠️  Build of ${{ env.GAR_IMAGE_NAME }}:${{ env.LATEST_TAG }} failed. Please review logs and correct issues."

Makefile

@@ -45,7 +45,7 @@ lint: $(INSTALL_STAMP)  ## Analyze code base
 	$(VENV)/bin/python bin/repo-python-versions.py
 
 test: $(INSTALL_STAMP)  ## Run unit tests
-	PYTHONPATH=. $(VENV)/bin/coverage run -m pytest kinto-remote-settings cronjobs
+	PYTHONPATH=. $(VENV)/bin/coverage run -m pytest kinto-remote-settings cronjobs git-reader
 	$(VENV)/bin/coverage report -m --fail-under 99
 
 browser-test:  ## Run browser tests using Docker

git-reader/Dockerfile

@@ -3,29 +3,33 @@ FROM python:3.13.2-slim-bullseye
 # Install uv.
 COPY --from=ghcr.io/astral-sh/uv:latest /uv /bin/uv
 
-RUN apt-get update && apt-get -y install git git-lfs
+RUN apt-get update && apt-get -y install git git-lfs rsync util-linux
 
 WORKDIR /app
 
 RUN chown 10001:10001 /app && \
     groupadd --gid 10001 app && \
     useradd --no-create-home --uid 10001 --gid 10001 --home-dir /app app
 
+RUN mkdir /app/.ssh && \
+    ssh-keyscan github.com >> /app/.ssh/known_hosts && \
+    chown -R app:app /app/.ssh
+
 COPY --chown=app:app pyproject.toml .
 COPY --chown=app:app uv.lock .
 
 RUN uv sync --locked --no-cache
 
+COPY --chown=app:app run.sh .
+RUN chmod +x run.sh
 COPY --chown=app:app main.py .
 
-RUN mkdir /app/.ssh && \
-    ssh-keyscan github.com >> /app/.ssh/known_hosts && \
-    chown -R app:app /app/.ssh
-
 USER app
 
-ENV GIT_REPO_PATH=/mnt/data
+ENV GIT_REPO_PATH=/mnt/data/latest
 ENV SELF_CONTAINED=false
 
 EXPOSE 8000
-CMD ["uv", "run", "uvicorn", "main:app", "--host", "0.0.0.0", "--port", "8000"]
+
+ENTRYPOINT [ "/app/run.sh" ]
+CMD [ "web" ]

git-reader/README.md

@@ -1,37 +1,45 @@
 # Remote Settings Over Git
 
-## Getting Started
-
-Clone a Remote Settings data repository into a folder:
+## Building the Docker image
 
 ```bash
-git clone git@github.com:leplatrem/remote-settings-data.git /mnt/git/remote-settings-data
+docker build -t remote-settings-git-reader .
 ```
 
-If the container is configured to serve attachments (see `SELF_CONTAINED` below), make sure to install Git LFS and pull the LFS files:
+## Settings
+
+- ``GIT_REPO_PATH``: the path to the Git repository to use.
+- ``SELF_CONTAINED`` (default: `false`): if set to `true`, the application will serve all necessary content from the Git repository, including
+  attachments and certificates chains.
+- ``ATTACHMENTS_BASE_URL`` (default: `None`): this URL will be used as the base URL for attachments. If `SELF_CONTAINED` is `false`, this URL is required. With self-contained, the current domain will be used by default (`Host` request header) if not set.
 
-```bash
-git lfs install
-git lfs pull
-git lfs fsck
-```
 
-Build the container:
+## Running the application
+
+The application needs access to a Git repository containing Remote Settings data (read-only);
 
 ```bash
-docker build -t remote-settings-git-reader .
+docker run --rm -p 8000:8000 \
+    -e GIT_REPO_PATH=/mnt/data/latest \
+    -e SELF_CONTAINED=true \
+    -v /mnt/git/remote-settings-data:/mnt/data:ro \
+    remote-settings-git-reader
 ```
 
-Then you can start the application with:
+But first, we will initialize the folder structure required to execute Git updates atomically.
+Use the ``init`` command and the ``GIT_REPO_URL`` environment variable to specify the repository to clone:
 
 ```bash
-docker run --rm -p 8000:8000 \
-    -e GIT_REPO_PATH=/mnt/data \
+docker run --rm \
+    -e GIT_REPO_URL=git@github.com:mozilla/remote-settings-data.git \
+    -e GIT_REPO_PATH=/mnt/data/latest \
     -e SELF_CONTAINED=true \
     -v /mnt/git/remote-settings-data:/mnt/data \
-    remote-settings-git-reader
+    remote-settings-git-reader init
 ```
 
+Unless you used an anonymous clone, this is likely to fail, as the container needs access to the Git repository via SSH.
+
 ### Using SSH keys
 
 When cloning the repository anonymously (from `https://...`) the Git LFS is rate-limited and it is very likely that you will hit the limit when pulling the LFS files.
@@ -45,13 +53,13 @@ Since the container is going to regularly run Git fetch commands to keep the rep
 This requires to have a SSH agent working on the host. It has the advantage of not requiring the container to have access to the actual key and passphrase (if any).
 
 ```bash
-docker run --rm -p 8000:8000 \
-    -e GIT_REPO_PATH=/mnt/data \
+docker run --rm \
+    -e GIT_REPO_PATH=/mnt/data/latest \
     -e SELF_CONTAINED=true \
     -v /mnt/git/remote-settings-data:/mnt/data \
     -e SSH_AUTH_SOCK=/app/ssh-agent \
     -v $SSH_AUTH_SOCK:/app/ssh-agent \
-    remote-settings-git-reader
+    remote-settings-git-reader init
 ```
 
 2. Or pass the private key file into the container.
@@ -73,12 +81,12 @@ EOF
 And then mount the SSH material directory into the container:
 
 ```bash
-docker run --rm -p 8000:8000 \
-    -e GIT_REPO_PATH=/mnt/data \
+docker run --rm \
+    -e GIT_REPO_PATH=/mnt/data/latest \
     -e SELF_CONTAINED=true \
     -v /mnt/git/remote-settings-data:/mnt/data \
     -v `pwd`/ssh-material:/app/.ssh \
-    remote-settings-git-reader
+    remote-settings-git-reader init
 ```
 
 You can test your SSH setup:
@@ -92,9 +100,19 @@ docker run \
 Hi <username>! You've successfully authenticated, but GitHub does not provide shell access.
 ```
 
-## Settings
+## Updating the repository
 
-- ``GIT_REPO_PATH``: the path to the Git repository to use.
-- ``SELF_CONTAINED`` (default: `false`): if set to `true`, the application will serve all necessary content from the Git repository, including
-  attachments and certificates chains.
-- ``ATTACHMENTS_BASE_URL`` (default: `None`): this URL will be used as the base URL for attachments. If `SELF_CONTAINED` is `false`, this URL is mandatory, otherwise the current domain will be used by default (`Host` request header).
+The container can be used to update the Git repository, by running the `update` command:
+
+```bash
+docker run --rm \
+    -e GIT_REPO_PATH=/mnt/data/latest \
+    -e SELF_CONTAINED=true \
+    -v /mnt/git/remote-settings-data:/mnt/data \
+    remote-settings-git-reader update
+```
+This command can be run periodically (e.g., via a cron job) to keep the repository up to date. For example, every 5 minutes:
+
+```bash
+*/5 * * * * docker run ...
+```