switch to UV wholesale

.github/workflows/dependabot-uv-lock.yml

@@ -0,0 +1,23 @@
+# Update UV lock after dependabot changes (until dependabot supports uv.lock)
+name: uv
+
+on:
+  pull_request:
+    paths:
+      - "pyproject.toml"
+
+permissions:
+  contents: write
+  pull-requests: write
+
+jobs:
+  lock:
+    if: ${{ github.actor == 'dependabot[bot]' }}
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+      - uses: astral-sh/setup-uv@v5
+      - run: uv lock
+      - uses: stefanzweifel/git-auto-commit-action@v5
+        with:
+          commit_message: Regenerate uv.lock

.github/workflows/lint_and_test.yml

@@ -71,20 +71,19 @@ jobs:
           - 6379:6379
     steps:
       - uses: actions/checkout@v4
-      - name: Set up Python
-        uses: actions/setup-python@v5
+      - uses: actions/setup-python@v5
         with:
           python-version: ${{ env.PYTHON_VERSION }}
-          cache: 'pip'
+      - uses: astral-sh/setup-uv@v3
+        with:
+          enable-cache: true
       - name: Install Dependencies
-        run: |
-          python -m pip install --upgrade pip
-          pip install -r dev-requirements.txt
+        run: uv sync --frozen
       - name: Run Tests
         env:
           DJANGO_DATABASE_USER: postgres
           DJANGO_DATABASE_PASSWORD: postgres_password
-        run: pytest -n auto --cov --cov-report=xml --junitxml=junit.xml -o junit_family=legacy
+        run: uv run --no-sync pytest -n auto --cov --cov-report=xml --junitxml=junit.xml -o junit_family=legacy
 
       - name: Upload results to Codecov
         if: ${{ !cancelled() }}

Dockerfile

@@ -2,10 +2,34 @@ FROM python:3.11-slim-bullseye AS build-python
 RUN apt-get update \
   # dependencies for building Python packages
   && apt-get install -y build-essential libpq-dev
-COPY ./requirements /requirements
-RUN pip wheel --no-cache-dir --no-deps --wheel-dir /wheels \
-    -r /requirements/requirements.txt \
-    -r /requirements/prod-requirements.txt
+
+# This approximately follows this guide: https://hynek.me/articles/docker-uv/
+# Which creates a standalone environment with the dependencies.
+# - Silence uv complaining about not being able to use hard links,
+# - tell uv to byte-compile packages for faster application startups,
+# - prevent uv from accidentally downloading isolated Python builds,
+# - pick a Python (use `/usr/bin/python3.12` on uv 0.5.0 and later),
+# - and finally declare `/app` as the target for `uv sync`.
+ENV UV_LINK_MODE=copy \
+    UV_COMPILE_BYTECODE=1 \
+    UV_PYTHON_DOWNLOADS=never \
+    UV_PROJECT_ENVIRONMENT=/code/.venv
+
+COPY --from=ghcr.io/astral-sh/uv:0.5.2 /uv /uvx /bin/
+
+# Since there's no point in shipping lock files, we move them
+# into a directory that is NOT copied into the runtime image.
+# The trailing slash makes COPY create `/_lock/` automagically.
+COPY pyproject.toml uv.lock /_lock/
+
+# Synchronize dependencies.
+# This layer is cached until uv.lock or pyproject.toml change.
+RUN --mount=type=cache,target=/root/.cache \
+    cd /_lock && \
+    uv sync \
+      --frozen \
+      --no-group dev \
+      --group prod
 
 FROM node:18-bullseye AS build-node
 RUN nodejs -v && npm -v
@@ -24,40 +48,38 @@ FROM python:3.11-slim-bullseye
 ENV PYTHONUNBUFFERED=1
 ENV DEBUG=0
 
-RUN apt-get update && apt-get install -y \
-  # psycopg2 dependencies
-  libpq-dev \
-  # Translations dependencies
-  gettext \
-  # audio/video dependencies
-  ffmpeg \
-  # Azure cognitive audio dependencies
-  build-essential libssl-dev ca-certificates libasound2 wget \
-  # curl for heroku log shipping
-  curl \
-  # mimetype detection (creates /etc/mime.types)
-  mailcap \
-  # mimetype detection from content
-  libmagic1 \
-  # psql client for dbshell
-  postgresql-client \
-  # cleaning up unused files
-  && apt-get purge -y --auto-remove -o APT::AutoRemove::RecommendsImportant=false \
-  && rm -rf /var/lib/apt/lists/*
+RUN --mount=target=/var/lib/apt/lists,type=cache,sharing=locked \
+    --mount=target=/var/cache/apt,type=cache,sharing=locked \
+    rm -f /etc/apt/apt.conf.d/docker-clean && \
+    apt-get update \
+    && apt-get install -y \
+    # psycopg2 dependencies
+    libpq-dev \
+    # Translations dependencies
+    gettext \
+    # audio/video dependencies
+    ffmpeg \
+    # Azure cognitive audio dependencies
+    build-essential libssl-dev ca-certificates libasound2 wget \
+    # curl for heroku log shipping
+    curl \
+    # mimetype detection (creates /etc/mime.types)
+    mailcap \
+    # mimetype detection from content
+    libmagic1 \
+    # psql client for dbshell
+    postgresql-client \
+    # cleaning up unused files
+    && apt-get purge -y --auto-remove -o APT::AutoRemove::RecommendsImportant=false
 
 RUN addgroup --system django \
     && adduser --system --ingroup django django
 
-COPY --from=build-node /code/static /code/static
-COPY --from=build-python /wheels /wheels
-COPY ./requirements /requirements
-RUN pip install --no-index --find-links=/wheels \
-    -r /requirements/requirements.txt \
-    -r /requirements/prod-requirements.txt \
-    && rm -rf /wheels \
-    && rm -rf /root/.cache/pip/*
-
 WORKDIR /code
+COPY --from=build-node /code/static /code/static
+COPY --from=build-python --chown=django:django /code /code
+# make sure we use the virtualenv python/gunicorn/celery by default
+ENV PATH="/code/.venv/bin:$PATH"
 
 COPY --chown=django:django . /code
 

README.md

@@ -6,8 +6,8 @@ Experiments with AI, GPT and LLMs. See [this wiki](https://dimagi.atlassian.net/
 
 ## Dev Environment Setup
 
-This project uses [Invoke](https://www.pyinvoke.org/) for dev automation. You can view the list of
-available commands with:
+This project uses [UV](https://docs.astral.sh/uv/getting-started/installation/) and
+uses [Invoke](https://www.pyinvoke.org/) for dev automation. You can view the list of available commands with:
 
 ```shell
 inv -l
@@ -19,8 +19,7 @@ New commands / updates can be made to the `tasks.py` file.
 Setup a virtualenv and install requirements:
 
 ```bash
-python -m venv venv
-pip install -r dev-requirements.txt
+uv sync
 ```
 
 Python 3.11 is recommended.

pyproject.toml

@@ -1,3 +1,93 @@
+[project]
+name = "open-chat-studio"
+version = "0.1.0"
+requires-python = ">=3.11"
+dependencies = [
+    "Django",
+    "Markdown",
+    "anthropic",
+    "azure-cognitiveservices-speech",
+    "boto3",
+    "celery",
+    "celery-progress",
+    "celery[redis]",
+    "django-allauth",
+    "django-allauth-2fa",
+    "django-anymail",
+    "django-cleanup",
+    # original django-cryptography is not compatible with Django 5 and does not seem to be maintained
+    # https://github.com/georgemarshall/django-cryptography/issues/74
+    "django-cryptography-django5",
+    "django-environ",
+    "django-field-audit>=1.2.7",
+    "django-health-check",
+    "django-hijack",
+    "django-redis",
+    "django-storages[s3]",
+    "django-tables2",
+    "django-taggit",
+    "django-template-partials",
+    "django-tz-detect",
+    "django-waffle",
+    "django_celery_beat",
+    "djangorestframework",
+    "djangorestframework-api-key",
+    "drf-spectacular",
+    "fbmessenger",
+    "ffmpeg",  # Audio transcription
+    "httpx",
+    "jinja2",
+    "langchain>=0.3,<0.4",
+    "langchain-core>=0.3.23,<0.4",
+    "langchain-anthropic",
+    "langchain-openai",
+    "langchain-community>=0.3,<0.4",
+    "langfuse",
+    "langgraph>=0.2.20,<0.3",
+    "loguru",
+    "openai",
+    "openapi_pydantic",
+    "pandas",
+    "psycopg[binary]",
+    "pyTelegramBotAPI==4.12.0",
+    "pydantic",
+    "pydub", # Audio transcription
+    "RestrictedPython",
+    "sentry-sdk",
+    "slack-bolt",
+    "taskbadger",
+    "tenacity",
+    "tiktoken",
+    "transformers",  # this is required by the tokenizer that langchain uses
+    "turn-python>=0.2.0",
+    "twilio",
+    "whitenoise[brotli]",
+    "phonenumberslite",
+    "emoji",
+    "python-magic",
+]
+
+[dependency-groups]
+dev = [
+    "uv",
+    "ruff",
+    "mock",
+    "freezegun",
+    "invoke",
+    "termcolor",
+    "watchfiles",
+    "pytest",
+    "pytest-django",
+    "factory-boy",
+    "pytest-httpx",
+    "pytest-cov",
+    "pytest-xdist",
+]
+prod = [
+    "gunicorn",
+    "gevent",
+]
+
 [tool.pytest.ini_options]
 addopts = "--ds=gpt_playground.settings --reuse-db --strict-markers --tb=short"
 python_files = "tests.py test_*.py *_tests.py"