Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
45 changes: 45 additions & 0 deletions .github/actions/install-dependencies/action.yaml
Original file line number Diff line number Diff line change
@@ -0,0 +1,45 @@
name: 'Install Dependencies'
description: 'Sets up common tools and dependencies used across workflows'
inputs:
install-just:
description: 'Install Just task runner'
required: false
default: 'true'
install-uv:
description: 'Install uv package manager'
required: false
default: 'true'
just-version:
description: 'Version of Just to install'
required: false
default: '1.21.0'
uv-cache-dependency-glob:
description: 'Glob pattern for uv cache dependency files'
required: false
default: ''
uv-version:
description: 'Version of uv to install'
required: false
default: '0.9.7'
python-version:
description: 'Python version'
required: false
default: '3.13'

runs:
using: 'composite'
steps:
- name: Install Just
if: ${{ inputs.install-just == 'true' }}
uses: extractions/setup-just@e33e0265a09d6d736e2ee1e0eb685ef1de4669ff # v3.0.0
with:
just-version: ${{ inputs.just-version }}

- name: Install uv
if: ${{ inputs.install-uv == 'true' }}
uses: astral-sh/setup-uv@681c641aba71e4a1c380be3ab5e12ad51f415867 # v7.1.6
with:
version: ${{ inputs.uv-version }}
enable-cache: ${{ inputs.uv-cache-dependency-glob != '' && 'true' || 'false' }}
cache-dependency-glob: ${{ inputs.uv-cache-dependency-glob }}
python-version: ${{ inputs.python-version }}
25 changes: 8 additions & 17 deletions .github/workflows/backend-lint-and-test.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -66,12 +66,10 @@ jobs:
with:
persist-credentials: false

- name: Install uv and set the Python version
uses: astral-sh/setup-uv@681c641aba71e4a1c380be3ab5e12ad51f415867 # v7.1.6
- name: Install dependencies
uses: ./.github/actions/install-dependencies
with:
version: "0.9.7"
enable-cache: false
python-version: "3.13"
uv-cache-dependency-glob: 'application/backend/uv.lock'

- name: Install OpenCV dependencies
run: |
Expand All @@ -80,37 +78,30 @@ jobs:
libgl1 \
libglib2.0-0

- name: Prepare venv and install Python dependencies
working-directory: application/backend
run: |
uv lock --check
uv sync --frozen --extra cpu --extra mqtt

- name: Check formatting with ruff
working-directory: application/backend
run: |
uv run ruff check --output-format=github .
uv run ruff format --check .
just ruff

- name: Check imports with import-linter
working-directory: application/backend
run: |
uv run lint-imports
just lint-imports

- name: Check source code with mypy
working-directory: application/backend
run: |
uv run mypy . --config-file=pyproject.toml
just mypy

- name: Run unit tests
working-directory: application/backend
run: |
uv run pytest tests/unit
just test-unit

- name: Run integration tests
working-directory: application/backend
run: |
uv run pytest tests/integration
just test-integration

required_check:
name: Required Check backend-lint-and-test
Expand Down
56 changes: 17 additions & 39 deletions .github/workflows/lib-lint-and-test.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -67,32 +67,24 @@ jobs:
with:
persist-credentials: false

- name: Install uv and set the Python version
uses: astral-sh/setup-uv@681c641aba71e4a1c380be3ab5e12ad51f415867 # v7.1.6
- name: Install dependencies
uses: ./.github/actions/install-dependencies
with:
version: "0.9.7"
enable-cache: false
uv-cache-dependency-glob: 'library/uv.lock'
python-version: "3.10" # Use the lowest supported Python version for linting/type-checking

- name: Installing dependencies
working-directory: library
run: |
uv lock --check
uv sync --frozen

- name: Check formatting with ruff
working-directory: library
run: |
uv run ruff check --output-format=github .
uv run ruff format --check .
just ruff

# TODO: remove 'continue-on-error' after addressing all mypy issues
# https://github.com/open-edge-platform/training_extensions/issues/5006
- name: Check source code with mypy
continue-on-error: true
working-directory: library
run: |
uv run mypy src/otx
just mypy

unit-tests:
runs-on: [self-hosted, linux, x64, dev, dmount]
Expand Down Expand Up @@ -122,32 +114,25 @@ jobs:
with:
toolchain: stable

- name: Install uv and set the Python version
uses: astral-sh/setup-uv@681c641aba71e4a1c380be3ab5e12ad51f415867 # v7.1.6
- name: Install dependencies
uses: ./.github/actions/install-dependencies
with:
version: "0.9.7"
enable-cache: false
uv-cache-dependency-glob: 'library/uv.lock'
python-version: ${{ matrix.python-version }}

- name: Installing dependencies
working-directory: library
run: |
uv lock --check
uv sync --frozen

- name: Unit testing (excluding models)
working-directory: library
env:
OMP_NUM_THREADS: "1"
run: uv run pytest tests/unit --ignore=tests/unit/backend/native/models --cov --cov-report=xml --cov-append
run: just test-unit

# Unit test models separately using --forked flag so that each is run in a separate process.
# Running these models after each other from the same process can cause segfaults
- name: Unit testing models
working-directory: library
env:
OMP_NUM_THREADS: "1"
run: uv run pytest tests/unit/backend/native/models --cov --cov-report=xml --cov-append --forked
run: just test-unit-native-models

- name: Upload coverage reports to Codecov
uses: codecov/codecov-action@671740ac38dd9b0130fbe1cec585b89eea48d3de # v5.5.2
Expand Down Expand Up @@ -182,19 +167,18 @@ jobs:
with:
persist-credentials: false

- name: Install dependencies
uses: ./.github/actions/install-dependencies
with:
uv-cache-dependency-glob: 'library/uv.lock'
python-version: ${{ matrix.python-version }}

# TODO remove after switching to an official Datumaro release with pre-built wheels
- name: Installing Rust toolchain
uses: dtolnay/rust-toolchain@6d653acede28d24f02e3cd41383119e8b1b35921
with:
toolchain: stable

- name: Install uv and set the Python version
uses: astral-sh/setup-uv@681c641aba71e4a1c380be3ab5e12ad51f415867 # v7.1.6
with:
version: "0.9.7"
enable-cache: false
python-version: ${{ matrix.python-version }}

- name: Installing dependencies
working-directory: library
run: |
Expand All @@ -206,13 +190,7 @@ jobs:
env:
CUBLAS_WORKSPACE_CONFIG: ":4096:8"
run: |
uv run --frozen --extra cuda pytest tests/integration \
-ra \
--showlocals \
--csv=${{ matrix.task }}_Python-${{ matrix.python-version }}.csv \
--task ${{ matrix.task }} \
--open-subprocess \
--run-category-only
just device='cuda' test-integration '${{ matrix.task }}_Python-${{ matrix.python-version }}.csv' '${{ matrix.task }}'

required-check:
name: Required Check lib-lint-and-test
Expand Down
13 changes: 4 additions & 9 deletions .github/workflows/ui-lint-and-test.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -69,12 +69,8 @@ jobs:
with:
persist-credentials: false

- name: Install uv and set the Python version
uses: astral-sh/setup-uv@681c641aba71e4a1c380be3ab5e12ad51f415867 # v7.1.6
with:
version: "0.9.7"
enable-cache: false
python-version: "3.13"
- name: Install dependencies
uses: ./.github/actions/install-dependencies

- name: Install OpenCV dependencies
run: |
Expand All @@ -90,13 +86,12 @@ jobs:
- name: Get OpenAPI spec
working-directory: application/backend
run: |
export PYTHONPATH=.
uv run app/cli.py gen-api --target-path openapi-spec.json
just gen-openapi

- uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6.0.0
with:
name: openapi-spec
path: application/backend/openapi-spec.json
path: application/backend/.build/openapi-spec.json

build:
name: Build UI
Expand Down
28 changes: 6 additions & 22 deletions .github/workflows/windows-installer.yml
Original file line number Diff line number Diff line change
Expand Up @@ -88,18 +88,10 @@ jobs:
persist-credentials: false

# Install and setup build tools

- name: Set up Python
uses: actions/setup-python@e797f83bcb11b83ae66e0230d6156d7c80228e7c # v6.0.0
with:
python-version: "3.13"

- name: Install uv
uses: astral-sh/setup-uv@a02a550bdd3185dba2ebb6aa98d77047ce54ad21 # v6.2.1
- name: Install dependencies
uses: ./.github/actions/install-dependencies
with:
version: "0.9.7"
enable-cache: true
cache-dependency-glob: "application/backend/uv.lock"
uv-cache-dependency-glob: 'application/backend/uv.lock'

- name: Install Rust
uses: dtolnay/rust-toolchain@6d9817901c499d6b02debbb57edb38d33daa680b
Expand All @@ -115,25 +107,17 @@ jobs:

# Build backend

- name: Build venv
working-directory: application/backend
env:
DEVICE: ${{ inputs.device || 'xpu' }}
run: |
uv lock --check
uv sync --extra installer --extra $env:DEVICE --extra mqtt --frozen --trusted-host pypi.org --trusted-host pypi.python.org --trusted-host files.pythonhosted.org

- name: Build OpenAPI spec
working-directory: application/backend
env:
PYTHONPATH: "."
DEVICE: ${{ inputs.device || 'xpu' }}
run: |
uv run --no-sync app/cli.py gen-api --target-path .build/openapi-spec.json
just --shell "${{ env.GIT_SH_PATH }}" --shell-arg -c device="$env:DEVICE" gen-openapi

- name: Build executable
working-directory: application/backend
run: |
uv run pyinstaller --noconfirm geti-tune.spec
just --shell "${{ env.GIT_SH_PATH }}" --shell-arg -c device="$env:DEVICE" pyinstaller

# Copy backend artifacts to UI

Expand Down
70 changes: 70 additions & 0 deletions application/Justfile
Original file line number Diff line number Diff line change
@@ -0,0 +1,70 @@
#!/usr/bin/env just --justfile
import "recipes.justfile"

# Use bash as the shell for evaluating backtick expressions
set shell := ["bash", "-cu"]

# -------------------------------------------------------------------------------------------------
# Runtime variables
# -------------------------------------------------------------------------------------------------
port := "7860"
ui-port := "3000"
webcam-device := "/dev/video0"
data-dir := ""
host := "0.0.0.0"
webrtc-ports := "50000-51000"
device := "cpu" # cpu, cuda, xpu

# -------------------------------------------------------------------------------------------------
# Variables used in the Justfile to build in Docker image
# -------------------------------------------------------------------------------------------------
docker-build-context := ""
build-target := "cpu" # options: cpu, gpu, xpu
component-name := "geti-tune"
container-registry := 'localhost:5000/open-edge-platform'
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Why this won't target our default ghcr registry that we will push images to? Or at least dev ghcr registry?

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

This is done similar to Geti Prompt. But I can change it if needed

version := "latest"
image-name := container-registry / component-name / build-target + ":" + version

host-ip := `uv run python -c "import socket; s=socket.socket(socket.AF_INET,socket.SOCK_DGRAM); s.connect(('8.8.8.8',80)); print(s.getsockname()[0]); s.close()"`

docker-build-args := "--build-arg http_proxy --build-arg https_proxy --build-arg no_proxy --build-arg HTTP_PROXY --build-arg HTTPS_PROXY --build-arg NO_PROXY"
docker-run-args := "--env http_proxy --env https_proxy --env no_proxy --env HTTP_PROXY --env HTTPS_PROXY --env NO_PROXY"
Comment on lines +30 to +31
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Why by default are we adding proxy args/envs to docker build cmd? Will our builders be behind some proxy?

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Similar to previous comment - this is similar approach with Geti Prompt


# Run both backend and frontend development servers
dev:
#!/usr/bin/env bash
set -euo pipefail

export PUBLIC_API_URL="http://{{ host-ip }}:{{ port }}"

(cd backend && just port={{ port }} host={{ host }} device={{ device }} dev) &
(cd ui && npm install && PORT={{ ui-port }} npm start) &

wait

# Build geti tune with optional port mapping
build-image:
@echo "Building docker image for component: {{ component-name }}"
@docker build \
{{ docker-build-args }} \
{{ docker-build-context }} \
--tag "{{ image-name }}" \
--file docker/Dockerfile . \
--target geti-tune-{{ build-target }}

# launch geti tune with optional webcam, volume and port mapping
run-image: build-image
@echo "Running docker image for component: {{ component-name }} with build-target: {{ build-target }}"
@echo "Proxy args: {{ docker-run-args }}"
@docker run --rm \
{{ docker-run-args }} \
--sysctl net.ipv4.ip_local_port_range="{{ replace(webrtc-ports, '-', ' ') }}" \
--publish {{ webrtc-ports }}:{{ webrtc-ports }}/udp \
--publish {{ port }}:{{ port }} \
--env PORT={{ port }} \
--env HOST={{ host }} \
--env WEBRTC_ADVERTISE_IP='{{ host-ip }}' \
{{ if data-dir != "" { " --volume " + data-dir + ":/geti_prompt/data" } else { "" } }} \
{{ if path_exists(webcam-device) == "true" { " --device " + webcam-device + ":" + webcam-device } else { "" } }} \
--name "{{ component-name }}-{{ version }}" \
"{{ image-name }}"
Loading
Loading