Run integration tests from llama-stack using local client #1
Workflow file for this run
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Integration Tests (Client-side) | |
| run-name: Run integration tests from llama-stack using local client | |
| on: | |
| push: | |
| branches: | |
| - main | |
| - 'release-[0-9]+.[0-9]+.x' | |
| pull_request: | |
| branches: | |
| - main | |
| - 'release-[0-9]+.[0-9]+.x' | |
| types: [opened, synchronize, reopened] | |
| paths: | |
| - 'src/llama_stack_client/**' | |
| - 'uv.lock' | |
| - 'pyproject.toml' | |
| - '.github/workflows/integration-tests.yml' | |
| workflow_dispatch: | |
| inputs: | |
| test-setup: | |
| description: 'Test against a specific setup' | |
| type: string | |
| default: 'ollama' | |
| concurrency: | |
| # Skip concurrency for pushes to main - each commit should be tested independently | |
| group: ${{ github.workflow }}-${{ github.ref == 'refs/heads/main' && github.run_id || github.ref }} | |
| cancel-in-progress: true | |
| jobs: | |
| generate-matrix: | |
| runs-on: ubuntu-latest | |
| outputs: | |
| matrix: ${{ steps.set-matrix.outputs.matrix }} | |
| steps: | |
| - name: Checkout llama-stack repository | |
| uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0 | |
| with: | |
| repository: llamastack/llama-stack | |
| ref: main | |
| - name: Generate test matrix | |
| id: set-matrix | |
| run: | | |
| # Generate matrix from CI_MATRIX in llama-stack tests/integration/ci_matrix.json | |
| MATRIX=$(PYTHONPATH=. python3 scripts/generate_ci_matrix.py \ | |
| --test-setup "${{ github.event.inputs.test-setup }}") | |
| echo "matrix=$MATRIX" >> $GITHUB_OUTPUT | |
| echo "Generated matrix: $MATRIX" | |
| run-replay-mode-tests: | |
| needs: generate-matrix | |
| runs-on: ubuntu-latest | |
| name: ${{ format('Integration Tests ({0}, {1}, {2}, {3})', matrix.client, matrix.config.setup, matrix.python-version, matrix.config.suite) }} | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| client: [library, docker, server] | |
| python-version: ['3.12'] | |
| # Test configurations: Generated from CI_MATRIX in llama-stack tests/integration/ci_matrix.json | |
| config: ${{ fromJSON(needs.generate-matrix.outputs.matrix).include }} | |
| steps: | |
| - name: Checkout llama-stack-client repository | |
| uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0 | |
| with: | |
| path: llama-stack-client-python | |
| - name: Checkout llama-stack repository | |
| uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0 | |
| with: | |
| repository: llamastack/llama-stack | |
| ref: main | |
| path: llama-stack | |
| - name: Install uv | |
| if: ${{ matrix.config.allowed_clients == null || contains(matrix.config.allowed_clients, matrix.client) }} | |
| uses: astral-sh/setup-uv@6b9c6063abd6010835644d4c2e1bef4cf5cd0fca # v6.0.1 | |
| with: | |
| python-version: ${{ matrix.python-version }} | |
| version: 0.7.6 | |
| - name: Install llama-stack dependencies | |
| if: ${{ matrix.config.allowed_clients == null || contains(matrix.config.allowed_clients, matrix.client) }} | |
| working-directory: llama-stack | |
| run: | | |
| echo "Installing llama-stack dependencies" | |
| uv sync --all-groups | |
| uv pip install faiss-cpu | |
| - name: Install local llama-stack-client | |
| if: ${{ matrix.config.allowed_clients == null || contains(matrix.config.allowed_clients, matrix.client) }} | |
| working-directory: llama-stack | |
| run: | | |
| echo "Installing local llama-stack-client from ../llama-stack-client-python" | |
| uv pip install -e ../llama-stack-client-python | |
| echo "Installed llama packages:" | |
| uv pip list | grep llama | |
| - name: Build Llama Stack | |
| if: ${{ matrix.config.allowed_clients == null || contains(matrix.config.allowed_clients, matrix.client) }} | |
| working-directory: llama-stack | |
| run: | | |
| echo "Building Llama Stack" | |
| LLAMA_STACK_DIR=. \ | |
| uv run --no-sync llama stack list-deps ci-tests | xargs -L1 uv pip install | |
| - name: Configure git for commits | |
| if: ${{ matrix.config.allowed_clients == null || contains(matrix.config.allowed_clients, matrix.client) }} | |
| working-directory: llama-stack | |
| run: | | |
| git config --local user.email "github-actions[bot]@users.noreply.github.com" | |
| git config --local user.name "github-actions[bot]" | |
| - name: Check Storage and Memory Available Before Tests | |
| if: ${{ (matrix.config.allowed_clients == null || contains(matrix.config.allowed_clients, matrix.client)) && always() }} | |
| run: | | |
| free -h | |
| df -h | |
| - name: Run Integration Tests (Replay Mode) | |
| if: ${{ matrix.config.allowed_clients == null || contains(matrix.config.allowed_clients, matrix.client) }} | |
| working-directory: llama-stack | |
| env: | |
| OPENAI_API_KEY: dummy | |
| run: | | |
| STACK_CONFIG="${{ matrix.config.stack_config || (matrix.client == 'library' && 'ci-tests') || (matrix.client == 'server' && 'server:ci-tests') || 'docker:ci-tests' }}" | |
| SCRIPT_ARGS="--stack-config $STACK_CONFIG --inference-mode replay" | |
| # Add optional arguments | |
| if [ -n '${{ matrix.config.setup }}' ]; then | |
| SCRIPT_ARGS="$SCRIPT_ARGS --setup ${{ matrix.config.setup }}" | |
| fi | |
| if [ -n '${{ matrix.config.suite }}' ]; then | |
| SCRIPT_ARGS="$SCRIPT_ARGS --suite ${{ matrix.config.suite }}" | |
| fi | |
| echo "=== Running command ===" | |
| echo "uv run --no-sync ./scripts/integration-tests.sh $SCRIPT_ARGS" | |
| echo "" | |
| uv run --no-sync ./scripts/integration-tests.sh $SCRIPT_ARGS | tee pytest-replay.log | |
| - name: Upload logs | |
| if: ${{ always() }} | |
| uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2 | |
| with: | |
| name: logs-${{ github.run_id }}-${{ github.run_attempt || '1' }}-${{ strategy.job-index || github.job }}-${{ github.action }} | |
| path: | | |
| llama-stack/*.log | |
| retention-days: 1 |