From e6c7bc9bcf8cb884b70b177dbb9531bc34e837b0 Mon Sep 17 00:00:00 2001 From: Test Runner Date: Thu, 26 Mar 2026 11:13:27 -0300 Subject: [PATCH] feat(gh-workflows): add supply chain security scan gate Add automated supply chain security scanning to the upstream sync pipeline, responding to the litellm v1.82.8 PyPI compromise (BerriAI/litellm#24512). New workflow (carto-supply-chain-security.yml) with 5 layers: - GuardDog (Datadog/OpenSSF): Semgrep-powered malicious pattern detection - Wheel allowlist audit: only litellm/ and dist-info/ files allowed - GuardDog wheel verify: scan built wheel for malicious code - pip-audit (PyPA): known vulnerability scanning (advisory) - Structural integrity: setup.py guard, dependency source check Additional hardening: - Pin all GitHub Actions in CARTO workflows by commit SHA (prevent tag hijacking) - Add wheel audit step to Dockerfile build (defense-in-depth at build time) - Add supply chain security checklist to upstream sync PR body - Add supply chain scan function to ci_cd/security_scans.sh Co-authored-by: Claude --- .github/workflows/carto-features-check.yml | 2 +- .github/workflows/carto-release.yaml | 6 +- .../workflows/carto-schema-sync-validator.yml | 2 +- .github/workflows/carto-slack-changelog.yml | 4 +- .../workflows/carto-supply-chain-security.yml | 202 ++++++++++++++++++ .../carto-upstream-sync-ci-fixer.yml | 4 +- ...-upstream-sync-customizations-analyzer.yml | 12 +- .../workflows/carto-upstream-sync-main.yml | 15 +- .../carto-upstream-sync-resolver.yml | 8 +- Dockerfile | 13 ++ ci_cd/security_scans.sh | 82 ++++++- 11 files changed, 323 insertions(+), 27 deletions(-) create mode 100644 .github/workflows/carto-supply-chain-security.yml diff --git a/.github/workflows/carto-features-check.yml b/.github/workflows/carto-features-check.yml index 22891bc6a0e..310df4bdbd7 100644 --- a/.github/workflows/carto-features-check.yml +++ b/.github/workflows/carto-features-check.yml @@ -14,7 +14,7 @@ jobs: runs-on: ubuntu-latest steps: - name: Checkout code - uses: actions/checkout@v4 + uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4 - name: Check manifest exists id: check-manifest diff --git a/.github/workflows/carto-release.yaml b/.github/workflows/carto-release.yaml index 339e9c111b6..3ffb3c6e3bb 100644 --- a/.github/workflows/carto-release.yaml +++ b/.github/workflows/carto-release.yaml @@ -118,7 +118,7 @@ jobs: packages: write steps: - name: Checkout repository - uses: actions/checkout@v4 + uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4 with: ref: ${{ github.event.inputs.branch }} fetch-depth: 0 # Fetch all history for tags @@ -281,7 +281,7 @@ jobs: - name: Generate intelligent release notes with Claude id: release-notes if: steps.manual-notes.outputs.notes_generated != 'true' - uses: anthropics/claude-code-action@v1 + uses: anthropics/claude-code-action@673eb13aa77026be5c507eda12322c1a58b80f0b # v1 with: use_vertex: "true" github_token: ${{ secrets.GITHUB_TOKEN }} @@ -531,7 +531,7 @@ jobs: git push origin "${{ steps.calc-version.outputs.release_tag }}" - name: Create GitHub Release - uses: softprops/action-gh-release@v1 + uses: softprops/action-gh-release@26994186c0ac3ef5cae75ac16aa32e8153525f77 # v1 with: tag_name: ${{ steps.calc-version.outputs.release_tag }} name: ${{ steps.calc-version.outputs.release_tag }} diff --git a/.github/workflows/carto-schema-sync-validator.yml b/.github/workflows/carto-schema-sync-validator.yml index cf133625e48..b245e24d9bf 100644 --- a/.github/workflows/carto-schema-sync-validator.yml +++ b/.github/workflows/carto-schema-sync-validator.yml @@ -31,7 +31,7 @@ jobs: steps: - name: Checkout PR - uses: actions/checkout@v4 + uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4 with: fetch-depth: 0 diff --git a/.github/workflows/carto-slack-changelog.yml b/.github/workflows/carto-slack-changelog.yml index 38899d97866..56b015ef046 100644 --- a/.github/workflows/carto-slack-changelog.yml +++ b/.github/workflows/carto-slack-changelog.yml @@ -34,7 +34,7 @@ jobs: steps: - name: Checkout repository - uses: actions/checkout@v4 + uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4 with: fetch-depth: 0 @@ -431,7 +431,7 @@ jobs: id: insight if: steps.slack_pr.outputs.success == 'true' || steps.slack_release.outputs.success == 'true' continue-on-error: true # If Claude fails, we still posted the main message - uses: anthropics/claude-code-action@v1 + uses: anthropics/claude-code-action@673eb13aa77026be5c507eda12322c1a58b80f0b # v1 with: use_vertex: "true" github_token: ${{ secrets.GITHUB_TOKEN }} diff --git a/.github/workflows/carto-supply-chain-security.yml b/.github/workflows/carto-supply-chain-security.yml new file mode 100644 index 00000000000..69924bd7445 --- /dev/null +++ b/.github/workflows/carto-supply-chain-security.yml @@ -0,0 +1,202 @@ +################################################################################ +# CARTO - Supply Chain Security Scan +################################################################################ +# +# Automated supply chain security gate for PRs targeting carto/main. +# Responds to the litellm v1.82.8 PyPI compromise (BerriAI/litellm#24512). +# +# Layers: +# 1. GuardDog (Datadog/OpenSSF) — Semgrep-powered malicious pattern detection +# 2. Wheel allowlist — only litellm/ and dist-info/ files allowed +# 3. GuardDog wheel verify — scan built wheel for malicious code +# 4. pip-audit (PyPA) — known vulnerability scanning (advisory) +# 5. Structural integrity — setup.py guard, dependency source check +# +# Tools are pinned by version to prevent self-compromise. +# +# Reference: https://github.com/BerriAI/litellm/issues/24512 +# Ticket: https://app.shortcut.com/cartoteam/story/543659 + +name: CARTO - Supply Chain Security Scan + +on: + pull_request: + branches: + - carto/main + workflow_dispatch: + +permissions: + contents: read + security-events: write # For SARIF upload to GitHub Security tab + +jobs: + supply-chain-scan: + name: Supply Chain Security Scan + runs-on: ubuntu-latest + steps: + - name: Checkout repository + uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4 + with: + fetch-depth: 0 + + - name: Setup Python + uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5 + with: + python-version: "3.13" + + # ── Tool Installation (pinned versions) ──────────────────────────────── + - name: Install security tools + run: | + set -eu + echo "::group::Installing security scanning tools" + pip install "guarddog==2.9.0" "pip-audit==2.10.0" build + echo "GuardDog version: $(guarddog --version 2>/dev/null || echo 'installed')" + echo "pip-audit version: $(pip-audit --version)" + echo "::endgroup::" + + # ── Layer 1: GuardDog source scan ────────────────────────────────────── + - name: "Scan source for malicious patterns (GuardDog)" + id: guarddog-source + run: | + set -eu + echo "::group::GuardDog source scan" + + # Generate SARIF report for GitHub Security tab + guarddog pypi scan . --output-format sarif > guarddog-source.sarif 2>&1 || true + + # Human-readable output for PR checks + echo "Running GuardDog scan on source code..." + guarddog pypi scan . --output-format text || { + echo "::error::GuardDog detected suspicious patterns in source code" + exit 1 + } + + echo "::endgroup::" + + # ── Layer 2: Wheel build + allowlist audit ───────────────────────────── + - name: "Build wheel and audit contents" + id: wheel-audit + run: | + set -eu + echo "::group::Wheel build and allowlist audit" + + # Build wheel from source (no network isolation — uses local source only) + python -m build --wheel --no-isolation 2>&1 || { + echo "::warning::Wheel build failed — skipping wheel audit" + echo "wheel_built=false" >> $GITHUB_OUTPUT + echo "::endgroup::" + exit 0 + } + + WHEEL=$(ls dist/*.whl | head -1) + echo "Built wheel: $WHEEL" + echo "wheel_built=true" >> $GITHUB_OUTPUT + + # Allowlist audit: only litellm/ and dist-info/ files should exist + python3 << 'PYEOF' + import zipfile, sys, glob + + whl = glob.glob("dist/*.whl")[0] + print(f"Auditing wheel: {whl}") + + with zipfile.ZipFile(whl) as z: + files = z.namelist() + print(f"Total files in wheel: {len(files)}") + + # Allowlisted prefixes — anything else is suspicious + allowed_prefixes = ("litellm/", "litellm-", "litellm_") + unexpected = [f for f in files if not any(f.startswith(p) for p in allowed_prefixes)] + + # Defense in depth: explicit .pth check + pth_files = [f for f in files if f.endswith(".pth")] + + if pth_files: + print("\n❌ CRITICAL: .pth files detected in wheel!") + print("This is the exact attack vector from litellm v1.82.8 (BerriAI/litellm#24512)") + for f in pth_files: + print(f" 🚨 {f}") + sys.exit(1) + + if unexpected: + print(f"\n❌ FAIL: {len(unexpected)} unexpected files in wheel:") + for f in unexpected: + print(f" ⚠️ {f}") + print("\nOnly files under litellm/ and litellm-*.dist-info/ are allowed.") + sys.exit(1) + + print(f"\n✅ Wheel OK: {len(files)} files, all under allowed prefixes") + PYEOF + + echo "::endgroup::" + + # ── Layer 3: GuardDog wheel scan ─────────────────────────────────────── + - name: "Scan built wheel for malicious patterns" + if: steps.wheel-audit.outputs.wheel_built == 'true' + run: | + set -eu + echo "::group::GuardDog wheel verification" + guarddog pypi verify dist/*.whl || { + echo "::error::GuardDog detected suspicious patterns in the built wheel" + exit 1 + } + echo "::endgroup::" + + # ── Layer 4: pip-audit (known CVEs) ──────────────────────────────────── + - name: "Audit dependencies for known vulnerabilities (advisory)" + continue-on-error: true # Advisory — deps may lag behind CVE fixes + run: | + set -eu + echo "::group::pip-audit dependency scan" + pip-audit -r requirements.txt --progress-spinner off --output-format columns || { + echo "::warning::pip-audit found known vulnerabilities in dependencies" + echo "This is advisory — review the findings above" + } + echo "::endgroup::" + + # ── Layer 5: Structural integrity checks ─────────────────────────────── + - name: "Check structural integrity" + run: | + set -eu + echo "::group::Structural integrity checks" + FAIL=0 + + # Check 1: setup.py should NOT exist (we use Poetry exclusively) + if [ -f setup.py ]; then + echo "❌ FAIL: setup.py detected — this repo uses Poetry exclusively" + echo "A setup.py file appearing is a supply chain attack indicator" + echo "::error::setup.py detected — possible supply chain injection" + FAIL=1 + else + echo "✅ No setup.py (Poetry-only repo)" + fi + + # Check 2: No non-PyPI dependency sources + if grep -qE '(git\+|https?://|file://)' requirements.txt 2>/dev/null; then + echo "❌ FAIL: Non-PyPI dependency sources detected in requirements.txt:" + grep -nE '(git\+|https?://|file://)' requirements.txt + echo "::error::Non-PyPI dependency sources in requirements.txt" + FAIL=1 + else + echo "✅ All dependencies use standard PyPI sources" + fi + + # Check 3: No suspicious build hooks in pyproject.toml + if grep -qiE '(cmdclass|post_install|pre_install|data_files)' pyproject.toml 2>/dev/null; then + echo "⚠️ WARN: Build hooks or data_files detected in pyproject.toml" + grep -niE '(cmdclass|post_install|pre_install|data_files)' pyproject.toml + echo "::warning::Build hooks or data_files in pyproject.toml — manual review required" + else + echo "✅ No suspicious build hooks in pyproject.toml" + fi + + echo "::endgroup::" + exit $FAIL + + # ── Upload SARIF for GitHub Security tab ─────────────────────────────── + - name: Upload GuardDog SARIF results + if: always() && hashFiles('guarddog-source.sarif') != '' + uses: github/codeql-action/upload-sarif@480db559a14342288b67e54bd959dd52dc3ee68f # v3 + with: + sarif_file: guarddog-source.sarif + category: guarddog-supply-chain + continue-on-error: true # Don't fail the workflow if SARIF upload fails diff --git a/.github/workflows/carto-upstream-sync-ci-fixer.yml b/.github/workflows/carto-upstream-sync-ci-fixer.yml index 0f3e8eb0394..ffee7e643ae 100644 --- a/.github/workflows/carto-upstream-sync-ci-fixer.yml +++ b/.github/workflows/carto-upstream-sync-ci-fixer.yml @@ -117,7 +117,7 @@ jobs: steps: - name: Checkout repository - uses: actions/checkout@v4 + uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4 with: ref: ${{ needs.check-ci-status.outputs.branch-name }} fetch-depth: 0 @@ -394,7 +394,7 @@ jobs: - name: Run Claude Code to fix failures id: claude-fix - uses: anthropics/claude-code-action@v1 + uses: anthropics/claude-code-action@673eb13aa77026be5c507eda12322c1a58b80f0b # v1 with: use_vertex: "true" github_token: ${{ secrets.X_GITHUB_SUPERCARTOFANTE }} diff --git a/.github/workflows/carto-upstream-sync-customizations-analyzer.yml b/.github/workflows/carto-upstream-sync-customizations-analyzer.yml index 5659c41dafd..47ddc791f33 100644 --- a/.github/workflows/carto-upstream-sync-customizations-analyzer.yml +++ b/.github/workflows/carto-upstream-sync-customizations-analyzer.yml @@ -250,7 +250,7 @@ jobs: fi - name: Checkout PR branch - uses: actions/checkout@v4 + uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4 with: ref: ${{ steps.pr-info.outputs.head-ref }} fetch-depth: 0 @@ -466,7 +466,7 @@ jobs: - name: Run Claude Code for feature analysis id: claude-analyze - uses: anthropics/claude-code-action@v1 + uses: anthropics/claude-code-action@673eb13aa77026be5c507eda12322c1a58b80f0b # v1 with: use_vertex: "true" github_token: ${{ secrets.X_GITHUB_SUPERCARTOFANTE }} @@ -692,7 +692,7 @@ jobs: - name: Upload analysis artifact if: steps.verify-analysis.outputs.analysis_exists == 'true' - uses: actions/upload-artifact@v4 + uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4 with: name: feature-analysis path: | @@ -919,7 +919,7 @@ jobs: echo "[Fixer] PR #${PR_NUMBER} on branch ${HEAD_REF}" - name: Checkout PR branch - uses: actions/checkout@v4 + uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4 with: ref: ${{ steps.fix-pr-info.outputs.head-ref }} fetch-depth: 0 @@ -1013,7 +1013,7 @@ jobs: - name: Download analysis artifacts (analyze-and-fix mode) if: needs.check-trigger.outputs.mode == 'analyze-and-fix' - uses: actions/download-artifact@v4 + uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4 with: name: feature-analysis path: /tmp/ @@ -1054,7 +1054,7 @@ jobs: - name: Run Claude Code to fix dropped features id: claude-fix - uses: anthropics/claude-code-action@v1 + uses: anthropics/claude-code-action@673eb13aa77026be5c507eda12322c1a58b80f0b # v1 with: use_vertex: "true" github_token: ${{ secrets.X_GITHUB_SUPERCARTOFANTE }} diff --git a/.github/workflows/carto-upstream-sync-main.yml b/.github/workflows/carto-upstream-sync-main.yml index 77f6be32ab1..3b056eaca03 100644 --- a/.github/workflows/carto-upstream-sync-main.yml +++ b/.github/workflows/carto-upstream-sync-main.yml @@ -52,7 +52,7 @@ jobs: # Checkout carto/main to check what version we currently have in production # NOT main (which has unstable upstream dev code that's always ahead of stable) - name: Checkout carto/main branch - uses: actions/checkout@v4 + uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4 with: ref: carto/main fetch-depth: 0 @@ -128,7 +128,7 @@ jobs: if: needs.check-new-release.outputs.has-new-release == 'true' steps: - name: Checkout main branch - uses: actions/checkout@v4 + uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4 with: ref: main fetch-depth: 0 @@ -288,7 +288,7 @@ jobs: pr-version: ${{ steps.check-pr.outputs.pr-version }} steps: - name: Checkout repository - uses: actions/checkout@v4 + uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4 with: fetch-depth: 0 @@ -375,7 +375,7 @@ jobs: sync-branch: ${{ steps.create-sync-branch.outputs.branch-name }} steps: - name: Checkout main branch - uses: actions/checkout@v4 + uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4 with: ref: main fetch-depth: 0 @@ -461,6 +461,13 @@ jobs: - [ ] CARTO customizations preserved - [ ] \`pyproject.toml\` version matches upstream + ### 🔒 Supply Chain Security + + - [ ] Supply chain security scan passed ([GuardDog](https://github.com/DataDog/guarddog) + wheel audit) + - [ ] No unexpected files in built wheel (allowlist: \`litellm/\` + \`dist-info/\` only) + - [ ] No suspicious dependency changes in \`requirements.txt\` + - [ ] \`pyproject.toml\` build config unchanged or reviewed + ---
diff --git a/.github/workflows/carto-upstream-sync-resolver.yml b/.github/workflows/carto-upstream-sync-resolver.yml index 3f2e17c6701..1360f5d97c9 100644 --- a/.github/workflows/carto-upstream-sync-resolver.yml +++ b/.github/workflows/carto-upstream-sync-resolver.yml @@ -305,7 +305,7 @@ jobs: # Then merge BASE branch (carto/main) into it to bring CARTO customizations # Resolution commits are pushed directly to this branch, updating the existing PR - name: Checkout repository (sync branch - upstream with history) - uses: actions/checkout@v4 + uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4 with: ref: ${{ steps.pr-info.outputs.head-ref }} fetch-depth: 0 @@ -710,7 +710,7 @@ jobs: - name: Run Claude Code for conflict resolution (GOAL-DRIVEN) id: claude-run - uses: anthropics/claude-code-action@v1 + uses: anthropics/claude-code-action@673eb13aa77026be5c507eda12322c1a58b80f0b # v1 with: use_vertex: "true" github_token: ${{ secrets.X_GITHUB_SUPERCARTOFANTE }} @@ -1457,7 +1457,7 @@ jobs: echo "[Verify] PR #${PR_NUMBER}: ${HEAD_REF} → ${BASE_REF}" - name: Checkout PR branch - uses: actions/checkout@v4 + uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4 with: ref: ${{ steps.pr-info.outputs.head-ref }} fetch-depth: 0 @@ -1590,7 +1590,7 @@ jobs: - name: Run Claude Code to verify CARTO features id: claude-verify - uses: anthropics/claude-code-action@v1 + uses: anthropics/claude-code-action@673eb13aa77026be5c507eda12322c1a58b80f0b # v1 with: use_vertex: "true" github_token: ${{ secrets.X_GITHUB_SUPERCARTOFANTE }} diff --git a/Dockerfile b/Dockerfile index 717ec2bcb77..fd1a2aec03d 100644 --- a/Dockerfile +++ b/Dockerfile @@ -27,6 +27,19 @@ RUN sed -i 's/\r$//' docker/build_admin_ui.sh && chmod +x docker/build_admin_ui. # Build the package RUN rm -rf dist/* && python -m build +# Supply chain security: audit wheel contents (responds to litellm v1.82.8 compromise) +# Only litellm/ and dist-info/ files should exist. Anything else (e.g., .pth files) fails the build. +RUN python -c "\ +import zipfile, sys, glob; \ +whl = glob.glob('dist/*.whl')[0]; \ +z = zipfile.ZipFile(whl); \ +files = z.namelist(); \ +bad = [f for f in files if not f.startswith(('litellm/', 'litellm-', 'litellm_'))]; \ +pth = [f for f in files if f.endswith('.pth')]; \ +[(print(f'CRITICAL: .pth file: {f}')) for f in pth]; \ +[(print(f'UNEXPECTED: {f}')) for f in bad]; \ +sys.exit(1) if (bad or pth) else print(f'Wheel audit OK: {len(files)} files verified')" + # There should be only one wheel file now, assume the build only creates one RUN ls -1 dist/*.whl | head -1 diff --git a/ci_cd/security_scans.sh b/ci_cd/security_scans.sh index 770610c2a3b..00d3abceaba 100755 --- a/ci_cd/security_scans.sh +++ b/ci_cd/security_scans.sh @@ -233,21 +233,95 @@ run_grype_scans() { echo "Grype scans completed successfully" } +# Function to install GuardDog +install_guarddog() { + echo "Installing GuardDog..." + pip3 install "guarddog==2.9.0" + echo "GuardDog installed successfully" +} + +# Function to run supply chain security scans +# Responds to litellm v1.82.8 PyPI compromise (BerriAI/litellm#24512) +run_supply_chain_scans() { + echo "Running supply chain security scans..." + + if ! command -v guarddog &> /dev/null; then + install_guarddog + fi + + # Scan 1: GuardDog source scan for malicious patterns + echo "Scanning source code for malicious patterns (GuardDog)..." + guarddog pypi scan . --output-format text || { + echo "" + echo "==========================================" + echo "ERROR: Supply Chain Security Scan Failed" + echo "==========================================" + echo "GuardDog detected suspicious patterns in the source code." + echo "Review the findings above before proceeding." + echo "==========================================" + echo "" + exit 1 + } + + # Scan 2: Wheel allowlist audit + echo "Auditing wheel contents..." + if ls dist/*.whl 1>/dev/null 2>&1; then + python3 -c " +import zipfile, sys, glob +whl = glob.glob('dist/*.whl')[0] +with zipfile.ZipFile(whl) as z: + files = z.namelist() + bad = [f for f in files if not f.startswith(('litellm/', 'litellm-', 'litellm_'))] + pth = [f for f in files if f.endswith('.pth')] + if pth: + print('CRITICAL: .pth files detected:', pth) + sys.exit(1) + if bad: + print('UNEXPECTED files in wheel:', bad) + sys.exit(1) + print(f'Wheel audit OK: {len(files)} files verified') +" || { + echo "ERROR: Wheel audit failed — unexpected files detected" + exit 1 + } + else + echo "No wheel found in dist/ — skipping wheel audit" + fi + + # Scan 3: Structural integrity + echo "Checking structural integrity..." + if [ -f setup.py ]; then + echo "ERROR: setup.py detected — this repo uses Poetry exclusively" + exit 1 + fi + + if grep -qE '(git\+|https?://|file://)' requirements.txt 2>/dev/null; then + echo "ERROR: Non-PyPI dependency sources detected in requirements.txt" + grep -nE '(git\+|https?://|file://)' requirements.txt + exit 1 + fi + + echo "Supply chain security scans completed successfully" +} + # Main execution main() { echo "Installing security scanning tools..." install_trivy install_grype - + # echo "Running secret detection scans..." # run_secret_detection - + echo "Running filesystem vulnerability scans..." run_trivy_scans - + echo "Running Docker image vulnerability scans..." run_grype_scans - + + echo "Running supply chain security scans..." + run_supply_chain_scans + echo "All security scans completed successfully!" }