sentry-auto-fix.yml

name: Sentry Auto-Fix

on:
  issues:
    types: [labeled]
  workflow_dispatch:
    inputs:
      issue_number:
        description: 'Issue number to triage'
        required: true
  pull_request_review:
    types: [submitted]
  issue_comment:
    types: [created, edited]
  schedule:
    - cron: '0 */2 * * *'

permissions:
  contents: write
  issues: write
  pull-requests: write
  id-token: write

jobs:
  triage-and-fix:
    concurrency:
      group: sentry-triage-${{ github.event.issue.number || github.event.inputs.issue_number }}
      cancel-in-progress: false
    if: >
      (
        (github.event_name == 'issues' &&
         github.event.action == 'labeled' &&
         (contains(github.event.issue.labels.*.name, 'sentry') ||
          contains(github.event.issue.labels.*.name, 'worker-crash')) &&
         !contains(github.event.issue.labels.*.name, 'needs-human'))
        ||
        github.event_name == 'workflow_dispatch'
      )
    runs-on: ubuntu-latest
    timeout-minutes: 30
    steps:
      - name: Get issue info
        id: issue_info
        env:
          GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
        run: |
          if [ "${{ github.event_name }}" = "workflow_dispatch" ]; then
            ISSUE_NUM="${{ github.event.inputs.issue_number }}"
          else
            ISSUE_NUM="${{ github.event.issue.number }}"
          fi
          echo "number=$ISSUE_NUM" >> $GITHUB_OUTPUT

          ISSUE_DATA=$(gh issue view "$ISSUE_NUM" --repo ${{ github.repository }} --json title,body)
          echo "title=$(echo "$ISSUE_DATA" | jq -r '.title')" >> $GITHUB_OUTPUT

          # Save body to file to avoid escaping issues

      - name: Verify worker-crash trigger source
        if: contains(github.event.issue.labels.*.name, 'worker-crash')
        env:
          GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
        run: |
          if [ "${{ github.actor }}" != "github-actions[bot]" ]; then
            gh issue comment ${{ steps.issue_info.outputs.number }} \
              --repo ${{ github.repository }} \
              --body "Skipped: worker-crash auto-fix can only be triggered by the automated watchdog (github-actions[bot])."
            echo "::error::Unauthorized actor ${{ github.actor }} added worker-crash label"
            exit 1
          fi
          echo "$ISSUE_DATA" | jq -r '.body' > /tmp/issue-body.txt

      - name: Checkout
        uses: actions/checkout@v4
        with:
          fetch-depth: 0
          ref: develop

      - uses: actions/setup-node@v4
        with:
          node-version: 18

      - name: Install backend deps
        run: cd run && npm install

      - name: Install frontend deps
        run: yarn install

      - uses: superfly/flyctl-actions/setup-flyctl@master

      - name: Setup SSH
        uses: webfactory/ssh-agent@v0.9.0
        with:
          ssh-private-key: ${{ secrets.SSH_PRIVATE_KEY }}

      - name: Add known hosts
        run: |
          mkdir -p ~/.ssh
          ssh-keyscan -H 157.90.154.200 >> ~/.ssh/known_hosts

      - name: Configure git
        run: |
          git config user.name "github-actions[bot]"
          git config user.email "github-actions[bot]@users.noreply.github.com"

      - name: Circuit breaker - check for active incident
        id: circuit_breaker
        env:
          GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
        run: |
          # Count sentry-labeled issues created in the last 30 minutes
          CUTOFF=$(date -u -d '30 minutes ago' +%Y-%m-%dT%H:%M:%SZ 2>/dev/null || date -u -v-30M +%Y-%m-%dT%H:%M:%SZ)
          RECENT_COUNT=$(gh issue list --repo ${{ github.repository }} --label sentry --state open --json number,createdAt \
            --jq "[.[] | select(.createdAt > \"$CUTOFF\")] | length")

          # Also check for incident-labeled issues (created by scanner correlation)
          HAS_INCIDENT=$(gh issue list --repo ${{ github.repository }} --label incident --state open --limit 1 --json number \
            --jq 'length')

          if [ "$HAS_INCIDENT" -gt 0 ]; then
            echo "tripped=true" >> $GITHUB_OUTPUT
            echo "Circuit breaker: active incident issue exists. Skipping auto-fix."
            gh issue edit ${{ steps.issue_info.outputs.number }} --add-label "needs-human" --repo ${{ github.repository }}
            gh issue comment ${{ steps.issue_info.outputs.number }} \
              --body "⚡ **Circuit breaker tripped**: An active incident issue exists. Auto-fix paused to avoid treating individual symptoms. See the open \`incident\` issue for context. @antoinedc please investigate the root cause." \
              --repo ${{ github.repository }}
          elif [ "$RECENT_COUNT" -ge 3 ]; then
            echo "tripped=true" >> $GITHUB_OUTPUT
            echo "Circuit breaker: $RECENT_COUNT sentry issues in last 30 min (threshold: 3). Escalating."
            gh issue edit ${{ steps.issue_info.outputs.number }} --add-label "needs-human" --repo ${{ github.repository }}
            gh issue comment ${{ steps.issue_info.outputs.number }} \
              --body "⚡ **Circuit breaker tripped**: $RECENT_COUNT sentry issues opened in the last 30 minutes, indicating a systemic incident. Auto-fix paused to avoid fixing symptoms individually. @antoinedc please investigate the root cause." \
              --repo ${{ github.repository }}
          else
            echo "tripped=false" >> $GITHUB_OUTPUT
          fi

      - name: Check for existing fix PR
        id: existing_pr
        if: steps.circuit_breaker.outputs.tripped != 'true'
        run: |
          EXISTING_PR=$(gh pr list --repo ${{ github.repository }} --head "fix/sentry-${{ steps.issue_info.outputs.number }}" --state open --json number -q '.[0].number' 2>/dev/null || true)
          if [ -n "$EXISTING_PR" ]; then
            echo "exists=true" >> $GITHUB_OUTPUT
            echo "PR #$EXISTING_PR already exists for this issue, skipping"
          else
            echo "exists=false" >> $GITHUB_OUTPUT
          fi
        env:
          GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}

      - name: Notify dashboard - triaging
        if: always()
        continue-on-error: true
        run: |
          curl -s -X POST "${{ secrets.APP_URL }}/webhooks/github-actions" \
            -H "Authorization: Bearer ${{ secrets.ETHERNAL_WEBHOOK_SECRET }}" \
            -H "Content-Type: application/json" \
            -d "{
              \"workflowRunId\": ${{ github.run_id }},
              \"githubIssueNumber\": ${{ steps.issue_info.outputs.number }},
              \"status\": \"triaging\",
              \"currentStep\": \"Claude is investigating the error\"
            }"

      - name: Check for regression
        id: regression
        env:
          SENTRY_API_TOKEN: ${{ secrets.SENTRY_API_TOKEN }}
          GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
        run: |
          # Extract Sentry issue ID from issue body
          SENTRY_ISSUE_ID=$(grep -oP 'sentry\.tryethernal\.com/organizations/sentry/issues/\K\d+' /tmp/issue-body.txt | head -1)
          CONTEXT=""

          if [ -n "$SENTRY_ISSUE_ID" ]; then
            # Check if Sentry marks this as a regression
            IS_REGRESSION=$(curl -s \
              "https://sentry.tryethernal.com/api/0/issues/$SENTRY_ISSUE_ID/" \
              -H "Authorization: Bearer $SENTRY_API_TOKEN" | jq -r '.isRegression // false')

            if [ "$IS_REGRESSION" = "true" ]; then
              # Search for previous closed issues with the same Sentry issue ID
              PREV_ISSUES=$(gh issue list --label sentry --state closed --limit 20 --json number,title,body \
                --jq "[.[] | select(.body | contains(\"issues/$SENTRY_ISSUE_ID/\")) | {number, title}]")

              # Find linked PRs for those issues
              PREV_PRS=""
              for ISSUE_NUM in $(echo "$PREV_ISSUES" | jq -r '.[].number'); do
                PR=$(gh pr list --state merged --search "Fixes #$ISSUE_NUM" --json number,title,url \
                  --jq '.[0] | "PR #\(.number): \(.title) (\(.url))"' 2>/dev/null)
                if [ -n "$PR" ]; then
                  PREV_PRS="$PREV_PRS\n- $PR (fixed issue #$ISSUE_NUM)"
                fi
              done

              if [ -n "$PREV_PRS" ]; then
                CONTEXT="⚠️ **REGRESSION**: This error was previously fixed and has recurred.

          Previous fix attempts:
          $(echo -e "$PREV_PRS")

          Review the previous PRs to understand what was tried before. The previous fix was insufficient — identify WHY it regressed and ensure your new fix addresses the root cause more thoroughly."
              else
                CONTEXT="⚠️ **REGRESSION**: Sentry reports this error was previously resolved but has recurred. Search git history for related fixes."
              fi
            fi
          fi

          # Save to file to avoid shell escaping issues
          echo "$CONTEXT" > /tmp/regression-context.txt
          if [ -n "$CONTEXT" ]; then
            echo "is_regression=true" >> $GITHUB_OUTPUT
          else
            echo "is_regression=false" >> $GITHUB_OUTPUT
          fi

      - name: Start conversation streamer
        if: steps.circuit_breaker.outputs.tripped != 'true' && steps.existing_pr.outputs.exists != 'true'
        continue-on-error: true
        run: |
          chmod +x .github/scripts/stream-conversation.sh
          .github/scripts/stream-conversation.sh &
          echo $! > /tmp/streamer-pid
        env:
          WEBHOOK_URL: ${{ secrets.APP_URL }}/webhooks/github-actions
          WEBHOOK_SECRET: ${{ secrets.ETHERNAL_WEBHOOK_SECRET }}
          WORKFLOW_RUN_ID: ${{ github.run_id }}
          GITHUB_ISSUE_NUMBER: ${{ steps.issue_info.outputs.number }}

      - name: Claude Code - Triage and Fix
        if: steps.circuit_breaker.outputs.tripped != 'true' && steps.existing_pr.outputs.exists != 'true'
        uses: anthropics/claude-code-action@v1
        with:
          claude_code_oauth_token: ${{ secrets.CLAUDE_CODE_OAUTH_TOKEN }}
          label_trigger: sentry
          allowed_bots: "ethernal-sentry[bot],claude[bot]"
          claude_args: "--model claude-sonnet-4-20250514 --max-turns 50 --allowedTools Bash,Read,Write,Edit,Glob,Grep"
          show_full_output: true
          prompt: |
            You are an autonomous error-fixing agent for the Ethernal project — an EVM block explorer.
            A Sentry error has been auto-reported as GitHub issue #${{ steps.issue_info.outputs.number }}.

            ## Issue Details
            Title: ${{ steps.issue_info.outputs.title }}
            Body:
            $(cat /tmp/issue-body.txt)

            ## Regression Context
            $(cat /tmp/regression-context.txt)

            ## Your Task

            ### 1. Investigate
            - Parse the Sentry issue from the body to extract error type, stack trace, and context
            - Use the Sentry API to fetch full event details:
              ```
              curl -H "Authorization: Bearer $SENTRY_API_TOKEN" \
                "https://sentry.tryethernal.com/api/0/issues/{sentry_issue_id}/events/latest/"
              ```
              The Sentry issue ID may be in the issue body URL. The org slug is "sentry".
              Backend project slug: "ethernal-backend", Frontend project slug: "ethernal-frontend"
            - Check production logs if needed: `flyctl logs -a ethernal --no-tail | tail -100`
            - Query the read-only database if the error involves data issues:
              ```
              psql "$PROD_DATABASE_URL_READONLY" -c "SELECT ..."
              ```
              NEVER write to this database. SELECT queries only.

            ### 2. Triage (choose ONE action)

            **IMPORTANT**: If the issue already has the `needs-human` label, it was flagged as a sustained infrastructure error (high volume of connection errors outside of a deploy). In this case, ALWAYS choose **AUTO-FIX** — investigate the root cause (check DB health, connection pool stats, Fly.io status) and attempt a fix. Also notify @antoinedc:
            ```
            gh issue comment ${{ steps.issue_info.outputs.number }} --body "🚨 Sustained infra errors detected (not correlated with a deploy). Investigating and attempting a fix. @antoinedc please monitor."
            ```

            **CLOSE** the issue if:
            - It's a duplicate of an existing issue
            - It's expected behavior (e.g., user input validation errors)
            - It's a third-party/network error we can't control
            - It's already fixed on the develop branch
            - It's a transient infrastructure error (database connection drops, ECONNRESET, "Connection terminated unexpectedly", SequelizeConnectionError, SequelizeConnectionRefusedError) — BUT first verify it correlates with a recent deploy/restart by checking: `flyctl releases -a ethernal --json | jq '.[0:3]'`. If the error timestamps align with a release within the last 10 minutes, it's transient → close. If there's NO recent deploy and connection errors persist, ESCALATE instead — it may be a real infra issue. Do NOT add retry logic or try-catch wrappers for these — BullMQ already handles retries at the job level.
            Close with: `gh issue close ${{ steps.issue_info.outputs.number }} --comment "Closing: [reason]"`

            **ESCALATE** if:
            - It involves security, data corruption, or data loss
            - It touches billing, auth, Stripe, or payment code
            - The root cause is unclear after investigation
            - The fix would require architectural changes
            Escalate with:
            ```
            gh issue edit ${{ steps.issue_info.outputs.number }} --add-label "needs-human"
            gh issue comment ${{ steps.issue_info.outputs.number }} --body "Escalating: [reason]. @antoinedc please review."
            ```

            **AUTO-FIX** if:
            - It's a null/undefined check, missing error handling, off-by-one error
            - It's a simple logic bug with clear root cause
            - The fix is contained (1-3 files) and low risk

            **ESCALATE** (not auto-fix) if the issue is an **infrastructure/config problem** (WebSocket failures, Fly.io service instability, DNS, routing, health checks). These require human review — do NOT create code PRs for infra issues.

            For **performance issues specifically**, apply this triage checklist BEFORE writing any code:
            1. **Check event volume thresholds FIRST — this is mandatory**:
               - **User-facing endpoints** (API routes, HTTP handlers): need **50+ events/24h**. Below that → CLOSE.
               - **Background jobs** (BullMQ workers, cron jobs): need **100+ events/24h**. Below that → CLOSE.
               If the event count is below the threshold, CLOSE the issue immediately with a comment explaining the threshold. Do NOT proceed to investigation or fix.
            2. **Check for missing indexes**: Run `\di` on the affected table(s) via the read-only DB. If the slow query would be fixed by an index, the fix is just a migration — don't restructure code.
            3. **Prefer simple fixes**: An index > a query tweak > code restructuring. Never split a working single query into multiple queries to "optimize" it unless you've confirmed the original query plan is actually bad.
            4. **Complexity budget**: If your fix requires > 20 lines of logic changes (not counting tests), ESCALATE instead of auto-fixing. Complex performance optimizations need human review.
            5. **Group related issues**: Before fixing, check if there are other open sentry issues touching the same code path. If so, note it in the issue comment — one coherent fix is better than 5 micro-PRs.

            ### 3. Fix (only if AUTO-FIX)
            - Before creating a branch, verify no PR already exists:
              ```bash
              if gh pr list --repo tryethernal/ethernal --head "fix/sentry-${{ steps.issue_info.outputs.number }}" --state open --json number -q '.[0].number' 2>/dev/null | grep -q .; then
                echo "A fix PR already exists for this issue. Exiting."
                exit 0
              fi
              ```
            - **Check for overlapping in-flight PRs**: Before writing code, identify which files you plan to change. Then check if any other open `fix/sentry-*` PRs touch those same files:
              ```bash
              # List files changed by other open sentry fix PRs (use jq filtering, not head: search which requires exact match)
              gh pr list --repo tryethernal/ethernal --state open --json number,headRefName \
                --jq '[.[] | select(.headRefName | startswith("fix/sentry-"))] | .[].number' \
                > /tmp/sentry-pr-nums.txt
              for PR_NUM in $(cat /tmp/sentry-pr-nums.txt); do
                gh pr diff "$PR_NUM" --repo tryethernal/ethernal --name-only 2>/dev/null
              done | sort -u > /tmp/other-pr-files.txt
              ```
              If any of your planned files appear in `/tmp/other-pr-files.txt`, **ESCALATE instead of fixing** — another fix is already in flight for the same code. Note which PR has the overlap in the escalation comment.
            - Create a branch: `git checkout -b fix/sentry-${{ steps.issue_info.outputs.number }}`
            - Implement the minimal fix needed
            - Run relevant tests:
              - Backend: `cd run && npm test -- --testPathPattern=<relevant_test>`
              - Frontend: `yarn test -- <relevant_test>`
            - If tests fail, fix them. If you can't fix them, escalate instead.
            - Commit with a clear message explaining the fix
            - Push: `git push origin fix/sentry-${{ steps.issue_info.outputs.number }}`
            - Create PR:
              ```
              gh pr create \
                --title "fix: [brief description]" \
                --body "## Summary
              Fixes #${{ steps.issue_info.outputs.number }}

              **Sentry Error:** [error type and message]
              **Root Cause:** [what caused it]
              **Fix:** [what was changed and why]
              **Regression:** [If regression, link to previous PR and explain why the previous fix was insufficient]

              ## Test plan
              - [ ] Relevant unit tests pass
              - [ ] Fix addresses the root cause, not just symptoms

              🤖 Generated with [Claude Code](https://claude.com/claude-code)" \
                --base develop
              ```

            ### Rules
            - NEVER modify files in: `run/api/stripe.js`, `run/webhooks/stripe.js`, `run/lib/stripe.js`, `run/middlewares/auth.js`, `run/lib/crypto.js`, `run/config/database.js`
            - ESCALATE (do not auto-fix) any change that modifies database connection pool settings, connection timeouts, or Sequelize pool configuration — these affect every query system-wide and require human review
            - NEVER write to the production database
            - NEVER skip running tests
            - NEVER close the issue manually — GitHub auto-closes it when the PR with "Fixes #N" is merged
            - Keep fixes minimal — don't refactor surrounding code
            - If uncertain about the fix, escalate rather than guess
            - NEVER add retry/reconnection logic for database connection errors — BullMQ already handles retries. Adding in-job retry creates double-retry complexity.
            - NEVER silently catch and swallow database errors with fallback values — this masks real issues. Let errors propagate so BullMQ retries the job.
            - NEVER split working database queries into multiple queries to "prevent timeouts" — timeouts are solved by indexes, not query splitting.
            - NEVER wrap entire job functions in try-catch just to log and re-throw — this adds no value.
            - For performance fixes: ALWAYS check existing indexes first (`\di tablename` on read-only DB). If the right index exists, the query is likely fine — close the issue or look for the real problem elsewhere.
            - For performance fixes: prefer adding an index (migration) over restructuring code. An index is 1 file, zero risk, and fixes the root cause. Code restructuring (caching, batching, multi-path strategies) adds complexity and bugs.
            - NEVER create a fix PR with > 20 lines of logic changes for a performance issue. Escalate instead — complex optimizations need human review.
            - NEVER claim a fix was made without verifying it appears in `git diff`. Run `git diff` before writing the review processing summary and confirm each claimed change is present.
            - If your fix introduces a new catch block, timeout handler, or fallback path, you MUST add a test that exercises that path. Untested error-handling code is where bugs hide — the overnight incident on 2026-03-30 had 3 bugs in auto-merged PRs, all in untested timeout/fallback paths. If you can't write a test for the new path, ESCALATE instead of shipping untested error handling.
        env:
          SENTRY_API_TOKEN: ${{ secrets.SENTRY_API_TOKEN }}
          PROD_DATABASE_URL_READONLY: ${{ secrets.PROD_DATABASE_URL_READONLY }}
          FLY_API_TOKEN: ${{ secrets.FLY_API_TOKEN }}

      - name: Stop conversation streamer
        if: always()
        continue-on-error: true
        run: |
          if [ -f /tmp/streamer-pid ]; then
            kill $(cat /tmp/streamer-pid) 2>/dev/null || true
          fi

      - name: Extract conversation log
        id: conversation
        if: always()
        continue-on-error: true
        run: |
          LOG_FILE="/home/runner/work/_temp/claude-execution-output.json"
          if [ ! -f "$LOG_FILE" ]; then
            echo "[]" > /tmp/conversation-log.json
            exit 0
          fi

          python3 << 'PYEOF'
          import json
          with open("/home/runner/work/_temp/claude-execution-output.json") as f:
              data = json.load(f)

          log = []
          messages = data if isinstance(data, list) else [data]
          for msg in messages:
              if not isinstance(msg, dict):
                  continue
              role = msg.get("role", msg.get("type", ""))
              if role in ("system", "preset"):
                  continue
              content = msg.get("content", [])
              if isinstance(content, str):
                  log.append({"role": role, "text": content})
                  continue
              if not isinstance(content, list):
                  content = [content] if isinstance(content, dict) else []
              for block in content:
                  if not isinstance(block, dict):
                      continue
                  btype = block.get("type", "")
                  if btype == "text":
                      log.append({"role": role, "text": block.get("text", "")})
                  elif btype == "tool_use":
                      inp = block.get("input", {})
                      log.append({
                          "role": role,
                          "tool": block.get("name", ""),
                          "input": (inp.get("command") or inp.get("description") or str(inp))[:500]
                      })
                  elif btype == "tool_result":
                      cv = block.get("content", "")
                      if isinstance(cv, list):
                          cv = " ".join(b.get("text", "") for b in cv if isinstance(b, dict))
                      log.append({"role": "tool", "output": str(cv)[:1000]})
          log = log[-200:]
          with open("/tmp/conversation-log.json", "w") as f:
              json.dump(log, f)
          print(f"Extracted {len(log)} conversation entries")
          PYEOF

      - name: Notify dashboard - post-triage
        if: always()
        continue-on-error: true
        run: |
          # Determine status based on whether a PR branch exists
          if git ls-remote --heads origin "fix/sentry-${{ steps.issue_info.outputs.number }}" | grep -q .; then
            STATUS="fixing"
            STEP="Claude created a fix branch"
            DECISION="auto-fix"
          else
            # Check if issue was closed
            ISSUE_STATE=$(gh issue view ${{ steps.issue_info.outputs.number }} --repo ${{ github.repository }} --json state -q '.state')
            if [ "$ISSUE_STATE" = "CLOSED" ]; then
              STATUS="closed"
              STEP="Issue was closed by Claude"
              DECISION="close"
            elif gh issue view ${{ steps.issue_info.outputs.number }} --repo ${{ github.repository }} --json labels -q '.labels[].name' | grep -q "needs-human"; then
              STATUS="escalated"
              STEP="Escalated to human"
              DECISION="escalate"
            else
              STATUS="failed"
              STEP="Triage completed without action"
              DECISION="unknown"
            fi
          fi

          # Build payload with conversation log
          CONV_LOG=$(cat /tmp/conversation-log.json 2>/dev/null || echo "[]")
          jq -n \
            --argjson runId "${{ github.run_id }}" \
            --argjson issueNum "${{ steps.issue_info.outputs.number }}" \
            --arg status "$STATUS" \
            --arg step "$STEP" \
            --arg decision "$DECISION" \
            --argjson convLog "$CONV_LOG" \
            '{
              workflowRunId: $runId,
              githubIssueNumber: $issueNum,
              status: $status,
              currentStep: $step,
              triageDecision: $decision,
              conversationLog: $convLog
            }' | curl -s -X POST "${{ secrets.APP_URL }}/webhooks/github-actions" \
            -H "Authorization: Bearer ${{ secrets.ETHERNAL_WEBHOOK_SECRET }}" \
            -H "Content-Type: application/json" \
            -d @-
        env:
          GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}

      - name: Notify dashboard - PR created
        if: always()
        continue-on-error: true
        run: |
          PR_NUMBER=$(gh pr list --head "fix/sentry-${{ steps.issue_info.outputs.number }}" --json number -q '.[0].number' 2>/dev/null)
          if [ -n "$PR_NUMBER" ]; then
            curl -s -X POST "${{ secrets.APP_URL }}/webhooks/github-actions" \
              -H "Authorization: Bearer ${{ secrets.ETHERNAL_WEBHOOK_SECRET }}" \
              -H "Content-Type: application/json" \
              -d "{
                \"workflowRunId\": ${{ github.run_id }},
                \"githubIssueNumber\": ${{ steps.issue_info.outputs.number }},
                \"githubPrNumber\": $PR_NUMBER,
                \"status\": \"reviewing\",
                \"currentStep\": \"PR #$PR_NUMBER awaiting review\"
              }"
          fi
        env:
          GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}

      - name: Notify dashboard - failure
        if: failure()
        continue-on-error: true
        run: |
          curl -s -X POST "${{ secrets.APP_URL }}/webhooks/github-actions" \
            -H "Authorization: Bearer ${{ secrets.ETHERNAL_WEBHOOK_SECRET }}" \
            -H "Content-Type: application/json" \
            -d "{
              \"workflowRunId\": ${{ github.run_id }},
              \"githubIssueNumber\": ${{ steps.issue_info.outputs.number }},
              \"status\": \"failed\",
              \"currentStep\": \"Workflow failed\"
            }"

  process-review:
    concurrency:
      group: sentry-review-${{ github.event.issue.number || github.event.pull_request.number }}
      cancel-in-progress: true
    if: >
      (
        (github.event_name == 'pull_request_review' &&
         startsWith(github.event.pull_request.head.ref, 'fix/sentry-'))
        ||
        (github.event_name == 'issue_comment' &&
         github.event.issue.pull_request &&
         contains(github.event.comment.body, '@claude'))
        ||
        (github.event_name == 'issue_comment' &&
         github.event.issue.pull_request &&
         github.event.sender.login == 'greptile-apps[bot]')
      ) &&
      github.event.sender.login != 'github-actions[bot]' &&
      github.event.sender.login != 'claude[bot]'
    runs-on: ubuntu-latest
    timeout-minutes: 30
    steps:
      - name: Debounce
        run: sleep 30

      - name: Get PR info
        id: pr_info
        run: |
          if [ "${{ github.event_name }}" = "issue_comment" ]; then
            PR_NUMBER="${{ github.event.issue.number }}"
            BRANCH=$(gh pr view "$PR_NUMBER" --repo ${{ github.repository }} --json headRefName -q '.headRefName')
          else
            PR_NUMBER="${{ github.event.pull_request.number }}"
            BRANCH="${{ github.event.pull_request.head.ref }}"
          fi
          echo "pr_number=$PR_NUMBER" >> $GITHUB_OUTPUT
          echo "branch=$BRANCH" >> $GITHUB_OUTPUT
        env:
          GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}

      - name: Check if sentry fix PR
        id: check_sentry
        run: |
          BRANCH="${{ steps.pr_info.outputs.branch }}"
          if [[ "$BRANCH" == fix/sentry-* ]]; then
            echo "is_sentry=true" >> $GITHUB_OUTPUT
          else
            echo "is_sentry=false" >> $GITHUB_OUTPUT
          fi

      - name: Checkout PR branch
        if: steps.check_sentry.outputs.is_sentry == 'true'
        uses: actions/checkout@v4
        with:
          fetch-depth: 0
          ref: ${{ steps.pr_info.outputs.branch }}

      - uses: actions/setup-node@v4
        if: steps.check_sentry.outputs.is_sentry == 'true'
        with:
          node-version: 18

      - name: Install backend deps
        if: steps.check_sentry.outputs.is_sentry == 'true'
        run: cd run && npm install

      - name: Install frontend deps
        if: steps.check_sentry.outputs.is_sentry == 'true'
        run: yarn install

      - uses: superfly/flyctl-actions/setup-flyctl@master
        if: steps.check_sentry.outputs.is_sentry == 'true'

      - name: Setup SSH
        if: steps.check_sentry.outputs.is_sentry == 'true'
        uses: webfactory/ssh-agent@v0.9.0
        with:
          ssh-private-key: ${{ secrets.SSH_PRIVATE_KEY }}

      - name: Add known hosts
        if: steps.check_sentry.outputs.is_sentry == 'true'
        run: |
          mkdir -p ~/.ssh
          ssh-keyscan -H 157.90.154.200 >> ~/.ssh/known_hosts

      - name: Configure git
        if: steps.check_sentry.outputs.is_sentry == 'true'
        run: |
          git config user.name "github-actions[bot]"
          git config user.email "github-actions[bot]@users.noreply.github.com"

      - name: Start conversation streamer
        if: steps.check_sentry.outputs.is_sentry == 'true'
        continue-on-error: true
        run: |
          chmod +x .github/scripts/stream-conversation.sh
          .github/scripts/stream-conversation.sh &
          echo $! > /tmp/streamer-pid
        env:
          WEBHOOK_URL: ${{ secrets.APP_URL }}/webhooks/github-actions
          WEBHOOK_SECRET: ${{ secrets.ETHERNAL_WEBHOOK_SECRET }}
          WORKFLOW_RUN_ID: ${{ github.run_id }}
          GITHUB_ISSUE_NUMBER: ${{ steps.pr_info.outputs.pr_number }}

      - name: Claude Code - Process Review
        id: claude_review
        if: steps.check_sentry.outputs.is_sentry == 'true'
        uses: anthropics/claude-code-action@v1
        with:
          claude_code_oauth_token: ${{ secrets.CLAUDE_CODE_OAUTH_TOKEN }}
          allowed_bots: "ethernal-sentry[bot],greptile-apps[bot]"
          claude_args: "--model claude-sonnet-4-20250514 --max-turns 50 --allowedTools Bash,Read,Write,Edit,Glob,Grep"
          show_full_output: true
          base_branch: develop
          prompt: |
            You are processing code review feedback on PR #${{ steps.pr_info.outputs.pr_number }} (branch: ${{ steps.pr_info.outputs.branch }}).

            ## Step 1: Fetch All Comments

            ```bash
            # Fetch formal PR reviews
            gh api repos/${{ github.repository }}/pulls/${{ steps.pr_info.outputs.pr_number }}/reviews
            # Fetch inline review comments
            gh api repos/${{ github.repository }}/pulls/${{ steps.pr_info.outputs.pr_number }}/comments
            # Fetch issue-level comments (where Greptile posts its summary)
            gh api repos/${{ github.repository }}/issues/${{ steps.pr_info.outputs.pr_number }}/comments
            ```

            ## Step 2: Verify and Process Review Comments

            For each review comment from `greptile-apps[bot]` that flags an issue:
            1. **FIRST verify** the comment is correct by reading the actual code it references. Check ALL commits on the branch, not just the diff — a later commit may have already addressed the issue.
            2. If it's a **valid issue** → fix it, commit, push, reply explaining the fix, and react with 👍:
               ```bash
               # For inline review comments:
               gh api repos/${{ github.repository }}/pulls/comments/{comment_id}/reactions -f content='+1'
               # For issue-level comments:
               gh api repos/${{ github.repository }}/issues/comments/{comment_id}/reactions -f content='+1'
               ```
            3. If it's **incorrect** → reply on the PR explaining why it's wrong, and react with 👎:
               ```bash
               # For inline review comments:
               gh api repos/${{ github.repository }}/pulls/comments/{comment_id}/reactions -f content='-1'
               # For issue-level comments:
               gh api repos/${{ github.repository }}/issues/comments/{comment_id}/reactions -f content='-1'
               ```
            4. If it was a **valid issue but already fixed** by a later commit on this branch → reply noting it was valid but already addressed, and react with 👍:
               ```bash
               # For inline review comments:
               gh api repos/${{ github.repository }}/pulls/comments/{comment_id}/reactions -f content='+1'
               # For issue-level comments:
               gh api repos/${{ github.repository }}/issues/comments/{comment_id}/reactions -f content='+1'
               ```
            5. If it's unclear → ask for clarification in a reply (no reaction yet)

            **IMPORTANT:** Do NOT react to comments before verifying them. Verify first, then react with the appropriate emoji based on your assessment.
            If the review summary is clean (no issues flagged), react with 👍 on the summary.

            Take comments seriously but verify they're correct before acting.
            NEVER remove working code to satisfy a review bot.
            NEVER modify: `run/api/stripe.js`, `run/webhooks/stripe.js`, `run/lib/stripe.js`, `run/middlewares/auth.js`, `run/lib/crypto.js`

            ## Step 3: Signal Merge Readiness

            After processing ALL comments, determine if the PR is ready to merge:
            - **READY**: All review issues have been addressed (fixed or correctly dismissed)
            - **NOT READY**: There are unresolved issues you couldn't fix, or you asked for clarification

            You MUST run this command at the very end:
            ```bash
            # If ALL issues are resolved:
            echo "READY_TO_MERGE" > /tmp/merge-status

            # If there are unresolved issues:
            echo "NOT_READY" > /tmp/merge-status
            ```

            Then write a short summary of what you did. If not ready, explain what remains.
        env:
          SENTRY_API_TOKEN: ${{ secrets.SENTRY_API_TOKEN }}
          PROD_DATABASE_URL_READONLY: ${{ secrets.PROD_DATABASE_URL_READONLY }}
          FLY_API_TOKEN: ${{ secrets.FLY_API_TOKEN }}

      - name: Stop conversation streamer
        if: always()
        continue-on-error: true
        run: |
          if [ -f /tmp/streamer-pid ]; then
            kill $(cat /tmp/streamer-pid) 2>/dev/null || true
          fi

      - name: Check merge readiness
        id: merge_check
        if: steps.check_sentry.outputs.is_sentry == 'true'
        run: |
          if [ -f /tmp/merge-status ] && grep -q "READY_TO_MERGE" /tmp/merge-status; then
            echo "ready=true" >> $GITHUB_OUTPUT
          else
            echo "ready=false" >> $GITHUB_OUTPUT
            echo "PR is not ready to merge. Skipping merge and deploy."
          fi

      - name: Check review confidence
        id: confidence_check
        if: steps.check_sentry.outputs.is_sentry == 'true' && steps.merge_check.outputs.ready == 'true'
        run: |
          # Fetch latest Greptile review comment
          GREPTILE_COMMENT=$(gh api repos/${{ github.repository }}/issues/${{ steps.pr_info.outputs.pr_number }}/comments \
            --jq '[.[] | select(.user.login == "greptile-apps[bot]")] | last | .body // ""')

          # Greptile formats as "<h3>Confidence Score: X/5</h3>" — anchor to that
          SCORE=$(echo "$GREPTILE_COMMENT" | grep -oP '(?i)confidence\s+score[:\s]*\K\d(?=/5)' | head -1)
          # Fallback: last X/5 occurrence (summary score appears after per-file scores)
          if [ -z "$SCORE" ]; then
            SCORE=$(echo "$GREPTILE_COMMENT" | grep -oP '\d(?=/5)' | tail -1)
          fi

          if [ -z "$SCORE" ]; then
            echo "No confidence score found, proceeding"
            echo "sufficient=true" >> $GITHUB_OUTPUT
          elif [ "$SCORE" -lt 3 ]; then
            echo "Low confidence: $SCORE/5. Flagging for human review."
            gh pr edit ${{ steps.pr_info.outputs.pr_number }} --add-label "needs-human" --repo ${{ github.repository }}
            gh pr comment ${{ steps.pr_info.outputs.pr_number }} --body "Review confidence $SCORE/5 (threshold: 3/5). Flagging for human review." --repo ${{ github.repository }}
            echo "sufficient=false" >> $GITHUB_OUTPUT
          else
            echo "Confidence: $SCORE/5 (sufficient)"
            echo "sufficient=true" >> $GITHUB_OUTPUT
          fi
        env:
          GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}

      - name: Wait for CI checks
        id: ci_check
        if: steps.check_sentry.outputs.is_sentry == 'true' && steps.merge_check.outputs.ready == 'true' && steps.confidence_check.outputs.sufficient == 'true'
        run: |
          echo "Waiting for CI checks to complete on PR #${{ steps.pr_info.outputs.pr_number }}..."
          for i in $(seq 1 60); do
            CHECKS_JSON=$(gh pr checks ${{ steps.pr_info.outputs.pr_number }} --repo ${{ github.repository }} --json name,state --jq '[.[] | select(.name != "process-review" and .name != "triage-and-fix" and .name != "Greptile Review")]')
            STATUS=$(echo "$CHECKS_JSON" | jq -r 'if length == 0 then "pass" elif [.[] | select(.state != "SUCCESS" and .state != "SKIPPED")] | length == 0 then "pass" elif [.[] | select(.state == "FAILURE")] | length > 0 then "fail" else "pending" end')
            echo "Check $i/60: $STATUS"
            if [ "$STATUS" = "pass" ]; then
              echo "passed=true" >> $GITHUB_OUTPUT
              exit 0
            elif [ "$STATUS" = "fail" ]; then
              echo "CI checks failed. Skipping merge."
              echo "passed=false" >> $GITHUB_OUTPUT
              exit 0
            fi
            sleep 30
          done
          echo "CI checks timed out after 30 minutes."
          echo "passed=false" >> $GITHUB_OUTPUT
        env:
          GH_TOKEN: ${{ secrets.GH_PAT }}

      - name: Merge PR
        id: merge_pr
        if: steps.check_sentry.outputs.is_sentry == 'true' && steps.merge_check.outputs.ready == 'true' && steps.confidence_check.outputs.sufficient == 'true' && steps.ci_check.outputs.passed == 'true'
        run: |
          gh pr merge ${{ steps.pr_info.outputs.pr_number }} \
            --squash --admin --delete-branch \
            --repo ${{ github.repository }}
        env:
          GH_TOKEN: ${{ secrets.GH_PAT }}

      - name: Check hotfix label
        id: hotfix_check
        if: steps.merge_pr.conclusion == 'success'
        run: |
          ISSUE_NUMBER=$(gh pr view ${{ steps.pr_info.outputs.pr_number }} --repo ${{ github.repository }} --json body -q '.body' | grep -oP 'Fixes #\K\d+' | head -1)
          IS_HOTFIX=0
          if [ -n "$ISSUE_NUMBER" ]; then
            IS_HOTFIX=$(gh issue view "$ISSUE_NUMBER" --repo ${{ github.repository }} --json labels -q '.labels[].name' | grep -c '^hotfix$' || true)
          fi
          echo "is_hotfix=${IS_HOTFIX}" >> "$GITHUB_OUTPUT"
          echo "issue_number=${ISSUE_NUMBER:-0}" >> "$GITHUB_OUTPUT"
          echo "Hotfix: ${IS_HOTFIX}, Issue: ${ISSUE_NUMBER:-none}"
        env:
          GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}

      # --- Non-hotfix path: notify merged status and stop ---

      - name: Notify dashboard - merged (batch deploy)
        if: >
          steps.merge_pr.conclusion == 'success' &&
          steps.hotfix_check.outputs.is_hotfix == '0'
        continue-on-error: true
        run: |
          curl -s -X POST "${{ secrets.APP_URL }}/webhooks/github-actions" \
            -H "Authorization: Bearer ${{ secrets.ETHERNAL_WEBHOOK_SECRET }}" \
            -H "Content-Type: application/json" \
            -d "{
              \"githubIssueNumber\": ${{ steps.hotfix_check.outputs.issue_number }},
              \"githubPrNumber\": ${{ steps.pr_info.outputs.pr_number }},
              \"status\": \"merged\",
              \"currentStep\": \"PR merged, awaiting batch deploy\"
            }"

      # --- Hotfix path: deploy inline (existing logic) ---

      - name: Notify dashboard - merging (hotfix)
        if: >
          steps.merge_pr.conclusion == 'success' &&
          steps.hotfix_check.outputs.is_hotfix == '1'
        continue-on-error: true
        run: |
          curl -s -X POST "${{ secrets.APP_URL }}/webhooks/github-actions" \
            -H "Authorization: Bearer ${{ secrets.ETHERNAL_WEBHOOK_SECRET }}" \
            -H "Content-Type: application/json" \
            -d "{
              \"githubIssueNumber\": ${{ steps.hotfix_check.outputs.issue_number }},
              \"githubPrNumber\": ${{ steps.pr_info.outputs.pr_number }},
              \"status\": \"merging\",
              \"currentStep\": \"PR merged, deploying (hotfix)\"
            }"

      - name: Deploy (hotfix)
        id: deploy_hotfix
        if: >
          steps.merge_pr.conclusion == 'success' &&
          steps.hotfix_check.outputs.is_hotfix == '1'
        env:
          GH_PAT: ${{ secrets.GH_PAT }}
        run: |
          git remote set-url origin "https://x-access-token:${GH_PAT}@github.com/${{ github.repository }}.git"
          git checkout develop
          git pull origin develop

          LAST_TAG=$(git describe --tags --abbrev=0)
          echo "Last tag: $LAST_TAG"
          echo "last_tag=$LAST_TAG" >> "$GITHUB_OUTPUT"

          # Get commits since last tag, filter out release machinery
          COMMITS=$(git log --oneline "$LAST_TAG..HEAD" | grep -viE '^[a-f0-9]+ (update changelog|[0-9]+\.[0-9]+\.[0-9]+|Merge (branch|develop|master))' || true)
          echo "Commits since $LAST_TAG:"
          echo "$COMMITS"

          if [ -z "$COMMITS" ]; then
            echo "No meaningful commits to deploy"
            exit 0
          fi

          # Generate changelog entry
          VERSION_DATE=$(date +%Y-%m-%d)
          CURRENT_VERSION=$(node -p "require('./package.json').version")
          IFS='.' read -r MAJOR MINOR PATCH <<< "$CURRENT_VERSION"
          NEW_VERSION="$MAJOR.$MINOR.$((PATCH + 1))"

          CHANGELOG_ENTRY="## [$NEW_VERSION] - $VERSION_DATE
          ### Fixed
          $(echo "$COMMITS" | sed 's/^[a-f0-9]* /- /')"

          # Insert after header (line 5)
          head -5 CHANGELOG.md > CHANGELOG.tmp
          echo "" >> CHANGELOG.tmp
          echo "$CHANGELOG_ENTRY" >> CHANGELOG.tmp
          echo "" >> CHANGELOG.tmp
          tail -n +6 CHANGELOG.md >> CHANGELOG.tmp
          mv CHANGELOG.tmp CHANGELOG.md

          git add CHANGELOG.md
          git commit -m "update changelog"

          npm version patch --message '%s'

          git push origin develop
          git push origin "v$NEW_VERSION"

          # Sync master
          git checkout master
          git pull origin master
          git merge develop --no-ff -m "Merge develop into master for v$NEW_VERSION"
          git push origin master
          git checkout develop

          echo "Deployed v$NEW_VERSION"

      - name: Notify dashboard - deploying (hotfix)
        if: >
          steps.deploy_hotfix.conclusion == 'success' &&
          steps.hotfix_check.outputs.is_hotfix == '1'
        continue-on-error: true
        run: |
          curl -s -X POST "${{ secrets.APP_URL }}/webhooks/github-actions" \
            -H "Authorization: Bearer ${{ secrets.ETHERNAL_WEBHOOK_SECRET }}" \
            -H "Content-Type: application/json" \
            -d "{
              \"githubIssueNumber\": ${{ steps.hotfix_check.outputs.issue_number }},
              \"githubPrNumber\": ${{ steps.pr_info.outputs.pr_number }},
              \"status\": \"deploying\",
              \"currentStep\": \"Deploying to production (hotfix)\"
            }"

      - name: Resolve Sentry issue (hotfix)
        if: >
          steps.deploy_hotfix.conclusion == 'success' &&
          steps.hotfix_check.outputs.is_hotfix == '1'
        env:
          SENTRY_API_TOKEN: ${{ secrets.SENTRY_API_TOKEN }}
          GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
        run: |
          ISSUE_NUMBER=${{ steps.hotfix_check.outputs.issue_number }}
          if [ "$ISSUE_NUMBER" = "0" ] || [ -z "$ISSUE_NUMBER" ]; then
            echo "No linked issue found, skipping Sentry resolution"
            exit 0
          fi

          # Extract Sentry issue ID from the GitHub issue body
          SENTRY_ISSUE_ID=$(gh issue view "$ISSUE_NUMBER" --repo ${{ github.repository }} --json body -q '.body' | grep -oP 'sentry\.tryethernal\.com/organizations/sentry/issues/\K\d+' | head -1)
          if [ -z "$SENTRY_ISSUE_ID" ]; then
            echo "No Sentry issue ID found in issue #$ISSUE_NUMBER body, skipping"
            exit 0
          fi

          echo "Resolving Sentry issue $SENTRY_ISSUE_ID (from GitHub issue #$ISSUE_NUMBER)"
          curl -s -X PUT \
            "https://sentry.tryethernal.com/api/0/issues/$SENTRY_ISSUE_ID/" \
            -H "Authorization: Bearer $SENTRY_API_TOKEN" \
            -H "Content-Type: application/json" \
            -d '{"status": "resolved"}' | jq '{status, statusDetails}'

      - name: Notify dashboard - completed (hotfix)
        if: >
          steps.deploy_hotfix.conclusion == 'success' &&
          steps.hotfix_check.outputs.is_hotfix == '1'
        continue-on-error: true
        run: |
          curl -s -X POST "${{ secrets.APP_URL }}/webhooks/github-actions" \
            -H "Authorization: Bearer ${{ secrets.ETHERNAL_WEBHOOK_SECRET }}" \
            -H "Content-Type: application/json" \
            -d "{
              \"githubIssueNumber\": ${{ steps.hotfix_check.outputs.issue_number }},
              \"githubPrNumber\": ${{ steps.pr_info.outputs.pr_number }},
              \"status\": \"completed\",
              \"currentStep\": \"Fix deployed and Sentry issue resolved (hotfix)\",
              \"completedAt\": \"$(date -u +%Y-%m-%dT%H:%M:%SZ)\"
            }"

      - name: Resolve swept-in non-hotfix Sentry issues
        if: >
          steps.deploy_hotfix.conclusion == 'success' &&
          steps.hotfix_check.outputs.is_hotfix == '1'
        continue-on-error: true
        env:
          GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
          SENTRY_API_TOKEN: ${{ secrets.SENTRY_API_TOKEN }}
        run: |
          # Use pre-deploy tag captured by the deploy step
          LAST_TAG=${{ steps.deploy_hotfix.outputs.last_tag }}
          HOTFIX_PR=${{ steps.pr_info.outputs.pr_number }}
          PR_NUMBERS=$(git log --oneline "$LAST_TAG..HEAD" | grep -viE '^[a-f0-9]+ (update changelog|[0-9]+\.[0-9]+\.[0-9]+|Merge (branch|develop|master))' | grep -oP '#\K\d+' | grep -v "^${HOTFIX_PR}$" || true)

          if [ -z "$PR_NUMBERS" ]; then
            echo "No other PRs swept into this hotfix release"
            exit 0
          fi

          echo "Found swept-in PRs: $PR_NUMBERS"

          for PR_NUM in $PR_NUMBERS; do
            echo "--- Processing swept-in PR #$PR_NUM ---"

            ISSUE_NUMBER=$(gh pr view "$PR_NUM" --repo ${{ github.repository }} --json body -q '.body' 2>/dev/null | grep -oP 'Fixes #\K\d+' | head -1 || true)
            if [ -z "$ISSUE_NUMBER" ]; then
              echo "No linked issue in PR #$PR_NUM, skipping"
              continue
            fi

            IS_SENTRY=$(gh issue view "$ISSUE_NUMBER" --repo ${{ github.repository }} --json labels -q '.labels[].name' 2>/dev/null | grep -c '^sentry$' || true)
            if [ "$IS_SENTRY" = "0" ]; then
              echo "Issue #$ISSUE_NUMBER is not a sentry issue, skipping"
              continue
            fi

            SENTRY_ISSUE_ID=$(gh issue view "$ISSUE_NUMBER" --repo ${{ github.repository }} --json body -q '.body' | grep -oP 'sentry\.tryethernal\.com/organizations/sentry/issues/\K\d+' | head -1 || true)
            if [ -z "$SENTRY_ISSUE_ID" ]; then
              echo "No Sentry issue ID in issue #$ISSUE_NUMBER, skipping"
              continue
            fi

            echo "Resolving swept-in Sentry issue $SENTRY_ISSUE_ID (issue #$ISSUE_NUMBER, PR #$PR_NUM)"
            curl -s -X PUT \
              "https://sentry.tryethernal.com/api/0/issues/$SENTRY_ISSUE_ID/" \
              -H "Authorization: Bearer $SENTRY_API_TOKEN" \
              -H "Content-Type: application/json" \
              -d '{"status": "resolved"}' | jq '{status, statusDetails}'

            curl -s -X POST "${{ secrets.APP_URL }}/webhooks/github-actions" \
              -H "Authorization: Bearer ${{ secrets.ETHERNAL_WEBHOOK_SECRET }}" \
              -H "Content-Type: application/json" \
              -d "{
                \"githubIssueNumber\": ${ISSUE_NUMBER},
                \"githubPrNumber\": ${PR_NUM},
                \"status\": \"completed\",
                \"currentStep\": \"Fix deployed via hotfix release\",
                \"completedAt\": \"$(date -u +%Y-%m-%dT%H:%M:%SZ)\"
              }"
          done

  recover-stuck:
    if: github.event_name == 'schedule'
    runs-on: ubuntu-latest
    timeout-minutes: 10
    steps:
      - name: Find and merge stuck sentry PRs
        run: |
          # Find open sentry fix PRs not updated in the last hour
          STUCK_PRS=$(gh pr list --repo ${{ github.repository }} \
            --search "head:fix/sentry- is:open" \
            --json number,updatedAt \
            --jq '[.[] | select((.updatedAt | fromdateiso8601) < (now - 3600))] | .[].number')

          if [ -z "$STUCK_PRS" ]; then
            echo "No stuck PRs found"
            exit 0
          fi

          for PR_NUM in $STUCK_PRS; do
            echo "--- Checking PR #$PR_NUM ---"

            # Check mergeable status
            MERGEABLE=$(gh pr view "$PR_NUM" --repo ${{ github.repository }} --json mergeable -q '.mergeable')
            if [ "$MERGEABLE" != "MERGEABLE" ]; then
              echo "PR #$PR_NUM is $MERGEABLE, skipping"
              continue
            fi

            # Skip PRs flagged for human review
            HAS_NEEDS_HUMAN=$(gh pr view "$PR_NUM" --repo ${{ github.repository }} --json labels \
              -q '[.labels[].name] | any(. == "needs-human")')
            if [ "$HAS_NEEDS_HUMAN" = "true" ]; then
              echo "PR #$PR_NUM has needs-human label, skipping"
              continue
            fi

            # Check CI status (exclude workflow-specific checks)
            FAILING=$(gh pr checks "$PR_NUM" --repo ${{ github.repository }} --json name,state \
              --jq '[.[] | select(.name != "process-review" and .name != "triage-and-fix" and .name != "recover-stuck" and .name != "Greptile Review") | select(.state != "SUCCESS" and .state != "SKIPPED")] | length')
            if [ "$FAILING" != "0" ]; then
              echo "PR #$PR_NUM has failing CI, skipping"
              continue
            fi

            # Check if Greptile reviewed but process-review failed or never ran
            GREPTILE_ISSUE_COMMENTS=$(gh api repos/${{ github.repository }}/issues/$PR_NUM/comments \
              --jq '[.[] | select(.user.login == "greptile-apps[bot]")] | length')
            GREPTILE_REVIEWS=$(gh api repos/${{ github.repository }}/pulls/$PR_NUM/reviews \
              --jq '[.[] | select(.user.login == "greptile-apps[bot]")] | length')
            GREPTILE_COMMENTED=$(( GREPTILE_ISSUE_COMMENTS + GREPTILE_REVIEWS ))
            REVIEW_STATE=$(gh pr checks "$PR_NUM" --repo ${{ github.repository }} --json name,state \
              --jq '[.[] | select(.name == "process-review")] | .[0].state // "NONE"')

            if [ "$GREPTILE_COMMENTED" != "0" ] && [ "$REVIEW_STATE" != "SUCCESS" ]; then
              # Check if we already retried (look for our retry comment)
              RETRY_COUNT=$(gh api repos/${{ github.repository }}/issues/$PR_NUM/comments \
                --jq '[.[] | select(.body | contains("@claude Please process the Greptile review"))] | length')
              if [ "$RETRY_COUNT" -ge 2 ]; then
                echo "PR #$PR_NUM: review processing failed after $RETRY_COUNT retries. Flagging for human review."
                gh pr edit "$PR_NUM" --add-label "needs-human" --repo ${{ github.repository }}
                gh pr comment "$PR_NUM" --body "Automated review processing failed after multiple retries. Flagging for human review. @${{ vars.ONCALL_HANDLE }}" --repo ${{ github.repository }}
                continue
              fi
              echo "PR #$PR_NUM has unprocessed Greptile review (process-review: $REVIEW_STATE). Re-triggering review."
              gh pr comment "$PR_NUM" --body "@claude Please process the Greptile review comments on this PR." --repo ${{ github.repository }}
              continue
            fi

            echo "Merging stuck PR #$PR_NUM"
            gh pr merge "$PR_NUM" --squash --admin --delete-branch --repo ${{ github.repository }} || echo "Failed to merge #$PR_NUM"
          done
        env:
          GH_TOKEN: ${{ secrets.GH_PAT }}