fix: add captureTurn transport watchdog and runTrackedJob hard timeout

Completes the defense-in-depth strategy for stuck jobs. The plugin's
job state machine is single-writer: only runTrackedJob writes terminal
status. But that writer sits behind a chain of async promises
(transport -> captureTurn -> runner), and any link in that chain can
break silently, freezing state.json at status:"running" forever.

Previous commits handled the read side (dead-PID reconciliation with
TOCTOU guards) and UX transparency (idle surfacing, relative log
timestamps). This commit closes the write side with two additional
layers:

Layer 1 — captureTurn transport watchdog (lib/codex.mjs):
  state.completion resolves only via turn/completed or an inferred
  250ms final_answer timer. If the app-server (direct or via broker)
  disconnects before either signal, the promise hangs forever. Race
  it against client.exitPromise: on transport close, treat a seen
  final_answer as inferred success, otherwise reject with a clear
  "app-server disconnected before the turn completed" error. This
  covers the captureTurn-hang failure mode reported in the wild where
  codex CLI exits cleanly (exit 0) without sending turn/completed and
  the companion waits forever.

Layer 2 — runTrackedJob hard timeout (lib/tracked-jobs.mjs):
  Even if every inner watchdog fails, Promise.race the runner against
  a configurable hard cap (default 15m, overridable via
  CODEX_JOB_TIMEOUT_MS env or per-call timeoutMs). On timeout, write
  status:"failed" with timedOut:true and a descriptive error. The job
  record also carries timeoutAt so /codex:status can render
  "hard timeout in 59s" when a job is also flagged staleLog —
  telling the user the escape hatch is imminent instead of leaving
  them guessing how long to wait.

Combined coverage:
  - worker process dies silently       -> dead-PID reconciliation
  - app-server disconnects mid-turn    -> captureTurn watchdog
  - runner hangs in any other way      -> hard timeout
  - user can always see what happened  -> UX layer

Refs upstream openai#243, openai#184, openai#222, openai#183, openai#164.

Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>

Author: audichuang

plugins/codex/scripts/lib/codex.mjs

@@ -597,6 +597,31 @@ async function captureTurn(client, threadId, startRequest, options = {}) {
       completeTurn(state, response.turn);
     }
 
+    // Transport-level watchdog: if the app-server (direct or via broker)
+    // disconnects before we see turn/completed or a final_answer-phase
+    // agentMessage, state.completion will never resolve on its own. Race
+    // against client.exitPromise so the companion always reaches a terminal
+    // state. This avoids the "captureTurn hangs indefinitely" failure mode
+    // that leaves a job stuck at status:running even though no worker exists.
+    const transportWatchdog = client.exitPromise.then(() => {
+      if (state.completed) return;
+      if (state.finalAnswerSeen) {
+        // final_answer was seen; the 250ms inferred-completion timer may or
+        // may not have fired, and the transport closed before turn/completed.
+        // Treat as inferred success — this is the same fallback the existing
+        // scheduleInferredCompletion path uses.
+        completeTurn(state, null, { inferred: true });
+        return;
+      }
+      const exitError =
+        client.exitError ?? state.error ??
+        new Error("Codex app-server disconnected before the turn completed (no turn/completed or final_answer received).");
+      state.error = state.error ?? exitError;
+      state.rejectCompletion(exitError);
+    });
+    // Swallow unhandled-rejection noise; we only care that the watchdog fires.
+    transportWatchdog.catch?.(() => {});
+
     return await state.completion;
   } finally {
     clearCompletionTimer(state);

plugins/codex/scripts/lib/job-control.mjs

@@ -203,6 +203,10 @@ export function enrichJob(job, options = {}) {
   const isActive = job.status === "queued" || job.status === "running";
   const idle = isActive ? computeIdleInfo(job.logFile) : null;
 
+  const timeoutAt = typeof job.timeoutAt === "string" ? Date.parse(job.timeoutAt) : null;
+  const timeoutRemainingMs =
+    isActive && Number.isFinite(timeoutAt) ? Math.max(0, timeoutAt - Date.now()) : null;
+
   const enriched = {
     ...job,
     kindLabel: getJobTypeLabel(job),
@@ -218,7 +222,10 @@ export function enrichJob(job, options = {}) {
     lastActivityAt: idle?.lastActivityAt ?? null,
     idleForMs: idle?.idleForMs ?? null,
     idleFor: idle?.idleFor ?? null,
-    staleLog: Boolean(idle && idle.idleForMs > STALE_LOG_THRESHOLD_MS)
+    staleLog: Boolean(idle && idle.idleForMs > STALE_LOG_THRESHOLD_MS),
+    timeoutRemainingMs,
+    timeoutRemaining:
+      timeoutRemainingMs == null ? null : formatRelativeAgo(timeoutRemainingMs)?.replace(/ ago$/, "")
   };
 
   return {

plugins/codex/scripts/lib/render.mjs

@@ -126,6 +126,11 @@ function pushJobDetails(lines, job, options = {}) {
   if (job.autoReconciled) {
     const pidSuffix = job.reconciledDeadPid ? ` (PID ${job.reconciledDeadPid})` : "";
     lines.push(`  ! Auto-reconciled as failed: worker process${pidSuffix} exited without reporting.`);
+  } else if (job.timedOut) {
+    lines.push(`  ! Hard timeout: runner watchdog aborted the job after exceeding the configured duration.`);
+    if (job.errorMessage) {
+      lines.push(`  Error: ${job.errorMessage}`);
+    }
   } else if (job.errorMessage && (job.status === "failed" || job.status === "cancelled")) {
     lines.push(`  Error: ${job.errorMessage}`);
   }
@@ -140,7 +145,10 @@ function pushJobDetails(lines, job, options = {}) {
   }
   if (options.showElapsed && job.idleFor) {
     const marker = job.staleLog ? "  ! " : "  ";
-    lines.push(`${marker}Last activity: ${job.idleFor}${job.staleLog ? " — no log output; process may be stuck" : ""}`);
+    const timeoutHint =
+      job.staleLog && job.timeoutRemaining ? ` (hard timeout in ${job.timeoutRemaining})` : "";
+    const stuckHint = job.staleLog ? " — no log output; process may be stuck" : "";
+    lines.push(`${marker}Last activity: ${job.idleFor}${stuckHint}${timeoutHint}`);
   }
   if (options.showDuration && job.duration) {
     lines.push(`  Duration: ${job.duration}`);

plugins/codex/scripts/lib/tracked-jobs.mjs

@@ -4,11 +4,30 @@ import process from "node:process";
 import { readJobFile, resolveJobFile, resolveJobLogFile, upsertJob, writeJobFile } from "./state.mjs";
 
 export const SESSION_ID_ENV = "CODEX_COMPANION_SESSION_ID";
+export const JOB_TIMEOUT_ENV = "CODEX_JOB_TIMEOUT_MS";
+export const DEFAULT_JOB_TIMEOUT_MS = 15 * 60 * 1000;
 
 export function nowIso() {
   return new Date().toISOString();
 }
 
+function resolveJobTimeoutMs(options) {
+  if (Number.isFinite(options.timeoutMs) && options.timeoutMs > 0) {
+    return options.timeoutMs;
+  }
+  const envValue = Number(process.env[JOB_TIMEOUT_ENV]);
+  if (Number.isFinite(envValue) && envValue > 0) {
+    return envValue;
+  }
+  return DEFAULT_JOB_TIMEOUT_MS;
+}
+
+function formatTimeoutHuman(ms) {
+  if (ms >= 60_000) return `${Math.round(ms / 60_000)}m`;
+  if (ms >= 1000) return `${Math.round(ms / 1000)}s`;
+  return `${ms}ms`;
+}
+
 function normalizeProgressEvent(value) {
   if (value && typeof value === "object" && !Array.isArray(value)) {
     return {
@@ -140,19 +159,41 @@ function readStoredJobOrNull(workspaceRoot, jobId) {
 }
 
 export async function runTrackedJob(job, runner, options = {}) {
+  const timeoutMs = resolveJobTimeoutMs(options);
   const runningRecord = {
     ...job,
     status: "running",
     startedAt: nowIso(),
     phase: "starting",
     pid: process.pid,
-    logFile: options.logFile ?? job.logFile ?? null
+    logFile: options.logFile ?? job.logFile ?? null,
+    timeoutAt: new Date(Date.now() + timeoutMs).toISOString(),
+    timeoutMs
   };
   writeJobFile(job.workspaceRoot, job.id, runningRecord);
   upsertJob(job.workspaceRoot, runningRecord);
 
+  // Layer-2 watchdog: no matter what goes wrong inside runner (captureTurn
+  // hang, broker wedge, internal deadlock), this timer guarantees the job
+  // reaches a terminal state. Layer 1 (captureTurn exitPromise watchdog) and
+  // layer 3 (listJobs dead-PID reconciliation) catch most cases — this is
+  // the backstop for the rest.
+  let timeoutHandle = null;
+  let timedOut = false;
+  const timeoutPromise = new Promise((_resolve, reject) => {
+    timeoutHandle = setTimeout(() => {
+      timedOut = true;
+      reject(new Error(`Tracked job ${job.id} exceeded the ${formatTimeoutHuman(timeoutMs)} hard timeout and was aborted by the runner watchdog.`));
+    }, timeoutMs);
+    timeoutHandle.unref?.();
+  });
+
   try {
-    const execution = await runner();
+    const execution = await Promise.race([runner(), timeoutPromise]);
+    if (timeoutHandle) {
+      clearTimeout(timeoutHandle);
+      timeoutHandle = null;
+    }
     const completionStatus = execution.exitStatus === 0 ? "completed" : "failed";
     const completedAt = nowIso();
     writeJobFile(job.workspaceRoot, job.id, {
@@ -179,25 +220,31 @@ export async function runTrackedJob(job, runner, options = {}) {
     appendLogBlock(options.logFile ?? job.logFile ?? null, "Final output", execution.rendered);
     return execution;
   } catch (error) {
+    if (timeoutHandle) {
+      clearTimeout(timeoutHandle);
+      timeoutHandle = null;
+    }
     const errorMessage = error instanceof Error ? error.message : String(error);
     const existing = readStoredJobOrNull(job.workspaceRoot, job.id) ?? runningRecord;
     const completedAt = nowIso();
-    writeJobFile(job.workspaceRoot, job.id, {
-      ...existing,
+    const failurePatch = {
       status: "failed",
       phase: "failed",
       errorMessage,
       pid: null,
-      completedAt,
+      completedAt
+    };
+    if (timedOut) {
+      failurePatch.timedOut = true;
+    }
+    writeJobFile(job.workspaceRoot, job.id, {
+      ...existing,
+      ...failurePatch,
       logFile: options.logFile ?? job.logFile ?? existing.logFile ?? null
     });
     upsertJob(job.workspaceRoot, {
       id: job.id,
-      status: "failed",
-      phase: "failed",
-      pid: null,
-      errorMessage,
-      completedAt
+      ...failurePatch
     });
     throw error;
   }