Skip to content

Commit f998c3f

Browse files
jrusso1020jrusso1020
committed
refactor(engine): type capture failures
1 parent fc3a9cb commit f998c3f

8 files changed

Lines changed: 362 additions & 113 deletions

File tree

‎packages/engine/src/index.ts‎

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -106,6 +106,13 @@ export {
106106
type BeforeCaptureHook,
107107
type DiscardWarmupInnerCapture,
108108
} from "./services/frameCapture.js";
109+
export {
110+
CaptureFailure,
111+
classifyCaptureFailure,
112+
isFatalCaptureFailure,
113+
type CaptureFailureKind,
114+
type CaptureWorkerDiagnostic,
115+
} from "./services/captureFailure.js";
109116

110117
// ── Screenshot (BeginFrame) ─────────────────────────────────────────────────────
111118
export {

‎packages/engine/src/services/captureFailure.test.ts‎

Lines changed: 68 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,68 @@
1+
import { describe, expect, it } from "vitest";
2+
import { CaptureFailure, classifyCaptureFailure, isFatalCaptureFailure } from "./captureFailure.js";
3+
4+
describe("classifyCaptureFailure", () => {
5+
it.each([
6+
["Target closed", "transient_browser"],
7+
["Runtime.callFunctionOn timed out after 30000ms", "protocol_timeout"],
8+
["Runtime.evaluate timed out", "protocol_timeout"],
9+
["Waiting failed: 30000ms exceeded", "protocol_timeout"],
10+
["JavaScript heap out of memory", "memory_exhaustion"],
11+
["drawElement self-verify failed", "verification"],
12+
["Composition has zero duration. Runtime ready: true", "authoring"],
13+
] as const)("classifies %s as %s", (message, kind) => {
14+
expect(classifyCaptureFailure(new Error(message)).kind).toBe(kind);
15+
});
16+
17+
it("lets the composed signal authoritatively classify cancellation", () => {
18+
const controller = new AbortController();
19+
controller.abort();
20+
expect(
21+
classifyCaptureFailure(new Error("Target closed"), { signal: controller.signal }).kind,
22+
).toBe("cancelled");
23+
});
24+
25+
it("lets a later cancellation override an already typed transient failure", () => {
26+
const controller = new AbortController();
27+
const transient = new CaptureFailure({
28+
kind: "transient_browser",
29+
message: "Target closed",
30+
workerDiagnostics: [
31+
{ workerId: 1, framesCaptured: 2, startFrame: 0, endFrame: 4, lines: ["Target closed"] },
32+
],
33+
});
34+
controller.abort();
35+
36+
const cancelled = classifyCaptureFailure(transient, { signal: controller.signal });
37+
38+
expect(cancelled.kind).toBe("cancelled");
39+
expect(cancelled.cause).toBe(transient);
40+
expect(cancelled.workerDiagnostics).toEqual(transient.workerDiagnostics);
41+
});
42+
43+
it("preserves cause and immutable worker diagnostics", () => {
44+
const cause = Object.assign(new Error("write failed"), { code: "ENOSPC" });
45+
const failure = classifyCaptureFailure(cause, {
46+
workerDiagnostics: [
47+
{ workerId: 2, framesCaptured: 3, startFrame: 0, endFrame: 10, lines: ["disk full"] },
48+
],
49+
});
50+
51+
expect(failure.kind).toBe("io");
52+
expect(failure.cause).toBe(cause);
53+
expect(failure.workerDiagnostics[0]?.workerId).toBe(2);
54+
expect(Object.isFrozen(failure.workerDiagnostics)).toBe(true);
55+
expect(Object.isFrozen(failure.workerDiagnostics[0]?.lines)).toBe(true);
56+
});
57+
58+
it("marks structural failures fatal but leaves retryable failures non-fatal", () => {
59+
expect(
60+
isFatalCaptureFailure(new CaptureFailure({ kind: "authoring", message: "bad source" })),
61+
).toBe(true);
62+
expect(
63+
isFatalCaptureFailure(
64+
new CaptureFailure({ kind: "protocol_timeout", message: "protocol timeout" }),
65+
),
66+
).toBe(false);
67+
});
68+
});

‎packages/engine/src/services/captureFailure.ts‎

Lines changed: 164 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,164 @@
1+
export type CaptureFailureKind =
2+
| "cancelled"
3+
| "transient_browser"
4+
| "protocol_timeout"
5+
| "memory_exhaustion"
6+
| "verification"
7+
| "authoring"
8+
| "io";
9+
10+
export interface CaptureWorkerDiagnostic {
11+
workerId: number;
12+
framesCaptured: number;
13+
startFrame: number;
14+
endFrame: number;
15+
lines: readonly string[];
16+
}
17+
18+
export class CaptureFailure extends Error {
19+
readonly kind: CaptureFailureKind;
20+
readonly cause: unknown;
21+
readonly workerDiagnostics: readonly CaptureWorkerDiagnostic[];
22+
23+
constructor(input: {
24+
kind: CaptureFailureKind;
25+
message: string;
26+
cause?: unknown;
27+
workerDiagnostics?: readonly CaptureWorkerDiagnostic[];
28+
}) {
29+
super(input.message);
30+
this.name = "CaptureFailure";
31+
this.kind = input.kind;
32+
this.cause = input.cause;
33+
this.workerDiagnostics = Object.freeze(
34+
(input.workerDiagnostics ?? []).map((diagnostic) =>
35+
Object.freeze({ ...diagnostic, lines: Object.freeze([...diagnostic.lines]) }),
36+
),
37+
);
38+
if (input.cause instanceof Error && input.cause.stack) this.stack = input.cause.stack;
39+
}
40+
}
41+
42+
const TRANSIENT_BROWSER_ERROR_PATTERNS = [
43+
/Navigating frame was detached/i,
44+
/Target closed/i,
45+
/Session closed/i,
46+
/browser has disconnected/i,
47+
/Page crashed/i,
48+
/Execution context was destroyed/i,
49+
/Cannot find context with specified id/i,
50+
/Failed to launch the browser process/i,
51+
/Navigation timeout of \d+ ms exceeded/i,
52+
/ECONNREFUSED/i,
53+
/Composition has zero duration[\s\S]*Runtime ready: false/,
54+
];
55+
56+
const PROTOCOL_TIMEOUT_PATTERNS = [
57+
/Runtime\.callFunctionOn timed out/i,
58+
/Runtime\.evaluate timed out/i,
59+
/HeadlessExperimental\.beginFrame timed out/i,
60+
/Protocol error[\s\S]*tim(?:ed|e) out/i,
61+
/Waiting failed:\s*\d+\s*ms exceeded/i,
62+
/Waiting failed[\s\S]*timeout/i,
63+
/timeout exceeded/i,
64+
];
65+
66+
const MEMORY_EXHAUSTION_ERROR_PATTERNS = [
67+
/Set maximum size exceeded/i,
68+
/Map maximum size exceeded/i,
69+
/Invalid (?:array|string) length/i,
70+
/Array buffer allocation failed/i,
71+
/Cannot create a string longer than/i,
72+
/Reached heap limit/i,
73+
/JavaScript heap out of memory/i,
74+
];
75+
76+
// Bun/JSC reports oversized allocations as the bare string "Out of memory".
77+
// Match only the complete message, or the complete worker segment produced by
78+
// parallel capture, so unrelated WebGL diagnostics are not misclassified.
79+
const BUN_MEMORY_EXHAUSTION_EXACT_MESSAGE = /^out of memory\.?$/i;
80+
const BUN_MEMORY_EXHAUSTION_WRAPPED_WORKER_MESSAGE = /\bworker \d+: out of memory\.?(?:;|$)/i;
81+
82+
const VERIFICATION_ERROR_PATTERNS = [
83+
/DrawElementVerificationError/i,
84+
/drawElement self-verify/i,
85+
/verification (?:failed|mismatch)/i,
86+
/blank drawElement frame/i,
87+
];
88+
89+
const AUTHORING_ERROR_PATTERNS = [
90+
/Composition has zero duration[\s\S]*Runtime ready: true/i,
91+
/data-duration/i,
92+
/No root \[data-composition-id\]/i,
93+
/failed to parse/i,
94+
/unparseable/i,
95+
];
96+
97+
function messageOf(error: unknown): string {
98+
return error instanceof Error ? error.message : String(error);
99+
}
100+
101+
function matchesAny(message: string, patterns: readonly RegExp[]): boolean {
102+
return patterns.some((pattern) => pattern.test(message));
103+
}
104+
105+
function ioError(error: unknown, message: string): boolean {
106+
const code = error instanceof Error ? (error as NodeJS.ErrnoException).code : undefined;
107+
return (
108+
Boolean(code && /^(?:EACCES|EEXIST|EIO|EMFILE|ENFILE|ENOENT|ENOSPC|EPERM|EROFS)$/.test(code)) ||
109+
/(?:read|write|rename|copy|open|file|directory).*(?:failed|error)/i.test(message)
110+
);
111+
}
112+
113+
export function classifyCaptureFailure(
114+
error: unknown,
115+
options: {
116+
signal?: AbortSignal;
117+
workerDiagnostics?: readonly CaptureWorkerDiagnostic[];
118+
} = {},
119+
): CaptureFailure {
120+
if (error instanceof CaptureFailure && !options.workerDiagnostics && !options.signal?.aborted) {
121+
return error;
122+
}
123+
const message = messageOf(error);
124+
let kind: CaptureFailureKind;
125+
if (options.signal?.aborted || /(?:render|capture)?_?cancelled|AbortError/i.test(message)) {
126+
kind = "cancelled";
127+
} else if (
128+
BUN_MEMORY_EXHAUSTION_EXACT_MESSAGE.test(message.trim()) ||
129+
BUN_MEMORY_EXHAUSTION_WRAPPED_WORKER_MESSAGE.test(message) ||
130+
matchesAny(message, MEMORY_EXHAUSTION_ERROR_PATTERNS)
131+
) {
132+
kind = "memory_exhaustion";
133+
} else if (matchesAny(message, VERIFICATION_ERROR_PATTERNS)) {
134+
kind = "verification";
135+
} else if (matchesAny(message, PROTOCOL_TIMEOUT_PATTERNS)) {
136+
kind = "protocol_timeout";
137+
} else if (matchesAny(message, TRANSIENT_BROWSER_ERROR_PATTERNS)) {
138+
kind = "transient_browser";
139+
} else if (matchesAny(message, AUTHORING_ERROR_PATTERNS)) {
140+
kind = "authoring";
141+
} else {
142+
kind = ioError(error, message) ? "io" : "authoring";
143+
}
144+
return new CaptureFailure({
145+
kind,
146+
message,
147+
cause: error,
148+
workerDiagnostics:
149+
options.workerDiagnostics ??
150+
(error instanceof CaptureFailure ? error.workerDiagnostics : undefined),
151+
});
152+
}
153+
154+
export function isTransientBrowserError(error: unknown): boolean {
155+
return classifyCaptureFailure(error).kind === "transient_browser";
156+
}
157+
158+
export function isMemoryExhaustionError(error: unknown): boolean {
159+
return classifyCaptureFailure(error).kind === "memory_exhaustion";
160+
}
161+
162+
export function isFatalCaptureFailure(failure: CaptureFailure): boolean {
163+
return !["cancelled", "transient_browser", "protocol_timeout"].includes(failure.kind);
164+
}

‎packages/engine/src/services/frameCapture.ts‎

Lines changed: 1 addition & 95 deletions
Original file line numberDiff line numberDiff line change
@@ -54,6 +54,7 @@ import type {
5454
CaptureWarning,
5555
SubTimelineWaitOutcome,
5656
} from "../types.js";
57+
export { isMemoryExhaustionError, isTransientBrowserError } from "./captureFailure.js";
5758

5859
export type { CaptureOptions, CaptureResult, CaptureBufferResult, CapturePerfSummary };
5960

@@ -3431,98 +3432,3 @@ export function getCapturePerfSummary(session: CaptureSession): CapturePerfSumma
34313432
deNcprFallbacks: session.deNcprFallbacks ?? 0,
34323433
};
34333434
}
3434-
3435-
// ── Transient browser error classification ─────────────────────────────────
3436-
// Puppeteer/Chrome can fail with transient errors that succeed on retry with a
3437-
// fresh browser session. These are infrastructure-level failures (frame
3438-
// detachment, connection drop, OOM kill, launch failure) — NOT composition bugs.
3439-
3440-
const TRANSIENT_BROWSER_ERROR_PATTERNS = [
3441-
/Navigating frame was detached/i,
3442-
/Target closed/i,
3443-
/Session closed/i,
3444-
/browser has disconnected/i,
3445-
/Page crashed/i,
3446-
/Execution context was destroyed/i,
3447-
/Cannot find context with specified id/i,
3448-
/Failed to launch the browser process/i,
3449-
/Navigation timeout of \d+ ms exceeded/i,
3450-
/ECONNREFUSED/i,
3451-
// pollHfReady's own timeout — thrown when window.__renderReady never flips
3452-
// true within playerReadyTimeout. "Runtime ready: false" means init simply
3453-
// didn't finish in time (commonly a slow/contended host, e.g. several
3454-
// concurrent renders), which a fresh session usually clears on retry. This
3455-
// is distinct from the "Runtime ready: true" fast-fail case a few lines up
3456-
// in pollHfReady (no timeline + no data-duration) — that's a genuine
3457-
// authoring bug and intentionally NOT matched here, so it still fails fast.
3458-
/Composition has zero duration[\s\S]*Runtime ready: false/,
3459-
];
3460-
3461-
export function isTransientBrowserError(error: unknown): boolean {
3462-
const message = error instanceof Error ? error.message : String(error);
3463-
return TRANSIENT_BROWSER_ERROR_PATTERNS.some((pattern) => pattern.test(message));
3464-
}
3465-
3466-
// ── Memory-exhaustion classification ────────────────────────────────────────
3467-
// A render can run the Node process (or a page-side allocation) out of memory
3468-
// on an oversized composition — huge canvas, thousands of frames, or a very
3469-
// large frame cache. These surface as cryptic V8 RangeErrors ("Set maximum
3470-
// size exceeded", "Invalid array length"/"string length", "Array buffer
3471-
// allocation failed") or a hard V8 heap-limit abort. They are NOT transient
3472-
// (a retry re-hits the same ceiling) and NOT composition-logic bugs — they're
3473-
// resource limits. Classify them so the caller can surface actionable guidance
3474-
// (lower resolution / fps / duration, or enable low-memory mode) instead of a
3475-
// raw RangeError.
3476-
3477-
// Deliberately specific: each pattern is a distinct V8/Node allocation-failure
3478-
// signature. We intentionally do NOT match a bare /out of memory/ — that
3479-
// substring appears in benign browser-console noise (WebGL `CONTEXT_LOST … out
3480-
// of memory`, GPU driver notes) that gets carried into the error path, and
3481-
// misclassifying it would replace the real failure message with generic OOM
3482-
// guidance.
3483-
const MEMORY_EXHAUSTION_ERROR_PATTERNS = [
3484-
/Set maximum size exceeded/i,
3485-
/Map maximum size exceeded/i,
3486-
/Invalid (?:array|string) length/i,
3487-
/Array buffer allocation failed/i,
3488-
/Cannot create a string longer than/i,
3489-
/Reached heap limit/i,
3490-
/JavaScript heap out of memory/i,
3491-
];
3492-
3493-
// The producer's deployed runtime is Bun (JavaScriptCore), not Node (V8) —
3494-
// see `packages/gcp-cloud-run/Dockerfile`'s `CMD ["bun", "dist/server.js"]`.
3495-
// JSC's own allocation-failure message for the equivalent single-oversized-
3496-
// allocation RangeErrors above is the bare string "Out of memory" (verified:
3497-
// `new Uint8Array(Number.MAX_SAFE_INTEGER)`, an unbounded `Set`, and
3498-
// `"x".repeat(2**53)` all throw exactly this under Bun) — none of the V8
3499-
// patterns above match it. This is exactly the substring the comment above
3500-
// says NOT to match anywhere in the message (benign browser-console noise
3501-
// like a WebGL `CONTEXT_LOST … out of memory` carries that phrase too), so
3502-
// this checks the ENTIRE (trimmed) message equals it, not merely contains
3503-
// it — a compound message with other text around the phrase still misses.
3504-
const BUN_MEMORY_EXHAUSTION_EXACT_MESSAGE = /^out of memory\.?$/i;
3505-
3506-
// The parallel-DE capture path — the exact cohort the OOM-aware retry in
3507-
// renderOrchestrator.ts targets — never reaches isMemoryExhaustionError with
3508-
// a bare message: `executeParallelCapture`/`formatWorkerFailure`
3509-
// (parallelCoordinator.ts) always wrap a worker's error as
3510-
// "Worker N: <message>", optionally suffixed "; diagnostics: ..." and joined
3511-
// with other failed workers' segments via "; ", all prefixed
3512-
// "[Parallel] Capture failed: ". The exact-match check above is defeated by
3513-
// that wrapping entirely (verified) — this pattern recovers the Bun OOM
3514-
// signal by requiring "out of memory" appear immediately after "Worker N: "
3515-
// and immediately before end-of-string, ";", or ".", i.e. as the WHOLE
3516-
// worker-segment content, not merely somewhere inside it. This preserves the
3517-
// exact-match property (no bare "out of memory" substring inside otherwise-
3518-
// unrelated worker text, e.g. "Worker 2: WebGL context lost, out of memory
3519-
// reported by driver" does NOT match) while surviving this codebase's own
3520-
// error-flattening.
3521-
const BUN_MEMORY_EXHAUSTION_WRAPPED_WORKER_MESSAGE = /\bworker \d+: out of memory\.?(?:;|$)/i;
3522-
3523-
export function isMemoryExhaustionError(error: unknown): boolean {
3524-
const message = error instanceof Error ? error.message : String(error);
3525-
if (BUN_MEMORY_EXHAUSTION_EXACT_MESSAGE.test(message.trim())) return true;
3526-
if (BUN_MEMORY_EXHAUSTION_WRAPPED_WORKER_MESSAGE.test(message)) return true;
3527-
return MEMORY_EXHAUSTION_ERROR_PATTERNS.some((pattern) => pattern.test(message));
3528-
}

0 commit comments

Comments
 (0)