Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
7 changes: 7 additions & 0 deletions packages/engine/src/index.ts
Original file line number Diff line number Diff line change
Expand Up @@ -106,6 +106,13 @@ export {
type BeforeCaptureHook,
type DiscardWarmupInnerCapture,
} from "./services/frameCapture.js";
export {
CaptureFailure,
classifyCaptureFailure,
isFatalCaptureFailure,
type CaptureFailureKind,
type CaptureWorkerDiagnostic,
} from "./services/captureFailure.js";

// ── Screenshot (BeginFrame) ─────────────────────────────────────────────────────
export {
Expand Down
68 changes: 68 additions & 0 deletions packages/engine/src/services/captureFailure.test.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,68 @@
import { describe, expect, it } from "vitest";
import { CaptureFailure, classifyCaptureFailure, isFatalCaptureFailure } from "./captureFailure.js";

describe("classifyCaptureFailure", () => {
it.each([
["Target closed", "transient_browser"],
["Runtime.callFunctionOn timed out after 30000ms", "protocol_timeout"],
["Runtime.evaluate timed out", "protocol_timeout"],
["Waiting failed: 30000ms exceeded", "protocol_timeout"],
["JavaScript heap out of memory", "memory_exhaustion"],
["drawElement self-verify failed", "verification"],
["Composition has zero duration. Runtime ready: true", "authoring"],
] as const)("classifies %s as %s", (message, kind) => {
expect(classifyCaptureFailure(new Error(message)).kind).toBe(kind);
});

it("lets the composed signal authoritatively classify cancellation", () => {
const controller = new AbortController();
controller.abort();
expect(
classifyCaptureFailure(new Error("Target closed"), { signal: controller.signal }).kind,
).toBe("cancelled");
});

it("lets a later cancellation override an already typed transient failure", () => {
const controller = new AbortController();
const transient = new CaptureFailure({
kind: "transient_browser",
message: "Target closed",
workerDiagnostics: [
{ workerId: 1, framesCaptured: 2, startFrame: 0, endFrame: 4, lines: ["Target closed"] },
],
});
controller.abort();

const cancelled = classifyCaptureFailure(transient, { signal: controller.signal });

expect(cancelled.kind).toBe("cancelled");
expect(cancelled.cause).toBe(transient);
expect(cancelled.workerDiagnostics).toEqual(transient.workerDiagnostics);
});

it("preserves cause and immutable worker diagnostics", () => {
const cause = Object.assign(new Error("write failed"), { code: "ENOSPC" });
const failure = classifyCaptureFailure(cause, {
workerDiagnostics: [
{ workerId: 2, framesCaptured: 3, startFrame: 0, endFrame: 10, lines: ["disk full"] },
],
});

expect(failure.kind).toBe("io");
expect(failure.cause).toBe(cause);
expect(failure.workerDiagnostics[0]?.workerId).toBe(2);
expect(Object.isFrozen(failure.workerDiagnostics)).toBe(true);
expect(Object.isFrozen(failure.workerDiagnostics[0]?.lines)).toBe(true);
});

it("marks structural failures fatal but leaves retryable failures non-fatal", () => {
expect(
isFatalCaptureFailure(new CaptureFailure({ kind: "authoring", message: "bad source" })),
).toBe(true);
expect(
isFatalCaptureFailure(
new CaptureFailure({ kind: "protocol_timeout", message: "protocol timeout" }),
),
).toBe(false);
});
});
164 changes: 164 additions & 0 deletions packages/engine/src/services/captureFailure.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,164 @@
export type CaptureFailureKind =
| "cancelled"
| "transient_browser"
| "protocol_timeout"
| "memory_exhaustion"
| "verification"
| "authoring"
| "io";

export interface CaptureWorkerDiagnostic {
workerId: number;
framesCaptured: number;
startFrame: number;
endFrame: number;
lines: readonly string[];
}

export class CaptureFailure extends Error {
readonly kind: CaptureFailureKind;
readonly cause: unknown;
readonly workerDiagnostics: readonly CaptureWorkerDiagnostic[];

constructor(input: {
kind: CaptureFailureKind;
message: string;
cause?: unknown;
workerDiagnostics?: readonly CaptureWorkerDiagnostic[];
}) {
super(input.message);
this.name = "CaptureFailure";
this.kind = input.kind;
this.cause = input.cause;
this.workerDiagnostics = Object.freeze(
(input.workerDiagnostics ?? []).map((diagnostic) =>
Object.freeze({ ...diagnostic, lines: Object.freeze([...diagnostic.lines]) }),
),
);
if (input.cause instanceof Error && input.cause.stack) this.stack = input.cause.stack;
}
}

const TRANSIENT_BROWSER_ERROR_PATTERNS = [
/Navigating frame was detached/i,
/Target closed/i,
/Session closed/i,
/browser has disconnected/i,
/Page crashed/i,
/Execution context was destroyed/i,
/Cannot find context with specified id/i,
/Failed to launch the browser process/i,
/Navigation timeout of \d+ ms exceeded/i,
/ECONNREFUSED/i,
/Composition has zero duration[\s\S]*Runtime ready: false/,
];

const PROTOCOL_TIMEOUT_PATTERNS = [
/Runtime\.callFunctionOn timed out/i,
/Runtime\.evaluate timed out/i,
/HeadlessExperimental\.beginFrame timed out/i,
/Protocol error[\s\S]*tim(?:ed|e) out/i,
/Waiting failed:\s*\d+\s*ms exceeded/i,
/Waiting failed[\s\S]*timeout/i,
/timeout exceeded/i,
];

const MEMORY_EXHAUSTION_ERROR_PATTERNS = [
/Set maximum size exceeded/i,
/Map maximum size exceeded/i,
/Invalid (?:array|string) length/i,
/Array buffer allocation failed/i,
/Cannot create a string longer than/i,
/Reached heap limit/i,
/JavaScript heap out of memory/i,
];

// Bun/JSC reports oversized allocations as the bare string "Out of memory".
// Match only the complete message, or the complete worker segment produced by
// parallel capture, so unrelated WebGL diagnostics are not misclassified.
const BUN_MEMORY_EXHAUSTION_EXACT_MESSAGE = /^out of memory\.?$/i;
const BUN_MEMORY_EXHAUSTION_WRAPPED_WORKER_MESSAGE = /\bworker \d+: out of memory\.?(?:;|$)/i;

const VERIFICATION_ERROR_PATTERNS = [
/DrawElementVerificationError/i,
/drawElement self-verify/i,
/verification (?:failed|mismatch)/i,
/blank drawElement frame/i,
];

const AUTHORING_ERROR_PATTERNS = [
/Composition has zero duration[\s\S]*Runtime ready: true/i,
/data-duration/i,
/No root \[data-composition-id\]/i,
/failed to parse/i,
/unparseable/i,
];

function messageOf(error: unknown): string {
return error instanceof Error ? error.message : String(error);
}

function matchesAny(message: string, patterns: readonly RegExp[]): boolean {
return patterns.some((pattern) => pattern.test(message));
}

function ioError(error: unknown, message: string): boolean {
const code = error instanceof Error ? (error as NodeJS.ErrnoException).code : undefined;
return (
Boolean(code && /^(?:EACCES|EEXIST|EIO|EMFILE|ENFILE|ENOENT|ENOSPC|EPERM|EROFS)$/.test(code)) ||
/(?:read|write|rename|copy|open|file|directory).*(?:failed|error)/i.test(message)
);
}

export function classifyCaptureFailure(
error: unknown,
options: {
signal?: AbortSignal;
workerDiagnostics?: readonly CaptureWorkerDiagnostic[];
} = {},
): CaptureFailure {
if (error instanceof CaptureFailure && !options.workerDiagnostics && !options.signal?.aborted) {
return error;
}
const message = messageOf(error);
let kind: CaptureFailureKind;
if (options.signal?.aborted || /(?:render|capture)?_?cancelled|AbortError/i.test(message)) {
kind = "cancelled";
} else if (
BUN_MEMORY_EXHAUSTION_EXACT_MESSAGE.test(message.trim()) ||
BUN_MEMORY_EXHAUSTION_WRAPPED_WORKER_MESSAGE.test(message) ||
matchesAny(message, MEMORY_EXHAUSTION_ERROR_PATTERNS)
) {
kind = "memory_exhaustion";
} else if (matchesAny(message, VERIFICATION_ERROR_PATTERNS)) {
kind = "verification";
} else if (matchesAny(message, PROTOCOL_TIMEOUT_PATTERNS)) {
kind = "protocol_timeout";
} else if (matchesAny(message, TRANSIENT_BROWSER_ERROR_PATTERNS)) {
kind = "transient_browser";
} else if (matchesAny(message, AUTHORING_ERROR_PATTERNS)) {
kind = "authoring";
} else {
kind = ioError(error, message) ? "io" : "authoring";
}
return new CaptureFailure({
kind,
message,
cause: error,
workerDiagnostics:
options.workerDiagnostics ??
(error instanceof CaptureFailure ? error.workerDiagnostics : undefined),
});
}

export function isTransientBrowserError(error: unknown): boolean {
return classifyCaptureFailure(error).kind === "transient_browser";
}

export function isMemoryExhaustionError(error: unknown): boolean {
return classifyCaptureFailure(error).kind === "memory_exhaustion";
}

export function isFatalCaptureFailure(failure: CaptureFailure): boolean {
return !["cancelled", "transient_browser", "protocol_timeout"].includes(failure.kind);
}
96 changes: 1 addition & 95 deletions packages/engine/src/services/frameCapture.ts
Original file line number Diff line number Diff line change
Expand Up @@ -54,6 +54,7 @@ import type {
CaptureWarning,
SubTimelineWaitOutcome,
} from "../types.js";
export { isMemoryExhaustionError, isTransientBrowserError } from "./captureFailure.js";

export type { CaptureOptions, CaptureResult, CaptureBufferResult, CapturePerfSummary };

Expand Down Expand Up @@ -3431,98 +3432,3 @@ export function getCapturePerfSummary(session: CaptureSession): CapturePerfSumma
deNcprFallbacks: session.deNcprFallbacks ?? 0,
};
}

// ── Transient browser error classification ─────────────────────────────────
// Puppeteer/Chrome can fail with transient errors that succeed on retry with a
// fresh browser session. These are infrastructure-level failures (frame
// detachment, connection drop, OOM kill, launch failure) — NOT composition bugs.

const TRANSIENT_BROWSER_ERROR_PATTERNS = [
/Navigating frame was detached/i,
/Target closed/i,
/Session closed/i,
/browser has disconnected/i,
/Page crashed/i,
/Execution context was destroyed/i,
/Cannot find context with specified id/i,
/Failed to launch the browser process/i,
/Navigation timeout of \d+ ms exceeded/i,
/ECONNREFUSED/i,
// pollHfReady's own timeout — thrown when window.__renderReady never flips
// true within playerReadyTimeout. "Runtime ready: false" means init simply
// didn't finish in time (commonly a slow/contended host, e.g. several
// concurrent renders), which a fresh session usually clears on retry. This
// is distinct from the "Runtime ready: true" fast-fail case a few lines up
// in pollHfReady (no timeline + no data-duration) — that's a genuine
// authoring bug and intentionally NOT matched here, so it still fails fast.
/Composition has zero duration[\s\S]*Runtime ready: false/,
];

export function isTransientBrowserError(error: unknown): boolean {
const message = error instanceof Error ? error.message : String(error);
return TRANSIENT_BROWSER_ERROR_PATTERNS.some((pattern) => pattern.test(message));
}

// ── Memory-exhaustion classification ────────────────────────────────────────
// A render can run the Node process (or a page-side allocation) out of memory
// on an oversized composition — huge canvas, thousands of frames, or a very
// large frame cache. These surface as cryptic V8 RangeErrors ("Set maximum
// size exceeded", "Invalid array length"/"string length", "Array buffer
// allocation failed") or a hard V8 heap-limit abort. They are NOT transient
// (a retry re-hits the same ceiling) and NOT composition-logic bugs — they're
// resource limits. Classify them so the caller can surface actionable guidance
// (lower resolution / fps / duration, or enable low-memory mode) instead of a
// raw RangeError.

// Deliberately specific: each pattern is a distinct V8/Node allocation-failure
// signature. We intentionally do NOT match a bare /out of memory/ — that
// substring appears in benign browser-console noise (WebGL `CONTEXT_LOST … out
// of memory`, GPU driver notes) that gets carried into the error path, and
// misclassifying it would replace the real failure message with generic OOM
// guidance.
const MEMORY_EXHAUSTION_ERROR_PATTERNS = [
/Set maximum size exceeded/i,
/Map maximum size exceeded/i,
/Invalid (?:array|string) length/i,
/Array buffer allocation failed/i,
/Cannot create a string longer than/i,
/Reached heap limit/i,
/JavaScript heap out of memory/i,
];

// The producer's deployed runtime is Bun (JavaScriptCore), not Node (V8) —
// see `packages/gcp-cloud-run/Dockerfile`'s `CMD ["bun", "dist/server.js"]`.
// JSC's own allocation-failure message for the equivalent single-oversized-
// allocation RangeErrors above is the bare string "Out of memory" (verified:
// `new Uint8Array(Number.MAX_SAFE_INTEGER)`, an unbounded `Set`, and
// `"x".repeat(2**53)` all throw exactly this under Bun) — none of the V8
// patterns above match it. This is exactly the substring the comment above
// says NOT to match anywhere in the message (benign browser-console noise
// like a WebGL `CONTEXT_LOST … out of memory` carries that phrase too), so
// this checks the ENTIRE (trimmed) message equals it, not merely contains
// it — a compound message with other text around the phrase still misses.
const BUN_MEMORY_EXHAUSTION_EXACT_MESSAGE = /^out of memory\.?$/i;

// The parallel-DE capture path — the exact cohort the OOM-aware retry in
// renderOrchestrator.ts targets — never reaches isMemoryExhaustionError with
// a bare message: `executeParallelCapture`/`formatWorkerFailure`
// (parallelCoordinator.ts) always wrap a worker's error as
// "Worker N: <message>", optionally suffixed "; diagnostics: ..." and joined
// with other failed workers' segments via "; ", all prefixed
// "[Parallel] Capture failed: ". The exact-match check above is defeated by
// that wrapping entirely (verified) — this pattern recovers the Bun OOM
// signal by requiring "out of memory" appear immediately after "Worker N: "
// and immediately before end-of-string, ";", or ".", i.e. as the WHOLE
// worker-segment content, not merely somewhere inside it. This preserves the
// exact-match property (no bare "out of memory" substring inside otherwise-
// unrelated worker text, e.g. "Worker 2: WebGL context lost, out of memory
// reported by driver" does NOT match) while surviving this codebase's own
// error-flattening.
const BUN_MEMORY_EXHAUSTION_WRAPPED_WORKER_MESSAGE = /\bworker \d+: out of memory\.?(?:;|$)/i;

export function isMemoryExhaustionError(error: unknown): boolean {
const message = error instanceof Error ? error.message : String(error);
if (BUN_MEMORY_EXHAUSTION_EXACT_MESSAGE.test(message.trim())) return true;
if (BUN_MEMORY_EXHAUSTION_WRAPPED_WORKER_MESSAGE.test(message)) return true;
return MEMORY_EXHAUSTION_ERROR_PATTERNS.some((pattern) => pattern.test(message));
}
Loading
Loading