refactor(engine): type capture failures

This commit is contained in:
James
2026-07-17 04:10:30 -04:00
parent c268f5ba85
commit acb3d81b99
8 changed files with 363 additions and 118 deletions
+7
View File
@@ -124,6 +124,13 @@ export {
type BeforeCaptureHook,
type DiscardWarmupInnerCapture,
} from "./services/frameCapture.js";
export {
CaptureFailure,
classifyCaptureFailure,
isFatalCaptureFailure,
type CaptureFailureKind,
type CaptureWorkerDiagnostic,
} from "./services/captureFailure.js";
// ── Screenshot (BeginFrame) ─────────────────────────────────────────────────────
export {
@@ -0,0 +1,68 @@
import { describe, expect, it } from "vitest";
import { CaptureFailure, classifyCaptureFailure, isFatalCaptureFailure } from "./captureFailure.js";
describe("classifyCaptureFailure", () => {
it.each([
["Target closed", "transient_browser"],
["Runtime.callFunctionOn timed out after 30000ms", "protocol_timeout"],
["Runtime.evaluate timed out", "protocol_timeout"],
["Waiting failed: 30000ms exceeded", "protocol_timeout"],
["JavaScript heap out of memory", "memory_exhaustion"],
["drawElement self-verify failed", "verification"],
["Composition has zero duration. Runtime ready: true", "authoring"],
] as const)("classifies %s as %s", (message, kind) => {
expect(classifyCaptureFailure(new Error(message)).kind).toBe(kind);
});
it("lets the composed signal authoritatively classify cancellation", () => {
const controller = new AbortController();
controller.abort();
expect(
classifyCaptureFailure(new Error("Target closed"), { signal: controller.signal }).kind,
).toBe("cancelled");
});
it("lets a later cancellation override an already typed transient failure", () => {
const controller = new AbortController();
const transient = new CaptureFailure({
kind: "transient_browser",
message: "Target closed",
workerDiagnostics: [
{ workerId: 1, framesCaptured: 2, startFrame: 0, endFrame: 4, lines: ["Target closed"] },
],
});
controller.abort();
const cancelled = classifyCaptureFailure(transient, { signal: controller.signal });
expect(cancelled.kind).toBe("cancelled");
expect(cancelled.cause).toBe(transient);
expect(cancelled.workerDiagnostics).toEqual(transient.workerDiagnostics);
});
it("preserves cause and immutable worker diagnostics", () => {
const cause = Object.assign(new Error("write failed"), { code: "ENOSPC" });
const failure = classifyCaptureFailure(cause, {
workerDiagnostics: [
{ workerId: 2, framesCaptured: 3, startFrame: 0, endFrame: 10, lines: ["disk full"] },
],
});
expect(failure.kind).toBe("io");
expect(failure.cause).toBe(cause);
expect(failure.workerDiagnostics[0]?.workerId).toBe(2);
expect(Object.isFrozen(failure.workerDiagnostics)).toBe(true);
expect(Object.isFrozen(failure.workerDiagnostics[0]?.lines)).toBe(true);
});
it("marks structural failures fatal but leaves retryable failures non-fatal", () => {
expect(
isFatalCaptureFailure(new CaptureFailure({ kind: "authoring", message: "bad source" })),
).toBe(true);
expect(
isFatalCaptureFailure(
new CaptureFailure({ kind: "protocol_timeout", message: "protocol timeout" }),
),
).toBe(false);
});
});
@@ -0,0 +1,165 @@
export type CaptureFailureKind =
| "cancelled"
| "transient_browser"
| "protocol_timeout"
| "memory_exhaustion"
| "verification"
| "authoring"
| "io";
export interface CaptureWorkerDiagnostic {
workerId: number;
framesCaptured: number;
startFrame: number;
endFrame: number;
lines: readonly string[];
}
export class CaptureFailure extends Error {
readonly kind: CaptureFailureKind;
readonly cause: unknown;
readonly workerDiagnostics: readonly CaptureWorkerDiagnostic[];
constructor(input: {
kind: CaptureFailureKind;
message: string;
cause?: unknown;
workerDiagnostics?: readonly CaptureWorkerDiagnostic[];
}) {
super(input.message);
this.name = "CaptureFailure";
this.kind = input.kind;
this.cause = input.cause;
this.workerDiagnostics = Object.freeze(
(input.workerDiagnostics ?? []).map((diagnostic) =>
Object.freeze({ ...diagnostic, lines: Object.freeze([...diagnostic.lines]) }),
),
);
if (input.cause instanceof Error && input.cause.stack) this.stack = input.cause.stack;
}
}
const TRANSIENT_BROWSER_ERROR_PATTERNS = [
/Navigating frame was detached/i,
/Target closed/i,
/Session closed/i,
/browser has disconnected/i,
/Page crashed/i,
/Execution context was destroyed/i,
/Cannot find context with specified id/i,
/Failed to launch the browser process/i,
/Navigation timeout of \d+ ms exceeded/i,
/ECONNREFUSED/i,
/net::ERR_NETWORK_CHANGED/i,
/Composition has zero duration[\s\S]*Runtime ready: false/,
];
const PROTOCOL_TIMEOUT_PATTERNS = [
/Runtime\.callFunctionOn timed out/i,
/Runtime\.evaluate timed out/i,
/HeadlessExperimental\.beginFrame timed out/i,
/Protocol error[\s\S]*tim(?:ed|e) out/i,
/Waiting failed:\s*\d+\s*ms exceeded/i,
/Waiting failed[\s\S]*timeout/i,
/timeout exceeded/i,
];
const MEMORY_EXHAUSTION_ERROR_PATTERNS = [
/Set maximum size exceeded/i,
/Map maximum size exceeded/i,
/Invalid (?:array|string) length/i,
/Array buffer allocation failed/i,
/Cannot create a string longer than/i,
/Reached heap limit/i,
/JavaScript heap out of memory/i,
];
// Bun/JSC reports oversized allocations as the bare string "Out of memory".
// Match only the complete message, or the complete worker segment produced by
// parallel capture, so unrelated WebGL diagnostics are not misclassified.
const BUN_MEMORY_EXHAUSTION_EXACT_MESSAGE = /^out of memory\.?$/i;
const BUN_MEMORY_EXHAUSTION_WRAPPED_WORKER_MESSAGE = /\bworker \d+: out of memory\.?(?:;|$)/i;
const VERIFICATION_ERROR_PATTERNS = [
/DrawElementVerificationError/i,
/drawElement self-verify/i,
/verification (?:failed|mismatch)/i,
/blank drawElement frame/i,
];
const AUTHORING_ERROR_PATTERNS = [
/Composition has zero duration[\s\S]*Runtime ready: true/i,
/data-duration/i,
/No root \[data-composition-id\]/i,
/failed to parse/i,
/unparseable/i,
];
function messageOf(error: unknown): string {
return error instanceof Error ? error.message : String(error);
}
function matchesAny(message: string, patterns: readonly RegExp[]): boolean {
return patterns.some((pattern) => pattern.test(message));
}
function ioError(error: unknown, message: string): boolean {
const code = error instanceof Error ? (error as NodeJS.ErrnoException).code : undefined;
return (
Boolean(code && /^(?:EACCES|EEXIST|EIO|EMFILE|ENFILE|ENOENT|ENOSPC|EPERM|EROFS)$/.test(code)) ||
/(?:read|write|rename|copy|open|file|directory).*(?:failed|error)/i.test(message)
);
}
export function classifyCaptureFailure(
error: unknown,
options: {
signal?: AbortSignal;
workerDiagnostics?: readonly CaptureWorkerDiagnostic[];
} = {},
): CaptureFailure {
if (error instanceof CaptureFailure && !options.workerDiagnostics && !options.signal?.aborted) {
return error;
}
const message = messageOf(error);
let kind: CaptureFailureKind;
if (options.signal?.aborted || /(?:render|capture)?_?cancelled|AbortError/i.test(message)) {
kind = "cancelled";
} else if (
BUN_MEMORY_EXHAUSTION_EXACT_MESSAGE.test(message.trim()) ||
BUN_MEMORY_EXHAUSTION_WRAPPED_WORKER_MESSAGE.test(message) ||
matchesAny(message, MEMORY_EXHAUSTION_ERROR_PATTERNS)
) {
kind = "memory_exhaustion";
} else if (matchesAny(message, VERIFICATION_ERROR_PATTERNS)) {
kind = "verification";
} else if (matchesAny(message, PROTOCOL_TIMEOUT_PATTERNS)) {
kind = "protocol_timeout";
} else if (matchesAny(message, TRANSIENT_BROWSER_ERROR_PATTERNS)) {
kind = "transient_browser";
} else if (matchesAny(message, AUTHORING_ERROR_PATTERNS)) {
kind = "authoring";
} else {
kind = ioError(error, message) ? "io" : "authoring";
}
return new CaptureFailure({
kind,
message,
cause: error,
workerDiagnostics:
options.workerDiagnostics ??
(error instanceof CaptureFailure ? error.workerDiagnostics : undefined),
});
}
export function isTransientBrowserError(error: unknown): boolean {
return classifyCaptureFailure(error).kind === "transient_browser";
}
export function isMemoryExhaustionError(error: unknown): boolean {
return classifyCaptureFailure(error).kind === "memory_exhaustion";
}
export function isFatalCaptureFailure(failure: CaptureFailure): boolean {
return !["cancelled", "transient_browser", "protocol_timeout"].includes(failure.kind);
}
+1 -100
View File
@@ -54,6 +54,7 @@ import type {
CaptureWarning,
SubTimelineWaitOutcome,
} from "../types.js";
export { isMemoryExhaustionError, isTransientBrowserError } from "./captureFailure.js";
export type { CaptureOptions, CaptureResult, CaptureBufferResult, CapturePerfSummary };
@@ -3698,103 +3699,3 @@ export function getCapturePerfSummary(session: CaptureSession): CapturePerfSumma
deNcprFallbacks: session.deNcprFallbacks ?? 0,
};
}
// ── Transient browser error classification ─────────────────────────────────
// Puppeteer/Chrome can fail with transient errors that succeed on retry with a
// fresh browser session. These are infrastructure-level failures (frame
// detachment, connection drop, OOM kill, launch failure) — NOT composition bugs.
const TRANSIENT_BROWSER_ERROR_PATTERNS = [
/Navigating frame was detached/i,
/Target closed/i,
/Session closed/i,
/browser has disconnected/i,
/Page crashed/i,
/Execution context was destroyed/i,
/Cannot find context with specified id/i,
/Failed to launch the browser process/i,
/Navigation timeout of \d+ ms exceeded/i,
/ECONNREFUSED/i,
// Chromium can briefly invalidate even a localhost connection when Windows
// reports an adapter/route change. A fresh capture session succeeds once the
// network stack settles, so treat this like the other bounded navigation
// retries instead of failing the render immediately.
/net::ERR_NETWORK_CHANGED/i,
// pollHfReady's own timeout — thrown when window.__renderReady never flips
// true within playerReadyTimeout. "Runtime ready: false" means init simply
// didn't finish in time (commonly a slow/contended host, e.g. several
// concurrent renders), which a fresh session usually clears on retry. This
// is distinct from the "Runtime ready: true" fast-fail case a few lines up
// in pollHfReady (no timeline + no data-duration) — that's a genuine
// authoring bug and intentionally NOT matched here, so it still fails fast.
/Composition has zero duration[\s\S]*Runtime ready: false/,
];
export function isTransientBrowserError(error: unknown): boolean {
const message = error instanceof Error ? error.message : String(error);
return TRANSIENT_BROWSER_ERROR_PATTERNS.some((pattern) => pattern.test(message));
}
// ── Memory-exhaustion classification ────────────────────────────────────────
// A render can run the Node process (or a page-side allocation) out of memory
// on an oversized composition — huge canvas, thousands of frames, or a very
// large frame cache. These surface as cryptic V8 RangeErrors ("Set maximum
// size exceeded", "Invalid array length"/"string length", "Array buffer
// allocation failed") or a hard V8 heap-limit abort. They are NOT transient
// (a retry re-hits the same ceiling) and NOT composition-logic bugs — they're
// resource limits. Classify them so the caller can surface actionable guidance
// (lower resolution / fps / duration, or enable low-memory mode) instead of a
// raw RangeError.
// Deliberately specific: each pattern is a distinct V8/Node allocation-failure
// signature. We intentionally do NOT match a bare /out of memory/ — that
// substring appears in benign browser-console noise (WebGL `CONTEXT_LOST … out
// of memory`, GPU driver notes) that gets carried into the error path, and
// misclassifying it would replace the real failure message with generic OOM
// guidance.
const MEMORY_EXHAUSTION_ERROR_PATTERNS = [
/Set maximum size exceeded/i,
/Map maximum size exceeded/i,
/Invalid (?:array|string) length/i,
/Array buffer allocation failed/i,
/Cannot create a string longer than/i,
/Reached heap limit/i,
/JavaScript heap out of memory/i,
];
// The producer's deployed runtime is Bun (JavaScriptCore), not Node (V8) —
// see `packages/gcp-cloud-run/Dockerfile`'s `CMD ["bun", "dist/server.js"]`.
// JSC's own allocation-failure message for the equivalent single-oversized-
// allocation RangeErrors above is the bare string "Out of memory" (verified:
// `new Uint8Array(Number.MAX_SAFE_INTEGER)`, an unbounded `Set`, and
// `"x".repeat(2**53)` all throw exactly this under Bun) — none of the V8
// patterns above match it. This is exactly the substring the comment above
// says NOT to match anywhere in the message (benign browser-console noise
// like a WebGL `CONTEXT_LOST … out of memory` carries that phrase too), so
// this checks the ENTIRE (trimmed) message equals it, not merely contains
// it — a compound message with other text around the phrase still misses.
const BUN_MEMORY_EXHAUSTION_EXACT_MESSAGE = /^out of memory\.?$/i;
// The parallel-DE capture path — the exact cohort the OOM-aware retry in
// renderOrchestrator.ts targets — never reaches isMemoryExhaustionError with
// a bare message: `executeParallelCapture`/`formatWorkerFailure`
// (parallelCoordinator.ts) always wrap a worker's error as
// "Worker N: <message>", optionally suffixed "; diagnostics: ..." and joined
// with other failed workers' segments via "; ", all prefixed
// "[Parallel] Capture failed: ". The exact-match check above is defeated by
// that wrapping entirely (verified) — this pattern recovers the Bun OOM
// signal by requiring "out of memory" appear immediately after "Worker N: "
// and immediately before end-of-string, ";", or ".", i.e. as the WHOLE
// worker-segment content, not merely somewhere inside it. This preserves the
// exact-match property (no bare "out of memory" substring inside otherwise-
// unrelated worker text, e.g. "Worker 2: WebGL context lost, out of memory
// reported by driver" does NOT match) while surviving this codebase's own
// error-flattening.
const BUN_MEMORY_EXHAUSTION_WRAPPED_WORKER_MESSAGE = /\bworker \d+: out of memory\.?(?:;|$)/i;
export function isMemoryExhaustionError(error: unknown): boolean {
const message = error instanceof Error ? error.message : String(error);
if (BUN_MEMORY_EXHAUSTION_EXACT_MESSAGE.test(message.trim())) return true;
if (BUN_MEMORY_EXHAUSTION_WRAPPED_WORKER_MESSAGE.test(message)) return true;
return MEMORY_EXHAUSTION_ERROR_PATTERNS.some((pattern) => pattern.test(message));
}
@@ -0,0 +1,63 @@
import { mkdtempSync, rmSync } from "node:fs";
import { tmpdir } from "node:os";
import { join } from "node:path";
import { afterEach, describe, expect, it, vi } from "vitest";
import type { CaptureSession } from "./frameCapture.js";
describe("executeParallelCapture peer abort", () => {
afterEach(() => {
vi.resetModules();
vi.doUnmock("./frameCapture.js");
});
it("aborts peer workers on the first fatal classified failure", async () => {
const root = mkdtempSync(join(tmpdir(), "hf-peer-abort-"));
const captureFrame = vi.fn().mockResolvedValue(undefined);
const closeCaptureSession = vi.fn().mockResolvedValue(undefined);
vi.doMock("./frameCapture.js", async (importOriginal) => {
const original = await importOriginal<typeof import("./frameCapture.js")>();
return {
...original,
createCaptureSession: vi.fn(async (_url: string, outputDir: string) => {
const workerId = outputDir.endsWith("worker-0") ? 0 : 1;
return {
workerId,
browserConsoleBuffer: workerId === 0 ? ["[Browser:ERROR] bad source"] : [],
} as unknown as CaptureSession & { workerId: number };
}),
initializeSession: vi.fn(async (session: CaptureSession & { workerId: number }) => {
if (session.workerId === 0) {
throw new Error("Composition has zero duration. Runtime ready: true");
}
await Promise.resolve();
}),
captureFrame,
closeCaptureSession,
getCapturePerfSummary: vi.fn(() => ({ frames: 0 })),
};
});
try {
const { executeParallelCapture } = await import("./parallelCoordinator.js");
const result = executeParallelCapture(
"http://127.0.0.1",
root,
[
{ workerId: 0, startFrame: 0, endFrame: 1, outputDir: join(root, "worker-0") },
{ workerId: 1, startFrame: 1, endFrame: 2, outputDir: join(root, "worker-1") },
],
{ width: 320, height: 180, fps: { num: 30, den: 1 } },
() => null,
);
await expect(result).rejects.toMatchObject({
name: "CaptureFailure",
kind: "authoring",
});
expect(captureFrame).not.toHaveBeenCalled();
expect(closeCaptureSession).toHaveBeenCalledTimes(2);
} finally {
rmSync(root, { recursive: true, force: true });
}
});
});
@@ -28,6 +28,12 @@ import { assertSwiftShader } from "../utils/assertSwiftShader.js";
import { readWebGlVendorInfoFromCanvas } from "../utils/readWebGlVendorInfoFromCanvas.js";
import { resolveHeadlessShellPath } from "./browserManager.js";
import { getSystemTotalMb } from "./systemMemory.js";
import {
CaptureFailure,
classifyCaptureFailure,
isFatalCaptureFailure,
type CaptureWorkerDiagnostic,
} from "./captureFailure.js";
export interface WorkerTask {
workerId: number;
@@ -61,6 +67,7 @@ export interface WorkerResult {
perf?: CapturePerfSummary;
error?: string;
diagnostics?: string[];
failure?: CaptureFailure;
}
export interface ParallelProgress {
@@ -423,6 +430,7 @@ async function executeWorkerTask(
onFrameBuffer?: (frameIndex: number, buffer: Buffer, session: CaptureSession) => Promise<void>,
config?: Partial<EngineConfig>,
parallel?: boolean,
onFailure?: (failure: CaptureFailure) => void,
): Promise<WorkerResult> {
const startTime = Date.now();
let framesCaptured = 0;
@@ -483,8 +491,19 @@ async function executeWorkerTask(
perf,
};
} catch (error) {
const errMsg = error instanceof Error ? error.message : String(error);
const diagnostics = session ? selectWorkerDiagnostics(session.browserConsoleBuffer) : [];
const workerDiagnostic: CaptureWorkerDiagnostic = {
workerId: task.workerId,
framesCaptured,
startFrame: task.startFrame,
endFrame: task.endFrame,
lines: diagnostics,
};
const failure = classifyCaptureFailure(error, {
signal,
workerDiagnostics: [workerDiagnostic],
});
onFailure?.(failure);
return {
workerId: task.workerId,
framesCaptured,
@@ -492,8 +511,9 @@ async function executeWorkerTask(
endFrame: task.endFrame,
durationMs: Date.now() - startTime,
perf,
error: errMsg,
error: failure.message,
diagnostics: diagnostics.length > 0 ? diagnostics : undefined,
failure,
};
} finally {
if (session) await closeCaptureSession(session).catch(() => {});
@@ -566,6 +586,16 @@ export async function executeParallelCapture(
deVerifySamples === captureOptions.deVerifySamples
? captureOptions
: { ...captureOptions, deVerifySamples };
const peerController = new AbortController();
const workerSignal = signal
? AbortSignal.any([signal, peerController.signal])
: peerController.signal;
let firstFatalFailure: CaptureFailure | undefined;
const onFailure = (failure: CaptureFailure): void => {
if (firstFatalFailure || !isFatalCaptureFailure(failure)) return;
firstFatalFailure = failure;
peerController.abort(failure);
};
const results = await Promise.all(
tasks.map((task) =>
executeWorkerTask(
@@ -573,11 +603,12 @@ export async function executeParallelCapture(
serverUrl,
workerCaptureOptions,
createBeforeCaptureHook,
signal,
workerSignal,
onFrameCaptured,
onFrameBuffer,
config,
parallel,
onFailure,
),
),
);
@@ -593,10 +624,17 @@ export async function executeParallelCapture(
}
}
const errors = results.filter((r) => r.error);
const errors = results.filter((r) => r.failure || r.error);
if (errors.length > 0) {
const errorMessages = errors.map(formatWorkerFailure).join("; ");
throw new Error(`[Parallel] Capture failed: ${errorMessages}`);
const representative = firstFatalFailure ?? errors.find((result) => result.failure)?.failure;
const workerDiagnostics = errors.flatMap((result) => result.failure?.workerDiagnostics ?? []);
throw new CaptureFailure({
kind: representative?.kind ?? "io",
message: `[Parallel] Capture failed: ${errorMessages}`,
cause: representative,
workerDiagnostics,
});
}
return results;
@@ -1,13 +1,18 @@
import { normalizeErrorMessage } from "../../utils/errorMessage.js";
import { CaptureFailure, classifyCaptureFailure } from "@hyperframes/engine";
export class CaptureStageError extends Error {
export class CaptureStageError extends CaptureFailure {
readonly browserConsole: string[];
readonly cause: unknown;
constructor(input: { cause: unknown; browserConsole: string[] }) {
super(normalizeErrorMessage(input.cause));
const classified = classifyCaptureFailure(input.cause);
super({
kind: classified.kind,
message: normalizeErrorMessage(input.cause),
cause: input.cause,
workerDiagnostics: classified.workerDiagnostics,
});
this.name = "CaptureStageError";
this.cause = input.cause;
this.browserConsole = input.browserConsole.slice();
if (input.cause instanceof Error && input.cause.stack) {
this.stack = input.cause.stack;
@@ -80,8 +80,8 @@ import {
resolveHeadlessShellPath,
applyConcreteGpuScreenshotClamp,
scaleProtocolTimeoutForComposition,
classifyCaptureFailure,
isMemoryExhaustionError,
isTransientBrowserError,
isDrawElementVerificationError,
getDrawElementVerificationDetails,
augmentProtocolTimeoutError,
@@ -816,12 +816,9 @@ export function resetCaptureAttemptProgress(job: { framesRendered?: number }): v
export function isRecoverableParallelCaptureError(error: unknown): boolean {
const message = normalizeErrorMessage(error);
return (
message.includes("[Parallel] Capture failed") &&
/Runtime\.callFunctionOn timed out|HeadlessExperimental\.beginFrame timed out|Waiting failed|timeout exceeded|timed out|Navigation timeout|Protocol error|Target closed/i.test(
message,
)
);
if (!message.includes("[Parallel] Capture failed")) return false;
const kind = classifyCaptureFailure(error).kind;
return kind === "transient_browser" || kind === "protocol_timeout";
}
/**
@@ -995,11 +992,12 @@ export async function executeDiskCaptureWithAdaptiveRetry(options: {
missingRanges = remaining;
attempt++;
} catch (error) {
const failure = classifyCaptureFailure(error, { signal: options.abortSignal });
// A cancelled render tears the browser down, which surfaces as a
// transient-looking `Target closed`. Rethrow immediately so cancellation
// never burns a retry (or logs a misleading transient-failure warning) —
// the caller's abort handling owns cancellation.
if (options.abortSignal?.aborted) {
if (failure.kind === "cancelled") {
throw error;
}
const remaining = findMissingFrameRanges(
@@ -1028,7 +1026,7 @@ export async function executeDiskCaptureWithAdaptiveRetry(options: {
// retry for the session-init phase they share.
if (
options.allowRetry &&
isTransientBrowserError(error) &&
failure.kind === "transient_browser" &&
transientRetriesUsed < MAX_TRANSIENT_CAPTURE_RETRIES
) {
transientRetriesUsed++;