fix(producer): harden capture against timeouts, transient tab deaths, and OOM (#1842)

Four independent capture-infra hardening changes for the P2-5 failure bucket (~15K err / ~7K users):

- protocolTimeout auto-scales by device-scaled output area (applied before probe launch, since it's immutable post ppt.launch()).
- Single bounded transient retry (MAX_TRANSIENT_CAPTURE_RETRIES=1) on Target closed / Page crashed in the parallel disk-capture path; abort short-circuits before retry.
- Narrow OOM classification (Set maximum size exceeded etc., disjoint from transient) → actionable guidance naming output dims.
- StreamingEncoder.getExitError() threads FFmpeg's real exit reason into frame-0 encoder-death errors.

Render-reliability workstream P2-5. Success measured on PostHog dashboard 1783183.

Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
This commit is contained in:
James Russo
2026-07-01 18:50:16 -07:00
committed by GitHub
co-authored by Claude Opus 4.8
parent 180f368af1
commit c0c3abf0f1
14 changed files with 584 additions and 14 deletions
+41 -1
View File
@@ -1,5 +1,5 @@
import { describe, it, expect, beforeEach, afterEach } from "vitest";
import { resolveConfig, DEFAULT_CONFIG } from "./config.js";
import { resolveConfig, DEFAULT_CONFIG, scaleProtocolTimeoutForComposition } from "./config.js";
import { isLowMemorySystem } from "./services/systemMemory.js";
describe("resolveConfig", () => {
@@ -211,3 +211,43 @@ describe("resolveConfig", () => {
});
});
});
describe("scaleProtocolTimeoutForComposition", () => {
const base = 300_000;
it("keeps the base timeout for a reference-or-smaller canvas", () => {
// 1080p == reference area → factor 1, no scale.
expect(scaleProtocolTimeoutForComposition(base, { width: 1920, height: 1080 })).toBe(base);
// Smaller than reference → still the base (never scales down).
expect(scaleProtocolTimeoutForComposition(base, { width: 1280, height: 720 })).toBe(base);
});
it("scales up proportionally with output pixel area", () => {
// 4K == 4× the reference area, which stays under the 30-minute ceiling.
const scaled = scaleProtocolTimeoutForComposition(base, { width: 3840, height: 2160 });
expect(scaled).toBeGreaterThan(base);
expect(scaled).toBe(base * 4);
});
it("clamps at the 30-minute ceiling for a pathological canvas", () => {
// 8K == 16× area → 4.8M ms, clamped to the 30-minute ceiling.
const scaled = scaleProtocolTimeoutForComposition(base, { width: 7680, height: 4320 });
expect(scaled).toBe(1_800_000);
});
it("never lowers a base timeout that already exceeds the ceiling", () => {
// Base above the 30-min ceiling + a large canvas: must not clamp below base.
const highBase = 2_400_000;
expect(
scaleProtocolTimeoutForComposition(highBase, { width: 3840, height: 2160 }),
).toBeGreaterThanOrEqual(highBase);
});
it("returns the base timeout for degenerate dimensions", () => {
expect(scaleProtocolTimeoutForComposition(base, { width: 0, height: 1080 })).toBe(base);
expect(scaleProtocolTimeoutForComposition(base, { width: 1920, height: 0 })).toBe(base);
expect(scaleProtocolTimeoutForComposition(base, { width: Number.NaN, height: 1080 })).toBe(
base,
);
});
});
+47
View File
@@ -252,6 +252,53 @@ export const DEFAULT_CONFIG: EngineConfig = {
debug: false,
};
/**
* Reference canvas area for the baseline `protocolTimeout`: 1080p. A single CDP
* call (`Runtime.callFunctionOn` seek+paint, or `Page.captureScreenshot`)
* scales with the *output pixel area* it has to render/serialize — NOT with the
* frame count (that governs total wall-clock, capped separately by the ffmpeg
* streaming inactivity timeout). A fixed 300s ceiling intermittently kills
* legitimate slow-but-valid renders on large canvases with
* `Runtime.callFunctionOn timed out`, so we scale the per-call ceiling with
* area.
*/
const PROTOCOL_TIMEOUT_REFERENCE_PIXELS = 1920 * 1080;
/**
* Absolute ceiling on the scaled protocol timeout (30 minutes). Bounds the
* blast radius: a genuinely wedged CDP call must still eventually fail rather
* than hang for an unbounded time on a pathologically large composition.
*/
const MAX_SCALED_PROTOCOL_TIMEOUT_MS = 1_800_000;
/**
* Scale a base `protocolTimeout` up for oversized compositions.
*
* Scales by output pixel area (`width*height / reference`) — where width/height
* are the *device-scaled output* dimensions (the pixels a single CDP call
* actually renders/serializes), not the CSS composition size. Clamped to
* `[baseTimeout, max(baseTimeout, MAX_SCALED_PROTOCOL_TIMEOUT_MS)]`: never
* scales DOWN (a small composition — or a base already above the ceiling —
* keeps the configured base), and only ever raises. Pure function; exported
* for tests.
*/
export function scaleProtocolTimeoutForComposition(
baseTimeoutMs: number,
dims: { width: number; height: number },
): number {
const { width, height } = dims;
if (!Number.isFinite(width) || !Number.isFinite(height) || width <= 0 || height <= 0) {
return baseTimeoutMs;
}
const factor = (width * height) / PROTOCOL_TIMEOUT_REFERENCE_PIXELS;
if (factor <= 1) return baseTimeoutMs;
const scaled = Math.ceil(baseTimeoutMs * factor);
// Ceiling is `max(base, MAX)` so an explicit base above the ceiling is never
// lowered (preserves the "only ever raise" contract for all callers).
const ceiling = Math.max(baseTimeoutMs, MAX_SCALED_PROTOCOL_TIMEOUT_MS);
return Math.min(ceiling, Math.max(baseTimeoutMs, scaled));
}
function memoryAdaptiveCacheLimit(): number {
const total = getSystemTotalMb();
if (total < 4096) return 32;
+7 -1
View File
@@ -43,7 +43,12 @@ export type {
} from "./types.js";
// ── Configuration ──────────────────────────────────────────────────────────────
export { resolveConfig, DEFAULT_CONFIG, type EngineConfig } from "./config.js";
export {
resolveConfig,
DEFAULT_CONFIG,
scaleProtocolTimeoutForComposition,
type EngineConfig,
} from "./config.js";
export {
DEFAULT_VP9_CPU_USED,
MAX_VP9_CPU_USED,
@@ -83,6 +88,7 @@ export {
prepareCaptureSessionForReuse,
type CaptureSession,
isTransientBrowserError,
isMemoryExhaustionError,
type BeforeCaptureHook,
type DiscardWarmupInnerCapture,
} from "./services/frameCapture.js";
@@ -1,5 +1,5 @@
import { describe, expect, it } from "vitest";
import { isTransientBrowserError } from "./frameCapture.js";
import { isMemoryExhaustionError, isTransientBrowserError } from "./frameCapture.js";
describe("isTransientBrowserError", () => {
it.each([
@@ -45,3 +45,47 @@ describe("isTransientBrowserError", () => {
expect(isTransientBrowserError(42)).toBe(false);
});
});
describe("isMemoryExhaustionError", () => {
it.each([
"Set maximum size exceeded",
"Map maximum size exceeded",
"Invalid array length",
"Invalid string length",
"Array buffer allocation failed",
"Cannot create a string longer than 0x1fffffe8 characters",
"FATAL ERROR: Reached heap limit Allocation failed - JavaScript heap out of memory",
"JavaScript heap out of memory",
])("returns true for memory-exhaustion error: %s", (message) => {
expect(isMemoryExhaustionError(new Error(message))).toBe(true);
});
it.each([
"Target closed",
"Runtime.callFunctionOn timed out",
"net::ERR_NAME_NOT_RESOLVED",
"Composition duration is 0",
"",
// Deliberately NOT matched — a bare "out of memory" substring appears in
// benign WebGL/GPU console noise; only the specific V8/Node allocation
// signatures (and "JavaScript heap out of memory") count.
"WebGL: CONTEXT_LOST_WEBGL loseContext: context out of memory",
"GL_OUT_OF_MEMORY: out of memory",
])("returns false for non-memory error: %s", (message) => {
expect(isMemoryExhaustionError(new Error(message))).toBe(false);
});
it("handles non-Error values", () => {
expect(isMemoryExhaustionError("Set maximum size exceeded")).toBe(true);
expect(isMemoryExhaustionError("some other string")).toBe(false);
expect(isMemoryExhaustionError(null)).toBe(false);
expect(isMemoryExhaustionError(undefined)).toBe(false);
});
// A memory-exhaustion error is a resource ceiling, not a flaky-tab hiccup —
// it must NOT be classified as transient (a retry re-hits the same wall).
it("is disjoint from transient classification", () => {
expect(isTransientBrowserError(new Error("Set maximum size exceeded"))).toBe(false);
expect(isMemoryExhaustionError(new Error("Target closed"))).toBe(false);
});
});
@@ -2006,3 +2006,35 @@ export function isTransientBrowserError(error: unknown): boolean {
const message = error instanceof Error ? error.message : String(error);
return TRANSIENT_BROWSER_ERROR_PATTERNS.some((pattern) => pattern.test(message));
}
// ── Memory-exhaustion classification ────────────────────────────────────────
// A render can run the Node process (or a page-side allocation) out of memory
// on an oversized composition — huge canvas, thousands of frames, or a very
// large frame cache. These surface as cryptic V8 RangeErrors ("Set maximum
// size exceeded", "Invalid array length"/"string length", "Array buffer
// allocation failed") or a hard V8 heap-limit abort. They are NOT transient
// (a retry re-hits the same ceiling) and NOT composition-logic bugs — they're
// resource limits. Classify them so the caller can surface actionable guidance
// (lower resolution / fps / duration, or enable low-memory mode) instead of a
// raw RangeError.
// Deliberately specific: each pattern is a distinct V8/Node allocation-failure
// signature. We intentionally do NOT match a bare /out of memory/ — that
// substring appears in benign browser-console noise (WebGL `CONTEXT_LOST … out
// of memory`, GPU driver notes) that gets carried into the error path, and
// misclassifying it would replace the real failure message with generic OOM
// guidance.
const MEMORY_EXHAUSTION_ERROR_PATTERNS = [
/Set maximum size exceeded/i,
/Map maximum size exceeded/i,
/Invalid (?:array|string) length/i,
/Array buffer allocation failed/i,
/Cannot create a string longer than/i,
/Reached heap limit/i,
/JavaScript heap out of memory/i,
];
export function isMemoryExhaustionError(error: unknown): boolean {
const message = error instanceof Error ? error.message : String(error);
return MEMORY_EXHAUSTION_ERROR_PATTERNS.some((pattern) => pattern.test(message));
}
@@ -526,6 +526,55 @@ describe("spawnStreamingEncoder lifecycle and cleanup", () => {
expect(result.error).toContain("Encoder error");
});
it("getExitError surfaces the ffmpeg failure reason after a non-zero exit", async () => {
const { spawn, calls } = createSpawnSpy();
vi.resetModules();
vi.doMock("child_process", () => ({ spawn }));
const { spawnStreamingEncoder } = await import("./streamingEncoder.js");
const dir = mkdtempSync(join(tmpdir(), "se-exiterr-"));
const encoder = await spawnStreamingEncoder(join(dir, "out.mp4"), baseOptions);
const proc = calls[0]!.proc;
// While running, there is no exit error to report.
expect(encoder.getExitError()).toBeUndefined();
proc.stderr.emit("data", Buffer.from("Unknown encoder 'libx264'\n"));
await new Promise<void>((resolve) => {
process.nextTick(() => {
proc.emit("close", 1);
resolve();
});
});
// After a non-zero exit, the reason is available synchronously — this is
// what `ensureFrameWritten` reads to turn "encoder exited before frame 0"
// into an actionable message.
const exitError = encoder.getExitError();
expect(exitError).toContain("FFmpeg exited with code 1");
expect(exitError).toContain("Unknown encoder 'libx264'");
});
it("getExitError returns undefined after a clean exit", async () => {
const { spawn, calls } = createSpawnSpy();
vi.resetModules();
vi.doMock("child_process", () => ({ spawn }));
const { spawnStreamingEncoder } = await import("./streamingEncoder.js");
const dir = mkdtempSync(join(tmpdir(), "se-exitok-"));
const encoder = await spawnStreamingEncoder(join(dir, "out.mp4"), baseOptions);
const proc = calls[0]!.proc;
await new Promise<void>((resolve) => {
process.nextTick(() => {
proc.emit("close", 0);
resolve();
});
});
expect(encoder.getExitError()).toBeUndefined();
});
it("returns a failure result (does NOT throw) when ffmpeg fails to spawn (ENOENT)", async () => {
const { spawn, calls } = createSpawnSpy();
vi.resetModules();
@@ -141,6 +141,13 @@ export interface StreamingEncoder {
writeFrame: (buffer: Buffer) => Promise<boolean>;
close: () => Promise<StreamingEncoderResult>;
getExitStatus: () => "running" | "success" | "error";
/**
* The FFmpeg failure reason (exit code + tail of stderr), or `undefined`
* while the process is still running / exited cleanly. Lets a `writeFrame`
* that returned `false` because FFmpeg died surface WHY it died (bad args,
* unsupported codec, disk full) instead of a bare "encoder exited" message.
*/
getExitError: () => string | undefined;
}
/**
@@ -600,6 +607,11 @@ export async function spawnStreamingEncoder(
},
getExitStatus: () => exitStatus,
getExitError: () => {
if (exitStatus !== "error") return undefined;
return formatFfmpegError(exitCode, stderr);
},
};
return encoder;
@@ -353,11 +353,21 @@ export async function captureTransitionFrameOnWorker(
* Streaming-encoder writes report `false` when FFmpeg is already gone.
* Continuing to capture into a dead encoder wastes the rest of the render,
* so every frame loop stops with a frame-indexed error instead.
*
* When the encoder is provided, the thrown error includes FFmpeg's own failure
* reason (exit code + stderr tail) — otherwise the message is just "exited
* before frame N", which for the frame-0 case (bad args / unsupported codec /
* missing binary quirk) is cryptic and unactionable.
*/
export function ensureFrameWritten(frameWritten: boolean, frameIndex: number): void {
if (!frameWritten) {
throw new Error(`Streaming encoder exited before frame ${frameIndex} was written`);
}
export function ensureFrameWritten(
frameWritten: boolean,
frameIndex: number,
encoder?: { getExitError: () => string | undefined },
): void {
if (frameWritten) return;
const reason = encoder?.getExitError();
const base = `Streaming encoder exited before frame ${frameIndex} was written`;
throw new Error(reason ? `${base}: ${reason}` : base);
}
// ─── HDR video raw-frame cleanup (sequential path only) ────────────────────
@@ -184,7 +184,7 @@ export async function runHybridLayeredFrameLoop(input: HybridLoopInput): Promise
const writeEncoded = async (frameIdx: number, buf: Buffer): Promise<void> => {
await reorderBuffer.waitForFrame(frameIdx);
const writeStart = Date.now();
ensureFrameWritten(await hdrEncoder.writeFrame(buf), frameIdx);
ensureFrameWritten(await hdrEncoder.writeFrame(buf), frameIdx, hdrEncoder);
addHdrTiming(hdrPerf, "encoderWriteMs", writeStart);
reorderBuffer.advanceTo(frameIdx + 1);
framesWritten += 1;
@@ -190,7 +190,7 @@ export async function runSequentialLayeredFrameLoop(input: SequentialLoopInput):
);
addHdrTiming(hdrPerf, "transitionCompositeMs", transitionTimingStart);
await timeHdrPhaseAsync(hdrPerf, "encoderWriteMs", async () =>
ensureFrameWritten(await hdrEncoder.writeFrame(transitionBuffers.output), i),
ensureFrameWritten(await hdrEncoder.writeFrame(transitionBuffers.output), i, hdrEncoder),
);
} else {
if (hdrPerf) hdrPerf.normalFrames += 1;
@@ -205,7 +205,7 @@ export async function runSequentialLayeredFrameLoop(input: SequentialLoopInput):
);
}
await timeHdrPhaseAsync(hdrPerf, "encoderWriteMs", async () =>
ensureFrameWritten(await hdrEncoder.writeFrame(normalCanvas), i),
ensureFrameWritten(await hdrEncoder.writeFrame(normalCanvas), i, hdrEncoder),
);
}
@@ -12,6 +12,7 @@ const spawnStreamingEncoder = mock(async () => ({
writeFrame,
close: closeEncoder,
getExitStatus: () => "success",
getExitError: () => undefined,
}));
let failCaptureFrameToBuffer = false;
let failInitializeSession = false;
@@ -209,7 +209,7 @@ export async function runCaptureStreamingStage(
const onFrameBuffer = async (frameIndex: number, buffer: Buffer): Promise<void> => {
await reorderBuffer.waitForFrame(frameIndex);
ensureFrameWritten(await currentEncoder.writeFrame(buffer), frameIndex);
ensureFrameWritten(await currentEncoder.writeFrame(buffer), frameIndex, currentEncoder);
reorderBuffer.advanceTo(frameIndex + 1);
};
@@ -280,7 +280,7 @@ export async function runCaptureStreamingStage(
const time = (i * job.config.fps.den) / job.config.fps.num;
const { buffer } = await captureFrameToBuffer(session, i, time);
await reorderBuffer.waitForFrame(i);
ensureFrameWritten(await currentEncoder.writeFrame(buffer), i);
ensureFrameWritten(await currentEncoder.writeFrame(buffer), i, currentEncoder);
reorderBuffer.advanceTo(i + 1);
job.framesRendered = i + 1;
@@ -17,6 +17,7 @@ vi.mock("@hyperframes/engine", async (importOriginal) => {
import {
buildMissingFrameRetryBatches,
captureAttemptMadeProgress,
describeMemoryExhaustion,
executeDiskCaptureWithAdaptiveRetry,
collectVideoMetadataHints,
collectVideoReadinessSkipIds,
@@ -24,10 +25,12 @@ import {
findMissingFrameRanges,
getNextRetryWorkerCount,
isRecoverableParallelCaptureError,
MAX_TRANSIENT_CAPTURE_RETRIES,
resolveCaptureForceScreenshotForPageSideCompositing,
shouldDiscardProbeSessionForPageSideCompositing,
shouldUseStreamingEncode,
} from "./renderOrchestrator.js";
import { ensureFrameWritten } from "./render/stages/captureHdrFrameShared.js";
import { resolveCompositeTransfer, shouldUseLayeredComposite } from "./hdrCompositor.js";
import {
createCaptureCalibrationConfig,
@@ -45,7 +48,7 @@ import {
resolveDeviceScaleFactor,
writeCompiledArtifacts,
} from "./render/shared.js";
import { toExternalAssetKey } from "../utils/paths.js";
import { formatCaptureFrameName, toExternalAssetKey } from "../utils/paths.js";
describe("extractStandaloneEntryFromIndex", () => {
it("reuses the index wrapper and keeps only the requested composition host", () => {
@@ -169,6 +172,205 @@ describe("executeDiskCaptureWithAdaptiveRetry — zero-progress bail (integratio
});
});
describe("executeDiskCaptureWithAdaptiveRetry — transient Target-closed single retry (integration)", () => {
const makeLog = () => ({ error: vi.fn(), warn: vi.fn(), info: vi.fn(), debug: vi.fn() });
const writeAllFrames = (framesDir: string, totalFrames: number): void => {
for (let i = 0; i < totalFrames; i++) {
writeFileSync(join(framesDir, formatCaptureFrameName(i, "jpg")), "x");
}
};
afterEach(() => {
vi.mocked(executeParallelCapture).mockReset();
vi.mocked(mergeWorkerFrames).mockReset();
});
it("retries ONCE at the same worker count on a transient Target closed with zero progress", async () => {
const workDir = mkdtempSync(join(tmpdir(), "hf-transient-work-"));
const framesDir = mkdtempSync(join(tmpdir(), "hf-transient-frames-"));
const log = makeLog();
let call = 0;
// First attempt: the tab dies before any frame is captured (frame 0) — zero
// forward progress, which the worker-halving retry deliberately bails on.
// The transient retry recovers it without changing the worker count.
vi.mocked(executeParallelCapture).mockImplementation(async () => {
call++;
if (call === 1) {
throw new Error("Protocol error (Page.captureScreenshot): Target closed");
}
writeAllFrames(framesDir, 4);
return [];
});
vi.mocked(mergeWorkerFrames).mockResolvedValue(undefined);
try {
const attempts = await executeDiskCaptureWithAdaptiveRetry({
serverUrl: "http://localhost:0",
workDir,
framesDir,
totalFrames: 4,
initialWorkerCount: 1,
allowRetry: true,
frameExt: "jpg",
captureOptions: {} as CaptureOptions,
createBeforeCaptureHook: () => null,
cfg: {} as EngineConfig,
log,
dedupPerfs: [],
});
expect(vi.mocked(executeParallelCapture)).toHaveBeenCalledTimes(2);
// Both attempts ran at the same worker count (transient retry doesn't halve).
expect(attempts.map((a) => a.workers)).toEqual([1, 1]);
expect(log.warn).toHaveBeenCalledWith(
expect.stringContaining("Transient browser failure"),
expect.objectContaining({ transientRetriesUsed: 1 }),
);
} finally {
rmSync(workDir, { recursive: true, force: true });
rmSync(framesDir, { recursive: true, force: true });
}
});
it("does NOT retry a transient error when the render was aborted", async () => {
const workDir = mkdtempSync(join(tmpdir(), "hf-transient-abort-work-"));
const framesDir = mkdtempSync(join(tmpdir(), "hf-transient-abort-frames-"));
const log = makeLog();
const controller = new AbortController();
// Cancellation tears the browser down, surfacing as a transient-looking
// "Target closed" — but an aborted render must fail immediately, not retry.
vi.mocked(executeParallelCapture).mockImplementation(async () => {
controller.abort();
throw new Error("Target closed");
});
vi.mocked(mergeWorkerFrames).mockResolvedValue(undefined);
try {
await expect(
executeDiskCaptureWithAdaptiveRetry({
serverUrl: "http://localhost:0",
workDir,
framesDir,
totalFrames: 4,
initialWorkerCount: 2,
allowRetry: true,
frameExt: "jpg",
captureOptions: {} as CaptureOptions,
createBeforeCaptureHook: () => null,
abortSignal: controller.signal,
cfg: {} as EngineConfig,
log,
dedupPerfs: [],
}),
).rejects.toThrow(/Target closed/);
// Exactly one attempt — no transient retry burned on a cancelled render.
expect(vi.mocked(executeParallelCapture)).toHaveBeenCalledTimes(1);
expect(log.warn).not.toHaveBeenCalledWith(
expect.stringContaining("Transient browser failure"),
expect.anything(),
);
} finally {
rmSync(workDir, { recursive: true, force: true });
rmSync(framesDir, { recursive: true, force: true });
}
});
it("gives up after MAX_TRANSIENT_CAPTURE_RETRIES when the tab keeps dying", async () => {
const workDir = mkdtempSync(join(tmpdir(), "hf-transient2-work-"));
const framesDir = mkdtempSync(join(tmpdir(), "hf-transient2-frames-"));
const log = makeLog();
vi.mocked(executeParallelCapture).mockRejectedValue(new Error("Session closed"));
vi.mocked(mergeWorkerFrames).mockResolvedValue(undefined);
try {
await expect(
executeDiskCaptureWithAdaptiveRetry({
serverUrl: "http://localhost:0",
workDir,
framesDir,
totalFrames: 4,
initialWorkerCount: 1,
allowRetry: true,
frameExt: "jpg",
captureOptions: {} as CaptureOptions,
createBeforeCaptureHook: () => null,
cfg: {} as EngineConfig,
log,
dedupPerfs: [],
}),
).rejects.toThrow(/Session closed/);
// 1 initial attempt + exactly MAX_TRANSIENT_CAPTURE_RETRIES retries.
expect(vi.mocked(executeParallelCapture)).toHaveBeenCalledTimes(
1 + MAX_TRANSIENT_CAPTURE_RETRIES,
);
} finally {
rmSync(workDir, { recursive: true, force: true });
rmSync(framesDir, { recursive: true, force: true });
}
});
});
describe("describeMemoryExhaustion", () => {
it("returns actionable guidance for a memory-exhaustion error", () => {
const msg = describeMemoryExhaustion(new Error("Set maximum size exceeded"), {
width: 3840,
height: 2160,
totalFrames: 5400,
});
expect(msg).not.toBeNull();
expect(msg).toContain("ran out of memory");
expect(msg).toContain("3840×2160");
expect(msg).toContain("5400 frames");
expect(msg).toContain("Set maximum size exceeded");
expect(msg).toContain("--low-memory-mode");
});
it("omits dimensions when they are unknown", () => {
const msg = describeMemoryExhaustion(new Error("JavaScript heap out of memory"), {});
expect(msg).not.toBeNull();
expect(msg).not.toContain("×");
});
it("returns null for a non-memory error (leaves the original message intact)", () => {
expect(
describeMemoryExhaustion(new Error("Target closed"), {
width: 1920,
height: 1080,
totalFrames: 100,
}),
).toBeNull();
});
});
describe("ensureFrameWritten", () => {
it("returns without throwing when the frame was written", () => {
expect(() => ensureFrameWritten(true, 0)).not.toThrow();
});
it("throws a bare frame-indexed error when no encoder context is supplied", () => {
expect(() => ensureFrameWritten(false, 7)).toThrow(
"Streaming encoder exited before frame 7 was written",
);
});
it("includes the ffmpeg exit reason when the encoder reports one", () => {
const encoder = { getExitError: () => "FFmpeg exited with code 1: Unknown encoder 'libx264'" };
expect(() => ensureFrameWritten(false, 0, encoder)).toThrow(
/Streaming encoder exited before frame 0 was written: FFmpeg exited with code 1: Unknown encoder 'libx264'/,
);
});
it("falls back to the bare message when the encoder has no exit reason yet", () => {
const encoder = { getExitError: () => undefined };
expect(() => ensureFrameWritten(false, 3, encoder)).toThrow(
"Streaming encoder exited before frame 3 was written",
);
});
});
describe("shouldUseStreamingEncode", () => {
const streamingEnabledConfig = {
enableStreamingEncode: true,
@@ -69,6 +69,9 @@ import {
type CapturePerfSummary,
resolveBrowserGpuMode,
resolveHeadlessShellPath,
scaleProtocolTimeoutForComposition,
isMemoryExhaustionError,
isTransientBrowserError,
} from "@hyperframes/engine";
import { join, dirname, resolve } from "path";
import { randomUUID } from "crypto";
@@ -551,6 +554,17 @@ export function getNextRetryWorkerCount(currentWorkers: number): number {
return Math.max(1, Math.floor(currentWorkers / 2));
}
/**
* Bounded number of retries for transient browser deaths (a `Target closed` /
* `Page crashed` — the tab died, not the composition). Distinct from the
* worker-count-halving retry: a transient death is often a one-off (contended
* host, OOM-killed tab, flaky CDP session) that clears on a fresh session, so
* we retry ONCE at the SAME worker count before falling through to the
* halving/structural-failure logic. Capped at 1 so a deterministically-dying
* tab can't loop.
*/
export const MAX_TRANSIENT_CAPTURE_RETRIES = 1;
/**
* A retry only pays off if the attempt that just finished captured at least one
* frame toward its target. When it captured nothing (frames still missing >=
@@ -578,6 +592,34 @@ export function isRecoverableParallelCaptureError(error: unknown): boolean {
);
}
/**
* Turn a cryptic memory-exhaustion failure (V8 `Set maximum size exceeded`,
* heap-limit abort, oversized allocation) into an actionable message. These
* come from oversized compositions — very high resolution, very long duration,
* or a huge frame count — not composition-logic bugs, and a retry re-hits the
* same ceiling. The guidance points at the levers that actually reduce memory
* pressure. Returns the original message unchanged for non-OOM errors.
*/
export function describeMemoryExhaustion(
error: unknown,
ctx: { width?: number; height?: number; totalFrames?: number },
): string | null {
if (!isMemoryExhaustionError(error)) return null;
const raw = normalizeErrorMessage(error);
const dims =
ctx.width && ctx.height
? ` (${ctx.width}×${ctx.height}${ctx.totalFrames ? `, ${ctx.totalFrames} frames` : ""})`
: "";
return (
`Render ran out of memory${dims}: ${raw}\n` +
"The composition is too large for the available memory. To reduce memory pressure:\n" +
" - Lower the output resolution or split the composition into shorter scenes.\n" +
" - Reduce the frame count (shorter duration or lower fps).\n" +
" - Run with fewer parallel workers (`--workers 1`).\n" +
" - Set PRODUCER_LOW_MEMORY_MODE=true (or `--low-memory-mode`) to use the low-memory render profile."
);
}
function countCapturedFrames(
totalFrames: number,
framesDir: string,
@@ -622,6 +664,7 @@ export async function executeDiskCaptureWithAdaptiveRetry(options: {
let currentWorkers = options.initialWorkerCount;
let missingRanges: FrameRange[] | null = null;
let attempt = 0;
let transientRetriesUsed = 0;
const rangeStart = options.frameRangeStart ?? 0;
while (true) {
@@ -715,6 +758,13 @@ export async function executeDiskCaptureWithAdaptiveRetry(options: {
missingRanges = remaining;
attempt++;
} catch (error) {
// A cancelled render tears the browser down, which surfaces as a
// transient-looking `Target closed`. Rethrow immediately so cancellation
// never burns a retry (or logs a misleading transient-failure warning) —
// the caller's abort handling owns cancellation.
if (options.abortSignal?.aborted) {
throw error;
}
const remaining = findMissingFrameRanges(
options.totalFrames,
options.framesDir,
@@ -725,6 +775,41 @@ export async function executeDiskCaptureWithAdaptiveRetry(options: {
}
const remainingCount = countFrameRanges(remaining);
const madeProgress = captureAttemptMadeProgress(frameCount, remainingCount);
// Single bounded retry for a transient browser death (`Target closed` /
// `Page crashed` / `Session closed`): the tab died mid-capture, not the
// composition. Unlike the worker-halving retry below, this keeps the same
// worker count (parallelism isn't the problem) and does NOT require
// forward progress — a tab that dies before frame 0 is the exact case we
// want to recover. Bounded by MAX_TRANSIENT_CAPTURE_RETRIES so a
// deterministically-dying tab still fails instead of looping.
//
// Scope: this covers the parallel disk-capture path (the multi-worker
// renders where a contended host most often drops a tab). The sequential
// and streaming capture paths run a single stateful session/encoder and
// don't route through here; probeStage already has its own transient
// retry for the session-init phase they share.
if (
options.allowRetry &&
isTransientBrowserError(error) &&
transientRetriesUsed < MAX_TRANSIENT_CAPTURE_RETRIES
) {
transientRetriesUsed++;
options.log.warn(
"[Render] Transient browser failure during capture; retrying once with a fresh session.",
{
attempt,
workers: currentWorkers,
missingFrames: remainingCount,
transientRetriesUsed,
error: error instanceof Error ? error.message : String(error),
},
);
missingRanges = remaining;
attempt++;
continue;
}
if (!madeProgress) {
options.log.warn(
"[Render] Capture attempt made no forward progress; composition is likely structurally broken — not retrying.",
@@ -891,6 +976,11 @@ export async function executeRenderJob(
let probeSession: CaptureSession | null = null;
let lastBrowserConsole: string[] = [];
let restoreLogger: (() => void) | null = null;
// Composition dimensions captured for the error path (OOM guidance). Assigned
// once the composition metadata / frame count are resolved inside the try.
let captureCompositionWidth: number | undefined;
let captureCompositionHeight: number | undefined;
let captureTotalFrames: number | undefined;
const perfStages: Record<string, number> = {};
const hdrDiagnostics: HdrDiagnostics = {
videoExtractionFailures: 0,
@@ -1043,6 +1133,11 @@ export async function executeRenderJob(
const composition = compileResult.composition;
const { deviceScaleFactor, outputWidth, outputHeight } = compileResult;
const { width, height } = composition;
// Capture the *output* (device-scaled) dimensions for the OOM error path —
// memory is allocated at output resolution, so the guidance must report the
// real pixel size that exhausted memory, not the smaller CSS composition.
captureCompositionWidth = outputWidth;
captureCompositionHeight = outputHeight;
perfStages.compileOnlyMs = compileResult.compileOnlyMs;
// Snapshot of `cfg.forceScreenshot` resolved by compileStage. The
// BeginFrame auto-worker calibration may flip this to `true` at
@@ -1087,6 +1182,31 @@ export async function executeRenderJob(
);
}
// Scale the CDP protocol timeout up for oversized compositions BEFORE the
// probe launches its browser. `protocolTimeout` is a Puppeteer
// connection-level setting baked in at `ppt.launch()` and immutable
// afterwards — and the probe browser is reused for capture on the common
// single-worker path — so this must be applied before the first launch, not
// after probe. A single CDP seek+capture call scales with *output* pixel
// area (device-scaled), so the fixed default intermittently kills
// legitimate slow-but-valid large renders with `Runtime.callFunctionOn
// timed out`. Only ever raises; small compositions keep the configured base.
const scaledProtocolTimeout = scaleProtocolTimeoutForComposition(cfg.protocolTimeout, {
width: outputWidth,
height: outputHeight,
});
if (scaledProtocolTimeout > cfg.protocolTimeout) {
log.info("[Render] Scaled CDP protocol timeout up for large composition.", {
from: cfg.protocolTimeout,
to: scaledProtocolTimeout,
outputWidth,
outputHeight,
deviceScaleFactor,
});
cfg.protocolTimeout = scaledProtocolTimeout;
updateCaptureObservability({ protocolTimeoutMs: scaledProtocolTimeout });
}
const probeResult = await observeRenderStage(
observability,
"browser_probe",
@@ -1122,6 +1242,8 @@ export async function executeRenderJob(
job.duration = probeResult.duration;
job.totalFrames = probeResult.totalFrames;
const totalFrames = probeResult.totalFrames;
captureTotalFrames = totalFrames;
perfStages.browserProbeMs = probeResult.browserProbeMs;
perfStages.compileMs = Date.now() - stage1Start;
observability.checkpoint("browser_probe", "duration resolved", {
@@ -1871,7 +1993,12 @@ export async function executeRenderJob(
? error
: new RenderCancelledError("render_cancelled");
}
const errorMessage = normalizeErrorMessage(error);
const memoryGuidance = describeMemoryExhaustion(error, {
width: captureCompositionWidth,
height: captureCompositionHeight,
totalFrames: captureTotalFrames,
});
const errorMessage = memoryGuidance ?? normalizeErrorMessage(error);
const carriedBrowserConsole = getCaptureStageBrowserConsole(error);
if (carriedBrowserConsole.length > 0) {
lastBrowserConsole = [...lastBrowserConsole, ...carriedBrowserConsole].slice(-200);