diff --git a/docs/packages/cli.mdx b/docs/packages/cli.mdx index 2ae1e4219..3a5446db5 100644 --- a/docs/packages/cli.mdx +++ b/docs/packages/cli.mdx @@ -718,6 +718,7 @@ Word-level transcripts (whisper output) are grouped into readable caption cues o | `--variables-file` | path | — | Path to a JSON file with variable overrides (alternative to `--variables`) | | `--strict-variables` | — | off | Fail render if any `--variables` key is undeclared or has a wrong type vs the composition's `data-composition-variables`. Without this flag, mismatches print as warnings and the render continues. | | `--browser-timeout` | seconds (0.001–86400) | 60 | Puppeteer page-navigation timeout for the entry HTML. Increase when heavy compositions (many videos, fonts, or asset requests) cannot reach `domcontentloaded` within the default 60 s. The flag takes **seconds**; the env fallback `PRODUCER_PAGE_NAVIGATION_TIMEOUT_MS` takes **milliseconds**. This controls `page.goto` only — very heavy compositions may also need `PRODUCER_PUPPETEER_PROTOCOL_TIMEOUT_MS` and/or `PRODUCER_PLAYER_READY_TIMEOUT_MS` bumped (post-navigation `window.__hf` readiness has its own 45 s budget). | + | `--protocol-timeout` | milliseconds (≥ 1000) | 300000 (5 min) | Puppeteer CDP protocol timeout — the per-call budget for `Runtime.callFunctionOn` seek/paint, `Page.captureScreenshot`, and other CDP round-trips. Raise on RAM-pressured hosts (≤ 8 GB), heavy-asset compositions (many videos + images), or when the render fails with `Runtime.callFunctionOn timed out` / `Target closed`. The default is auto-scaled per composition by output pixel area (a 4K comp bumps the ceiling proportionally, capped at 30 min); an explicit override sets the floor and disables scaling below it. Env fallback `PRODUCER_PUPPETEER_PROTOCOL_TIMEOUT_MS` (also **milliseconds**). | CRF and target bitrate default to the `--quality` preset. Use `--crf` or `--video-bitrate` for fine-grained overrides; `RenderConfig.crf` and `RenderConfig.videoBitrate` accept the same overrides programmatically. Use `--video-frame-format png` when source videos are UI recordings, screen captures, or other color-sensitive clips that should avoid JPEG frame extraction. diff --git a/packages/engine/src/index.ts b/packages/engine/src/index.ts index b539cb097..461606101 100644 --- a/packages/engine/src/index.ts +++ b/packages/engine/src/index.ts @@ -79,6 +79,10 @@ export { type CaptureMode, type AcquiredBrowser, } from "./services/browserManager.js"; +export { + augmentProtocolTimeoutError, + isProtocolTimeoutError, +} from "./services/protocolTimeoutErrorHint.js"; // ── Frame capture pipeline ────────────────────────────────────────────────────── export { diff --git a/packages/engine/src/services/protocolTimeoutErrorHint.test.ts b/packages/engine/src/services/protocolTimeoutErrorHint.test.ts new file mode 100644 index 000000000..5d1008a40 --- /dev/null +++ b/packages/engine/src/services/protocolTimeoutErrorHint.test.ts @@ -0,0 +1,80 @@ +import { describe, expect, it } from "vitest"; +import { augmentProtocolTimeoutError, isProtocolTimeoutError } from "./protocolTimeoutErrorHint.js"; + +describe("augmentProtocolTimeoutError", () => { + it("passes non-timeout errors through unchanged (same instance)", () => { + const original = new Error("V8 heap exhausted"); + const result = augmentProtocolTimeoutError(original, 300_000); + expect(result).toBe(original); + expect(result.message).toBe("V8 heap exhausted"); + }); + + it("augments Runtime.callFunctionOn timed out with the effective timeout", () => { + const original = new Error( + "Runtime.callFunctionOn timed out. Increase the 'protocolTimeout' setting.", + ); + const result = augmentProtocolTimeoutError(original, 600_000); + expect(result).not.toBe(original); + expect(result.message).toContain(original.message); + expect(result.message).toContain("HyperFrames effective protocolTimeout: 600000 ms"); + }); + + it("includes both env and CLI hints", () => { + const original = new Error("Runtime.callFunctionOn timed out"); + const result = augmentProtocolTimeoutError(original, 300_000); + expect(result.message).toContain("PRODUCER_PUPPETEER_PROTOCOL_TIMEOUT_MS"); + expect(result.message).toContain("--protocol-timeout"); + }); + + it("preserves err.cause on the augmented error", () => { + const original = new Error("Runtime.callFunctionOn timed out"); + const result = augmentProtocolTimeoutError(original, 300_000); + expect((result as Error & { cause?: unknown }).cause).toBe(original); + }); + + it("augments Target closed errors", () => { + const original = new Error("Protocol error (Runtime.callFunctionOn): Target closed."); + const result = augmentProtocolTimeoutError(original, 300_000); + expect(result).not.toBe(original); + expect(result.message).toContain("HyperFrames effective protocolTimeout"); + }); + + it("matches the protocolTimeout keyword case-insensitively", () => { + const original = new Error("some upstream saying PROTOCOLTIMEOUT was hit"); + const result = augmentProtocolTimeoutError(original, 300_000); + expect(result).not.toBe(original); + }); + + it("coerces non-Error thrown values into Error without augmenting", () => { + const result = augmentProtocolTimeoutError("plain string failure", 300_000); + expect(result).toBeInstanceOf(Error); + expect(result.message).toBe("plain string failure"); + // Not augmented: coerced string doesn't match the protocol-timeout regex. + expect(result.message).not.toContain("HyperFrames effective protocolTimeout"); + }); + + it("mentions the field-signal shape reporters hit", () => { + const original = new Error("Runtime.callFunctionOn timed out"); + const result = augmentProtocolTimeoutError(original, 300_000); + expect(result.message).toContain("ts=1784047847"); + expect(result.message).toContain("FFmpeg-only encoding"); + }); +}); + +describe("isProtocolTimeoutError", () => { + it("returns true for matching messages", () => { + expect(isProtocolTimeoutError(new Error("Runtime.callFunctionOn timed out"))).toBe(true); + expect(isProtocolTimeoutError(new Error("Target closed"))).toBe(true); + expect(isProtocolTimeoutError("protocolTimeout exceeded")).toBe(true); + }); + + it("returns false for non-matching messages", () => { + expect(isProtocolTimeoutError(new Error("V8 heap exhausted"))).toBe(false); + expect(isProtocolTimeoutError(new Error("Navigation timeout of 60000 ms exceeded"))).toBe( + false, + ); + expect(isProtocolTimeoutError(null)).toBe(false); + expect(isProtocolTimeoutError(undefined)).toBe(false); + expect(isProtocolTimeoutError(42)).toBe(false); + }); +}); diff --git a/packages/engine/src/services/protocolTimeoutErrorHint.ts b/packages/engine/src/services/protocolTimeoutErrorHint.ts new file mode 100644 index 000000000..04b48f634 --- /dev/null +++ b/packages/engine/src/services/protocolTimeoutErrorHint.ts @@ -0,0 +1,47 @@ +/** + * Augment Puppeteer CDP protocol-timeout errors with actionable guidance that + * points at the HyperFrames-specific knobs. Puppeteer's stock error text + * ("Runtime.callFunctionOn timed out. Increase the 'protocolTimeout' setting") + * doesn't tell the user which env var / CLI flag raises this timeout in + * HyperFrames, or what the currently-applied effective value is — so field + * reporters have hit this class of failure, given up, and switched to + * FFmpeg-only encoding rather than raise a knob they didn't know existed + * (field signal ts=1784047847). + * + * The helper is deliberately conservative: + * - Only augments errors whose message matches known protocol-timeout + * strings (`Runtime.callFunctionOn timed out`, `Target closed`, + * `protocolTimeout`). + * - Non-matching errors are returned unchanged (same instance). + * - Non-Error inputs are coerced with `new Error(String(err))` so callers + * get a well-typed `Error` back regardless of what was thrown. + * - The original error is preserved via `err.cause`, so stack introspection + * and downstream logging still see the raw Puppeteer message. + */ + +const PROTOCOL_TIMEOUT_MATCHER = /Runtime\.callFunctionOn timed out|Target closed|protocolTimeout/i; + +export function augmentProtocolTimeoutError(err: unknown, effectiveTimeoutMs: number): Error { + if (!(err instanceof Error)) return new Error(String(err)); + if (!PROTOCOL_TIMEOUT_MATCHER.test(err.message)) return err; + const augmented = new Error( + `${err.message}\n\n` + + `HyperFrames effective protocolTimeout: ${effectiveTimeoutMs} ms.\n\n` + + `To raise the timeout:\n` + + ` Env: PRODUCER_PUPPETEER_PROTOCOL_TIMEOUT_MS=\n` + + ` CLI: --protocol-timeout \n\n` + + `Field signal ts=1784047847: this class of failure appears on RAM-pressured hosts with heavy-asset compositions (9+ videos + 20+ images). If raising the timeout doesn't help, consider FFmpeg-only encoding.`, + ); + (augmented as Error & { cause?: unknown }).cause = err; + return augmented; +} + +/** + * Predicate variant: exposed for callers that only need to classify an error + * (e.g. observability, tests) without materialising an augmented Error. Uses + * the same matcher as the augmentation path so the two never drift. + */ +export function isProtocolTimeoutError(err: unknown): boolean { + const message = err instanceof Error ? err.message : typeof err === "string" ? err : ""; + return PROTOCOL_TIMEOUT_MATCHER.test(message); +} diff --git a/packages/producer/src/services/renderOrchestrator.ts b/packages/producer/src/services/renderOrchestrator.ts index 2899c6098..1ca68f8ba 100644 --- a/packages/producer/src/services/renderOrchestrator.ts +++ b/packages/producer/src/services/renderOrchestrator.ts @@ -78,6 +78,7 @@ import { isTransientBrowserError, isDrawElementVerificationError, getDrawElementVerificationDetails, + augmentProtocolTimeoutError, } from "@hyperframes/engine"; import { join, dirname, resolve } from "path"; import { totalmem } from "node:os"; @@ -3143,7 +3144,18 @@ export async function executeRenderJob( // Retry burn on a render that STILL failed — the actionable signal for tuning // MAX_TRANSIENT_CAPTURE_RETRIES (mirrors the success-path record above). recordTransientRetryObservability(); - const errorMessage = memoryGuidance ?? normalizeErrorMessage(error); + // Surface HyperFrames' PRODUCER_PUPPETEER_PROTOCOL_TIMEOUT_MS env + + // --protocol-timeout CLI in Puppeteer CDP protocol-timeout errors. Puppeteer's + // stock "Runtime.callFunctionOn timed out. Increase the 'protocolTimeout' + // setting" text doesn't name the HyperFrames knob and doesn't state the + // effective timeout that was already applied (300000 ms base + auto-scaling + // via `scaleProtocolTimeoutForComposition`). Field signal ts=1784047847 + // reporter gave up on HF and switched to FFmpeg because the error didn't + // point them at the lever. `augmentProtocolTimeoutError` returns the input + // unchanged when the message doesn't match, so non-timeout failures (memory + // exhaustion, other CDP errors) flow through with no change. + const protocolTimeoutError = augmentProtocolTimeoutError(error, cfg.protocolTimeout); + const errorMessage = memoryGuidance ?? normalizeErrorMessage(protocolTimeoutError); const carriedBrowserConsole = getCaptureStageBrowserConsole(error); if (carriedBrowserConsole.length > 0) { lastBrowserConsole = [...lastBrowserConsole, ...carriedBrowserConsole].slice(-200);