Merge pull request #2504 from heygen-com/via/protocol-timeout-discoverability

feat(engine): surface protocolTimeout env + flag in Puppeteer timeout errors

docs/packages/cli.mdx

@@ -718,6 +718,7 @@ Word-level transcripts (whisper output) are grouped into readable caption cues o
     | `--variables-file` | path | — | Path to a JSON file with variable overrides (alternative to `--variables`) |
     | `--strict-variables` | — | off | Fail render if any `--variables` key is undeclared or has a wrong type vs the composition's `data-composition-variables`. Without this flag, mismatches print as warnings and the render continues. |
     | `--browser-timeout` | seconds (0.001–86400) | 60 | Puppeteer page-navigation timeout for the entry HTML. Increase when heavy compositions (many videos, fonts, or asset requests) cannot reach `domcontentloaded` within the default 60 s. The flag takes **seconds**; the env fallback `PRODUCER_PAGE_NAVIGATION_TIMEOUT_MS` takes **milliseconds**. This controls `page.goto` only — very heavy compositions may also need `PRODUCER_PUPPETEER_PROTOCOL_TIMEOUT_MS` and/or `PRODUCER_PLAYER_READY_TIMEOUT_MS` bumped (post-navigation `window.__hf` readiness has its own 45 s budget). |
+    | `--protocol-timeout` | milliseconds (≥ 1000) | 300000 (5 min) | Puppeteer CDP protocol timeout — the per-call budget for `Runtime.callFunctionOn` seek/paint, `Page.captureScreenshot`, and other CDP round-trips. Raise on RAM-pressured hosts (≤ 8 GB), heavy-asset compositions (many videos + images), or when the render fails with `Runtime.callFunctionOn timed out` / `Target closed`. The default is auto-scaled per composition by output pixel area (a 4K comp bumps the ceiling proportionally, capped at 30 min); an explicit override sets the floor and disables scaling below it. Env fallback `PRODUCER_PUPPETEER_PROTOCOL_TIMEOUT_MS` (also **milliseconds**). |
 
     CRF and target bitrate default to the `--quality` preset. Use `--crf` or `--video-bitrate` for fine-grained overrides; `RenderConfig.crf` and `RenderConfig.videoBitrate` accept the same overrides programmatically. Use `--video-frame-format png` when source videos are UI recordings, screen captures, or other color-sensitive clips that should avoid JPEG frame extraction.
 

packages/engine/src/index.ts

@@ -79,6 +79,10 @@ export {
   type CaptureMode,
   type AcquiredBrowser,
 } from "./services/browserManager.js";
+export {
+  augmentProtocolTimeoutError,
+  isProtocolTimeoutError,
+} from "./services/protocolTimeoutErrorHint.js";
 
 // ── Frame capture pipeline ──────────────────────────────────────────────────────
 export {

packages/engine/src/services/protocolTimeoutErrorHint.test.ts

@@ -0,0 +1,80 @@
+import { describe, expect, it } from "vitest";
+import { augmentProtocolTimeoutError, isProtocolTimeoutError } from "./protocolTimeoutErrorHint.js";
+
+describe("augmentProtocolTimeoutError", () => {
+  it("passes non-timeout errors through unchanged (same instance)", () => {
+    const original = new Error("V8 heap exhausted");
+    const result = augmentProtocolTimeoutError(original, 300_000);
+    expect(result).toBe(original);
+    expect(result.message).toBe("V8 heap exhausted");
+  });
+
+  it("augments Runtime.callFunctionOn timed out with the effective timeout", () => {
+    const original = new Error(
+      "Runtime.callFunctionOn timed out. Increase the 'protocolTimeout' setting.",
+    );
+    const result = augmentProtocolTimeoutError(original, 600_000);
+    expect(result).not.toBe(original);
+    expect(result.message).toContain(original.message);
+    expect(result.message).toContain("HyperFrames effective protocolTimeout: 600000 ms");
+  });
+
+  it("includes both env and CLI hints", () => {
+    const original = new Error("Runtime.callFunctionOn timed out");
+    const result = augmentProtocolTimeoutError(original, 300_000);
+    expect(result.message).toContain("PRODUCER_PUPPETEER_PROTOCOL_TIMEOUT_MS");
+    expect(result.message).toContain("--protocol-timeout");
+  });
+
+  it("preserves err.cause on the augmented error", () => {
+    const original = new Error("Runtime.callFunctionOn timed out");
+    const result = augmentProtocolTimeoutError(original, 300_000);
+    expect((result as Error & { cause?: unknown }).cause).toBe(original);
+  });
+
+  it("augments Target closed errors", () => {
+    const original = new Error("Protocol error (Runtime.callFunctionOn): Target closed.");
+    const result = augmentProtocolTimeoutError(original, 300_000);
+    expect(result).not.toBe(original);
+    expect(result.message).toContain("HyperFrames effective protocolTimeout");
+  });
+
+  it("matches the protocolTimeout keyword case-insensitively", () => {
+    const original = new Error("some upstream saying PROTOCOLTIMEOUT was hit");
+    const result = augmentProtocolTimeoutError(original, 300_000);
+    expect(result).not.toBe(original);
+  });
+
+  it("coerces non-Error thrown values into Error without augmenting", () => {
+    const result = augmentProtocolTimeoutError("plain string failure", 300_000);
+    expect(result).toBeInstanceOf(Error);
+    expect(result.message).toBe("plain string failure");
+    // Not augmented: coerced string doesn't match the protocol-timeout regex.
+    expect(result.message).not.toContain("HyperFrames effective protocolTimeout");
+  });
+
+  it("mentions the field-signal shape reporters hit", () => {
+    const original = new Error("Runtime.callFunctionOn timed out");
+    const result = augmentProtocolTimeoutError(original, 300_000);
+    expect(result.message).toContain("ts=1784047847");
+    expect(result.message).toContain("FFmpeg-only encoding");
+  });
+});
+
+describe("isProtocolTimeoutError", () => {
+  it("returns true for matching messages", () => {
+    expect(isProtocolTimeoutError(new Error("Runtime.callFunctionOn timed out"))).toBe(true);
+    expect(isProtocolTimeoutError(new Error("Target closed"))).toBe(true);
+    expect(isProtocolTimeoutError("protocolTimeout exceeded")).toBe(true);
+  });
+
+  it("returns false for non-matching messages", () => {
+    expect(isProtocolTimeoutError(new Error("V8 heap exhausted"))).toBe(false);
+    expect(isProtocolTimeoutError(new Error("Navigation timeout of 60000 ms exceeded"))).toBe(
+      false,
+    );
+    expect(isProtocolTimeoutError(null)).toBe(false);
+    expect(isProtocolTimeoutError(undefined)).toBe(false);
+    expect(isProtocolTimeoutError(42)).toBe(false);
+  });
+});

packages/engine/src/services/protocolTimeoutErrorHint.ts

@@ -0,0 +1,47 @@
+/**
+ * Augment Puppeteer CDP protocol-timeout errors with actionable guidance that
+ * points at the HyperFrames-specific knobs. Puppeteer's stock error text
+ * ("Runtime.callFunctionOn timed out. Increase the 'protocolTimeout' setting")
+ * doesn't tell the user which env var / CLI flag raises this timeout in
+ * HyperFrames, or what the currently-applied effective value is — so field
+ * reporters have hit this class of failure, given up, and switched to
+ * FFmpeg-only encoding rather than raise a knob they didn't know existed
+ * (field signal ts=1784047847).
+ *
+ * The helper is deliberately conservative:
+ *   - Only augments errors whose message matches known protocol-timeout
+ *     strings (`Runtime.callFunctionOn timed out`, `Target closed`,
+ *     `protocolTimeout`).
+ *   - Non-matching errors are returned unchanged (same instance).
+ *   - Non-Error inputs are coerced with `new Error(String(err))` so callers
+ *     get a well-typed `Error` back regardless of what was thrown.
+ *   - The original error is preserved via `err.cause`, so stack introspection
+ *     and downstream logging still see the raw Puppeteer message.
+ */
+
+const PROTOCOL_TIMEOUT_MATCHER = /Runtime\.callFunctionOn timed out|Target closed|protocolTimeout/i;
+
+export function augmentProtocolTimeoutError(err: unknown, effectiveTimeoutMs: number): Error {
+  if (!(err instanceof Error)) return new Error(String(err));
+  if (!PROTOCOL_TIMEOUT_MATCHER.test(err.message)) return err;
+  const augmented = new Error(
+    `${err.message}\n\n` +
+      `HyperFrames effective protocolTimeout: ${effectiveTimeoutMs} ms.\n\n` +
+      `To raise the timeout:\n` +
+      `  Env: PRODUCER_PUPPETEER_PROTOCOL_TIMEOUT_MS=<higher-ms>\n` +
+      `  CLI: --protocol-timeout <higher-ms>\n\n` +
+      `Field signal ts=1784047847: this class of failure appears on RAM-pressured hosts with heavy-asset compositions (9+ videos + 20+ images). If raising the timeout doesn't help, consider FFmpeg-only encoding.`,
+  );
+  (augmented as Error & { cause?: unknown }).cause = err;
+  return augmented;
+}
+
+/**
+ * Predicate variant: exposed for callers that only need to classify an error
+ * (e.g. observability, tests) without materialising an augmented Error. Uses
+ * the same matcher as the augmentation path so the two never drift.
+ */
+export function isProtocolTimeoutError(err: unknown): boolean {
+  const message = err instanceof Error ? err.message : typeof err === "string" ? err : "";
+  return PROTOCOL_TIMEOUT_MATCHER.test(message);
+}

packages/producer/src/services/renderOrchestrator.ts

@@ -78,6 +78,7 @@ import {
   isTransientBrowserError,
   isDrawElementVerificationError,
   getDrawElementVerificationDetails,
+  augmentProtocolTimeoutError,
 } from "@hyperframes/engine";
 import { join, dirname, resolve } from "path";
 import { totalmem } from "node:os";
@@ -3143,7 +3144,18 @@ export async function executeRenderJob(
     // Retry burn on a render that STILL failed — the actionable signal for tuning
     // MAX_TRANSIENT_CAPTURE_RETRIES (mirrors the success-path record above).
     recordTransientRetryObservability();
-    const errorMessage = memoryGuidance ?? normalizeErrorMessage(error);
+    // Surface HyperFrames' PRODUCER_PUPPETEER_PROTOCOL_TIMEOUT_MS env +
+    // --protocol-timeout CLI in Puppeteer CDP protocol-timeout errors. Puppeteer's
+    // stock "Runtime.callFunctionOn timed out. Increase the 'protocolTimeout'
+    // setting" text doesn't name the HyperFrames knob and doesn't state the
+    // effective timeout that was already applied (300000 ms base + auto-scaling
+    // via `scaleProtocolTimeoutForComposition`). Field signal ts=1784047847
+    // reporter gave up on HF and switched to FFmpeg because the error didn't
+    // point them at the lever. `augmentProtocolTimeoutError` returns the input
+    // unchanged when the message doesn't match, so non-timeout failures (memory
+    // exhaustion, other CDP errors) flow through with no change.
+    const protocolTimeoutError = augmentProtocolTimeoutError(error, cfg.protocolTimeout);
+    const errorMessage = memoryGuidance ?? normalizeErrorMessage(protocolTimeoutError);
     const carriedBrowserConsole = getCaptureStageBrowserConsole(error);
     if (carriedBrowserConsole.length > 0) {
       lastBrowserConsole = [...lastBrowserConsole, ...carriedBrowserConsole].slice(-200);