/** * Frame Capture Service * * Uses Puppeteer to capture frames from any web page implementing the * window.__hf seek protocol. Navigates to a file server URL, waits for * the page to expose window.__hf, then captures frames deterministically * via Chrome's BeginFrame API or Page.captureScreenshot fallback. */ import { type Browser, type Page, type Viewport, type ConsoleMessage } from "puppeteer-core"; import { existsSync, mkdirSync, writeFileSync } from "fs"; import { join } from "path"; import { quantizeTimeToFrame } from "@hyperframes/core"; // ── Extracted modules ─────────────────────────────────────────────────────── import { acquireBrowser, releaseBrowser, forceReleaseBrowser, buildChromeArgs, resolveHeadlessShellPath, type CaptureMode, } from "./browserManager.js"; import { beginFrameCapture, getCdpSession, pageScreenshotCapture, initTransparentBackground, } from "./screenshotService.js"; import { DEFAULT_CONFIG, type EngineConfig } from "../config.js"; import type { CaptureOptions, CaptureVideoMetadataHint, CaptureResult, CaptureBufferResult, CapturePerfSummary, } from "../types.js"; export type { CaptureOptions, CaptureResult, CaptureBufferResult, CapturePerfSummary }; /** Called after seeking, before screenshot. Use for video frame injection or other pre-capture work. */ export type BeforeCaptureHook = (page: Page, time: number) => Promise; export interface CaptureSession { browser: Browser; page: Page; options: CaptureOptions; serverUrl: string; outputDir: string; onBeforeCapture: BeforeCaptureHook | null; isInitialized: boolean; // Tracks whether the page/browser handles have already been released by // closeCaptureSession. Used to make closeCaptureSession idempotent under // browser-pool semantics (see the function body for the full invariant). pageReleased?: boolean; browserReleased?: boolean; browserConsoleBuffer: string[]; capturePerf: { frames: number; seekMs: number; beforeCaptureMs: number; screenshotMs: number; totalMs: number; }; captureMode: CaptureMode; // BeginFrame state beginFrameTimeTicks: number; beginFrameIntervalMs: number; beginFrameHasDamageCount: number; beginFrameNoDamageCount: number; /** Optional producer config — when set, overrides module-level env var constants. */ config?: Partial; } // Circular buffer for browser console messages dumped on render failure diagnostics. // Complex compositions produce 100+ messages; 50 was too small to capture relevant errors. const BROWSER_CONSOLE_BUFFER_SIZE = 200; const CAPTURE_SESSION_CLOSE_TIMEOUT_MS = 5_000; async function waitForCloseWithTimeout(promise: Promise): Promise { let timedOut = false; let timer: ReturnType | undefined; await Promise.race([ promise.then( () => undefined, () => undefined, ), new Promise((resolve) => { timer = setTimeout(() => { timedOut = true; resolve(); }, CAPTURE_SESSION_CLOSE_TIMEOUT_MS); }), ]); if (timer) clearTimeout(timer); return !timedOut; } export async function createCaptureSession( serverUrl: string, outputDir: string, options: CaptureOptions, onBeforeCapture: BeforeCaptureHook | null = null, config?: Partial, ): Promise { if (!existsSync(outputDir)) mkdirSync(outputDir, { recursive: true }); // Determine capture mode before building args — BeginFrame flags only apply on Linux. // BeginFrame's compositor does not preserve alpha; callers that pass // `options.format === "png"` for transparent capture should also set // `config.forceScreenshot = true` (the producer's renderOrchestrator does this // automatically when `RenderConfig.format` is an alpha-capable value). const headlessShell = resolveHeadlessShellPath(config); const isLinux = process.platform === "linux"; const forceScreenshot = config?.forceScreenshot ?? DEFAULT_CONFIG.forceScreenshot; const preMode: CaptureMode = headlessShell && isLinux && !forceScreenshot ? "beginframe" : "screenshot"; const chromeArgs = buildChromeArgs( { width: options.width, height: options.height, captureMode: preMode }, config, ); const { browser, captureMode } = await acquireBrowser(chromeArgs, config); const page = await browser.newPage(); // Polyfill esbuild's keepNames helper inside the page. // // The engine is published as raw TypeScript (`packages/engine/package.json` // points `main`/`exports` at `./src/index.ts`) and downstream consumers // execute it through transpilers that may inject `__name(fn, "name")` // wrappers around named functions. Empirically, this happens with: // - tsx (its esbuild loader runs with keepNames=true), used by the // producer's parity-harness, ad-hoc dev scripts, and the // `bun run --filter @hyperframes/engine test` Vitest path. // - any tsup/esbuild build that explicitly enables keepNames. // // The HeyGen CLI (`packages/cli`) bundles this engine via tsup with // keepNames left at its default (false) — verified by grepping // `packages/cli/dist/cli.js`, where `__name(...)` call sites are absent. // Bun's TS loader also does not currently inject `__name`. Even so, // anything that calls `page.evaluate(fn)` with a nested named function // under tsx (most local development and tests) will serialize bodies // like `__name(nested,"nested")` and crash with `__name is not defined` // in the browser. The shim makes such calls a no-op. // // An alternative is to load browser-side code as raw text and inject it // via `page.addScriptTag({ content: ... })` — see // `packages/cli/src/commands/contrast-audit.browser.js` for that pattern. // Until every `page.evaluate(fn)` call site migrates, this polyfill is // the single line of defense. The companion regression test in // `frameCapture-namePolyfill.test.ts` verifies the shim stays wired up. await page.evaluateOnNewDocument(() => { const w = window as unknown as { __name?: (fn: T, _name: string) => T }; if (typeof w.__name !== "function") { w.__name = (fn: T, _name: string): T => fn; } }); const browserVersion = await browser.version(); const expectedMajor = config?.expectedChromiumMajor; if (Number.isFinite(expectedMajor)) { const actualChromiumMajor = Number.parseInt( (browserVersion.match(/(\d+)\./) || [])[1] || "", 10, ); if (Number.isFinite(actualChromiumMajor) && actualChromiumMajor !== expectedMajor) { throw new Error( `[FrameCapture] Chromium major mismatch expected=${expectedMajor} actual=${actualChromiumMajor} raw=${browserVersion}`, ); } } const viewport: Viewport = { width: options.width, height: options.height, deviceScaleFactor: options.deviceScaleFactor || 1, }; await page.setViewport(viewport); // Transparent-background setup is intentionally NOT done here. Chrome resets // the default-background-color override on navigation, and the // `[data-composition-id]{background:transparent}` stylesheet that // `initTransparentBackground` injects must land in a real `document.head`. // See `initializeSession()` below — it calls `initTransparentBackground` for // PNG captures after `page.goto(...)` and the `window.__hf` readiness poll. return { browser, page, options, serverUrl, outputDir, onBeforeCapture, isInitialized: false, browserConsoleBuffer: [], capturePerf: { frames: 0, seekMs: 0, beforeCaptureMs: 0, screenshotMs: 0, totalMs: 0, }, captureMode, beginFrameTimeTicks: 0, beginFrameIntervalMs: 1000 / Math.max(1, options.fps), beginFrameHasDamageCount: 0, beginFrameNoDamageCount: 0, config, }; } /** * Classify a console "Failed to load resource" error as a font-load failure. * * These are expected when deterministic font injection replaces Google Fonts * @import URLs with embedded base64 — or when the render environment has no * network access to Google Fonts. Suppressing them reduces noise in render * output without hiding real asset failures (images, videos, scripts, etc.). * * Chrome's `msg.text()` for a failed resource is typically just * `"Failed to load resource: net::ERR_FAILED"` — the URL is only on * `msg.location().url`. We match against both so the filter works regardless * of which form Chrome emits. */ export function isFontResourceError(type: string, text: string, locationUrl: string): boolean { if (type !== "error") return false; if (!text.startsWith("Failed to load resource")) return false; return /fonts\.googleapis|fonts\.gstatic|\.(woff2?|ttf|otf)(\b|$)/i.test( `${locationUrl} ${text}`, ); } async function pollPageExpression( page: Page, expression: string, timeoutMs: number, intervalMs: number = 100, ): Promise { const deadline = Date.now() + timeoutMs; while (Date.now() < deadline) { const ready = Boolean(await page.evaluate(expression)); if (ready) return true; await new Promise((resolve) => setTimeout(resolve, intervalMs)); } return Boolean(await page.evaluate(expression)); } async function applyVideoMetadataHints( page: Page, hints: readonly CaptureVideoMetadataHint[] | undefined, ): Promise { if (!hints || hints.length === 0) return; await page.evaluate( (metadataHints: CaptureVideoMetadataHint[]) => { for (const hint of metadataHints) { if ( !hint.id || !Number.isFinite(hint.width) || !Number.isFinite(hint.height) || hint.width <= 0 || hint.height <= 0 ) { continue; } const video = document.getElementById(hint.id) as HTMLVideoElement | null; if (!video) continue; if (!video.hasAttribute("width")) video.setAttribute("width", String(hint.width)); if (!video.hasAttribute("height")) video.setAttribute("height", String(hint.height)); const computed = window.getComputedStyle(video); if ( !video.style.aspectRatio && (!computed.aspectRatio || computed.aspectRatio === "auto") ) { video.style.aspectRatio = `${hint.width} / ${hint.height}`; } } }, [...hints], ); } async function waitForOptionalTailwindReady(page: Page, timeoutMs: number): Promise { const hasTailwindReady = await page.evaluate( `(() => { const ready = window.__tailwindReady; return !!ready && typeof ready.then === "function"; })()`, ); if (!hasTailwindReady) return; const ready = await Promise.race([ page.evaluate( `Promise.resolve(window.__tailwindReady).then(() => true, () => false)`, ) as Promise, new Promise((resolve) => setTimeout(() => resolve(false), timeoutMs)), ]); if (!ready) { throw new Error( `[FrameCapture] window.__tailwindReady not resolved after ${timeoutMs}ms. Tailwind browser runtime must finish before frame capture starts.`, ); } } export async function initializeSession(session: CaptureSession): Promise { const { page, serverUrl } = session; // Forward browser console to host with [Browser] prefix page.on("console", (msg: ConsoleMessage) => { const type = msg.type(); const text = msg.text(); const locationUrl = msg.location()?.url ?? ""; const isFontLoadError = isFontResourceError(type, text, locationUrl); // Other "Failed to load resource" 404s are typically non-blocking (e.g. // favicon, sourcemaps, optional assets). Prefix them so users know they // are harmless and don't confuse them with real render errors. const isResourceLoadError = type === "error" && text.startsWith("Failed to load resource") && !isFontLoadError; const prefix = isResourceLoadError ? "[non-blocking]" : type === "error" ? "[Browser:ERROR]" : type === "warn" ? "[Browser:WARN]" : "[Browser]"; if (!isFontLoadError) { console.log(`${prefix} ${text}`); } session.browserConsoleBuffer.push(`${prefix} ${text}`); if (session.browserConsoleBuffer.length > BROWSER_CONSOLE_BUFFER_SIZE) { session.browserConsoleBuffer.shift(); } }); page.on("pageerror", (err) => { const message = err instanceof Error ? err.message : String(err); const text = `[Browser:PAGEERROR] ${message}`; // Benign play/pause race during frame capture — suppress terminal noise, keep in buffer. const isPlayAbort = /^AbortError:/.test(message) && message.includes("play()") && message.includes("pause()"); if (!isPlayAbort) { console.error(text); } session.browserConsoleBuffer.push(text); if (session.browserConsoleBuffer.length > BROWSER_CONSOLE_BUFFER_SIZE) { session.browserConsoleBuffer.shift(); } }); // Navigate to the file server const url = `${serverUrl}/index.html`; if (session.captureMode === "screenshot") { // Screenshot mode: standard navigation, rAF works normally await page.goto(url, { waitUntil: "domcontentloaded", timeout: 60000 }); const pageReadyTimeout = session.config?.playerReadyTimeout ?? DEFAULT_CONFIG.playerReadyTimeout; const pageReady = await pollPageExpression( page, `!!(window.__hf && typeof window.__hf.seek === "function" && window.__hf.duration > 0)`, pageReadyTimeout, ); if (!pageReady) { throw new Error( `[FrameCapture] window.__hf not ready after ${pageReadyTimeout}ms. Page must expose window.__hf = { duration, seek }.`, ); } await applyVideoMetadataHints(page, session.options.videoMetadataHints); // Wait for all video elements to have loaded metadata (dimensions + duration) // Without this, frame 0 captures videos at their 300x150 default size. // skipReadinessVideoIds excludes natively-extracted videos (e.g. HDR HEVC // sources) whose frames come from ffmpeg out-of-band. videoMetadataHints // supply intrinsic dimensions for skipped videos whose layout depends on // aspect ratio, while Chromium may still fail to decode/load metadata. const skipIdsLiteral = JSON.stringify(session.options.skipReadinessVideoIds ?? []); const videosReady = await pollPageExpression( page, `(() => { const skip = new Set(${skipIdsLiteral}); const vids = Array.from(document.querySelectorAll("video")).filter(v => !skip.has(v.id)); return vids.length === 0 || vids.every(v => v.readyState >= 1); })()`, pageReadyTimeout, ); if (!videosReady) { throw new Error( `[FrameCapture] video metadata not ready after ${pageReadyTimeout}ms. Video elements must load metadata before capture starts.`, ); } await page.evaluate(`document.fonts?.ready`); await waitForOptionalTailwindReady(page, pageReadyTimeout); // For PNG captures, force the page background fully transparent so the // captured screenshots carry a real alpha channel. Must run AFTER // navigation (Chrome resets the override on every goto) and AFTER the // page is loaded (the injected stylesheet needs a real document.head). // The override is overridden by `body { background: ... }` and // `#root { background: ... }` rules — the helper handles that with a // `[data-composition-id]{background:transparent !important}` injection. if (session.options.format === "png") { await initTransparentBackground(session.page); } session.isInitialized = true; return; } // In BeginFrame mode, Chrome's event loop is paused until we issue frames. // Start a warmup loop to drive rAF/setTimeout callbacks during page load. let warmupRunning = true; let warmupTicks = 0; let warmupFrameTime = 0; const warmupIntervalMs = 33; // ~30fps let warmupClient: import("puppeteer-core").CDPSession | null = null; const warmupLoop = async () => { try { warmupClient = await getCdpSession(page); await warmupClient.send("HeadlessExperimental.enable"); } catch { /* page not ready yet */ } while (warmupRunning) { if (warmupClient) { try { await warmupClient.send("HeadlessExperimental.beginFrame", { frameTimeTicks: warmupFrameTime, interval: warmupIntervalMs, noDisplayUpdates: true, }); warmupFrameTime += warmupIntervalMs; warmupTicks++; } catch { /* ignore warmup errors */ } } await new Promise((r) => setTimeout(r, warmupIntervalMs)); } }; warmupLoop().catch(() => {}); await page.goto(url, { waitUntil: "domcontentloaded", timeout: 60000 }); // Poll for window.__hf readiness using manual evaluate loop (waitForFunction // uses rAF polling internally, which won't fire in beginFrame mode). const pageReadyTimeout = session.config?.playerReadyTimeout ?? DEFAULT_CONFIG.playerReadyTimeout; const pollDeadline = Date.now() + pageReadyTimeout; while (Date.now() < pollDeadline) { const ready = await page.evaluate( `!!(window.__hf && typeof window.__hf.seek === "function" && window.__hf.duration > 0)`, ); if (ready) break; await new Promise((r) => setTimeout(r, 100)); } const pageReady = await page.evaluate( `!!(window.__hf && typeof window.__hf.seek === "function" && window.__hf.duration > 0)`, ); if (!pageReady) { warmupRunning = false; throw new Error( `[FrameCapture] window.__hf not ready after ${pageReadyTimeout}ms. Page must expose window.__hf = { duration, seek }.`, ); } await applyVideoMetadataHints(page, session.options.videoMetadataHints); // Wait for all video elements to have loaded metadata (dimensions + duration). // Without this, frame 0 captures videos at their 300x150 default size. // See screenshot-mode comment above for why skipReadinessVideoIds and // videoMetadataHints are paired. const beginframeSkipIdsLiteral = JSON.stringify(session.options.skipReadinessVideoIds ?? []); const videoDeadline = Date.now() + (session.config?.playerReadyTimeout ?? DEFAULT_CONFIG.playerReadyTimeout); while (Date.now() < videoDeadline) { const videosReady = await page.evaluate( `(() => { const skip = new Set(${beginframeSkipIdsLiteral}); const vids = Array.from(document.querySelectorAll("video")).filter(v => !skip.has(v.id)); return vids.length === 0 || vids.every(v => v.readyState >= 1); })()`, ); if (videosReady) break; await new Promise((r) => setTimeout(r, 100)); } // Font check (no rAF dependency — uses fonts.ready API directly) await page.evaluate(`document.fonts?.ready`); await waitForOptionalTailwindReady(page, pageReadyTimeout); // Stop warmup warmupRunning = false; // Set base frame time ticks past warmup range session.beginFrameTimeTicks = (warmupTicks + 10) * session.beginFrameIntervalMs; // For PNG captures, inject the transparent-background override + stylesheet // (see the screenshot-mode branch above for the rationale). BeginFrame mode // does not actually preserve alpha through its compositor — callers that // need transparent output should set `forceScreenshot: true` so this branch // is bypassed entirely. The call is left here as defense-in-depth for any // future BeginFrame alpha support. if (session.options.format === "png") { await initTransparentBackground(session.page); } session.isInitialized = true; } async function captureFrameErrorDiagnostics( session: CaptureSession, frameIndex: number, time: number, error: Error, ): Promise { try { const diagnosticsDir = join(session.outputDir, "diagnostics"); if (!existsSync(diagnosticsDir)) mkdirSync(diagnosticsDir, { recursive: true }); const base = join(diagnosticsDir, `frame-error-${frameIndex}`); await session.page.screenshot({ path: `${base}.png`, type: "png", fullPage: true }); const html = await session.page.content(); writeFileSync(`${base}.html`, html, "utf-8"); writeFileSync( `${base}.json`, JSON.stringify( { frameIndex, time, error: error.message, stack: error.stack, browserConsoleTail: session.browserConsoleBuffer.slice(-30), }, null, 2, ), "utf-8", ); return `${base}.json`; } catch { return null; } } /** * Internal helper: seek timeline and inject video frames. * Shared by captureFrame (disk) and captureFrameToBuffer (buffer). * Returns timing breakdown for perf tracking. */ async function prepareFrameForCapture( session: CaptureSession, frameIndex: number, time: number, ): Promise<{ quantizedTime: number; seekMs: number; beforeCaptureMs: number; }> { const { page, options } = session; if (!session.isInitialized) { throw new Error("[FrameCapture] Session not initialized"); } const quantizedTime = quantizeTimeToFrame(time, options.fps); const seekStart = Date.now(); // Seek via the __hf protocol. The page's seek() implementation handles // all framework-specific logic (GSAP stepping, CSS animation sync, etc.) await page.evaluate((t: number) => { if (window.__hf && typeof window.__hf.seek === "function") { window.__hf.seek(t); } }, quantizedTime); const seekMs = Date.now() - seekStart; // Before-capture hook (e.g. video frame injection) const beforeCaptureStart = Date.now(); if (session.onBeforeCapture) { await session.onBeforeCapture(page, quantizedTime); } const beforeCaptureMs = Date.now() - beforeCaptureStart; return { quantizedTime, seekMs, beforeCaptureMs }; } /** * Internal core: prepare, screenshot, and track perf. * Shared by captureFrame (disk) and captureFrameToBuffer (buffer). * Returns the screenshot buffer, quantized time, and total capture time. */ async function captureFrameCore( session: CaptureSession, frameIndex: number, time: number, ): Promise<{ buffer: Buffer; quantizedTime: number; captureTimeMs: number }> { const { page, options } = session; const startTime = Date.now(); try { const { quantizedTime, seekMs, beforeCaptureMs } = await prepareFrameForCapture( session, frameIndex, time, ); const screenshotStart = Date.now(); let screenshotBuffer: Buffer; if (session.captureMode === "beginframe") { const frameTimeTicks = session.beginFrameTimeTicks + frameIndex * session.beginFrameIntervalMs; const result = await beginFrameCapture( page, options, frameTimeTicks, session.beginFrameIntervalMs, ); if (result.hasDamage) session.beginFrameHasDamageCount++; else session.beginFrameNoDamageCount++; screenshotBuffer = result.buffer; } else { screenshotBuffer = await pageScreenshotCapture(page, options); } const screenshotMs = Date.now() - screenshotStart; const captureTimeMs = Date.now() - startTime; session.capturePerf.frames += 1; session.capturePerf.seekMs += seekMs; session.capturePerf.beforeCaptureMs += beforeCaptureMs; session.capturePerf.screenshotMs += screenshotMs; session.capturePerf.totalMs += captureTimeMs; return { buffer: screenshotBuffer, quantizedTime, captureTimeMs }; } catch (captureError) { if (session.isInitialized) { await captureFrameErrorDiagnostics( session, frameIndex, time, captureError instanceof Error ? captureError : new Error(String(captureError)), ); } throw captureError; } } export async function captureFrame( session: CaptureSession, frameIndex: number, time: number, ): Promise { const { options, outputDir } = session; const { buffer, quantizedTime, captureTimeMs } = await captureFrameCore( session, frameIndex, time, ); const ext = options.format === "png" ? "png" : "jpg"; const frameName = `frame_${String(frameIndex).padStart(6, "0")}.${ext}`; const framePath = join(outputDir, frameName); writeFileSync(framePath, buffer); return { frameIndex, time: quantizedTime, path: framePath, captureTimeMs }; } /** * Capture a frame and return the screenshot as a Buffer instead of writing to disk. * Used by the streaming encode pipeline to pipe frames directly to FFmpeg stdin. */ export async function captureFrameToBuffer( session: CaptureSession, frameIndex: number, time: number, ): Promise { const { buffer, captureTimeMs } = await captureFrameCore(session, frameIndex, time); return { buffer, captureTimeMs }; } export async function closeCaptureSession(session: CaptureSession): Promise { // INVARIANT: closeCaptureSession is idempotent. The renderOrchestrator HDR // cleanup path tracks a `domSessionClosed` flag and may still re-call this // in the outer finally if the inner cleanup raised before the flag flipped. // // Naive idempotency would be unsafe under pool semantics: releaseBrowser // decrements pooledBrowserRefCount, so calling it twice for the same // acquire could close a browser that another session still holds. We make // it safe by gating each release behind a per-session "released" flag — // the second call sees the flag already set and skips the release. // // We set the flag AFTER (not before) the await so that if a release throws // midway, the unreleased resource is retried by the outer defensive call. // Example: page release succeeds, browser release throws → pageReleased=true // but browserReleased=false → second call no-ops on page and retries browser. // This matches the orchestrator's intent for HDR cleanup. if (!session.pageReleased && session.page) { const pageClosed = await waitForCloseWithTimeout(session.page.close()); if (!pageClosed) { console.warn("[FrameCapture] Timed out closing page; forcing browser process shutdown"); forceReleaseBrowser(session.browser); session.browserReleased = true; } session.pageReleased = true; } if (!session.browserReleased && session.browser) { const browserClosed = await waitForCloseWithTimeout( releaseBrowser(session.browser, session.config), ); if (!browserClosed) { console.warn("[FrameCapture] Timed out closing browser; forcing browser process shutdown"); forceReleaseBrowser(session.browser); } session.browserReleased = true; } session.isInitialized = false; } export function prepareCaptureSessionForReuse( session: CaptureSession, outputDir: string, onBeforeCapture: BeforeCaptureHook | null, ): void { if (!existsSync(outputDir)) { mkdirSync(outputDir, { recursive: true }); } session.outputDir = outputDir; session.onBeforeCapture = onBeforeCapture; session.capturePerf = { frames: 0, seekMs: 0, beforeCaptureMs: 0, screenshotMs: 0, totalMs: 0, }; session.beginFrameHasDamageCount = 0; session.beginFrameNoDamageCount = 0; } export async function getCompositionDuration(session: CaptureSession): Promise { if (!session.isInitialized) throw new Error("[FrameCapture] Session not initialized"); return session.page.evaluate(() => { return window.__hf?.duration ?? 0; }); } export function getCapturePerfSummary(session: CaptureSession): CapturePerfSummary { const frames = Math.max(1, session.capturePerf.frames); return { frames: session.capturePerf.frames, avgTotalMs: Math.round(session.capturePerf.totalMs / frames), avgSeekMs: Math.round(session.capturePerf.seekMs / frames), avgBeforeCaptureMs: Math.round(session.capturePerf.beforeCaptureMs / frames), avgScreenshotMs: Math.round(session.capturePerf.screenshotMs / frames), }; }