diff --git a/packages/engine/src/services/frameCapture-constructionOwnership.test.ts b/packages/engine/src/services/frameCapture-constructionOwnership.test.ts index 9eba72b486..6d57bc20d5 100644 --- a/packages/engine/src/services/frameCapture-constructionOwnership.test.ts +++ b/packages/engine/src/services/frameCapture-constructionOwnership.test.ts @@ -3,7 +3,7 @@ import { afterEach, describe, expect, it, vi } from "vitest"; import { mkdtempSync, rmSync } from "node:fs"; import { tmpdir } from "node:os"; import { join } from "node:path"; -import type { Browser, Page, PuppeteerNode } from "puppeteer-core"; +import { CDPSession, type Browser, type Page, type PuppeteerNode } from "puppeteer-core"; import { _resetBrowserPoolForTests, _setPuppeteerForTests, @@ -18,154 +18,78 @@ describe("createCaptureSession construction ownership", () => { _setPuppeteerForTests(undefined); }); - it("closes the page and releases its exact browser lease when bootstrap fails", async () => { - const outputDir = mkdtempSync(join(tmpdir(), "hf-session-owner-")); - const page = { - evaluateOnNewDocument: vi.fn().mockRejectedValue(new Error("bootstrap failed")), - close: vi.fn().mockResolvedValue(undefined), - } as unknown as Page; - const browser = { - connected: true, - newPage: vi.fn().mockResolvedValue(page), - version: vi.fn().mockResolvedValue("HeadlessChrome/150.0.0.0"), - close: vi.fn().mockResolvedValue(undefined), - disconnect: vi.fn(), - process: () => null, - } as unknown as Browser; - _setPuppeteerForTests({ - launch: vi.fn().mockResolvedValue(browser), - } as unknown as PuppeteerNode); + it.each([ + { stage: "network setup", blocked: "none" }, + { stage: "bootstrap", blocked: "none" }, + { stage: "bootstrap", blocked: "page" }, + { stage: "bootstrap", blocked: "browser" }, + ])( + "releases the exact lease after $stage failure with $blocked close blocked", + async ({ stage, blocked }) => { + if (blocked !== "none") vi.useFakeTimers(); + const outputDir = mkdtempSync(join(tmpdir(), "hf-session-owner-")); + const send = vi.fn().mockResolvedValue(undefined); + if (stage === "network setup") send.mockRejectedValue(new Error("network setup failed")); + const closeResult = (resource: string) => + blocked === resource ? new Promise(() => {}) : Promise.resolve(); + const page = { + mainFrame: () => ({ client: Object.setPrototypeOf({ send }, CDPSession.prototype) }), + evaluateOnNewDocument: vi.fn().mockRejectedValue(new Error("bootstrap failed")), + close: vi.fn(() => closeResult("page")), + } as unknown as Page; + const browser = { + connected: true, + newPage: vi.fn().mockResolvedValue(page), + version: vi.fn().mockResolvedValue("HeadlessChrome/150.0.0.0"), + close: vi.fn(() => closeResult("browser")), + disconnect: vi.fn(), + process: () => null, + } as unknown as Browser; + _setPuppeteerForTests({ + launch: vi.fn().mockResolvedValue(browser), + } as unknown as PuppeteerNode); - try { - await expect( - createCaptureSession( - "http://127.0.0.1:3000", - outputDir, - { - width: 320, - height: 180, - fps: { num: 30, den: 1 }, - format: "jpeg", - requiresWebGpu: false, - }, - null, - { - browserGpuMode: "software", - enableBrowserPool: true, - forceScreenshot: true, - }, - ), - ).rejects.toThrow("bootstrap failed"); + try { + const creating = expect( + createCaptureSession( + "http://127.0.0.1:3000", + outputDir, + { + width: 320, + height: 180, + fps: { num: 30, den: 1 }, + format: "jpeg", + requiresWebGpu: false, + }, + null, + { + browserGpuMode: "software", + enableBrowserPool: true, + forceScreenshot: true, + }, + ), + ).rejects.toThrow(`${stage} failed`); + if (blocked !== "none") await vi.runAllTimersAsync(); + await creating; - expect(page.close).toHaveBeenCalledTimes(1); - expect(browser.close).toHaveBeenCalledTimes(1); - } finally { - rmSync(outputDir, { recursive: true, force: true }); - } - }); - - it("force-releases its browser lease when rollback page close never settles", async () => { - vi.useFakeTimers(); - const outputDir = mkdtempSync(join(tmpdir(), "hf-session-owner-timeout-")); - const page = { - evaluateOnNewDocument: vi.fn().mockRejectedValue(new Error("bootstrap failed")), - close: vi.fn().mockReturnValue(new Promise(() => {})), - } as unknown as Page; - const disconnect = vi.fn(); - const browser = { - connected: true, - newPage: vi.fn().mockResolvedValue(page), - version: vi.fn().mockResolvedValue("HeadlessChrome/150.0.0.0"), - close: vi.fn().mockResolvedValue(undefined), - disconnect, - process: () => null, - } as unknown as Browser; - _setPuppeteerForTests({ - launch: vi.fn().mockResolvedValue(browser), - } as unknown as PuppeteerNode); - - try { - const creating = expect( - createCaptureSession( - "http://127.0.0.1:3000", - outputDir, - { - width: 320, - height: 180, - fps: { num: 30, den: 1 }, - format: "jpeg", - requiresWebGpu: false, - }, - null, - { - browserGpuMode: "software", - enableBrowserPool: true, - forceScreenshot: true, - }, - ), - ).rejects.toThrow("bootstrap failed"); - - await vi.runAllTimersAsync(); - await creating; - - expect(page.close).toHaveBeenCalledTimes(1); - expect(disconnect).toHaveBeenCalledTimes(1); - } finally { - vi.useRealTimers(); - rmSync(outputDir, { recursive: true, force: true }); - } - }); - - it("force-releases its browser lease when rollback browser close never settles", async () => { - vi.useFakeTimers(); - const outputDir = mkdtempSync(join(tmpdir(), "hf-session-browser-timeout-")); - const page = { - evaluateOnNewDocument: vi.fn().mockRejectedValue(new Error("bootstrap failed")), - close: vi.fn().mockResolvedValue(undefined), - } as unknown as Page; - const disconnect = vi.fn(); - const browser = { - connected: true, - newPage: vi.fn().mockResolvedValue(page), - version: vi.fn().mockResolvedValue("HeadlessChrome/150.0.0.0"), - close: vi.fn().mockReturnValue(new Promise(() => {})), - disconnect, - process: () => null, - } as unknown as Browser; - _setPuppeteerForTests({ - launch: vi.fn().mockResolvedValue(browser), - } as unknown as PuppeteerNode); - - try { - const creating = expect( - createCaptureSession( - "http://127.0.0.1:3000", - outputDir, - { - width: 320, - height: 180, - fps: { num: 30, den: 1 }, - format: "jpeg", - requiresWebGpu: false, - }, - null, - { - browserGpuMode: "software", - enableBrowserPool: true, - forceScreenshot: true, - }, - ), - ).rejects.toThrow("bootstrap failed"); - - await vi.runAllTimersAsync(); - await creating; - - expect(page.close).toHaveBeenCalledTimes(1); - expect(browser.close).toHaveBeenCalledTimes(1); - expect(disconnect).toHaveBeenCalledTimes(1); - } finally { - vi.useRealTimers(); - rmSync(outputDir, { recursive: true, force: true }); - } - }); + expect(send).toHaveBeenCalledWith("Network.enable", { + maxTotalBufferSize: 32 * 1024 * 1024, + maxResourceBufferSize: 16 * 1024 * 1024, + }); + if (stage === "network setup") { + expect(page.evaluateOnNewDocument).not.toHaveBeenCalled(); + } else { + expect(send.mock.invocationCallOrder[0]).toBeLessThan( + vi.mocked(page.evaluateOnNewDocument).mock.invocationCallOrder[0] ?? 0, + ); + } + expect(page.close).toHaveBeenCalledTimes(1); + if (blocked !== "page") expect(browser.close).toHaveBeenCalledTimes(1); + if (blocked !== "none") expect(browser.disconnect).toHaveBeenCalledTimes(1); + } finally { + vi.useRealTimers(); + rmSync(outputDir, { recursive: true, force: true }); + } + }, + ); }); diff --git a/packages/engine/src/services/frameCapture-freshFallback.test.ts b/packages/engine/src/services/frameCapture-freshFallback.test.ts index 6b3470d51e..8cd9639351 100644 --- a/packages/engine/src/services/frameCapture-freshFallback.test.ts +++ b/packages/engine/src/services/frameCapture-freshFallback.test.ts @@ -25,6 +25,7 @@ vi.mock("./browserManager.js", async (importOriginal) => ({ browser: { version: async () => "Chrome/150.0.0.0", newPage: async () => ({ + mainFrame: () => ({}), evaluateOnNewDocument: async () => {}, setViewport: async () => {}, evaluate: async () => undefined, diff --git a/packages/engine/src/services/frameCapture-softwareWebGpu.test.ts b/packages/engine/src/services/frameCapture-softwareWebGpu.test.ts index b155f16e8f..cde4f69f68 100644 --- a/packages/engine/src/services/frameCapture-softwareWebGpu.test.ts +++ b/packages/engine/src/services/frameCapture-softwareWebGpu.test.ts @@ -21,6 +21,7 @@ vi.mock("./browserManager.js", async (importOriginal) => ({ browser: { version: async () => "Chrome/150.0.0.0", newPage: async () => ({ + mainFrame: () => ({}), on: () => {}, goto: async () => null, createCDPSession: async () => ({ on: () => {}, send: async () => ({}) }), diff --git a/packages/engine/src/services/frameCapture.ts b/packages/engine/src/services/frameCapture.ts index 4ce71a25a6..0ffe036bee 100644 --- a/packages/engine/src/services/frameCapture.ts +++ b/packages/engine/src/services/frameCapture.ts @@ -9,6 +9,7 @@ */ import { + CDPSession, type Browser, type Page, type Protocol, @@ -1414,6 +1415,15 @@ async function constructCaptureSession( const page = await browser.newPage(); onPageCreated(page); + // Cap the existing session's inspection buffer; new sessions have separate buffers. + const frame = page.mainFrame(); + // Frame.client is runtime-only, so keep capture working when it is absent. + if ("client" in frame && frame.client instanceof CDPSession) { + await frame.client.send("Network.enable", { + maxTotalBufferSize: 32 * 1024 * 1024, + maxResourceBufferSize: 16 * 1024 * 1024, + }); + } // Polyfill esbuild's keepNames helper inside the page. // // The engine is published as raw TypeScript (`packages/engine/package.json` diff --git a/packages/engine/src/services/screenshotService.test.ts b/packages/engine/src/services/screenshotService.test.ts index 4f25011927..9fd1d9a0bb 100644 --- a/packages/engine/src/services/screenshotService.test.ts +++ b/packages/engine/src/services/screenshotService.test.ts @@ -17,6 +17,15 @@ import { DOM_LAYER_MASK_STYLE_ID, } from "./screenshotService.js"; +// These DOM tests isolate layout/decode from the CDP transport, exercised in Chrome separately. +vi.mock("./videoFrameTransport.js", () => ({ + withVideoFrameSources: async ( + _page: Page, + updates: Array<{ videoId: string; dataUri: string }>, + inject: (resolved: Array<{ videoId: string; dataUri: string }>) => Promise, + ) => inject(updates), +})); + // Stub a Page + CDPSession just enough that pageScreenshotCapture can call // `client.send("Page.captureScreenshot", ...)` and we can inspect the args. function makeFakePageWithCdp(send: (method: string, params: object) => Promise<{ data: string }>) { diff --git a/packages/engine/src/services/screenshotService.ts b/packages/engine/src/services/screenshotService.ts index 23cdffb088..d702f8bf17 100644 --- a/packages/engine/src/services/screenshotService.ts +++ b/packages/engine/src/services/screenshotService.ts @@ -7,6 +7,7 @@ // fallow-ignore-file code-duplication import { type Page } from "puppeteer-core"; +import { withVideoFrameSources, type VideoFrameUpdate } from "./videoFrameTransport.js"; import { type CaptureOptions } from "../types.js"; import { COLOR_GRADING_SOURCE_HIDDEN_ATTR } from "@hyperframes/core/color-grading"; import { @@ -643,10 +644,19 @@ export async function injectVideoFramesBatch( updates: Array<{ videoId: string; dataUri: string }>, ): Promise { if (updates.length === 0) return []; + return withVideoFrameSources(page, updates, (resolved) => + injectResolvedVideoFramesBatch(page, resolved), + ); +} + +async function injectResolvedVideoFramesBatch( + page: Page, + updates: VideoFrameUpdate[], +): Promise { return await page.evaluate( // fallow-ignore-next-line complexity async ( - items: Array<{ videoId: string; dataUri: string; frameId: string }>, + items: Array, visualProperties: string[], colorGradingSourceHiddenAttr: string, ) => { @@ -795,8 +805,11 @@ export async function injectVideoFramesBatch( img.decoding = "sync"; if (img.getAttribute("src") !== item.dataUri) { + if (item.crossOrigin) img.crossOrigin = item.crossOrigin; + else img.removeAttribute("crossorigin"); img.src = item.dataUri; - const source = item.dataUri.startsWith("data:") ? "inline frame" : item.dataUri; + const source = + item.sourceLabel ?? (item.dataUri.startsWith("data:") ? "inline frame" : item.dataUri); pendingDecodes.push( img.decode().catch((error: unknown) => { throw new Error( diff --git a/packages/engine/src/services/videoFrameTransport.browser.test.ts b/packages/engine/src/services/videoFrameTransport.browser.test.ts new file mode 100644 index 0000000000..398b8f3587 --- /dev/null +++ b/packages/engine/src/services/videoFrameTransport.browser.test.ts @@ -0,0 +1,162 @@ +import { spawnSync } from "node:child_process"; +import { createServer } from "node:http"; +import { afterAll, beforeAll, beforeEach, describe, expect, it } from "vitest"; +import puppeteer, { type Browser, type Page } from "puppeteer-core"; +import { resolveHeadlessShellPath } from "./browserManager.js"; +import { injectVideoFramesBatch } from "./screenshotService.js"; + +const chromePath = resolveHeadlessShellPath(); +const hasBrowser = chromePath !== undefined && spawnSync(chromePath, ["--version"]).status === 0; + +describe.skipIf(!hasBrowser)("video frame transport in Chrome", () => { + let browser: Browser; + let page: Page; + let origin: string; + let redPng: string; + let blueJpeg: string; + let unexpectedFrameRequests = 0; + const server = createServer((req, res) => { + if (req.url?.startsWith("/slow")) return; + if (req.url === "/external.png") { + res.writeHead(200, { "Content-Type": "image/png", "Cache-Control": "max-age=600" }); + res.end(Buffer.from(redPng.split(",")[1] ?? "", "base64")); + return; + } + if (req.url?.startsWith("/__hyperframes_video_frames__/")) unexpectedFrameRequests++; + res.writeHead(200, { "Content-Type": "text/html" }); + res.end( + '
', + ); + }); + + beforeAll(async () => { + await new Promise((resolve) => server.listen(0, "127.0.0.1", resolve)); + const address = server.address(); + if (!address || typeof address === "string") throw new Error("No test server address"); + origin = `http://127.0.0.1:${address.port}`; + browser = await puppeteer.launch({ + executablePath: chromePath, + headless: true, + args: ["--no-sandbox", "--disable-dev-shm-usage"], + }); + }); + beforeEach(async () => { + if (page) await page.close(); + page = await browser.newPage(); + await page.goto(origin); + [redPng, blueJpeg] = await Promise.all([ + frameDataUri("red"), + frameDataUri("blue", "image/jpeg"), + ]); + unexpectedFrameRequests = 0; + }); + afterAll(async () => { + await browser?.close(); + server.closeAllConnections(); + await new Promise((resolve) => server.close(() => resolve())); + }); + + const frameDataUri = (color: string, format = "image/png") => + page.evaluate( + (fill, mime) => { + const canvas = document.createElement("canvas"); + canvas.width = canvas.height = 4; + const ctx = canvas.getContext("2d"); + if (!ctx) throw new Error("No 2D context"); + ctx.fillStyle = fill; + ctx.fillRect(0, 0, 4, 4); + return canvas.toDataURL(mime, 1); + }, + color, + format, + ); + + const pixels = () => + page.evaluate(() => { + const canvas = new OffscreenCanvas(4, 4); + const ctx = canvas.getContext("2d"); + if (!ctx) throw new Error("No 2D context"); + return Array.from( + document.querySelectorAll("img.__render_frame__"), + (img) => { + ctx.clearRect(0, 0, 4, 4); + ctx.drawImage(img, 0, 0); + return { src: img.src, rgba: Array.from(ctx.getImageData(0, 0, 1, 1).data) }; + }, + ); + }); + + it("delivers independent decoded frames without retaining inline image URLs", async () => { + const headers: string[] = []; + page.on("response", (response) => { + if (response.request().resourceType() === "image") + headers.push(response.headers()["cache-control"] ?? ""); + }); + await injectVideoFramesBatch(page, [ + { videoId: "a", dataUri: redPng }, + { videoId: "b", dataUri: blueJpeg }, + ]); + const first = await pixels(); + expect(first.map((img) => img.rgba)).toEqual([ + [255, 0, 0, 255], + [0, 0, 254, 255], + ]); + expect(first.every((img) => img.src.startsWith(origin))).toBe(true); + expect(headers).toEqual(["no-store", "no-store"]); + await injectVideoFramesBatch(page, [{ videoId: "a", dataUri: blueJpeg }]); + expect((await pixels())[0]?.rgba).toEqual([0, 0, 254, 255]); + expect(await page.evaluate(async (url) => (await fetch(url)).status, first[0]?.src ?? "")).toBe( + 404, + ); + expect(unexpectedFrameRequests).toBe(0); + }); + + it("loads frames even while all same-origin download connections are busy", async () => { + await page.evaluate(() => { + for (let i = 0; i < 6; i++) void fetch(`/slow${i}.mp4`); + }); + await injectVideoFramesBatch(page, [{ videoId: "a", dataUri: redPng }]); + expect((await pixels())[0]?.rgba).toEqual([255, 0, 0, 255]); + }, 5000); + + it.each(["HTTP", "about:blank"])( + "preserves PNG alpha and GPU uploads from %s", + async (source) => { + if (source === "about:blank") { + const content = await page.content(); + await page.close(); + page = await browser.newPage(); + await page.setContent(content); + expect(page.url()).toBe("about:blank"); + } + const translucent = await frameDataUri("rgba(255,0,0,0.5)"); + await injectVideoFramesBatch(page, [{ videoId: "a", dataUri: translucent }]); + expect((await pixels())[0]?.rgba).toEqual([255, 0, 0, 128]); + expect( + await page.evaluate(() => { + const canvas = document.createElement("canvas"); + const gl = canvas.getContext("webgl"); + const img = document.querySelector("img"); + if (!gl || !img) throw new Error("No GPU texture source"); + gl.bindTexture(gl.TEXTURE_2D, gl.createTexture()); + gl.texImage2D(gl.TEXTURE_2D, 0, gl.RGBA, gl.RGBA, gl.UNSIGNED_BYTE, img); + return gl.getError(); + }), + ).toBe(0); + }, + ); + + it("releases failed frames and preserves the video-specific decode error", async () => { + await expect( + injectVideoFramesBatch(page, [{ videoId: "a", dataUri: "data:image/png;base64,AAAA" }]), + ).rejects.toThrow('Video frame for "a" failed to load (inline frame)'); + const src = await page.evaluate(() => document.querySelector("img")?.src ?? ""); + expect(await page.evaluate(async (url) => (await fetch(url)).status, src)).toBe(404); + }); + + it("keeps caller-provided frame URLs unchanged", async () => { + const src = `${origin}/external.png`; + await injectVideoFramesBatch(page, [{ videoId: "a", dataUri: src }]); + expect((await pixels())[0]).toEqual({ src, rgba: [255, 0, 0, 255] }); + }); +}); diff --git a/packages/engine/src/services/videoFrameTransport.test.ts b/packages/engine/src/services/videoFrameTransport.test.ts new file mode 100644 index 0000000000..ec5228ba4f --- /dev/null +++ b/packages/engine/src/services/videoFrameTransport.test.ts @@ -0,0 +1,115 @@ +import { EventEmitter } from "node:events"; +import { describe, expect, it, vi } from "vitest"; +import { type Page } from "puppeteer-core"; +import { withVideoFrameSources, type VideoFrameUpdate } from "./videoFrameTransport.js"; + +const update = { videoId: "clip", dataUri: "data:image/png;base64,AAAA" }; + +function fixture() { + const events = new EventEmitter(); + const send = vi.fn(async (_method: string, _params: unknown) => ({})); + const detach = vi.fn(async () => {}); + const close = new EventEmitter(); + const createCDPSession = vi.fn(async () => ({ on: events.on.bind(events), send, detach })); + // The transport only needs these Page methods; no browser protocol is simulated elsewhere. + const page = { + url: () => "http://localhost:4000/index.html", + createCDPSession, + once: close.once.bind(close), + } as unknown as Page; + const request = async (url: string) => { + events.emit("Fetch.requestPaused", { requestId: "request", request: { url } }); + await new Promise((resolve) => setImmediate(resolve)); + return send.mock.calls.findLast(([method]) => method === "Fetch.fulfillRequest")?.[1]; + }; + return { page, send, detach, createCDPSession, close, request }; +} + +describe("video frame transport ownership", () => { + it.each(["/frames/0001.png", "data:frame-test", "data:image/png,not-base64"])( + "does not intercept an unsupported source %s", + async (dataUri) => { + const f = fixture(); + const updates = [{ videoId: "clip", dataUri }]; + expect(await withVideoFrameSources(f.page, updates, async (resolved) => resolved)).toBe( + updates, + ); + expect(f.createCDPSession).not.toHaveBeenCalled(); + }, + ); + + it("releases success, failure and skipped batches without retaining response bytes", async () => { + const f = fixture(); + for (const failure of [false, true]) { + let url = ""; + const result = withVideoFrameSources(f.page, [update], async (resolved) => { + url = resolved[0]?.dataUri ?? ""; + expect(await f.request(url)).toMatchObject({ responseCode: 200, body: "AAAA" }); + if (failure) throw new Error("decode failed"); + return ["clip"]; + }); + if (failure) await expect(result).rejects.toThrow("decode failed"); + else await expect(result).resolves.toEqual(["clip"]); + expect(await f.request(url)).toMatchObject({ responseCode: 404, body: "" }); + } + let skipped = ""; + await withVideoFrameSources(f.page, [update], async (resolved) => { + skipped = resolved[0]?.dataUri ?? ""; + return []; + }); + expect(await f.request(skipped)).toMatchObject({ responseCode: 404, body: "" }); + expect(f.createCDPSession).toHaveBeenCalledTimes(1); + }); + + it("keeps overlapping batches isolated and initializes the page only once", async () => { + const f = fixture(); + let finish: () => void = () => {}; + const hold = new Promise((resolve) => { + finish = resolve; + }); + let first: VideoFrameUpdate[] = []; + const pending = withVideoFrameSources(f.page, [update], async (resolved) => { + first = resolved; + await hold; + }); + await withVideoFrameSources(f.page, [update], async (resolved) => { + expect(resolved[0]?.dataUri).not.toBe(first[0]?.dataUri); + expect(await f.request(resolved[0]?.dataUri ?? "")).toMatchObject({ responseCode: 200 }); + }); + expect(await f.request(first[0]?.dataUri ?? "")).toMatchObject({ responseCode: 200 }); + finish(); + await pending; + expect(await f.request(first[0]?.dataUri ?? "")).toMatchObject({ responseCode: 404 }); + expect(f.createCDPSession).toHaveBeenCalledTimes(1); + }); + + it("unblocks a failed protocol response and surfaces its cause", async () => { + const f = fixture(); + const error = new Error("protocol response failed"); + f.send.mockImplementation(async (method) => { + if (method === "Fetch.fulfillRequest") throw error; + return {}; + }); + await expect( + withVideoFrameSources(f.page, [update], async (resolved) => { + await f.request(resolved[0]?.dataUri ?? ""); + throw new Error("image decode failed"); + }), + ).rejects.toBe(error); + expect(f.send).toHaveBeenCalledWith("Fetch.failRequest", { + requestId: "request", + errorReason: "Failed", + }); + }); + + it("detaches failed initialization and permits a retry", async () => { + const f = fixture(); + f.send.mockRejectedValueOnce(new Error("cannot intercept")); + await expect(withVideoFrameSources(f.page, [update], async () => {})).rejects.toThrow( + "cannot intercept", + ); + expect(f.detach).toHaveBeenCalledTimes(1); + await withVideoFrameSources(f.page, [update], async () => {}); + expect(f.createCDPSession).toHaveBeenCalledTimes(2); + }); +}); diff --git a/packages/engine/src/services/videoFrameTransport.ts b/packages/engine/src/services/videoFrameTransport.ts new file mode 100644 index 0000000000..f692644c01 --- /dev/null +++ b/packages/engine/src/services/videoFrameTransport.ts @@ -0,0 +1,121 @@ +import { randomUUID } from "node:crypto"; +import { type CDPSession, type Page } from "puppeteer-core"; + +export interface VideoFrameUpdate { + videoId: string; + dataUri: string; + crossOrigin?: "anonymous"; + sourceLabel?: string; +} + +interface FrameResponse { + body: string; + contentType: string; + error?: unknown; +} + +interface FrameTransport { + prefix: string; + nextId: number; + responses: Map; +} + +const base64ImagePrefix = /^data:(image\/[\w.+-]+);base64,/; +const transports = new WeakMap>(); + +async function createTransport(page: Page): Promise { + const pageUrl = new URL(page.url()); + const origin = /^https?:$/.test(pageUrl.protocol) + ? pageUrl.origin + : "https://hyperframes-render.invalid"; + const prefix = `${origin}/__hyperframes_video_frames__/${randomUUID()}/`; + const responses = new Map(); + const client = await page.createCDPSession(); + client.on("Fetch.requestPaused", (event) => { + const frame = responses.get(event.request.url); + void fulfillFrame(client, event.requestId, frame); + }); + try { + await client.send("Fetch.enable", { + patterns: [{ urlPattern: `${prefix}*`, requestStage: "Request" }], + }); + } catch (error) { + await client.detach().catch(() => {}); + throw error; + } + page.once("close", () => responses.clear()); + return { prefix, nextId: 0, responses }; +} + +async function fulfillFrame( + client: CDPSession, + requestId: string, + frame: FrameResponse | undefined, +): Promise { + try { + await client.send("Fetch.fulfillRequest", { + requestId, + responseCode: frame ? 200 : 404, + responseHeaders: [ + { name: "Content-Type", value: frame?.contentType ?? "text/plain" }, + { name: "Cache-Control", value: "no-store" }, + { name: "Access-Control-Allow-Origin", value: "*" }, + ], + body: frame?.body ?? "", + }); + } catch (error) { + if (frame) frame.error = error; + // A failed protocol response must unblock img.decode, including on page closure. + await client.send("Fetch.failRequest", { requestId, errorReason: "Failed" }).catch(() => {}); + } +} + +/** + * Inline image URLs outlive the Node LRU in Chrome's resource cache. Deliver + * their bytes as no-store responses over CDP instead: the page sees short URLs, + * and frame requests never wait for the source videos' HTTP connections. + */ +export async function withVideoFrameSources( + page: Page, + updates: VideoFrameUpdate[], + inject: (resolved: VideoFrameUpdate[]) => Promise, +): Promise { + if (!updates.some((update) => base64ImagePrefix.test(update.dataUri))) return inject(updates); + let pending = transports.get(page); + if (!pending) { + pending = createTransport(page); + transports.set(page, pending); + } + let transport: FrameTransport; + try { + transport = await pending; + } catch (error) { + transports.delete(page); + throw error; + } + const registered: Array<{ url: string; frame: FrameResponse }> = []; + try { + const resolved = updates.map((update) => { + const match = base64ImagePrefix.exec(update.dataUri); + if (!match) return update; + const frame = { + contentType: match[1] ?? "image/png", + body: update.dataUri.slice(match[0].length), + }; + const url = `${transport.prefix}${transport.nextId++}`; + transport.responses.set(url, frame); + registered.push({ url, frame }); + return { + ...update, + dataUri: url, + crossOrigin: "anonymous" as const, + sourceLabel: "inline frame", + }; + }); + return await inject(resolved); + } catch (error) { + throw registered.find(({ frame }) => frame.error !== undefined)?.frame.error ?? error; + } finally { + for (const { url } of registered) transport.responses.delete(url); + } +} diff --git a/packages/producer/src/services/renderOrchestrator.ts b/packages/producer/src/services/renderOrchestrator.ts index fce6e022ab..f24a81e1ae 100644 --- a/packages/producer/src/services/renderOrchestrator.ts +++ b/packages/producer/src/services/renderOrchestrator.ts @@ -3517,36 +3517,11 @@ async function executeRenderPipeline(input: { }); }, }); - // The URL-served frame path (PR #596) hands each injected `` a - // fileServer URL instead of a base64 data URI, on the theory that - // shipping a short URL through `page.evaluate` beats shipping a - // multi-MB base64 string per frame. That holds when the fileServer - // is otherwise idle — but on video-heavy compositions, the same - // fileServer also serves every `