mirror of
https://github.com/heygen-com/hyperframes.git
synced 2026-10-30 09:16:55 +00:00
fix(engine): bound injected video frame resource retention (#5494)
* fix(engine): bound injected video frame resource retention * perf(engine): bound capture inspection response buffers * chore(engine): condense capture buffer explanation * test(engine): cover cross-origin frame GPU uploads
This commit is contained in:
Miguel Ángel
@@ -3,7 +3,7 @@ import { afterEach, describe, expect, it, vi } from "vitest";
|
||||
import { mkdtempSync, rmSync } from "node:fs";
|
||||
import { tmpdir } from "node:os";
|
||||
import { join } from "node:path";
|
||||
import type { Browser, Page, PuppeteerNode } from "puppeteer-core";
|
||||
import { CDPSession, type Browser, type Page, type PuppeteerNode } from "puppeteer-core";
|
||||
import {
|
||||
_resetBrowserPoolForTests,
|
||||
_setPuppeteerForTests,
|
||||
@@ -18,154 +18,78 @@ describe("createCaptureSession construction ownership", () => {
|
||||
_setPuppeteerForTests(undefined);
|
||||
});
|
||||
|
||||
it("closes the page and releases its exact browser lease when bootstrap fails", async () => {
|
||||
const outputDir = mkdtempSync(join(tmpdir(), "hf-session-owner-"));
|
||||
const page = {
|
||||
evaluateOnNewDocument: vi.fn().mockRejectedValue(new Error("bootstrap failed")),
|
||||
close: vi.fn().mockResolvedValue(undefined),
|
||||
} as unknown as Page;
|
||||
const browser = {
|
||||
connected: true,
|
||||
newPage: vi.fn().mockResolvedValue(page),
|
||||
version: vi.fn().mockResolvedValue("HeadlessChrome/150.0.0.0"),
|
||||
close: vi.fn().mockResolvedValue(undefined),
|
||||
disconnect: vi.fn(),
|
||||
process: () => null,
|
||||
} as unknown as Browser;
|
||||
_setPuppeteerForTests({
|
||||
launch: vi.fn().mockResolvedValue(browser),
|
||||
} as unknown as PuppeteerNode);
|
||||
it.each([
|
||||
{ stage: "network setup", blocked: "none" },
|
||||
{ stage: "bootstrap", blocked: "none" },
|
||||
{ stage: "bootstrap", blocked: "page" },
|
||||
{ stage: "bootstrap", blocked: "browser" },
|
||||
])(
|
||||
"releases the exact lease after $stage failure with $blocked close blocked",
|
||||
async ({ stage, blocked }) => {
|
||||
if (blocked !== "none") vi.useFakeTimers();
|
||||
const outputDir = mkdtempSync(join(tmpdir(), "hf-session-owner-"));
|
||||
const send = vi.fn().mockResolvedValue(undefined);
|
||||
if (stage === "network setup") send.mockRejectedValue(new Error("network setup failed"));
|
||||
const closeResult = (resource: string) =>
|
||||
blocked === resource ? new Promise<void>(() => {}) : Promise.resolve();
|
||||
const page = {
|
||||
mainFrame: () => ({ client: Object.setPrototypeOf({ send }, CDPSession.prototype) }),
|
||||
evaluateOnNewDocument: vi.fn().mockRejectedValue(new Error("bootstrap failed")),
|
||||
close: vi.fn(() => closeResult("page")),
|
||||
} as unknown as Page;
|
||||
const browser = {
|
||||
connected: true,
|
||||
newPage: vi.fn().mockResolvedValue(page),
|
||||
version: vi.fn().mockResolvedValue("HeadlessChrome/150.0.0.0"),
|
||||
close: vi.fn(() => closeResult("browser")),
|
||||
disconnect: vi.fn(),
|
||||
process: () => null,
|
||||
} as unknown as Browser;
|
||||
_setPuppeteerForTests({
|
||||
launch: vi.fn().mockResolvedValue(browser),
|
||||
} as unknown as PuppeteerNode);
|
||||
|
||||
try {
|
||||
await expect(
|
||||
createCaptureSession(
|
||||
"http://127.0.0.1:3000",
|
||||
outputDir,
|
||||
{
|
||||
width: 320,
|
||||
height: 180,
|
||||
fps: { num: 30, den: 1 },
|
||||
format: "jpeg",
|
||||
requiresWebGpu: false,
|
||||
},
|
||||
null,
|
||||
{
|
||||
browserGpuMode: "software",
|
||||
enableBrowserPool: true,
|
||||
forceScreenshot: true,
|
||||
},
|
||||
),
|
||||
).rejects.toThrow("bootstrap failed");
|
||||
try {
|
||||
const creating = expect(
|
||||
createCaptureSession(
|
||||
"http://127.0.0.1:3000",
|
||||
outputDir,
|
||||
{
|
||||
width: 320,
|
||||
height: 180,
|
||||
fps: { num: 30, den: 1 },
|
||||
format: "jpeg",
|
||||
requiresWebGpu: false,
|
||||
},
|
||||
null,
|
||||
{
|
||||
browserGpuMode: "software",
|
||||
enableBrowserPool: true,
|
||||
forceScreenshot: true,
|
||||
},
|
||||
),
|
||||
).rejects.toThrow(`${stage} failed`);
|
||||
if (blocked !== "none") await vi.runAllTimersAsync();
|
||||
await creating;
|
||||
|
||||
expect(page.close).toHaveBeenCalledTimes(1);
|
||||
expect(browser.close).toHaveBeenCalledTimes(1);
|
||||
} finally {
|
||||
rmSync(outputDir, { recursive: true, force: true });
|
||||
}
|
||||
});
|
||||
|
||||
it("force-releases its browser lease when rollback page close never settles", async () => {
|
||||
vi.useFakeTimers();
|
||||
const outputDir = mkdtempSync(join(tmpdir(), "hf-session-owner-timeout-"));
|
||||
const page = {
|
||||
evaluateOnNewDocument: vi.fn().mockRejectedValue(new Error("bootstrap failed")),
|
||||
close: vi.fn().mockReturnValue(new Promise<void>(() => {})),
|
||||
} as unknown as Page;
|
||||
const disconnect = vi.fn();
|
||||
const browser = {
|
||||
connected: true,
|
||||
newPage: vi.fn().mockResolvedValue(page),
|
||||
version: vi.fn().mockResolvedValue("HeadlessChrome/150.0.0.0"),
|
||||
close: vi.fn().mockResolvedValue(undefined),
|
||||
disconnect,
|
||||
process: () => null,
|
||||
} as unknown as Browser;
|
||||
_setPuppeteerForTests({
|
||||
launch: vi.fn().mockResolvedValue(browser),
|
||||
} as unknown as PuppeteerNode);
|
||||
|
||||
try {
|
||||
const creating = expect(
|
||||
createCaptureSession(
|
||||
"http://127.0.0.1:3000",
|
||||
outputDir,
|
||||
{
|
||||
width: 320,
|
||||
height: 180,
|
||||
fps: { num: 30, den: 1 },
|
||||
format: "jpeg",
|
||||
requiresWebGpu: false,
|
||||
},
|
||||
null,
|
||||
{
|
||||
browserGpuMode: "software",
|
||||
enableBrowserPool: true,
|
||||
forceScreenshot: true,
|
||||
},
|
||||
),
|
||||
).rejects.toThrow("bootstrap failed");
|
||||
|
||||
await vi.runAllTimersAsync();
|
||||
await creating;
|
||||
|
||||
expect(page.close).toHaveBeenCalledTimes(1);
|
||||
expect(disconnect).toHaveBeenCalledTimes(1);
|
||||
} finally {
|
||||
vi.useRealTimers();
|
||||
rmSync(outputDir, { recursive: true, force: true });
|
||||
}
|
||||
});
|
||||
|
||||
it("force-releases its browser lease when rollback browser close never settles", async () => {
|
||||
vi.useFakeTimers();
|
||||
const outputDir = mkdtempSync(join(tmpdir(), "hf-session-browser-timeout-"));
|
||||
const page = {
|
||||
evaluateOnNewDocument: vi.fn().mockRejectedValue(new Error("bootstrap failed")),
|
||||
close: vi.fn().mockResolvedValue(undefined),
|
||||
} as unknown as Page;
|
||||
const disconnect = vi.fn();
|
||||
const browser = {
|
||||
connected: true,
|
||||
newPage: vi.fn().mockResolvedValue(page),
|
||||
version: vi.fn().mockResolvedValue("HeadlessChrome/150.0.0.0"),
|
||||
close: vi.fn().mockReturnValue(new Promise<void>(() => {})),
|
||||
disconnect,
|
||||
process: () => null,
|
||||
} as unknown as Browser;
|
||||
_setPuppeteerForTests({
|
||||
launch: vi.fn().mockResolvedValue(browser),
|
||||
} as unknown as PuppeteerNode);
|
||||
|
||||
try {
|
||||
const creating = expect(
|
||||
createCaptureSession(
|
||||
"http://127.0.0.1:3000",
|
||||
outputDir,
|
||||
{
|
||||
width: 320,
|
||||
height: 180,
|
||||
fps: { num: 30, den: 1 },
|
||||
format: "jpeg",
|
||||
requiresWebGpu: false,
|
||||
},
|
||||
null,
|
||||
{
|
||||
browserGpuMode: "software",
|
||||
enableBrowserPool: true,
|
||||
forceScreenshot: true,
|
||||
},
|
||||
),
|
||||
).rejects.toThrow("bootstrap failed");
|
||||
|
||||
await vi.runAllTimersAsync();
|
||||
await creating;
|
||||
|
||||
expect(page.close).toHaveBeenCalledTimes(1);
|
||||
expect(browser.close).toHaveBeenCalledTimes(1);
|
||||
expect(disconnect).toHaveBeenCalledTimes(1);
|
||||
} finally {
|
||||
vi.useRealTimers();
|
||||
rmSync(outputDir, { recursive: true, force: true });
|
||||
}
|
||||
});
|
||||
expect(send).toHaveBeenCalledWith("Network.enable", {
|
||||
maxTotalBufferSize: 32 * 1024 * 1024,
|
||||
maxResourceBufferSize: 16 * 1024 * 1024,
|
||||
});
|
||||
if (stage === "network setup") {
|
||||
expect(page.evaluateOnNewDocument).not.toHaveBeenCalled();
|
||||
} else {
|
||||
expect(send.mock.invocationCallOrder[0]).toBeLessThan(
|
||||
vi.mocked(page.evaluateOnNewDocument).mock.invocationCallOrder[0] ?? 0,
|
||||
);
|
||||
}
|
||||
expect(page.close).toHaveBeenCalledTimes(1);
|
||||
if (blocked !== "page") expect(browser.close).toHaveBeenCalledTimes(1);
|
||||
if (blocked !== "none") expect(browser.disconnect).toHaveBeenCalledTimes(1);
|
||||
} finally {
|
||||
vi.useRealTimers();
|
||||
rmSync(outputDir, { recursive: true, force: true });
|
||||
}
|
||||
},
|
||||
);
|
||||
});
|
||||
|
||||
@@ -25,6 +25,7 @@ vi.mock("./browserManager.js", async (importOriginal) => ({
|
||||
browser: {
|
||||
version: async () => "Chrome/150.0.0.0",
|
||||
newPage: async () => ({
|
||||
mainFrame: () => ({}),
|
||||
evaluateOnNewDocument: async () => {},
|
||||
setViewport: async () => {},
|
||||
evaluate: async () => undefined,
|
||||
|
||||
@@ -21,6 +21,7 @@ vi.mock("./browserManager.js", async (importOriginal) => ({
|
||||
browser: {
|
||||
version: async () => "Chrome/150.0.0.0",
|
||||
newPage: async () => ({
|
||||
mainFrame: () => ({}),
|
||||
on: () => {},
|
||||
goto: async () => null,
|
||||
createCDPSession: async () => ({ on: () => {}, send: async () => ({}) }),
|
||||
|
||||
@@ -9,6 +9,7 @@
|
||||
*/
|
||||
|
||||
import {
|
||||
CDPSession,
|
||||
type Browser,
|
||||
type Page,
|
||||
type Protocol,
|
||||
@@ -1414,6 +1415,15 @@ async function constructCaptureSession(
|
||||
|
||||
const page = await browser.newPage();
|
||||
onPageCreated(page);
|
||||
// Cap the existing session's inspection buffer; new sessions have separate buffers.
|
||||
const frame = page.mainFrame();
|
||||
// Frame.client is runtime-only, so keep capture working when it is absent.
|
||||
if ("client" in frame && frame.client instanceof CDPSession) {
|
||||
await frame.client.send("Network.enable", {
|
||||
maxTotalBufferSize: 32 * 1024 * 1024,
|
||||
maxResourceBufferSize: 16 * 1024 * 1024,
|
||||
});
|
||||
}
|
||||
// Polyfill esbuild's keepNames helper inside the page.
|
||||
//
|
||||
// The engine is published as raw TypeScript (`packages/engine/package.json`
|
||||
|
||||
@@ -17,6 +17,15 @@ import {
|
||||
DOM_LAYER_MASK_STYLE_ID,
|
||||
} from "./screenshotService.js";
|
||||
|
||||
// These DOM tests isolate layout/decode from the CDP transport, exercised in Chrome separately.
|
||||
vi.mock("./videoFrameTransport.js", () => ({
|
||||
withVideoFrameSources: async (
|
||||
_page: Page,
|
||||
updates: Array<{ videoId: string; dataUri: string }>,
|
||||
inject: (resolved: Array<{ videoId: string; dataUri: string }>) => Promise<string[]>,
|
||||
) => inject(updates),
|
||||
}));
|
||||
|
||||
// Stub a Page + CDPSession just enough that pageScreenshotCapture can call
|
||||
// `client.send("Page.captureScreenshot", ...)` and we can inspect the args.
|
||||
function makeFakePageWithCdp(send: (method: string, params: object) => Promise<{ data: string }>) {
|
||||
|
||||
@@ -7,6 +7,7 @@
|
||||
|
||||
// fallow-ignore-file code-duplication
|
||||
import { type Page } from "puppeteer-core";
|
||||
import { withVideoFrameSources, type VideoFrameUpdate } from "./videoFrameTransport.js";
|
||||
import { type CaptureOptions } from "../types.js";
|
||||
import { COLOR_GRADING_SOURCE_HIDDEN_ATTR } from "@hyperframes/core/color-grading";
|
||||
import {
|
||||
@@ -643,10 +644,19 @@ export async function injectVideoFramesBatch(
|
||||
updates: Array<{ videoId: string; dataUri: string }>,
|
||||
): Promise<string[]> {
|
||||
if (updates.length === 0) return [];
|
||||
return withVideoFrameSources(page, updates, (resolved) =>
|
||||
injectResolvedVideoFramesBatch(page, resolved),
|
||||
);
|
||||
}
|
||||
|
||||
async function injectResolvedVideoFramesBatch(
|
||||
page: Page,
|
||||
updates: VideoFrameUpdate[],
|
||||
): Promise<string[]> {
|
||||
return await page.evaluate(
|
||||
// fallow-ignore-next-line complexity
|
||||
async (
|
||||
items: Array<{ videoId: string; dataUri: string; frameId: string }>,
|
||||
items: Array<VideoFrameUpdate & { frameId: string }>,
|
||||
visualProperties: string[],
|
||||
colorGradingSourceHiddenAttr: string,
|
||||
) => {
|
||||
@@ -795,8 +805,11 @@ export async function injectVideoFramesBatch(
|
||||
|
||||
img.decoding = "sync";
|
||||
if (img.getAttribute("src") !== item.dataUri) {
|
||||
if (item.crossOrigin) img.crossOrigin = item.crossOrigin;
|
||||
else img.removeAttribute("crossorigin");
|
||||
img.src = item.dataUri;
|
||||
const source = item.dataUri.startsWith("data:") ? "inline frame" : item.dataUri;
|
||||
const source =
|
||||
item.sourceLabel ?? (item.dataUri.startsWith("data:") ? "inline frame" : item.dataUri);
|
||||
pendingDecodes.push(
|
||||
img.decode().catch((error: unknown) => {
|
||||
throw new Error(
|
||||
|
||||
@@ -0,0 +1,162 @@
|
||||
import { spawnSync } from "node:child_process";
|
||||
import { createServer } from "node:http";
|
||||
import { afterAll, beforeAll, beforeEach, describe, expect, it } from "vitest";
|
||||
import puppeteer, { type Browser, type Page } from "puppeteer-core";
|
||||
import { resolveHeadlessShellPath } from "./browserManager.js";
|
||||
import { injectVideoFramesBatch } from "./screenshotService.js";
|
||||
|
||||
const chromePath = resolveHeadlessShellPath();
|
||||
const hasBrowser = chromePath !== undefined && spawnSync(chromePath, ["--version"]).status === 0;
|
||||
|
||||
describe.skipIf(!hasBrowser)("video frame transport in Chrome", () => {
|
||||
let browser: Browser;
|
||||
let page: Page;
|
||||
let origin: string;
|
||||
let redPng: string;
|
||||
let blueJpeg: string;
|
||||
let unexpectedFrameRequests = 0;
|
||||
const server = createServer((req, res) => {
|
||||
if (req.url?.startsWith("/slow")) return;
|
||||
if (req.url === "/external.png") {
|
||||
res.writeHead(200, { "Content-Type": "image/png", "Cache-Control": "max-age=600" });
|
||||
res.end(Buffer.from(redPng.split(",")[1] ?? "", "base64"));
|
||||
return;
|
||||
}
|
||||
if (req.url?.startsWith("/__hyperframes_video_frames__/")) unexpectedFrameRequests++;
|
||||
res.writeHead(200, { "Content-Type": "text/html" });
|
||||
res.end(
|
||||
'<html><body><div style="position:relative;width:4px;height:4px"><video id="a" style="width:4px;height:4px"></video><video id="b" style="width:4px;height:4px"></video></div></body></html>',
|
||||
);
|
||||
});
|
||||
|
||||
beforeAll(async () => {
|
||||
await new Promise<void>((resolve) => server.listen(0, "127.0.0.1", resolve));
|
||||
const address = server.address();
|
||||
if (!address || typeof address === "string") throw new Error("No test server address");
|
||||
origin = `http://127.0.0.1:${address.port}`;
|
||||
browser = await puppeteer.launch({
|
||||
executablePath: chromePath,
|
||||
headless: true,
|
||||
args: ["--no-sandbox", "--disable-dev-shm-usage"],
|
||||
});
|
||||
});
|
||||
beforeEach(async () => {
|
||||
if (page) await page.close();
|
||||
page = await browser.newPage();
|
||||
await page.goto(origin);
|
||||
[redPng, blueJpeg] = await Promise.all([
|
||||
frameDataUri("red"),
|
||||
frameDataUri("blue", "image/jpeg"),
|
||||
]);
|
||||
unexpectedFrameRequests = 0;
|
||||
});
|
||||
afterAll(async () => {
|
||||
await browser?.close();
|
||||
server.closeAllConnections();
|
||||
await new Promise<void>((resolve) => server.close(() => resolve()));
|
||||
});
|
||||
|
||||
const frameDataUri = (color: string, format = "image/png") =>
|
||||
page.evaluate(
|
||||
(fill, mime) => {
|
||||
const canvas = document.createElement("canvas");
|
||||
canvas.width = canvas.height = 4;
|
||||
const ctx = canvas.getContext("2d");
|
||||
if (!ctx) throw new Error("No 2D context");
|
||||
ctx.fillStyle = fill;
|
||||
ctx.fillRect(0, 0, 4, 4);
|
||||
return canvas.toDataURL(mime, 1);
|
||||
},
|
||||
color,
|
||||
format,
|
||||
);
|
||||
|
||||
const pixels = () =>
|
||||
page.evaluate(() => {
|
||||
const canvas = new OffscreenCanvas(4, 4);
|
||||
const ctx = canvas.getContext("2d");
|
||||
if (!ctx) throw new Error("No 2D context");
|
||||
return Array.from(
|
||||
document.querySelectorAll<HTMLImageElement>("img.__render_frame__"),
|
||||
(img) => {
|
||||
ctx.clearRect(0, 0, 4, 4);
|
||||
ctx.drawImage(img, 0, 0);
|
||||
return { src: img.src, rgba: Array.from(ctx.getImageData(0, 0, 1, 1).data) };
|
||||
},
|
||||
);
|
||||
});
|
||||
|
||||
it("delivers independent decoded frames without retaining inline image URLs", async () => {
|
||||
const headers: string[] = [];
|
||||
page.on("response", (response) => {
|
||||
if (response.request().resourceType() === "image")
|
||||
headers.push(response.headers()["cache-control"] ?? "");
|
||||
});
|
||||
await injectVideoFramesBatch(page, [
|
||||
{ videoId: "a", dataUri: redPng },
|
||||
{ videoId: "b", dataUri: blueJpeg },
|
||||
]);
|
||||
const first = await pixels();
|
||||
expect(first.map((img) => img.rgba)).toEqual([
|
||||
[255, 0, 0, 255],
|
||||
[0, 0, 254, 255],
|
||||
]);
|
||||
expect(first.every((img) => img.src.startsWith(origin))).toBe(true);
|
||||
expect(headers).toEqual(["no-store", "no-store"]);
|
||||
await injectVideoFramesBatch(page, [{ videoId: "a", dataUri: blueJpeg }]);
|
||||
expect((await pixels())[0]?.rgba).toEqual([0, 0, 254, 255]);
|
||||
expect(await page.evaluate(async (url) => (await fetch(url)).status, first[0]?.src ?? "")).toBe(
|
||||
404,
|
||||
);
|
||||
expect(unexpectedFrameRequests).toBe(0);
|
||||
});
|
||||
|
||||
it("loads frames even while all same-origin download connections are busy", async () => {
|
||||
await page.evaluate(() => {
|
||||
for (let i = 0; i < 6; i++) void fetch(`/slow${i}.mp4`);
|
||||
});
|
||||
await injectVideoFramesBatch(page, [{ videoId: "a", dataUri: redPng }]);
|
||||
expect((await pixels())[0]?.rgba).toEqual([255, 0, 0, 255]);
|
||||
}, 5000);
|
||||
|
||||
it.each(["HTTP", "about:blank"])(
|
||||
"preserves PNG alpha and GPU uploads from %s",
|
||||
async (source) => {
|
||||
if (source === "about:blank") {
|
||||
const content = await page.content();
|
||||
await page.close();
|
||||
page = await browser.newPage();
|
||||
await page.setContent(content);
|
||||
expect(page.url()).toBe("about:blank");
|
||||
}
|
||||
const translucent = await frameDataUri("rgba(255,0,0,0.5)");
|
||||
await injectVideoFramesBatch(page, [{ videoId: "a", dataUri: translucent }]);
|
||||
expect((await pixels())[0]?.rgba).toEqual([255, 0, 0, 128]);
|
||||
expect(
|
||||
await page.evaluate(() => {
|
||||
const canvas = document.createElement("canvas");
|
||||
const gl = canvas.getContext("webgl");
|
||||
const img = document.querySelector("img");
|
||||
if (!gl || !img) throw new Error("No GPU texture source");
|
||||
gl.bindTexture(gl.TEXTURE_2D, gl.createTexture());
|
||||
gl.texImage2D(gl.TEXTURE_2D, 0, gl.RGBA, gl.RGBA, gl.UNSIGNED_BYTE, img);
|
||||
return gl.getError();
|
||||
}),
|
||||
).toBe(0);
|
||||
},
|
||||
);
|
||||
|
||||
it("releases failed frames and preserves the video-specific decode error", async () => {
|
||||
await expect(
|
||||
injectVideoFramesBatch(page, [{ videoId: "a", dataUri: "data:image/png;base64,AAAA" }]),
|
||||
).rejects.toThrow('Video frame for "a" failed to load (inline frame)');
|
||||
const src = await page.evaluate(() => document.querySelector("img")?.src ?? "");
|
||||
expect(await page.evaluate(async (url) => (await fetch(url)).status, src)).toBe(404);
|
||||
});
|
||||
|
||||
it("keeps caller-provided frame URLs unchanged", async () => {
|
||||
const src = `${origin}/external.png`;
|
||||
await injectVideoFramesBatch(page, [{ videoId: "a", dataUri: src }]);
|
||||
expect((await pixels())[0]).toEqual({ src, rgba: [255, 0, 0, 255] });
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,115 @@
|
||||
import { EventEmitter } from "node:events";
|
||||
import { describe, expect, it, vi } from "vitest";
|
||||
import { type Page } from "puppeteer-core";
|
||||
import { withVideoFrameSources, type VideoFrameUpdate } from "./videoFrameTransport.js";
|
||||
|
||||
const update = { videoId: "clip", dataUri: "data:image/png;base64,AAAA" };
|
||||
|
||||
function fixture() {
|
||||
const events = new EventEmitter();
|
||||
const send = vi.fn(async (_method: string, _params: unknown) => ({}));
|
||||
const detach = vi.fn(async () => {});
|
||||
const close = new EventEmitter();
|
||||
const createCDPSession = vi.fn(async () => ({ on: events.on.bind(events), send, detach }));
|
||||
// The transport only needs these Page methods; no browser protocol is simulated elsewhere.
|
||||
const page = {
|
||||
url: () => "http://localhost:4000/index.html",
|
||||
createCDPSession,
|
||||
once: close.once.bind(close),
|
||||
} as unknown as Page;
|
||||
const request = async (url: string) => {
|
||||
events.emit("Fetch.requestPaused", { requestId: "request", request: { url } });
|
||||
await new Promise<void>((resolve) => setImmediate(resolve));
|
||||
return send.mock.calls.findLast(([method]) => method === "Fetch.fulfillRequest")?.[1];
|
||||
};
|
||||
return { page, send, detach, createCDPSession, close, request };
|
||||
}
|
||||
|
||||
describe("video frame transport ownership", () => {
|
||||
it.each(["/frames/0001.png", "data:frame-test", "data:image/png,not-base64"])(
|
||||
"does not intercept an unsupported source %s",
|
||||
async (dataUri) => {
|
||||
const f = fixture();
|
||||
const updates = [{ videoId: "clip", dataUri }];
|
||||
expect(await withVideoFrameSources(f.page, updates, async (resolved) => resolved)).toBe(
|
||||
updates,
|
||||
);
|
||||
expect(f.createCDPSession).not.toHaveBeenCalled();
|
||||
},
|
||||
);
|
||||
|
||||
it("releases success, failure and skipped batches without retaining response bytes", async () => {
|
||||
const f = fixture();
|
||||
for (const failure of [false, true]) {
|
||||
let url = "";
|
||||
const result = withVideoFrameSources(f.page, [update], async (resolved) => {
|
||||
url = resolved[0]?.dataUri ?? "";
|
||||
expect(await f.request(url)).toMatchObject({ responseCode: 200, body: "AAAA" });
|
||||
if (failure) throw new Error("decode failed");
|
||||
return ["clip"];
|
||||
});
|
||||
if (failure) await expect(result).rejects.toThrow("decode failed");
|
||||
else await expect(result).resolves.toEqual(["clip"]);
|
||||
expect(await f.request(url)).toMatchObject({ responseCode: 404, body: "" });
|
||||
}
|
||||
let skipped = "";
|
||||
await withVideoFrameSources(f.page, [update], async (resolved) => {
|
||||
skipped = resolved[0]?.dataUri ?? "";
|
||||
return [];
|
||||
});
|
||||
expect(await f.request(skipped)).toMatchObject({ responseCode: 404, body: "" });
|
||||
expect(f.createCDPSession).toHaveBeenCalledTimes(1);
|
||||
});
|
||||
|
||||
it("keeps overlapping batches isolated and initializes the page only once", async () => {
|
||||
const f = fixture();
|
||||
let finish: () => void = () => {};
|
||||
const hold = new Promise<void>((resolve) => {
|
||||
finish = resolve;
|
||||
});
|
||||
let first: VideoFrameUpdate[] = [];
|
||||
const pending = withVideoFrameSources(f.page, [update], async (resolved) => {
|
||||
first = resolved;
|
||||
await hold;
|
||||
});
|
||||
await withVideoFrameSources(f.page, [update], async (resolved) => {
|
||||
expect(resolved[0]?.dataUri).not.toBe(first[0]?.dataUri);
|
||||
expect(await f.request(resolved[0]?.dataUri ?? "")).toMatchObject({ responseCode: 200 });
|
||||
});
|
||||
expect(await f.request(first[0]?.dataUri ?? "")).toMatchObject({ responseCode: 200 });
|
||||
finish();
|
||||
await pending;
|
||||
expect(await f.request(first[0]?.dataUri ?? "")).toMatchObject({ responseCode: 404 });
|
||||
expect(f.createCDPSession).toHaveBeenCalledTimes(1);
|
||||
});
|
||||
|
||||
it("unblocks a failed protocol response and surfaces its cause", async () => {
|
||||
const f = fixture();
|
||||
const error = new Error("protocol response failed");
|
||||
f.send.mockImplementation(async (method) => {
|
||||
if (method === "Fetch.fulfillRequest") throw error;
|
||||
return {};
|
||||
});
|
||||
await expect(
|
||||
withVideoFrameSources(f.page, [update], async (resolved) => {
|
||||
await f.request(resolved[0]?.dataUri ?? "");
|
||||
throw new Error("image decode failed");
|
||||
}),
|
||||
).rejects.toBe(error);
|
||||
expect(f.send).toHaveBeenCalledWith("Fetch.failRequest", {
|
||||
requestId: "request",
|
||||
errorReason: "Failed",
|
||||
});
|
||||
});
|
||||
|
||||
it("detaches failed initialization and permits a retry", async () => {
|
||||
const f = fixture();
|
||||
f.send.mockRejectedValueOnce(new Error("cannot intercept"));
|
||||
await expect(withVideoFrameSources(f.page, [update], async () => {})).rejects.toThrow(
|
||||
"cannot intercept",
|
||||
);
|
||||
expect(f.detach).toHaveBeenCalledTimes(1);
|
||||
await withVideoFrameSources(f.page, [update], async () => {});
|
||||
expect(f.createCDPSession).toHaveBeenCalledTimes(2);
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,121 @@
|
||||
import { randomUUID } from "node:crypto";
|
||||
import { type CDPSession, type Page } from "puppeteer-core";
|
||||
|
||||
export interface VideoFrameUpdate {
|
||||
videoId: string;
|
||||
dataUri: string;
|
||||
crossOrigin?: "anonymous";
|
||||
sourceLabel?: string;
|
||||
}
|
||||
|
||||
interface FrameResponse {
|
||||
body: string;
|
||||
contentType: string;
|
||||
error?: unknown;
|
||||
}
|
||||
|
||||
interface FrameTransport {
|
||||
prefix: string;
|
||||
nextId: number;
|
||||
responses: Map<string, FrameResponse>;
|
||||
}
|
||||
|
||||
const base64ImagePrefix = /^data:(image\/[\w.+-]+);base64,/;
|
||||
const transports = new WeakMap<Page, Promise<FrameTransport>>();
|
||||
|
||||
async function createTransport(page: Page): Promise<FrameTransport> {
|
||||
const pageUrl = new URL(page.url());
|
||||
const origin = /^https?:$/.test(pageUrl.protocol)
|
||||
? pageUrl.origin
|
||||
: "https://hyperframes-render.invalid";
|
||||
const prefix = `${origin}/__hyperframes_video_frames__/${randomUUID()}/`;
|
||||
const responses = new Map<string, FrameResponse>();
|
||||
const client = await page.createCDPSession();
|
||||
client.on("Fetch.requestPaused", (event) => {
|
||||
const frame = responses.get(event.request.url);
|
||||
void fulfillFrame(client, event.requestId, frame);
|
||||
});
|
||||
try {
|
||||
await client.send("Fetch.enable", {
|
||||
patterns: [{ urlPattern: `${prefix}*`, requestStage: "Request" }],
|
||||
});
|
||||
} catch (error) {
|
||||
await client.detach().catch(() => {});
|
||||
throw error;
|
||||
}
|
||||
page.once("close", () => responses.clear());
|
||||
return { prefix, nextId: 0, responses };
|
||||
}
|
||||
|
||||
async function fulfillFrame(
|
||||
client: CDPSession,
|
||||
requestId: string,
|
||||
frame: FrameResponse | undefined,
|
||||
): Promise<void> {
|
||||
try {
|
||||
await client.send("Fetch.fulfillRequest", {
|
||||
requestId,
|
||||
responseCode: frame ? 200 : 404,
|
||||
responseHeaders: [
|
||||
{ name: "Content-Type", value: frame?.contentType ?? "text/plain" },
|
||||
{ name: "Cache-Control", value: "no-store" },
|
||||
{ name: "Access-Control-Allow-Origin", value: "*" },
|
||||
],
|
||||
body: frame?.body ?? "",
|
||||
});
|
||||
} catch (error) {
|
||||
if (frame) frame.error = error;
|
||||
// A failed protocol response must unblock img.decode, including on page closure.
|
||||
await client.send("Fetch.failRequest", { requestId, errorReason: "Failed" }).catch(() => {});
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Inline image URLs outlive the Node LRU in Chrome's resource cache. Deliver
|
||||
* their bytes as no-store responses over CDP instead: the page sees short URLs,
|
||||
* and frame requests never wait for the source videos' HTTP connections.
|
||||
*/
|
||||
export async function withVideoFrameSources<T>(
|
||||
page: Page,
|
||||
updates: VideoFrameUpdate[],
|
||||
inject: (resolved: VideoFrameUpdate[]) => Promise<T>,
|
||||
): Promise<T> {
|
||||
if (!updates.some((update) => base64ImagePrefix.test(update.dataUri))) return inject(updates);
|
||||
let pending = transports.get(page);
|
||||
if (!pending) {
|
||||
pending = createTransport(page);
|
||||
transports.set(page, pending);
|
||||
}
|
||||
let transport: FrameTransport;
|
||||
try {
|
||||
transport = await pending;
|
||||
} catch (error) {
|
||||
transports.delete(page);
|
||||
throw error;
|
||||
}
|
||||
const registered: Array<{ url: string; frame: FrameResponse }> = [];
|
||||
try {
|
||||
const resolved = updates.map((update) => {
|
||||
const match = base64ImagePrefix.exec(update.dataUri);
|
||||
if (!match) return update;
|
||||
const frame = {
|
||||
contentType: match[1] ?? "image/png",
|
||||
body: update.dataUri.slice(match[0].length),
|
||||
};
|
||||
const url = `${transport.prefix}${transport.nextId++}`;
|
||||
transport.responses.set(url, frame);
|
||||
registered.push({ url, frame });
|
||||
return {
|
||||
...update,
|
||||
dataUri: url,
|
||||
crossOrigin: "anonymous" as const,
|
||||
sourceLabel: "inline frame",
|
||||
};
|
||||
});
|
||||
return await inject(resolved);
|
||||
} catch (error) {
|
||||
throw registered.find(({ frame }) => frame.error !== undefined)?.frame.error ?? error;
|
||||
} finally {
|
||||
for (const { url } of registered) transport.responses.delete(url);
|
||||
}
|
||||
}
|
||||
@@ -3517,36 +3517,11 @@ async function executeRenderPipeline(input: {
|
||||
});
|
||||
},
|
||||
});
|
||||
// The URL-served frame path (PR #596) hands each injected `<img>` a
|
||||
// fileServer URL instead of a base64 data URI, on the theory that
|
||||
// shipping a short URL through `page.evaluate` beats shipping a
|
||||
// multi-MB base64 string per frame. That holds when the fileServer
|
||||
// is otherwise idle — but on video-heavy compositions, the same
|
||||
// fileServer also serves every `<video>.src`. The runtime's
|
||||
// drift-recovery branch (`runtime/media.ts:294-302`) issues
|
||||
// `el.load()` on the underlying `<video>` during seeks, kicking off
|
||||
// full-file downloads that occupy the fileServer's single Node
|
||||
// event loop (it uses `readFileSync` and offers no `Accept-Ranges`).
|
||||
// The injector's `<img>.decode()` then queues behind those video
|
||||
// fetches and is never serviced before puppeteer's protocol timeout
|
||||
// fires (`Runtime.callFunctionOn timed out`).
|
||||
//
|
||||
// Repro: synth 30 × 32 MB videos / 90 s comp on an 8-core / 30 GB
|
||||
// host = 537 s wall (broken corpus) / 428 s (corpus-fixed), every
|
||||
// render fails. Disabling the resolver (force base64-inline) gives
|
||||
// 1:59 (119 s) wall and a clean MP4 on the same comp, with no
|
||||
// regression on the 30 × 1.6 MB control corpus (137 s vs 135 s
|
||||
// baseline).
|
||||
//
|
||||
// Until this is properly gated (e.g. only enable URL-served when the
|
||||
// page has zero fileServer-bound `<video>.src` traffic), the inline
|
||||
// path is the safe default. The cache memory ceiling
|
||||
// (`frameDataUriCacheBytesLimitMb`, default 1500 MB above 8 GB
|
||||
// hosts) already bounds the cost. `createCompiledFrameSrcResolver`
|
||||
// and the `frameSrcResolver` option remain in their respective
|
||||
// modules (`packages/producer/src/services/render/shared.ts`,
|
||||
// `packages/engine/src/services/videoFrameInjector.ts`); the gating
|
||||
// PR will re-import the builder here.
|
||||
// The engine delivers cached frame bytes as no-store CDP responses.
|
||||
// Frame requests bypass the file server's source-video downloads, which
|
||||
// can occupy all HTTP connections during native-video drift recovery.
|
||||
// The byte ceiling bounds the Node LRU; Chrome receives short URLs so
|
||||
// its resource cache does not retain the corresponding inline strings.
|
||||
const createRenderVideoFrameInjector = (): BeforeCaptureHook | null =>
|
||||
createVideoFrameInjector(frameLookup, {
|
||||
frameDataUriCacheLimit: cfg.frameDataUriCacheLimit,
|
||||
|
||||
Reference in New Issue
Block a user