mirror of
https://github.com/heygen-com/hyperframes.git
synced 2026-11-15 22:08:03 +00:00
fix(engine): render workers stop loading the videos whose frames come from FFmpeg (#5795)
* fix(engine): render workers stop loading the videos whose frames come from FFmpeg * fix(engine): keep the media of a skipped video no poster sizes
This commit is contained in:
@@ -0,0 +1,18 @@
|
|||||||
|
import type { Server } from "node:http";
|
||||||
|
import puppeteer, { type Browser } from "puppeteer-core";
|
||||||
|
|
||||||
|
/** Listens on a free loopback port and returns the server's origin. */
|
||||||
|
export async function listenOnLoopback(server: Server): Promise<string> {
|
||||||
|
await new Promise<void>((resolve) => server.listen(0, "127.0.0.1", resolve));
|
||||||
|
const address = server.address();
|
||||||
|
if (!address || typeof address === "string") throw new Error("No test server address");
|
||||||
|
return `http://127.0.0.1:${address.port}`;
|
||||||
|
}
|
||||||
|
|
||||||
|
export function launchTestChrome(executablePath: string | undefined): Promise<Browser> {
|
||||||
|
return puppeteer.launch({
|
||||||
|
executablePath,
|
||||||
|
headless: true,
|
||||||
|
args: ["--no-sandbox", "--disable-dev-shm-usage"],
|
||||||
|
});
|
||||||
|
}
|
||||||
@@ -0,0 +1,91 @@
|
|||||||
|
import { spawnSync } from "node:child_process";
|
||||||
|
import { createServer, type ServerResponse } from "node:http";
|
||||||
|
import { afterAll, beforeAll, describe, expect, it } from "vitest";
|
||||||
|
import type { Browser } from "puppeteer-core";
|
||||||
|
import { resolveHeadlessShellPath } from "./browserManager.js";
|
||||||
|
import {
|
||||||
|
applyVideoMetadataHints,
|
||||||
|
releasableVideoIds,
|
||||||
|
releaseInjectedVideoMedia,
|
||||||
|
} from "./frameCapture.js";
|
||||||
|
import { listenOnLoopback, launchTestChrome } from "./__fixtures__/loopback.js";
|
||||||
|
|
||||||
|
const chromePath = resolveHeadlessShellPath();
|
||||||
|
const hasBrowser = chromePath !== undefined && spawnSync(chromePath, ["--version"]).status === 0;
|
||||||
|
|
||||||
|
describe("releasableVideoIds", () => {
|
||||||
|
it("keeps the media of a skipped video no poster sizes, such as a native HDR one", () => {
|
||||||
|
expect(
|
||||||
|
releasableVideoIds({
|
||||||
|
skipReadinessVideoIds: ["hdr", "sdr"],
|
||||||
|
videoMetadataHints: [{ id: "sdr", width: 640, height: 360 }],
|
||||||
|
} as Parameters<typeof releasableVideoIds>[0]),
|
||||||
|
).toEqual(["sdr"]);
|
||||||
|
});
|
||||||
|
});
|
||||||
|
|
||||||
|
describe.skipIf(!hasBrowser)("releaseInjectedVideoMedia in Chrome", () => {
|
||||||
|
const openMedia = new Set<ServerResponse>();
|
||||||
|
let origin = "";
|
||||||
|
let browser: Browser;
|
||||||
|
const server = createServer((req, res) => {
|
||||||
|
if (req.url?.startsWith("/media/")) {
|
||||||
|
res.writeHead(200, { "Content-Type": "video/mp4" });
|
||||||
|
res.write(Buffer.alloc(1024));
|
||||||
|
openMedia.add(res);
|
||||||
|
res.on("close", () => openMedia.delete(res));
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
const videos = ["a", "b", "c", "live"].map(
|
||||||
|
(id) =>
|
||||||
|
`<video id="${id}" preload="auto" src="/media/${id}.mp4" style="width:200px"></video>`,
|
||||||
|
);
|
||||||
|
res.writeHead(200, { "Content-Type": "text/html" });
|
||||||
|
res.end(`<html><body style="margin:0">${videos.join("")}</body></html>`);
|
||||||
|
});
|
||||||
|
|
||||||
|
beforeAll(async () => {
|
||||||
|
origin = await listenOnLoopback(server);
|
||||||
|
browser = await launchTestChrome(chromePath);
|
||||||
|
});
|
||||||
|
afterAll(async () => {
|
||||||
|
for (const res of openMedia) res.destroy();
|
||||||
|
await browser?.close();
|
||||||
|
await new Promise<void>((resolve) => server.close(() => resolve()));
|
||||||
|
});
|
||||||
|
|
||||||
|
it("drops the media of videos whose frames come from FFmpeg and keeps their layout", async () => {
|
||||||
|
const page = await browser.newPage();
|
||||||
|
await page.goto(`${origin}/`);
|
||||||
|
const deadline = Date.now() + 10_000;
|
||||||
|
while (openMedia.size < 4 && Date.now() < deadline) {
|
||||||
|
await new Promise((resolve) => setTimeout(resolve, 50));
|
||||||
|
}
|
||||||
|
expect(openMedia.size).toBe(4);
|
||||||
|
const injected = ["a", "b", "c"];
|
||||||
|
await applyVideoMetadataHints(
|
||||||
|
page,
|
||||||
|
injected.map((id) => ({ id, width: 640, height: 360 })),
|
||||||
|
);
|
||||||
|
|
||||||
|
await releaseInjectedVideoMedia(page, injected);
|
||||||
|
|
||||||
|
const released = await page.evaluate((ids) => {
|
||||||
|
return ids.map((id) => {
|
||||||
|
const video = document.getElementById(id) as HTMLVideoElement;
|
||||||
|
const box = video.getBoundingClientRect();
|
||||||
|
return `${id}:${video.networkState}:${box.width}x${box.height}`;
|
||||||
|
});
|
||||||
|
}, injected);
|
||||||
|
expect(released).toEqual(["a:0:200x112.5", "b:0:200x112.5", "c:0:200x112.5"]);
|
||||||
|
const releaseDeadline = Date.now() + 10_000;
|
||||||
|
while (openMedia.size > 1 && Date.now() < releaseDeadline) {
|
||||||
|
await new Promise((resolve) => setTimeout(resolve, 50));
|
||||||
|
}
|
||||||
|
expect(openMedia.size).toBe(1);
|
||||||
|
expect(await page.$eval("#live", (video) => (video as HTMLVideoElement).currentSrc)).toBe(
|
||||||
|
`${origin}/media/live.mp4`,
|
||||||
|
);
|
||||||
|
await page.close();
|
||||||
|
}, 30_000);
|
||||||
|
});
|
||||||
@@ -2217,6 +2217,33 @@ async function decodeAllImages(page: Page): Promise<void> {
|
|||||||
});
|
});
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/** Out-of-band videos a poster now sizes, so dropping their media cannot move layout. */
|
||||||
|
export function releasableVideoIds(options: CaptureOptions): string[] {
|
||||||
|
const hinted = new Set((options.videoMetadataHints ?? []).map((hint) => hint.id));
|
||||||
|
return (options.skipReadinessVideoIds ?? []).filter((id) => hinted.has(id));
|
||||||
|
}
|
||||||
|
|
||||||
|
/** FFmpeg supplies these videos' frames, so their own players would only hold decoders and buffers in every worker. */
|
||||||
|
export async function releaseInjectedVideoMedia(
|
||||||
|
page: Page,
|
||||||
|
videoIds: readonly string[],
|
||||||
|
): Promise<void> {
|
||||||
|
if (videoIds.length === 0) return;
|
||||||
|
await page.evaluate(
|
||||||
|
(ids: string[]) => {
|
||||||
|
for (const id of ids) {
|
||||||
|
const video = window.__hfMediaEl?.(id) ?? document.getElementById(id);
|
||||||
|
if (!(video instanceof HTMLVideoElement)) continue;
|
||||||
|
video.pause();
|
||||||
|
for (const source of video.querySelectorAll("source")) source.remove();
|
||||||
|
video.removeAttribute("src");
|
||||||
|
video.load();
|
||||||
|
}
|
||||||
|
},
|
||||||
|
[...videoIds],
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
export async function applyVideoMetadataHints(
|
export async function applyVideoMetadataHints(
|
||||||
page: Page,
|
page: Page,
|
||||||
hints: readonly CaptureVideoMetadataHint[] | undefined,
|
hints: readonly CaptureVideoMetadataHint[] | undefined,
|
||||||
@@ -2505,6 +2532,7 @@ export async function initializeSession(session: CaptureSession): Promise<void>
|
|||||||
recordSubTimelineWarning(session, pageReadyTimeout);
|
recordSubTimelineWarning(session, pageReadyTimeout);
|
||||||
|
|
||||||
await applyVideoMetadataHints(page, session.options.videoMetadataHints);
|
await applyVideoMetadataHints(page, session.options.videoMetadataHints);
|
||||||
|
await releaseInjectedVideoMedia(page, releasableVideoIds(session.options));
|
||||||
logInitPhase("applyVideoMetadataHints complete");
|
logInitPhase("applyVideoMetadataHints complete");
|
||||||
|
|
||||||
// Run independent readiness checks in parallel — videos, images, fonts,
|
// Run independent readiness checks in parallel — videos, images, fonts,
|
||||||
@@ -2661,6 +2689,7 @@ export async function initializeSession(session: CaptureSession): Promise<void>
|
|||||||
recordSubTimelineWarning(session, pageReadyTimeout);
|
recordSubTimelineWarning(session, pageReadyTimeout);
|
||||||
|
|
||||||
await applyVideoMetadataHints(page, session.options.videoMetadataHints);
|
await applyVideoMetadataHints(page, session.options.videoMetadataHints);
|
||||||
|
await releaseInjectedVideoMedia(page, releasableVideoIds(session.options));
|
||||||
logInitPhase("applyVideoMetadataHints complete");
|
logInitPhase("applyVideoMetadataHints complete");
|
||||||
|
|
||||||
// Run independent readiness checks in parallel — videos, images, fonts,
|
// Run independent readiness checks in parallel — videos, images, fonts,
|
||||||
@@ -3455,6 +3484,7 @@ export async function createStaticVerificationPage(session: CaptureSession): Pro
|
|||||||
knownUnregisteredIds: session.subTimelineWaitMemo.unregisteredIds,
|
knownUnregisteredIds: session.subTimelineWaitMemo.unregisteredIds,
|
||||||
});
|
});
|
||||||
await applyVideoMetadataHints(page, session.options.videoMetadataHints);
|
await applyVideoMetadataHints(page, session.options.videoMetadataHints);
|
||||||
|
await releaseInjectedVideoMedia(page, releasableVideoIds(session.options));
|
||||||
|
|
||||||
const skipVideoIds = session.options.skipReadinessVideoIds ?? [];
|
const skipVideoIds = session.options.skipReadinessVideoIds ?? [];
|
||||||
await Promise.all([
|
await Promise.all([
|
||||||
|
|||||||
@@ -3,6 +3,7 @@ import { createServer, type ServerResponse } from "node:http";
|
|||||||
import { afterAll, beforeAll, describe, expect, it } from "vitest";
|
import { afterAll, beforeAll, describe, expect, it } from "vitest";
|
||||||
import { acquireBrowser, drainBrowserPool, resolveHeadlessShellPath } from "./browserManager.js";
|
import { acquireBrowser, drainBrowserPool, resolveHeadlessShellPath } from "./browserManager.js";
|
||||||
import { shouldDisableBrowserPoolForParallelWorker } from "./parallelCoordinator.js";
|
import { shouldDisableBrowserPoolForParallelWorker } from "./parallelCoordinator.js";
|
||||||
|
import { listenOnLoopback } from "./__fixtures__/loopback.js";
|
||||||
|
|
||||||
const chromePath = resolveHeadlessShellPath();
|
const chromePath = resolveHeadlessShellPath();
|
||||||
const hasBrowser = chromePath !== undefined && spawnSync(chromePath, ["--version"]).status === 0;
|
const hasBrowser = chromePath !== undefined && spawnSync(chromePath, ["--version"]).status === 0;
|
||||||
@@ -35,10 +36,7 @@ describe.skipIf(!hasBrowser)("parallel worker browsers", () => {
|
|||||||
});
|
});
|
||||||
|
|
||||||
beforeAll(async () => {
|
beforeAll(async () => {
|
||||||
await new Promise<void>((resolve) => server.listen(0, "127.0.0.1", resolve));
|
origin = await listenOnLoopback(server);
|
||||||
const address = server.address();
|
|
||||||
if (!address || typeof address === "string") throw new Error("No test server address");
|
|
||||||
origin = `http://127.0.0.1:${address.port}`;
|
|
||||||
});
|
});
|
||||||
|
|
||||||
afterAll(async () => {
|
afterAll(async () => {
|
||||||
|
|||||||
@@ -1,9 +1,10 @@
|
|||||||
import { spawnSync } from "node:child_process";
|
import { spawnSync } from "node:child_process";
|
||||||
import { createServer } from "node:http";
|
import { createServer } from "node:http";
|
||||||
import { afterAll, beforeAll, beforeEach, describe, expect, it } from "vitest";
|
import { afterAll, beforeAll, beforeEach, describe, expect, it } from "vitest";
|
||||||
import puppeteer, { type Browser, type Page } from "puppeteer-core";
|
import type { Browser, Page } from "puppeteer-core";
|
||||||
import { resolveHeadlessShellPath } from "./browserManager.js";
|
import { resolveHeadlessShellPath } from "./browserManager.js";
|
||||||
import { injectVideoFramesBatch } from "./screenshotService.js";
|
import { injectVideoFramesBatch } from "./screenshotService.js";
|
||||||
|
import { listenOnLoopback, launchTestChrome } from "./__fixtures__/loopback.js";
|
||||||
|
|
||||||
const chromePath = resolveHeadlessShellPath();
|
const chromePath = resolveHeadlessShellPath();
|
||||||
const hasBrowser = chromePath !== undefined && spawnSync(chromePath, ["--version"]).status === 0;
|
const hasBrowser = chromePath !== undefined && spawnSync(chromePath, ["--version"]).status === 0;
|
||||||
@@ -30,15 +31,8 @@ describe.skipIf(!hasBrowser)("video frame transport in Chrome", () => {
|
|||||||
});
|
});
|
||||||
|
|
||||||
beforeAll(async () => {
|
beforeAll(async () => {
|
||||||
await new Promise<void>((resolve) => server.listen(0, "127.0.0.1", resolve));
|
origin = await listenOnLoopback(server);
|
||||||
const address = server.address();
|
browser = await launchTestChrome(chromePath);
|
||||||
if (!address || typeof address === "string") throw new Error("No test server address");
|
|
||||||
origin = `http://127.0.0.1:${address.port}`;
|
|
||||||
browser = await puppeteer.launch({
|
|
||||||
executablePath: chromePath,
|
|
||||||
headless: true,
|
|
||||||
args: ["--no-sandbox", "--disable-dev-shm-usage"],
|
|
||||||
});
|
|
||||||
});
|
});
|
||||||
beforeEach(async () => {
|
beforeEach(async () => {
|
||||||
if (page) await page.close();
|
if (page) await page.close();
|
||||||
|
|||||||
Reference in New Issue
Block a user