mirror of
https://github.com/heygen-com/hyperframes.git
synced 2026-11-19 19:36:04 +00:00
Merge pull request #3029 from heygen-com/fix-browser-gpu-verify
fix(engine): verify explicit browserGpuMode=hardware instead of trusting it
This commit is contained in:
@@ -241,11 +241,68 @@ describe("resolveBrowserGpuMode", () => {
|
||||
expect(mode).toBe("software");
|
||||
});
|
||||
|
||||
it("passes 'hardware' through unchanged without probing", async () => {
|
||||
it("passes 'hardware' through unchanged", async () => {
|
||||
setMockWebGlProbe({ hasWebGL: true, vendor: "NVIDIA", renderer: "NVIDIA GeForce RTX 3070" });
|
||||
const mode = await resolveBrowserGpuMode("hardware");
|
||||
expect(mode).toBe("hardware");
|
||||
});
|
||||
|
||||
it("warns when explicit 'hardware' probes to software, but still honours it", async () => {
|
||||
// heygen-com/hyperframes#2967: `--browser-gpu` inside a container with no
|
||||
// GPU passthrough rendered 19186 frames on CPU with no diagnostic.
|
||||
setMockWebGlProbe({
|
||||
hasWebGL: true,
|
||||
vendor: "Google Inc. (Google)",
|
||||
renderer: "ANGLE (Google, Vulkan 1.3.0 (SwiftShader Device))",
|
||||
});
|
||||
const warn = vi.spyOn(console, "warn").mockImplementation(() => {});
|
||||
const mode = await resolveBrowserGpuMode("hardware", { platform: "linux" });
|
||||
expect(mode).toBe("hardware");
|
||||
const warning = warn.mock.calls.map((call) => String(call[0])).join("\n");
|
||||
expect(warning).toContain("browserGpuMode=hardware was requested");
|
||||
expect(warning).toContain("--gpus all");
|
||||
// Once per process, not once per worker — the render path resolves the
|
||||
// mode for the probe browser plus every parallel worker.
|
||||
await resolveBrowserGpuMode("hardware", { platform: "linux" });
|
||||
await resolveBrowserGpuMode("hardware", { platform: "linux" });
|
||||
expect(warn).toHaveBeenCalledTimes(1);
|
||||
});
|
||||
|
||||
it("stays quiet when explicit 'hardware' probes to hardware", async () => {
|
||||
setMockWebGlProbe({ hasWebGL: true, vendor: "NVIDIA", renderer: "NVIDIA GeForce RTX 3070" });
|
||||
const warn = vi.spyOn(console, "warn").mockImplementation(() => {});
|
||||
expect(await resolveBrowserGpuMode("hardware")).toBe("hardware");
|
||||
expect(warn).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it("gives non-linux hosts the generic remediation, not the Docker one", async () => {
|
||||
setMockWebGlProbe({
|
||||
hasWebGL: true,
|
||||
vendor: "Google Inc. (Google)",
|
||||
renderer: "ANGLE (Google, Vulkan 1.3.0 (SwiftShader Device))",
|
||||
});
|
||||
const warn = vi.spyOn(console, "warn").mockImplementation(() => {});
|
||||
expect(await resolveBrowserGpuMode("hardware", { platform: "darwin" })).toBe("hardware");
|
||||
const warning = String(warn.mock.calls[0]?.[0]);
|
||||
expect(warning).toContain("host exposes a GPU");
|
||||
expect(warning).not.toContain("--gpus all");
|
||||
});
|
||||
|
||||
it("does not blame the GPU when the probe itself failed to launch", async () => {
|
||||
// A probe that could not run is NO evidence about the GPU. Sending this
|
||||
// operator to `--gpus all` would hide a broken Chrome install behind a
|
||||
// phantom passthrough problem.
|
||||
_setPuppeteerForTests({
|
||||
launch: vi.fn().mockRejectedValue(new Error("spawn ENOENT /bad/chrome")),
|
||||
} as unknown as PuppeteerNode);
|
||||
const warn = vi.spyOn(console, "warn").mockImplementation(() => {});
|
||||
expect(await resolveBrowserGpuMode("hardware", { platform: "linux" })).toBe("hardware");
|
||||
const warning = String(warn.mock.calls[0]?.[0]);
|
||||
expect(warning).toContain("GPU probe could not run");
|
||||
expect(warning).toContain("hyperframes doctor");
|
||||
expect(warning).not.toContain("--gpus all");
|
||||
});
|
||||
|
||||
it("falls back to 'software' when the probe browser cannot launch", async () => {
|
||||
// No chromePath, env unset, and (in the test env) no system Chrome to find
|
||||
// → puppeteer.launch will throw → caller catches → software fallback.
|
||||
@@ -272,31 +329,45 @@ describe("resolveBrowserGpuMode", () => {
|
||||
expect(second).toBe("software");
|
||||
// Reset and re-probe to confirm the test-only reset works.
|
||||
_resetAutoBrowserGpuModeCacheForTests();
|
||||
const third = await resolveBrowserGpuMode("hardware");
|
||||
setMockWebGlProbe({ hasWebGL: true, vendor: "NVIDIA", renderer: "NVIDIA GeForce RTX 3070" });
|
||||
const third = await resolveBrowserGpuMode("auto");
|
||||
expect(third).toBe("hardware");
|
||||
});
|
||||
|
||||
it("deduplicates concurrent auto-mode probes by caching the in-flight Promise", async () => {
|
||||
it("deduplicates concurrent probes so only one Chrome launches", async () => {
|
||||
// Parallel coordinator fires N workers via Promise.all — without Promise-
|
||||
// level caching, a `--workers 4` render against a no-GPU host would launch
|
||||
// 4 simultaneous probe Chromes. Verify all concurrent callers get the
|
||||
// exact same Promise reference (proving the probe runs once, not N times).
|
||||
const p1 = resolveBrowserGpuMode("auto", {
|
||||
chromePath: "/definitely/not/a/real/chrome/binary",
|
||||
browserTimeout: 2000,
|
||||
// 4 simultaneous probe Chromes. Assert the launch count directly rather
|
||||
// than Promise identity: `"auto"` and `"hardware"` now each adapt the
|
||||
// shared cached Promise via `.then`, so identity is no longer the
|
||||
// invariant — "the probe browser starts exactly once" is.
|
||||
const { launch } = setMockWebGlProbe({
|
||||
hasWebGL: true,
|
||||
vendor: "Google Inc. (Google)",
|
||||
renderer: "ANGLE (Google, Vulkan 1.3.0 (SwiftShader Device))",
|
||||
});
|
||||
const p2 = resolveBrowserGpuMode("auto", {
|
||||
chromePath: "/definitely/not/a/real/chrome/binary",
|
||||
browserTimeout: 2000,
|
||||
});
|
||||
const p3 = resolveBrowserGpuMode("auto", {
|
||||
chromePath: "/definitely/not/a/real/chrome/binary",
|
||||
browserTimeout: 2000,
|
||||
});
|
||||
expect(p1).toBe(p2);
|
||||
expect(p2).toBe(p3);
|
||||
const results = await Promise.all([p1, p2, p3]);
|
||||
const results = await Promise.all([
|
||||
resolveBrowserGpuMode("auto", { browserTimeout: 2000 }),
|
||||
resolveBrowserGpuMode("auto", { browserTimeout: 2000 }),
|
||||
resolveBrowserGpuMode("auto", { browserTimeout: 2000 }),
|
||||
]);
|
||||
expect(results).toEqual(["software", "software", "software"]);
|
||||
expect(launch).toHaveBeenCalledTimes(1);
|
||||
});
|
||||
|
||||
it("shares the one probe across mixed 'auto' and 'hardware' callers", async () => {
|
||||
const { launch } = setMockWebGlProbe({
|
||||
hasWebGL: true,
|
||||
vendor: "NVIDIA",
|
||||
renderer: "NVIDIA GeForce RTX 3070",
|
||||
});
|
||||
const results = await Promise.all([
|
||||
resolveBrowserGpuMode("auto"),
|
||||
resolveBrowserGpuMode("hardware"),
|
||||
resolveBrowserGpuMode("auto"),
|
||||
]);
|
||||
expect(results).toEqual(["hardware", "hardware", "hardware"]);
|
||||
expect(launch).toHaveBeenCalledTimes(1);
|
||||
});
|
||||
|
||||
it.each([
|
||||
|
||||
@@ -422,7 +422,25 @@ export const _probeBeginFrameSupportForTests = probeBeginFrameSupport;
|
||||
export const _closeBrowserAfterFailedProbeForTests = closeBrowserAfterFailedProbe;
|
||||
|
||||
/**
|
||||
* Cached *in-flight or resolved* probe Promise for `resolveBrowserGpuMode("auto", ...)`.
|
||||
* Outcome of the one-shot WebGL probe.
|
||||
*
|
||||
* `cause` distinguishes the two ways a probe lands on `"software"`, because
|
||||
* they need OPPOSITE remediation:
|
||||
* - `"no-gpu"` — the probe ran and Chrome reported a software renderer
|
||||
* (SwiftShader / llvmpipe). Remediation: GPU passthrough.
|
||||
* - `"probe-error"` — the probe itself failed (Chrome couldn't launch, bad
|
||||
* executable path, sandbox denied). We have NO evidence
|
||||
* about the GPU either way; telling the operator to fix GPU
|
||||
* passthrough would send them chasing the wrong problem.
|
||||
*/
|
||||
interface GpuProbeOutcome {
|
||||
mode: "software" | "hardware";
|
||||
cause?: "no-gpu" | "probe-error";
|
||||
}
|
||||
|
||||
/**
|
||||
* Cached *in-flight or resolved* probe Promise, shared by BOTH the `"auto"`
|
||||
* and explicit `"hardware"` entry points of `resolveBrowserGpuMode`.
|
||||
*
|
||||
* Caching the Promise (rather than the resolved value) deduplicates concurrent
|
||||
* callers — the parallel coordinator runs N workers via `Promise.all`, so a
|
||||
@@ -430,14 +448,13 @@ export const _closeBrowserAfterFailedProbeForTests = closeBrowserAfterFailedProb
|
||||
* simultaneous probe Chromes. The first call assigns the Promise and every
|
||||
* other concurrent caller awaits the same one, paying the ~240 ms probe cost
|
||||
* exactly once per process lifetime.
|
||||
*
|
||||
* Exported for tests; production callers go through `resolveBrowserGpuMode`.
|
||||
*/
|
||||
let _autoBrowserGpuModeCache: Promise<"software" | "hardware"> | undefined;
|
||||
let _autoBrowserGpuModeCache: Promise<GpuProbeOutcome> | undefined;
|
||||
|
||||
/** Test-only: reset the cached probe result. */
|
||||
export function _resetAutoBrowserGpuModeCacheForTests(): void {
|
||||
_autoBrowserGpuModeCache = undefined;
|
||||
_unverifiedHardwareGpuWarned = false;
|
||||
}
|
||||
|
||||
async function getPuppeteerOrNull(): Promise<PuppeteerNode | null> {
|
||||
@@ -478,7 +495,7 @@ async function probeAutoBrowserGpuMode(options: {
|
||||
chromePath?: string;
|
||||
browserTimeout?: number;
|
||||
platform?: NodeJS.Platform;
|
||||
}): Promise<"software" | "hardware"> {
|
||||
}): Promise<GpuProbeOutcome> {
|
||||
const platform = options.platform ?? process.platform;
|
||||
const browserTimeout = options.browserTimeout ?? DEFAULT_CONFIG.browserTimeout;
|
||||
const executablePath = options.chromePath ?? resolveHeadlessShellPath({});
|
||||
@@ -486,7 +503,7 @@ async function probeAutoBrowserGpuMode(options: {
|
||||
|
||||
if (ppt === null) {
|
||||
logResolvedBrowserGpuMode("software", "puppeteer unavailable");
|
||||
return "software";
|
||||
return { mode: "software", cause: "probe-error" };
|
||||
}
|
||||
|
||||
try {
|
||||
@@ -497,24 +514,35 @@ async function probeAutoBrowserGpuMode(options: {
|
||||
});
|
||||
const resolved = resolveWebGlProbeMode(info);
|
||||
logResolvedBrowserGpuMode(resolved, describeWebGlProbe(info));
|
||||
return resolved;
|
||||
return resolved === "hardware" ? { mode: "hardware" } : { mode: "software", cause: "no-gpu" };
|
||||
} catch (err) {
|
||||
logResolvedBrowserGpuMode("software", formatProbeFailure(err));
|
||||
return "software";
|
||||
return { mode: "software", cause: "probe-error" };
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Resolve `browserGpuMode` to a concrete `"software" | "hardware"` answer.
|
||||
*
|
||||
* For `"software"` / `"hardware"` this is a pure pass-through. For `"auto"`
|
||||
* it launches a tiny Chrome with the platform's hardware GPU args, runs a
|
||||
* one-shot WebGL availability probe, and falls back to `"software"` if
|
||||
* hardware-mode WebGL is unavailable. The Promise is cached for the process
|
||||
* lifetime, so concurrent callers (parallel workers) share the same probe.
|
||||
* For `"software"` this is a pure pass-through. For `"auto"` it launches a
|
||||
* tiny Chrome with the platform's hardware GPU args, runs a one-shot WebGL
|
||||
* availability probe, and falls back to `"software"` if hardware-mode WebGL
|
||||
* is unavailable. The Promise is cached for the process lifetime, so
|
||||
* concurrent callers (parallel workers) share the same probe.
|
||||
*
|
||||
* Any failure (Chrome launch error, navigation timeout, missing canvas API,
|
||||
* etc.) is treated as a `"software"` fallback. The render path with
|
||||
* `"hardware"` (an explicit `--browser-gpu` / `PRODUCER_BROWSER_GPU_MODE=
|
||||
* hardware`) is honoured verbatim — the operator asked for it — but runs the
|
||||
* SAME probe to VERIFY it, because Chrome's hardware GL args are advisory:
|
||||
* with no usable GPU in the sandbox (no `/dev/dri`, no NVIDIA container
|
||||
* runtime, missing EGL/driver libraries) Chrome silently falls back to
|
||||
* software WebGL and the render just runs at CPU speed. Without this check
|
||||
* the only trace is a buried `Automatic fallback to software WebGL` browser
|
||||
* warning — heygen-com/hyperframes#2967 rendered 19186 frames on CPU while
|
||||
* `--browser-gpu` was set and nothing said so. The probe result never
|
||||
* changes the returned mode; it only makes the fallback loud.
|
||||
*
|
||||
* Any probe failure (Chrome launch error, navigation timeout, missing canvas
|
||||
* API, etc.) is treated as a `"software"` result. The render path with
|
||||
* SwiftShader always works, so a misclassification toward software is the
|
||||
* safe failure mode; misclassifying toward hardware would error on the real
|
||||
* render.
|
||||
@@ -527,21 +555,84 @@ export function resolveBrowserGpuMode(
|
||||
platform?: NodeJS.Platform;
|
||||
} = {},
|
||||
): Promise<"software" | "hardware"> {
|
||||
if (mode !== "auto") return Promise.resolve(mode);
|
||||
if (_autoBrowserGpuModeCache) return _autoBrowserGpuModeCache;
|
||||
if (mode === "software") return Promise.resolve(mode);
|
||||
|
||||
_autoBrowserGpuModeCache = probeAutoBrowserGpuMode(options);
|
||||
return _autoBrowserGpuModeCache;
|
||||
_autoBrowserGpuModeCache ??= probeAutoBrowserGpuMode(options);
|
||||
if (mode === "auto") return _autoBrowserGpuModeCache.then((probed) => probed.mode);
|
||||
|
||||
return _autoBrowserGpuModeCache.then((probed) => {
|
||||
// Warn once per cache lifetime, not once per caller: `createCaptureSession`
|
||||
// resolves the mode for the probe browser AND every parallel worker, so
|
||||
// an un-deduplicated warning prints N+1 times and buries itself.
|
||||
if (probed.mode === "software" && !_unverifiedHardwareGpuWarned) {
|
||||
_unverifiedHardwareGpuWarned = true;
|
||||
console.warn(
|
||||
buildUnverifiedHardwareGpuWarning(options.platform ?? process.platform, probed.cause),
|
||||
);
|
||||
}
|
||||
return "hardware";
|
||||
});
|
||||
}
|
||||
|
||||
/**
|
||||
* Single observability surface for the auto-detect outcome. Logged exactly
|
||||
* Latch for the explicit-hardware-probed-to-software warning: fires once per
|
||||
* cache lifetime (re-armed by `_resetAutoBrowserGpuModeCacheForTests`).
|
||||
*/
|
||||
let _unverifiedHardwareGpuWarned = false;
|
||||
|
||||
/**
|
||||
* Warning text for "you asked for hardware GPU and we could not confirm it".
|
||||
*
|
||||
* Splits on `cause` because the two failure shapes need opposite remediation.
|
||||
* A probe that RAN and saw SwiftShader is a GPU-passthrough problem. A probe
|
||||
* that could not run tells us nothing about the GPU — pointing that operator
|
||||
* at `--gpus all` would send them chasing a phantom while their Chrome
|
||||
* install is the actual fault.
|
||||
*/
|
||||
function buildUnverifiedHardwareGpuWarning(
|
||||
platform: NodeJS.Platform | string,
|
||||
cause: GpuProbeOutcome["cause"],
|
||||
): string {
|
||||
if (cause === "probe-error") {
|
||||
return (
|
||||
"[hyperframes] browserGpuMode=hardware was requested, but the GPU probe could not run, " +
|
||||
"so hardware acceleration is UNVERIFIED — if Chrome falls back to software WebGL the " +
|
||||
"capture will run at CPU speed. Honouring the explicit request anyway.\n" +
|
||||
" This is a probe failure, not evidence of a missing GPU: see the " +
|
||||
"`browserGpuMode probe → software (probe failed ...)` line above for the underlying " +
|
||||
"error, which usually means Chrome could not launch (bad HYPERFRAMES_BROWSER_PATH, " +
|
||||
"missing shared libraries, or a denied sandbox) rather than a GPU problem.\n" +
|
||||
" Run `hyperframes doctor` to check the Chrome install."
|
||||
);
|
||||
}
|
||||
const remediation =
|
||||
platform === "linux"
|
||||
? "Inside Docker, the container needs GPU passthrough: `--gpus all` with the NVIDIA " +
|
||||
"Container Toolkit installed, or `--device /dev/dri` for Mesa/AMD/Intel. The image " +
|
||||
"also needs the matching userspace driver + libEGL. Verify with " +
|
||||
"`hyperframes render --browser-gpu` and watch for this warning disappearing."
|
||||
: "Check that the host exposes a GPU to this process and that the graphics drivers are " +
|
||||
"installed.";
|
||||
return (
|
||||
"[hyperframes] browserGpuMode=hardware was requested, but the WebGL probe found no " +
|
||||
"hardware GPU — Chrome will silently fall back to software WebGL and the capture will " +
|
||||
"run at CPU speed. Honouring the explicit request anyway.\n" +
|
||||
` ${remediation}\n` +
|
||||
" Pass --no-browser-gpu to select deterministic SwiftShader instead of waiting on a " +
|
||||
"hardware path that is not there."
|
||||
);
|
||||
}
|
||||
|
||||
/**
|
||||
* Single observability surface for the GPU probe outcome. Logged exactly
|
||||
* once per process (the probe runs once); without this line, a regression
|
||||
* to "always software even with a GPU present" would be invisible in
|
||||
* production. Goes to stderr to stay out of stdout pipelines.
|
||||
* production. Goes to stderr to stay out of stdout pipelines. Says "probe"
|
||||
* rather than "auto" because explicit `browserGpuMode=hardware` runs the
|
||||
* same probe to verify itself.
|
||||
*/
|
||||
function logResolvedBrowserGpuMode(resolved: "hardware" | "software", reason: string): void {
|
||||
console.error(`[hyperframes] browserGpuMode auto → ${resolved} (${reason})`);
|
||||
console.error(`[hyperframes] browserGpuMode probe → ${resolved} (${reason})`);
|
||||
}
|
||||
|
||||
function createBrowserLaunchFingerprint(
|
||||
|
||||
Reference in New Issue
Block a user