Merge pull request #3029 from heygen-com/fix-browser-gpu-verify

fix(engine): verify explicit browserGpuMode=hardware instead of trusting it
This commit is contained in:
Miguel Ángel
2026-08-04 13:19:51 -07:00
committed by GitHub
2 changed files with 203 additions and 41 deletions
@@ -241,11 +241,68 @@ describe("resolveBrowserGpuMode", () => {
expect(mode).toBe("software");
});
it("passes 'hardware' through unchanged without probing", async () => {
it("passes 'hardware' through unchanged", async () => {
setMockWebGlProbe({ hasWebGL: true, vendor: "NVIDIA", renderer: "NVIDIA GeForce RTX 3070" });
const mode = await resolveBrowserGpuMode("hardware");
expect(mode).toBe("hardware");
});
it("warns when explicit 'hardware' probes to software, but still honours it", async () => {
// heygen-com/hyperframes#2967: `--browser-gpu` inside a container with no
// GPU passthrough rendered 19186 frames on CPU with no diagnostic.
setMockWebGlProbe({
hasWebGL: true,
vendor: "Google Inc. (Google)",
renderer: "ANGLE (Google, Vulkan 1.3.0 (SwiftShader Device))",
});
const warn = vi.spyOn(console, "warn").mockImplementation(() => {});
const mode = await resolveBrowserGpuMode("hardware", { platform: "linux" });
expect(mode).toBe("hardware");
const warning = warn.mock.calls.map((call) => String(call[0])).join("\n");
expect(warning).toContain("browserGpuMode=hardware was requested");
expect(warning).toContain("--gpus all");
// Once per process, not once per worker — the render path resolves the
// mode for the probe browser plus every parallel worker.
await resolveBrowserGpuMode("hardware", { platform: "linux" });
await resolveBrowserGpuMode("hardware", { platform: "linux" });
expect(warn).toHaveBeenCalledTimes(1);
});
it("stays quiet when explicit 'hardware' probes to hardware", async () => {
setMockWebGlProbe({ hasWebGL: true, vendor: "NVIDIA", renderer: "NVIDIA GeForce RTX 3070" });
const warn = vi.spyOn(console, "warn").mockImplementation(() => {});
expect(await resolveBrowserGpuMode("hardware")).toBe("hardware");
expect(warn).not.toHaveBeenCalled();
});
it("gives non-linux hosts the generic remediation, not the Docker one", async () => {
setMockWebGlProbe({
hasWebGL: true,
vendor: "Google Inc. (Google)",
renderer: "ANGLE (Google, Vulkan 1.3.0 (SwiftShader Device))",
});
const warn = vi.spyOn(console, "warn").mockImplementation(() => {});
expect(await resolveBrowserGpuMode("hardware", { platform: "darwin" })).toBe("hardware");
const warning = String(warn.mock.calls[0]?.[0]);
expect(warning).toContain("host exposes a GPU");
expect(warning).not.toContain("--gpus all");
});
it("does not blame the GPU when the probe itself failed to launch", async () => {
// A probe that could not run is NO evidence about the GPU. Sending this
// operator to `--gpus all` would hide a broken Chrome install behind a
// phantom passthrough problem.
_setPuppeteerForTests({
launch: vi.fn().mockRejectedValue(new Error("spawn ENOENT /bad/chrome")),
} as unknown as PuppeteerNode);
const warn = vi.spyOn(console, "warn").mockImplementation(() => {});
expect(await resolveBrowserGpuMode("hardware", { platform: "linux" })).toBe("hardware");
const warning = String(warn.mock.calls[0]?.[0]);
expect(warning).toContain("GPU probe could not run");
expect(warning).toContain("hyperframes doctor");
expect(warning).not.toContain("--gpus all");
});
it("falls back to 'software' when the probe browser cannot launch", async () => {
// No chromePath, env unset, and (in the test env) no system Chrome to find
// → puppeteer.launch will throw → caller catches → software fallback.
@@ -272,31 +329,45 @@ describe("resolveBrowserGpuMode", () => {
expect(second).toBe("software");
// Reset and re-probe to confirm the test-only reset works.
_resetAutoBrowserGpuModeCacheForTests();
const third = await resolveBrowserGpuMode("hardware");
setMockWebGlProbe({ hasWebGL: true, vendor: "NVIDIA", renderer: "NVIDIA GeForce RTX 3070" });
const third = await resolveBrowserGpuMode("auto");
expect(third).toBe("hardware");
});
it("deduplicates concurrent auto-mode probes by caching the in-flight Promise", async () => {
it("deduplicates concurrent probes so only one Chrome launches", async () => {
// Parallel coordinator fires N workers via Promise.all — without Promise-
// level caching, a `--workers 4` render against a no-GPU host would launch
// 4 simultaneous probe Chromes. Verify all concurrent callers get the
// exact same Promise reference (proving the probe runs once, not N times).
const p1 = resolveBrowserGpuMode("auto", {
chromePath: "/definitely/not/a/real/chrome/binary",
browserTimeout: 2000,
// 4 simultaneous probe Chromes. Assert the launch count directly rather
// than Promise identity: `"auto"` and `"hardware"` now each adapt the
// shared cached Promise via `.then`, so identity is no longer the
// invariant — "the probe browser starts exactly once" is.
const { launch } = setMockWebGlProbe({
hasWebGL: true,
vendor: "Google Inc. (Google)",
renderer: "ANGLE (Google, Vulkan 1.3.0 (SwiftShader Device))",
});
const p2 = resolveBrowserGpuMode("auto", {
chromePath: "/definitely/not/a/real/chrome/binary",
browserTimeout: 2000,
});
const p3 = resolveBrowserGpuMode("auto", {
chromePath: "/definitely/not/a/real/chrome/binary",
browserTimeout: 2000,
});
expect(p1).toBe(p2);
expect(p2).toBe(p3);
const results = await Promise.all([p1, p2, p3]);
const results = await Promise.all([
resolveBrowserGpuMode("auto", { browserTimeout: 2000 }),
resolveBrowserGpuMode("auto", { browserTimeout: 2000 }),
resolveBrowserGpuMode("auto", { browserTimeout: 2000 }),
]);
expect(results).toEqual(["software", "software", "software"]);
expect(launch).toHaveBeenCalledTimes(1);
});
it("shares the one probe across mixed 'auto' and 'hardware' callers", async () => {
const { launch } = setMockWebGlProbe({
hasWebGL: true,
vendor: "NVIDIA",
renderer: "NVIDIA GeForce RTX 3070",
});
const results = await Promise.all([
resolveBrowserGpuMode("auto"),
resolveBrowserGpuMode("hardware"),
resolveBrowserGpuMode("auto"),
]);
expect(results).toEqual(["hardware", "hardware", "hardware"]);
expect(launch).toHaveBeenCalledTimes(1);
});
it.each([
+113 -22
View File
@@ -422,7 +422,25 @@ export const _probeBeginFrameSupportForTests = probeBeginFrameSupport;
export const _closeBrowserAfterFailedProbeForTests = closeBrowserAfterFailedProbe;
/**
* Cached *in-flight or resolved* probe Promise for `resolveBrowserGpuMode("auto", ...)`.
* Outcome of the one-shot WebGL probe.
*
* `cause` distinguishes the two ways a probe lands on `"software"`, because
* they need OPPOSITE remediation:
* - `"no-gpu"` — the probe ran and Chrome reported a software renderer
* (SwiftShader / llvmpipe). Remediation: GPU passthrough.
* - `"probe-error"` — the probe itself failed (Chrome couldn't launch, bad
* executable path, sandbox denied). We have NO evidence
* about the GPU either way; telling the operator to fix GPU
* passthrough would send them chasing the wrong problem.
*/
interface GpuProbeOutcome {
mode: "software" | "hardware";
cause?: "no-gpu" | "probe-error";
}
/**
* Cached *in-flight or resolved* probe Promise, shared by BOTH the `"auto"`
* and explicit `"hardware"` entry points of `resolveBrowserGpuMode`.
*
* Caching the Promise (rather than the resolved value) deduplicates concurrent
* callers — the parallel coordinator runs N workers via `Promise.all`, so a
@@ -430,14 +448,13 @@ export const _closeBrowserAfterFailedProbeForTests = closeBrowserAfterFailedProb
* simultaneous probe Chromes. The first call assigns the Promise and every
* other concurrent caller awaits the same one, paying the ~240 ms probe cost
* exactly once per process lifetime.
*
* Exported for tests; production callers go through `resolveBrowserGpuMode`.
*/
let _autoBrowserGpuModeCache: Promise<"software" | "hardware"> | undefined;
let _autoBrowserGpuModeCache: Promise<GpuProbeOutcome> | undefined;
/** Test-only: reset the cached probe result. */
export function _resetAutoBrowserGpuModeCacheForTests(): void {
_autoBrowserGpuModeCache = undefined;
_unverifiedHardwareGpuWarned = false;
}
async function getPuppeteerOrNull(): Promise<PuppeteerNode | null> {
@@ -478,7 +495,7 @@ async function probeAutoBrowserGpuMode(options: {
chromePath?: string;
browserTimeout?: number;
platform?: NodeJS.Platform;
}): Promise<"software" | "hardware"> {
}): Promise<GpuProbeOutcome> {
const platform = options.platform ?? process.platform;
const browserTimeout = options.browserTimeout ?? DEFAULT_CONFIG.browserTimeout;
const executablePath = options.chromePath ?? resolveHeadlessShellPath({});
@@ -486,7 +503,7 @@ async function probeAutoBrowserGpuMode(options: {
if (ppt === null) {
logResolvedBrowserGpuMode("software", "puppeteer unavailable");
return "software";
return { mode: "software", cause: "probe-error" };
}
try {
@@ -497,24 +514,35 @@ async function probeAutoBrowserGpuMode(options: {
});
const resolved = resolveWebGlProbeMode(info);
logResolvedBrowserGpuMode(resolved, describeWebGlProbe(info));
return resolved;
return resolved === "hardware" ? { mode: "hardware" } : { mode: "software", cause: "no-gpu" };
} catch (err) {
logResolvedBrowserGpuMode("software", formatProbeFailure(err));
return "software";
return { mode: "software", cause: "probe-error" };
}
}
/**
* Resolve `browserGpuMode` to a concrete `"software" | "hardware"` answer.
*
* For `"software"` / `"hardware"` this is a pure pass-through. For `"auto"`
* it launches a tiny Chrome with the platform's hardware GPU args, runs a
* one-shot WebGL availability probe, and falls back to `"software"` if
* hardware-mode WebGL is unavailable. The Promise is cached for the process
* lifetime, so concurrent callers (parallel workers) share the same probe.
* For `"software"` this is a pure pass-through. For `"auto"` it launches a
* tiny Chrome with the platform's hardware GPU args, runs a one-shot WebGL
* availability probe, and falls back to `"software"` if hardware-mode WebGL
* is unavailable. The Promise is cached for the process lifetime, so
* concurrent callers (parallel workers) share the same probe.
*
* Any failure (Chrome launch error, navigation timeout, missing canvas API,
* etc.) is treated as a `"software"` fallback. The render path with
* `"hardware"` (an explicit `--browser-gpu` / `PRODUCER_BROWSER_GPU_MODE=
* hardware`) is honoured verbatim — the operator asked for it — but runs the
* SAME probe to VERIFY it, because Chrome's hardware GL args are advisory:
* with no usable GPU in the sandbox (no `/dev/dri`, no NVIDIA container
* runtime, missing EGL/driver libraries) Chrome silently falls back to
* software WebGL and the render just runs at CPU speed. Without this check
* the only trace is a buried `Automatic fallback to software WebGL` browser
* warning — heygen-com/hyperframes#2967 rendered 19186 frames on CPU while
* `--browser-gpu` was set and nothing said so. The probe result never
* changes the returned mode; it only makes the fallback loud.
*
* Any probe failure (Chrome launch error, navigation timeout, missing canvas
* API, etc.) is treated as a `"software"` result. The render path with
* SwiftShader always works, so a misclassification toward software is the
* safe failure mode; misclassifying toward hardware would error on the real
* render.
@@ -527,21 +555,84 @@ export function resolveBrowserGpuMode(
platform?: NodeJS.Platform;
} = {},
): Promise<"software" | "hardware"> {
if (mode !== "auto") return Promise.resolve(mode);
if (_autoBrowserGpuModeCache) return _autoBrowserGpuModeCache;
if (mode === "software") return Promise.resolve(mode);
_autoBrowserGpuModeCache = probeAutoBrowserGpuMode(options);
return _autoBrowserGpuModeCache;
_autoBrowserGpuModeCache ??= probeAutoBrowserGpuMode(options);
if (mode === "auto") return _autoBrowserGpuModeCache.then((probed) => probed.mode);
return _autoBrowserGpuModeCache.then((probed) => {
// Warn once per cache lifetime, not once per caller: `createCaptureSession`
// resolves the mode for the probe browser AND every parallel worker, so
// an un-deduplicated warning prints N+1 times and buries itself.
if (probed.mode === "software" && !_unverifiedHardwareGpuWarned) {
_unverifiedHardwareGpuWarned = true;
console.warn(
buildUnverifiedHardwareGpuWarning(options.platform ?? process.platform, probed.cause),
);
}
return "hardware";
});
}
/**
* Single observability surface for the auto-detect outcome. Logged exactly
* Latch for the explicit-hardware-probed-to-software warning: fires once per
* cache lifetime (re-armed by `_resetAutoBrowserGpuModeCacheForTests`).
*/
let _unverifiedHardwareGpuWarned = false;
/**
* Warning text for "you asked for hardware GPU and we could not confirm it".
*
* Splits on `cause` because the two failure shapes need opposite remediation.
* A probe that RAN and saw SwiftShader is a GPU-passthrough problem. A probe
* that could not run tells us nothing about the GPU — pointing that operator
* at `--gpus all` would send them chasing a phantom while their Chrome
* install is the actual fault.
*/
function buildUnverifiedHardwareGpuWarning(
platform: NodeJS.Platform | string,
cause: GpuProbeOutcome["cause"],
): string {
if (cause === "probe-error") {
return (
"[hyperframes] browserGpuMode=hardware was requested, but the GPU probe could not run, " +
"so hardware acceleration is UNVERIFIED — if Chrome falls back to software WebGL the " +
"capture will run at CPU speed. Honouring the explicit request anyway.\n" +
" This is a probe failure, not evidence of a missing GPU: see the " +
"`browserGpuMode probe → software (probe failed ...)` line above for the underlying " +
"error, which usually means Chrome could not launch (bad HYPERFRAMES_BROWSER_PATH, " +
"missing shared libraries, or a denied sandbox) rather than a GPU problem.\n" +
" Run `hyperframes doctor` to check the Chrome install."
);
}
const remediation =
platform === "linux"
? "Inside Docker, the container needs GPU passthrough: `--gpus all` with the NVIDIA " +
"Container Toolkit installed, or `--device /dev/dri` for Mesa/AMD/Intel. The image " +
"also needs the matching userspace driver + libEGL. Verify with " +
"`hyperframes render --browser-gpu` and watch for this warning disappearing."
: "Check that the host exposes a GPU to this process and that the graphics drivers are " +
"installed.";
return (
"[hyperframes] browserGpuMode=hardware was requested, but the WebGL probe found no " +
"hardware GPU — Chrome will silently fall back to software WebGL and the capture will " +
"run at CPU speed. Honouring the explicit request anyway.\n" +
` ${remediation}\n` +
" Pass --no-browser-gpu to select deterministic SwiftShader instead of waiting on a " +
"hardware path that is not there."
);
}
/**
* Single observability surface for the GPU probe outcome. Logged exactly
* once per process (the probe runs once); without this line, a regression
* to "always software even with a GPU present" would be invisible in
* production. Goes to stderr to stay out of stdout pipelines.
* production. Goes to stderr to stay out of stdout pipelines. Says "probe"
* rather than "auto" because explicit `browserGpuMode=hardware` runs the
* same probe to verify itself.
*/
function logResolvedBrowserGpuMode(resolved: "hardware" | "software", reason: string): void {
console.error(`[hyperframes] browserGpuMode auto → ${resolved} (${reason})`);
console.error(`[hyperframes] browserGpuMode probe → ${resolved} (${reason})`);
}
function createBrowserLaunchFingerprint(