diff --git a/packages/engine/src/services/browserManager.test.ts b/packages/engine/src/services/browserManager.test.ts index db6e7e4686..dfa068e5d8 100644 --- a/packages/engine/src/services/browserManager.test.ts +++ b/packages/engine/src/services/browserManager.test.ts @@ -241,11 +241,68 @@ describe("resolveBrowserGpuMode", () => { expect(mode).toBe("software"); }); - it("passes 'hardware' through unchanged without probing", async () => { + it("passes 'hardware' through unchanged", async () => { + setMockWebGlProbe({ hasWebGL: true, vendor: "NVIDIA", renderer: "NVIDIA GeForce RTX 3070" }); const mode = await resolveBrowserGpuMode("hardware"); expect(mode).toBe("hardware"); }); + it("warns when explicit 'hardware' probes to software, but still honours it", async () => { + // heygen-com/hyperframes#2967: `--browser-gpu` inside a container with no + // GPU passthrough rendered 19186 frames on CPU with no diagnostic. + setMockWebGlProbe({ + hasWebGL: true, + vendor: "Google Inc. (Google)", + renderer: "ANGLE (Google, Vulkan 1.3.0 (SwiftShader Device))", + }); + const warn = vi.spyOn(console, "warn").mockImplementation(() => {}); + const mode = await resolveBrowserGpuMode("hardware", { platform: "linux" }); + expect(mode).toBe("hardware"); + const warning = warn.mock.calls.map((call) => String(call[0])).join("\n"); + expect(warning).toContain("browserGpuMode=hardware was requested"); + expect(warning).toContain("--gpus all"); + // Once per process, not once per worker — the render path resolves the + // mode for the probe browser plus every parallel worker. + await resolveBrowserGpuMode("hardware", { platform: "linux" }); + await resolveBrowserGpuMode("hardware", { platform: "linux" }); + expect(warn).toHaveBeenCalledTimes(1); + }); + + it("stays quiet when explicit 'hardware' probes to hardware", async () => { + setMockWebGlProbe({ hasWebGL: true, vendor: "NVIDIA", renderer: "NVIDIA GeForce RTX 3070" }); + const warn = vi.spyOn(console, "warn").mockImplementation(() => {}); + expect(await resolveBrowserGpuMode("hardware")).toBe("hardware"); + expect(warn).not.toHaveBeenCalled(); + }); + + it("gives non-linux hosts the generic remediation, not the Docker one", async () => { + setMockWebGlProbe({ + hasWebGL: true, + vendor: "Google Inc. (Google)", + renderer: "ANGLE (Google, Vulkan 1.3.0 (SwiftShader Device))", + }); + const warn = vi.spyOn(console, "warn").mockImplementation(() => {}); + expect(await resolveBrowserGpuMode("hardware", { platform: "darwin" })).toBe("hardware"); + const warning = String(warn.mock.calls[0]?.[0]); + expect(warning).toContain("host exposes a GPU"); + expect(warning).not.toContain("--gpus all"); + }); + + it("does not blame the GPU when the probe itself failed to launch", async () => { + // A probe that could not run is NO evidence about the GPU. Sending this + // operator to `--gpus all` would hide a broken Chrome install behind a + // phantom passthrough problem. + _setPuppeteerForTests({ + launch: vi.fn().mockRejectedValue(new Error("spawn ENOENT /bad/chrome")), + } as unknown as PuppeteerNode); + const warn = vi.spyOn(console, "warn").mockImplementation(() => {}); + expect(await resolveBrowserGpuMode("hardware", { platform: "linux" })).toBe("hardware"); + const warning = String(warn.mock.calls[0]?.[0]); + expect(warning).toContain("GPU probe could not run"); + expect(warning).toContain("hyperframes doctor"); + expect(warning).not.toContain("--gpus all"); + }); + it("falls back to 'software' when the probe browser cannot launch", async () => { // No chromePath, env unset, and (in the test env) no system Chrome to find // → puppeteer.launch will throw → caller catches → software fallback. @@ -272,31 +329,45 @@ describe("resolveBrowserGpuMode", () => { expect(second).toBe("software"); // Reset and re-probe to confirm the test-only reset works. _resetAutoBrowserGpuModeCacheForTests(); - const third = await resolveBrowserGpuMode("hardware"); + setMockWebGlProbe({ hasWebGL: true, vendor: "NVIDIA", renderer: "NVIDIA GeForce RTX 3070" }); + const third = await resolveBrowserGpuMode("auto"); expect(third).toBe("hardware"); }); - it("deduplicates concurrent auto-mode probes by caching the in-flight Promise", async () => { + it("deduplicates concurrent probes so only one Chrome launches", async () => { // Parallel coordinator fires N workers via Promise.all — without Promise- // level caching, a `--workers 4` render against a no-GPU host would launch - // 4 simultaneous probe Chromes. Verify all concurrent callers get the - // exact same Promise reference (proving the probe runs once, not N times). - const p1 = resolveBrowserGpuMode("auto", { - chromePath: "/definitely/not/a/real/chrome/binary", - browserTimeout: 2000, + // 4 simultaneous probe Chromes. Assert the launch count directly rather + // than Promise identity: `"auto"` and `"hardware"` now each adapt the + // shared cached Promise via `.then`, so identity is no longer the + // invariant — "the probe browser starts exactly once" is. + const { launch } = setMockWebGlProbe({ + hasWebGL: true, + vendor: "Google Inc. (Google)", + renderer: "ANGLE (Google, Vulkan 1.3.0 (SwiftShader Device))", }); - const p2 = resolveBrowserGpuMode("auto", { - chromePath: "/definitely/not/a/real/chrome/binary", - browserTimeout: 2000, - }); - const p3 = resolveBrowserGpuMode("auto", { - chromePath: "/definitely/not/a/real/chrome/binary", - browserTimeout: 2000, - }); - expect(p1).toBe(p2); - expect(p2).toBe(p3); - const results = await Promise.all([p1, p2, p3]); + const results = await Promise.all([ + resolveBrowserGpuMode("auto", { browserTimeout: 2000 }), + resolveBrowserGpuMode("auto", { browserTimeout: 2000 }), + resolveBrowserGpuMode("auto", { browserTimeout: 2000 }), + ]); expect(results).toEqual(["software", "software", "software"]); + expect(launch).toHaveBeenCalledTimes(1); + }); + + it("shares the one probe across mixed 'auto' and 'hardware' callers", async () => { + const { launch } = setMockWebGlProbe({ + hasWebGL: true, + vendor: "NVIDIA", + renderer: "NVIDIA GeForce RTX 3070", + }); + const results = await Promise.all([ + resolveBrowserGpuMode("auto"), + resolveBrowserGpuMode("hardware"), + resolveBrowserGpuMode("auto"), + ]); + expect(results).toEqual(["hardware", "hardware", "hardware"]); + expect(launch).toHaveBeenCalledTimes(1); }); it.each([ diff --git a/packages/engine/src/services/browserManager.ts b/packages/engine/src/services/browserManager.ts index 762626eb3c..899c3498b1 100644 --- a/packages/engine/src/services/browserManager.ts +++ b/packages/engine/src/services/browserManager.ts @@ -422,7 +422,25 @@ export const _probeBeginFrameSupportForTests = probeBeginFrameSupport; export const _closeBrowserAfterFailedProbeForTests = closeBrowserAfterFailedProbe; /** - * Cached *in-flight or resolved* probe Promise for `resolveBrowserGpuMode("auto", ...)`. + * Outcome of the one-shot WebGL probe. + * + * `cause` distinguishes the two ways a probe lands on `"software"`, because + * they need OPPOSITE remediation: + * - `"no-gpu"` — the probe ran and Chrome reported a software renderer + * (SwiftShader / llvmpipe). Remediation: GPU passthrough. + * - `"probe-error"` — the probe itself failed (Chrome couldn't launch, bad + * executable path, sandbox denied). We have NO evidence + * about the GPU either way; telling the operator to fix GPU + * passthrough would send them chasing the wrong problem. + */ +interface GpuProbeOutcome { + mode: "software" | "hardware"; + cause?: "no-gpu" | "probe-error"; +} + +/** + * Cached *in-flight or resolved* probe Promise, shared by BOTH the `"auto"` + * and explicit `"hardware"` entry points of `resolveBrowserGpuMode`. * * Caching the Promise (rather than the resolved value) deduplicates concurrent * callers — the parallel coordinator runs N workers via `Promise.all`, so a @@ -430,14 +448,13 @@ export const _closeBrowserAfterFailedProbeForTests = closeBrowserAfterFailedProb * simultaneous probe Chromes. The first call assigns the Promise and every * other concurrent caller awaits the same one, paying the ~240 ms probe cost * exactly once per process lifetime. - * - * Exported for tests; production callers go through `resolveBrowserGpuMode`. */ -let _autoBrowserGpuModeCache: Promise<"software" | "hardware"> | undefined; +let _autoBrowserGpuModeCache: Promise | undefined; /** Test-only: reset the cached probe result. */ export function _resetAutoBrowserGpuModeCacheForTests(): void { _autoBrowserGpuModeCache = undefined; + _unverifiedHardwareGpuWarned = false; } async function getPuppeteerOrNull(): Promise { @@ -478,7 +495,7 @@ async function probeAutoBrowserGpuMode(options: { chromePath?: string; browserTimeout?: number; platform?: NodeJS.Platform; -}): Promise<"software" | "hardware"> { +}): Promise { const platform = options.platform ?? process.platform; const browserTimeout = options.browserTimeout ?? DEFAULT_CONFIG.browserTimeout; const executablePath = options.chromePath ?? resolveHeadlessShellPath({}); @@ -486,7 +503,7 @@ async function probeAutoBrowserGpuMode(options: { if (ppt === null) { logResolvedBrowserGpuMode("software", "puppeteer unavailable"); - return "software"; + return { mode: "software", cause: "probe-error" }; } try { @@ -497,24 +514,35 @@ async function probeAutoBrowserGpuMode(options: { }); const resolved = resolveWebGlProbeMode(info); logResolvedBrowserGpuMode(resolved, describeWebGlProbe(info)); - return resolved; + return resolved === "hardware" ? { mode: "hardware" } : { mode: "software", cause: "no-gpu" }; } catch (err) { logResolvedBrowserGpuMode("software", formatProbeFailure(err)); - return "software"; + return { mode: "software", cause: "probe-error" }; } } /** * Resolve `browserGpuMode` to a concrete `"software" | "hardware"` answer. * - * For `"software"` / `"hardware"` this is a pure pass-through. For `"auto"` - * it launches a tiny Chrome with the platform's hardware GPU args, runs a - * one-shot WebGL availability probe, and falls back to `"software"` if - * hardware-mode WebGL is unavailable. The Promise is cached for the process - * lifetime, so concurrent callers (parallel workers) share the same probe. + * For `"software"` this is a pure pass-through. For `"auto"` it launches a + * tiny Chrome with the platform's hardware GPU args, runs a one-shot WebGL + * availability probe, and falls back to `"software"` if hardware-mode WebGL + * is unavailable. The Promise is cached for the process lifetime, so + * concurrent callers (parallel workers) share the same probe. * - * Any failure (Chrome launch error, navigation timeout, missing canvas API, - * etc.) is treated as a `"software"` fallback. The render path with + * `"hardware"` (an explicit `--browser-gpu` / `PRODUCER_BROWSER_GPU_MODE= + * hardware`) is honoured verbatim — the operator asked for it — but runs the + * SAME probe to VERIFY it, because Chrome's hardware GL args are advisory: + * with no usable GPU in the sandbox (no `/dev/dri`, no NVIDIA container + * runtime, missing EGL/driver libraries) Chrome silently falls back to + * software WebGL and the render just runs at CPU speed. Without this check + * the only trace is a buried `Automatic fallback to software WebGL` browser + * warning — heygen-com/hyperframes#2967 rendered 19186 frames on CPU while + * `--browser-gpu` was set and nothing said so. The probe result never + * changes the returned mode; it only makes the fallback loud. + * + * Any probe failure (Chrome launch error, navigation timeout, missing canvas + * API, etc.) is treated as a `"software"` result. The render path with * SwiftShader always works, so a misclassification toward software is the * safe failure mode; misclassifying toward hardware would error on the real * render. @@ -527,21 +555,84 @@ export function resolveBrowserGpuMode( platform?: NodeJS.Platform; } = {}, ): Promise<"software" | "hardware"> { - if (mode !== "auto") return Promise.resolve(mode); - if (_autoBrowserGpuModeCache) return _autoBrowserGpuModeCache; + if (mode === "software") return Promise.resolve(mode); - _autoBrowserGpuModeCache = probeAutoBrowserGpuMode(options); - return _autoBrowserGpuModeCache; + _autoBrowserGpuModeCache ??= probeAutoBrowserGpuMode(options); + if (mode === "auto") return _autoBrowserGpuModeCache.then((probed) => probed.mode); + + return _autoBrowserGpuModeCache.then((probed) => { + // Warn once per cache lifetime, not once per caller: `createCaptureSession` + // resolves the mode for the probe browser AND every parallel worker, so + // an un-deduplicated warning prints N+1 times and buries itself. + if (probed.mode === "software" && !_unverifiedHardwareGpuWarned) { + _unverifiedHardwareGpuWarned = true; + console.warn( + buildUnverifiedHardwareGpuWarning(options.platform ?? process.platform, probed.cause), + ); + } + return "hardware"; + }); } /** - * Single observability surface for the auto-detect outcome. Logged exactly + * Latch for the explicit-hardware-probed-to-software warning: fires once per + * cache lifetime (re-armed by `_resetAutoBrowserGpuModeCacheForTests`). + */ +let _unverifiedHardwareGpuWarned = false; + +/** + * Warning text for "you asked for hardware GPU and we could not confirm it". + * + * Splits on `cause` because the two failure shapes need opposite remediation. + * A probe that RAN and saw SwiftShader is a GPU-passthrough problem. A probe + * that could not run tells us nothing about the GPU — pointing that operator + * at `--gpus all` would send them chasing a phantom while their Chrome + * install is the actual fault. + */ +function buildUnverifiedHardwareGpuWarning( + platform: NodeJS.Platform | string, + cause: GpuProbeOutcome["cause"], +): string { + if (cause === "probe-error") { + return ( + "[hyperframes] browserGpuMode=hardware was requested, but the GPU probe could not run, " + + "so hardware acceleration is UNVERIFIED — if Chrome falls back to software WebGL the " + + "capture will run at CPU speed. Honouring the explicit request anyway.\n" + + " This is a probe failure, not evidence of a missing GPU: see the " + + "`browserGpuMode probe → software (probe failed ...)` line above for the underlying " + + "error, which usually means Chrome could not launch (bad HYPERFRAMES_BROWSER_PATH, " + + "missing shared libraries, or a denied sandbox) rather than a GPU problem.\n" + + " Run `hyperframes doctor` to check the Chrome install." + ); + } + const remediation = + platform === "linux" + ? "Inside Docker, the container needs GPU passthrough: `--gpus all` with the NVIDIA " + + "Container Toolkit installed, or `--device /dev/dri` for Mesa/AMD/Intel. The image " + + "also needs the matching userspace driver + libEGL. Verify with " + + "`hyperframes render --browser-gpu` and watch for this warning disappearing." + : "Check that the host exposes a GPU to this process and that the graphics drivers are " + + "installed."; + return ( + "[hyperframes] browserGpuMode=hardware was requested, but the WebGL probe found no " + + "hardware GPU — Chrome will silently fall back to software WebGL and the capture will " + + "run at CPU speed. Honouring the explicit request anyway.\n" + + ` ${remediation}\n` + + " Pass --no-browser-gpu to select deterministic SwiftShader instead of waiting on a " + + "hardware path that is not there." + ); +} + +/** + * Single observability surface for the GPU probe outcome. Logged exactly * once per process (the probe runs once); without this line, a regression * to "always software even with a GPU present" would be invisible in - * production. Goes to stderr to stay out of stdout pipelines. + * production. Goes to stderr to stay out of stdout pipelines. Says "probe" + * rather than "auto" because explicit `browserGpuMode=hardware` runs the + * same probe to verify itself. */ function logResolvedBrowserGpuMode(resolved: "hardware" | "software", reason: string): void { - console.error(`[hyperframes] browserGpuMode auto → ${resolved} (${reason})`); + console.error(`[hyperframes] browserGpuMode probe → ${resolved} (${reason})`); } function createBrowserLaunchFingerprint(