Merge pull request #642 from heygen-com/feat/auto-detect-browser-gpu

feat(engine): browserGpuMode "auto" — probe-once WebGL detection with software fallback

packages/cli/src/commands/render.test.ts

@@ -60,7 +60,7 @@ describe("renderLocal browser GPU config", () => {
       quality: "standard",
       format: "mp4",
       gpu: false,
-      browserGpu: false,
+      browserGpuMode: "software",
       hdrMode: "auto",
       quiet: true,
     });
@@ -72,6 +72,25 @@ describe("renderLocal browser GPU config", () => {
     });
   });
 
+  it("forwards browserGpuMode='auto' into producer config (probe-then-choose)", async () => {
+    const { renderLocal } = await import("./render.js");
+    await renderLocal("/tmp/project", "/tmp/out.mp4", {
+      fps: 30,
+      quality: "standard",
+      format: "mp4",
+      gpu: false,
+      browserGpuMode: "auto",
+      hdrMode: "auto",
+      quiet: true,
+    });
+
+    expect(producerState.resolveConfigCalls).toContainEqual({ browserGpuMode: "auto" });
+    expect(producerState.createdJobs[0]?.producerConfig).toMatchObject({
+      browserGpuMode: "auto",
+      resolved: true,
+    });
+  });
+
   it("passes an explicit hardware override for default local browser GPU", async () => {
     const { renderLocal } = await import("./render.js");
     await renderLocal("/tmp/project", "/tmp/out.mp4", {
@@ -79,7 +98,7 @@ describe("renderLocal browser GPU config", () => {
       quality: "standard",
       format: "mp4",
       gpu: false,
-      browserGpu: true,
+      browserGpuMode: "hardware",
       hdrMode: "auto",
       quiet: true,
     });
@@ -94,12 +113,18 @@ describe("renderLocal browser GPU config", () => {
   it("resolves browser GPU from CLI flags, Docker mode, and env fallback", async () => {
     const { resolveBrowserGpuForCli } = await import("./render.js");
 
-    expect(resolveBrowserGpuForCli(false, undefined, undefined)).toBe(true);
-    expect(resolveBrowserGpuForCli(false, undefined, "hardware")).toBe(true);
-    expect(resolveBrowserGpuForCli(false, undefined, "software")).toBe(false);
-    expect(resolveBrowserGpuForCli(false, true, "software")).toBe(true);
-    expect(resolveBrowserGpuForCli(false, false, "hardware")).toBe(false);
-    expect(resolveBrowserGpuForCli(true, undefined, "hardware")).toBe(false);
+    // Default (no flag, no env): auto — engine probes and chooses.
+    expect(resolveBrowserGpuForCli(false, undefined, undefined)).toBe("auto");
+    // Env override
+    expect(resolveBrowserGpuForCli(false, undefined, "hardware")).toBe("hardware");
+    expect(resolveBrowserGpuForCli(false, undefined, "software")).toBe("software");
+    expect(resolveBrowserGpuForCli(false, undefined, "auto")).toBe("auto");
+    // Explicit CLI flag wins over env
+    expect(resolveBrowserGpuForCli(false, true, "software")).toBe("hardware");
+    expect(resolveBrowserGpuForCli(false, false, "hardware")).toBe("software");
+    // Docker forces software regardless of flags/env
+    expect(resolveBrowserGpuForCli(true, undefined, "hardware")).toBe("software");
+    expect(resolveBrowserGpuForCli(true, undefined, "auto")).toBe("software");
   });
 
   it("forwards parsed --variables payload to createRenderJob", async () => {
@@ -109,7 +134,7 @@ describe("renderLocal browser GPU config", () => {
       quality: "standard",
       format: "mp4",
       gpu: false,
-      browserGpu: false,
+      browserGpuMode: "software",
       hdrMode: "auto",
       quiet: true,
       variables: { title: "Hello", count: 3 },
@@ -125,7 +150,7 @@ describe("renderLocal browser GPU config", () => {
       quality: "standard",
       format: "mp4",
       gpu: false,
-      browserGpu: false,
+      browserGpuMode: "software",
       hdrMode: "auto",
       quiet: true,
     });
@@ -147,7 +172,7 @@ describe("renderLocal browser GPU config", () => {
       quality: "standard",
       format: "mp4",
       gpu: false,
-      browserGpu: true,
+      browserGpuMode: "hardware",
       hdrMode: "auto",
       quiet: true,
       exitAfterComplete: true,

packages/cli/src/commands/render.ts

@@ -118,7 +118,7 @@ export default defineCommand({
     "browser-gpu": {
       type: "boolean",
       description:
-        "Use host GPU acceleration for Chrome/WebGL capture. Enabled by default for local renders; use --no-browser-gpu to opt out.",
+        "Force host GPU acceleration for Chrome/WebGL capture. Default: auto (probe on first launch; fall back to software if no GPU). Use --no-browser-gpu to force software (SwiftShader).",
     },
     quiet: {
       type: "boolean",
@@ -224,7 +224,7 @@ export default defineCommand({
     const useDocker = args.docker ?? false;
     const useGpu = args.gpu ?? false;
     const browserGpuArg = args["browser-gpu"];
-    const useBrowserGpu = resolveBrowserGpuForCli(useDocker, browserGpuArg);
+    const browserGpuMode = resolveBrowserGpuForCli(useDocker, browserGpuArg);
     const quiet = args.quiet ?? false;
     const strictAll = args["strict-all"] ?? false;
     const strictErrors = (args.strict ?? false) || strictAll;
@@ -275,10 +275,14 @@ export default defineCommand({
           c.dim(" \u2192 " + outputPath),
       );
       console.log(c.dim("   " + fps + "fps \u00B7 " + quality + " \u00B7 " + workerLabel));
-      if (useGpu || useBrowserGpu) {
+      if (useGpu || browserGpuMode !== "software") {
         const gpuModes = [
           useGpu ? "encoder GPU" : null,
-          useBrowserGpu ? "browser GPU (auto)" : null,
+          browserGpuMode === "hardware"
+            ? "browser GPU (forced)"
+            : browserGpuMode === "auto"
+              ? "browser GPU (auto-detect)"
+              : null,
         ].filter(Boolean);
         console.log(c.dim("   GPU: " + gpuModes.join(" + ")));
       }
@@ -397,7 +401,7 @@ export default defineCommand({
         format,
         workers,
         gpu: useGpu,
-        browserGpu: useBrowserGpu,
+        browserGpuMode,
         hdrMode: args.sdr ? "force-sdr" : args.hdr ? "force-hdr" : "auto",
         crf,
         videoBitrate,
@@ -412,7 +416,7 @@ export default defineCommand({
         format,
         workers,
         gpu: useGpu,
-        browserGpu: useBrowserGpu,
+        browserGpuMode,
         hdrMode: args.sdr ? "force-sdr" : args.hdr ? "force-hdr" : "auto",
         crf,
         videoBitrate,
@@ -431,7 +435,12 @@ interface RenderOptions {
   format: "mp4" | "webm" | "mov";
   workers?: number;
   gpu: boolean;
-  browserGpu: boolean;
+  /**
+   * Chrome WebGL backend mode. "auto" probes on first launch and falls back
+   * to "software" if no usable GPU. Defaults to "software" when omitted to
+   * stay backwards-compatible with callers that pre-date the tri-state.
+   */
+  browserGpuMode?: "auto" | "hardware" | "software";
   hdrMode: "auto" | "force-hdr" | "force-sdr";
   crf?: number;
   videoBitrate?: string;
@@ -579,15 +588,33 @@ export function validateVariablesAgainstProject(
   return validateVariables(values, meta.variables);
 }
 
+/**
+ * Resolve the browser-GPU mode for a CLI render invocation.
+ *
+ * Priority (highest first):
+ *   1. Docker mode → always "software" (docker has no portable GPU
+ *      passthrough; the engine's render path uses SwiftShader).
+ *   2. Explicit CLI flag — `--browser-gpu` → "hardware",
+ *      `--no-browser-gpu` → "software".
+ *   3. Env var `PRODUCER_BROWSER_GPU_MODE` accepts "hardware" / "software" /
+ *      "auto".
+ *   4. Default = "auto" — engine probes WebGL availability on first launch
+ *      and falls back to software if the host lacks a usable GPU.
+ *
+ * Returning "auto" by default lets local renders Just Work whether or not the
+ * host has a GPU, while preserving the explicit overrides for CI / power
+ * users who want failure-on-misconfig.
+ */
 export function resolveBrowserGpuForCli(
   useDocker: boolean,
   browserGpuArg: boolean | undefined,
   envMode = process.env.PRODUCER_BROWSER_GPU_MODE,
-): boolean {
-  if (useDocker) return false;
-  if (browserGpuArg !== undefined) return browserGpuArg;
-  if (envMode === "software") return false;
-  return true;
+): "auto" | "hardware" | "software" {
+  if (useDocker) return "software";
+  if (browserGpuArg === true) return "hardware";
+  if (browserGpuArg === false) return "software";
+  if (envMode === "hardware" || envMode === "software" || envMode === "auto") return envMode;
+  return "auto";
 }
 
 const DOCKER_IMAGE_PREFIX = "hyperframes-renderer";
@@ -707,7 +734,7 @@ async function renderDocker(
       format: options.format,
       workers: options.workers,
       gpu: options.gpu,
-      browserGpu: options.browserGpu,
+      browserGpu: options.browserGpuMode === "hardware",
       hdrMode: options.hdrMode,
       crf: options.crf,
       videoBitrate: options.videoBitrate,
@@ -777,7 +804,7 @@ export async function renderLocal(
     workers: options.workers,
     useGpu: options.gpu,
     producerConfig: producer.resolveConfig({
-      browserGpuMode: options.browserGpu ? "hardware" : "software",
+      browserGpuMode: options.browserGpuMode ?? "software",
     }),
     hdrMode: options.hdrMode,
     crf: options.crf,

packages/cli/src/docs/rendering.md

@@ -20,13 +20,13 @@ Requires: Docker installed and running.
 - `--crf` — Override encoder CRF (mutually exclusive with `--video-bitrate`)
 - `--video-bitrate` — Target video bitrate such as `10M` (mutually exclusive with `--crf`)
 - `--gpu` — Use GPU encoding (NVENC, VideoToolbox, VAAPI, QSV)
-- `--browser-gpu` / `--no-browser-gpu` — Use or opt out of host GPU acceleration for local Chrome/WebGL capture (enabled by default for local renders, disabled in Docker)
+- `--browser-gpu` / `--no-browser-gpu` — Force host GPU or software (SwiftShader) for Chrome/WebGL capture. Default for local renders is `auto` — probe WebGL availability on first launch and fall back to software if no GPU is reachable. Docker mode always uses software.
 - `-o, --output` — Custom output path
 
 ## Tips
 
 - Use `draft` quality for fast previews during development
-- Local renders use browser GPU capture automatically; use `--no-browser-gpu` to compare against the software-browser path
+- Local renders auto-detect GPU on first launch; use `--browser-gpu` to force hardware (errors if no GPU) or `--no-browser-gpu` to force SwiftShader
 - Use `--gpu` when a local render also benefits from hardware FFmpeg encoding
 - Use `npx hyperframes benchmark` to find optimal settings
 - 4 workers is usually the sweet spot for most compositions

packages/engine/src/config.test.ts

@@ -97,6 +97,13 @@ describe("resolveConfig", () => {
     expect(config.browserGpuMode).toBe("hardware");
   });
 
+  it("accepts 'auto' as a valid browser GPU mode env value", () => {
+    setEnv("PRODUCER_BROWSER_GPU_MODE", "auto");
+
+    const config = resolveConfig();
+    expect(config.browserGpuMode).toBe("auto");
+  });
+
   it("falls back to software browser GPU mode for invalid env values", () => {
     setEnv("PRODUCER_BROWSER_GPU_MODE", "native");
 

packages/engine/src/config.ts

@@ -32,10 +32,15 @@ export interface EngineConfig {
   chromePath?: string;
   disableGpu: boolean;
   /**
-   * Chrome/WebGL rendering backend. "software" keeps the existing SwiftShader
-   * path for reproducible output; "hardware" lets Chrome use the host GPU.
+   * Chrome/WebGL rendering backend.
+   * - "software": SwiftShader (CPU-only). Always works; ~5-50× slower than GPU.
+   * - "hardware": host GPU via platform-native ANGLE backend (Metal/D3D11/EGL).
+   *   Errors if no usable GPU is reachable from Chrome.
+   * - "auto": probe Chrome for WebGL availability on first launch in this
+   *   process; fall back to software if hardware-mode WebGL is unavailable.
+   *   Cost: one extra Chrome launch (~1-2 s) per process; result cached.
    */
-  browserGpuMode: "software" | "hardware";
+  browserGpuMode: "software" | "hardware" | "auto";
   enableBrowserPool: boolean;
   browserTimeout: number;
   protocolTimeout: number;
@@ -173,7 +178,7 @@ export function resolveConfig(overrides?: Partial<EngineConfig>): EngineConfig {
   };
   const envBrowserGpuMode = (): EngineConfig["browserGpuMode"] => {
     const raw = env("PRODUCER_BROWSER_GPU_MODE");
-    if (raw === "hardware" || raw === "software") return raw;
+    if (raw === "hardware" || raw === "software" || raw === "auto") return raw;
     return DEFAULT_CONFIG.browserGpuMode;
   };
 

packages/engine/src/index.ts

@@ -50,6 +50,7 @@ export {
   acquireBrowser,
   releaseBrowser,
   resolveHeadlessShellPath,
+  resolveBrowserGpuMode,
   buildChromeArgs,
   ENABLE_BROWSER_POOL,
   type BuildChromeArgsOptions,

packages/engine/src/services/browserManager.test.ts

@@ -1,6 +1,11 @@
-import { describe, expect, it, vi } from "vitest";
+import { afterEach, beforeEach, describe, expect, it, vi } from "vitest";
 
-import { buildChromeArgs, forceReleaseBrowser } from "./browserManager.js";
+import {
+  _resetAutoBrowserGpuModeCacheForTests,
+  buildChromeArgs,
+  forceReleaseBrowser,
+  resolveBrowserGpuMode,
+} from "./browserManager.js";
 
 describe("buildChromeArgs browser GPU mode", () => {
   const base = { width: 1920, height: 1080 };
@@ -10,6 +15,7 @@ describe("buildChromeArgs browser GPU mode", () => {
     expect(args).toContain("--enable-features=CanvasDrawElement");
     expect(args).toContain("--use-gl=angle");
     expect(args).toContain("--use-angle=swiftshader");
+    expect(args).toContain("--enable-unsafe-swiftshader");
     expect(args).not.toContain("--enable-gpu-rasterization");
   });
 
@@ -48,6 +54,80 @@ describe("buildChromeArgs browser GPU mode", () => {
   });
 });
 
+describe("resolveBrowserGpuMode", () => {
+  beforeEach(() => {
+    _resetAutoBrowserGpuModeCacheForTests();
+  });
+
+  afterEach(() => {
+    vi.restoreAllMocks();
+    _resetAutoBrowserGpuModeCacheForTests();
+  });
+
+  it("passes 'software' through unchanged without probing", async () => {
+    const mode = await resolveBrowserGpuMode("software");
+    expect(mode).toBe("software");
+  });
+
+  it("passes 'hardware' through unchanged without probing", async () => {
+    const mode = await resolveBrowserGpuMode("hardware");
+    expect(mode).toBe("hardware");
+  });
+
+  it("falls back to 'software' when the probe browser cannot launch", async () => {
+    // No chromePath, env unset, and (in the test env) no system Chrome to find
+    // → puppeteer.launch will throw → caller catches → software fallback.
+    // Force a definitely-missing chrome binary so the launch path errors fast.
+    const mode = await resolveBrowserGpuMode("auto", {
+      chromePath: "/definitely/not/a/real/chrome/binary",
+      browserTimeout: 2000,
+    });
+    expect(mode).toBe("software");
+  });
+
+  it("caches the probe result across calls", async () => {
+    const first = await resolveBrowserGpuMode("auto", {
+      chromePath: "/definitely/not/a/real/chrome/binary",
+      browserTimeout: 2000,
+    });
+    // Second call uses cache — no new launch. Assert the same answer comes back
+    // even with a different chromePath that would have a different probe outcome.
+    const second = await resolveBrowserGpuMode("auto", {
+      chromePath: "/another/definitely/missing/path",
+      browserTimeout: 2000,
+    });
+    expect(first).toBe("software");
+    expect(second).toBe("software");
+    // Reset and re-probe to confirm the test-only reset works.
+    _resetAutoBrowserGpuModeCacheForTests();
+    const third = await resolveBrowserGpuMode("hardware");
+    expect(third).toBe("hardware");
+  });
+
+  it("deduplicates concurrent auto-mode probes by caching the in-flight Promise", async () => {
+    // Parallel coordinator fires N workers via Promise.all — without Promise-
+    // level caching, a `--workers 4` render against a no-GPU host would launch
+    // 4 simultaneous probe Chromes. Verify all concurrent callers get the
+    // exact same Promise reference (proving the probe runs once, not N times).
+    const p1 = resolveBrowserGpuMode("auto", {
+      chromePath: "/definitely/not/a/real/chrome/binary",
+      browserTimeout: 2000,
+    });
+    const p2 = resolveBrowserGpuMode("auto", {
+      chromePath: "/definitely/not/a/real/chrome/binary",
+      browserTimeout: 2000,
+    });
+    const p3 = resolveBrowserGpuMode("auto", {
+      chromePath: "/definitely/not/a/real/chrome/binary",
+      browserTimeout: 2000,
+    });
+    expect(p1).toBe(p2);
+    expect(p2).toBe(p3);
+    const results = await Promise.all([p1, p2, p3]);
+    expect(results).toEqual(["software", "software", "software"]);
+  });
+});
+
 describe("forceReleaseBrowser", () => {
   it("kills the browser process and disconnects", () => {
     const killFn = vi.fn(() => true);

packages/engine/src/services/browserManager.ts

@@ -136,6 +136,119 @@ async function probeBeginFrameSupport(browser: Browser): Promise<boolean> {
   }
 }
 
+/**
+ * Cached *in-flight or resolved* probe Promise for `resolveBrowserGpuMode("auto", ...)`.
+ *
+ * Caching the Promise (rather than the resolved value) deduplicates concurrent
+ * callers — the parallel coordinator runs N workers via `Promise.all`, so a
+ * `--workers 4` render against a no-GPU host would otherwise fire 4
+ * simultaneous probe Chromes. The first call assigns the Promise and every
+ * other concurrent caller awaits the same one, paying the ~240 ms probe cost
+ * exactly once per process lifetime.
+ *
+ * Exported for tests; production callers go through `resolveBrowserGpuMode`.
+ */
+export let _autoBrowserGpuModeCache: Promise<"software" | "hardware"> | undefined;
+
+/** Test-only: reset the cached probe result. */
+export function _resetAutoBrowserGpuModeCacheForTests(): void {
+  _autoBrowserGpuModeCache = undefined;
+}
+
+/**
+ * Resolve `browserGpuMode` to a concrete `"software" | "hardware"` answer.
+ *
+ * For `"software"` / `"hardware"` this is a pure pass-through. For `"auto"`
+ * it launches a tiny Chrome with the platform's hardware GPU args, runs a
+ * one-shot WebGL availability probe, and falls back to `"software"` if
+ * hardware-mode WebGL is unavailable. The Promise is cached for the process
+ * lifetime, so concurrent callers (parallel workers) share the same probe.
+ *
+ * Any failure (Chrome launch error, navigation timeout, missing canvas API,
+ * etc.) is treated as a `"software"` fallback. The render path with
+ * SwiftShader always works, so a misclassification toward software is the
+ * safe failure mode; misclassifying toward hardware would error on the real
+ * render.
+ */
+export function resolveBrowserGpuMode(
+  mode: EngineConfig["browserGpuMode"],
+  options: {
+    chromePath?: string;
+    browserTimeout?: number;
+    platform?: NodeJS.Platform;
+  } = {},
+): Promise<"software" | "hardware"> {
+  if (mode !== "auto") return Promise.resolve(mode);
+  if (_autoBrowserGpuModeCache) return _autoBrowserGpuModeCache;
+
+  _autoBrowserGpuModeCache = (async () => {
+    const platform = options.platform ?? process.platform;
+    const browserTimeout = options.browserTimeout ?? DEFAULT_CONFIG.browserTimeout;
+    const executablePath = options.chromePath ?? resolveHeadlessShellPath({});
+
+    const probeArgs = [
+      "--no-sandbox",
+      "--disable-setuid-sandbox",
+      "--disable-dev-shm-usage",
+      "--enable-webgl",
+      "--ignore-gpu-blocklist",
+      ...getBrowserGpuArgs("hardware", platform),
+    ];
+
+    const ppt = await getPuppeteer().catch(() => null);
+    if (!ppt) {
+      logResolvedBrowserGpuMode("software", "puppeteer unavailable");
+      return "software" as const;
+    }
+
+    let probeBrowser: Browser | undefined;
+    try {
+      probeBrowser = await ppt.launch({
+        headless: true,
+        args: probeArgs,
+        defaultViewport: { width: 64, height: 64 },
+        executablePath,
+        timeout: browserTimeout,
+      });
+      const page = await probeBrowser.newPage();
+      const hasWebGL = await page.evaluate(() => {
+        try {
+          const c = document.createElement("canvas");
+          const gl =
+            c.getContext("webgl") ||
+            (c.getContext("experimental-webgl") as RenderingContext | null);
+          return gl !== null;
+        } catch {
+          return false;
+        }
+      });
+      const resolved = hasWebGL ? ("hardware" as const) : ("software" as const);
+      logResolvedBrowserGpuMode(resolved, hasWebGL ? "WebGL probe succeeded" : "WebGL unavailable");
+      return resolved;
+    } catch (err) {
+      logResolvedBrowserGpuMode(
+        "software",
+        `probe failed (${err instanceof Error ? err.message : String(err)})`,
+      );
+      return "software" as const;
+    } finally {
+      await probeBrowser?.close().catch(() => {});
+    }
+  })();
+
+  return _autoBrowserGpuModeCache;
+}
+
+/**
+ * Single observability surface for the auto-detect outcome. Logged exactly
+ * once per process (the probe runs once); without this line, a regression
+ * to "always software even with a GPU present" would be invisible in
+ * production. Goes to stderr to stay out of stdout pipelines.
+ */
+function logResolvedBrowserGpuMode(resolved: "hardware" | "software", reason: string): void {
+  console.error(`[hyperframes] browserGpuMode auto → ${resolved} (${reason})`);
+}
+
 export async function acquireBrowser(
   chromeArgs: string[],
   config?: Partial<
@@ -344,7 +457,20 @@ function getBrowserGpuArgs(
   platform: NodeJS.Platform,
 ): string[] {
   if (mode === "software") {
-    return ["--use-gl=angle", "--use-angle=swiftshader"];
+    // Chrome 120+ deprecated implicit SwiftShader fallback; the explicit
+    // path (--use-angle=swiftshader) keeps working but Chrome emits a
+    // deprecation warning unless --enable-unsafe-swiftshader is also set.
+    // Despite the name, this is exactly the behaviour Chrome had before;
+    // the flag exists to make CPU rasterisation an explicit opt-in rather
+    // than an implicit fallback for end users on the open web.
+    return ["--use-gl=angle", "--use-angle=swiftshader", "--enable-unsafe-swiftshader"];
+  }
+
+  if (mode === "auto") {
+    // Should not reach here — `resolveBrowserGpuMode` collapses "auto" to
+    // "software" or "hardware" before args are built. Be defensive: software
+    // is the always-safe fallback.
+    return ["--use-gl=angle", "--use-angle=swiftshader", "--enable-unsafe-swiftshader"];
   }
 
   switch (platform) {

packages/engine/src/services/frameCapture.ts

@@ -18,6 +18,7 @@ import {
   releaseBrowser,
   forceReleaseBrowser,
   buildChromeArgs,
+  resolveBrowserGpuMode,
   resolveHeadlessShellPath,
   type CaptureMode,
 } from "./browserManager.js";
@@ -115,9 +116,14 @@ export async function createCaptureSession(
   const forceScreenshot = config?.forceScreenshot ?? DEFAULT_CONFIG.forceScreenshot;
   const preMode: CaptureMode =
     headlessShell && isLinux && !forceScreenshot ? "beginframe" : "screenshot";
+  const requestedGpuMode = config?.browserGpuMode ?? DEFAULT_CONFIG.browserGpuMode;
+  const resolvedGpuMode = await resolveBrowserGpuMode(requestedGpuMode, {
+    chromePath: headlessShell ?? undefined,
+    browserTimeout: config?.browserTimeout,
+  });
   const chromeArgs = buildChromeArgs(
     { width: options.width, height: options.height, captureMode: preMode },
-    config,
+    { ...config, browserGpuMode: resolvedGpuMode },
   );
 
   const { browser, captureMode } = await acquireBrowser(chromeArgs, config);