From 400a4e22853906e531ab76d1b45792e70cb79ace Mon Sep 17 00:00:00 2001 From: James Russo Date: Thu, 12 Nov 2026 23:16:45 +0000 Subject: [PATCH] fix(engine): keep Chrome's sandbox on for local renders where the host supports it (#7625) * fix(engine): keep Chrome's sandbox on for local renders where the host supports it Chrome launched with --no-sandbox on every host. Keep the sandbox on macOS, Windows and non-root Linux outside containers; drop it as root, in Docker/Podman/Kubernetes, on Lambda and Cloud Run, and where Linux restricts unprivileged user namespaces. --no-zygote now ships only with --no-sandbox, since Chrome refuses it while sandboxed. A sandboxed Linux launch that fails at startup retries once without the sandbox and warns. HYPERFRAMES_CHROME_SANDBOX=1|0 overrides the choice. Co-Authored-By: Claude Opus 5.5 * test(cli): cover the Linux sandbox retry in managed browser launches The off-macOS closed-pipe case now passes --no-sandbox, so it still pins "no seatbelt relaunch" rather than the new Linux sandbox retry, which gets its own case. Co-Authored-By: Claude Opus 5.5 * fix(engine): honor HYPERFRAMES_CHROME_SANDBOX=1 and latch the fallback on clear evidence only With the override set to 1, a sandboxed launch Chrome refuses now fails instead of retrying unsandboxed. A bare startup exit (TargetCloseError) retries that launch without the sandbox but only latches the sandbox off after a second one, so a single unrelated early kill no longer disables it for the rest of a long-lived process. Chrome naming the sandbox in its error still latches at once. Co-Authored-By: Claude Opus 5.5 --------- Co-authored-by: Claude Opus 5.5 --- docs/packages/cli.mdx | 8 + packages/cli/src/browser/launch.ts | 6 +- .../cli/src/browser/sandboxDenied.test.ts | 26 ++- packages/engine/src/index.ts | 1 + .../engine/src/services/browserManager.ts | 28 +-- .../engine/src/services/chromeSandbox.test.ts | 166 ++++++++++++++++++ packages/engine/src/services/chromeSandbox.ts | 119 +++++++++++++ 7 files changed, 339 insertions(+), 15 deletions(-) create mode 100644 packages/engine/src/services/chromeSandbox.test.ts create mode 100644 packages/engine/src/services/chromeSandbox.ts diff --git a/docs/packages/cli.mdx b/docs/packages/cli.mdx index 22812d4f97..636660cc63 100644 --- a/docs/packages/cli.mdx +++ b/docs/packages/cli.mdx @@ -1894,6 +1894,14 @@ Low-memory detection reads **host** RAM, not cgroup or container limits, so containerised callers — including `--docker` — should set `PRODUCER_LOW_MEMORY_MODE` explicitly. +Render Chrome keeps its sandbox on macOS, on Windows, and on Linux for a +non-root user outside a container. It runs without one as root, in Docker, +Podman or Kubernetes, on AWS Lambda and Cloud Run, and where Linux restricts +unprivileged user namespaces (Ubuntu 23.10 and later). If a sandboxed Linux +launch still fails, the render prints a warning and continues without the +sandbox. Set `HYPERFRAMES_CHROME_SANDBOX=0` to always run without it, or `1` to +require it: with `1`, a launch Chrome refuses fails instead of falling back. + #### When a render times out Three separate budgets can expire, and the error message tells you which. All diff --git a/packages/cli/src/browser/launch.ts b/packages/cli/src/browser/launch.ts index db2d3799aa..79176280f0 100644 --- a/packages/cli/src/browser/launch.ts +++ b/packages/cli/src/browser/launch.ts @@ -2,6 +2,7 @@ import type { ChildProcess } from "node:child_process"; import { inAgentSandbox, isOneProcessOverPipe, + launchWithChromeSandboxFallback, machServiceDenied, oneProcessOverPipe, pipeLaunchClosedOnMac, @@ -108,8 +109,11 @@ export async function launchManagedBrowser( { diagnose = true }: { diagnose?: boolean } = {}, ): Promise { const sandboxed = isOneProcessOverPipe(options) || inAgentSandbox(); + const prepared = sandboxed ? oneProcessOverPipe(options) : options; try { - return await launchOwnedBrowser(puppeteer, sandboxed ? oneProcessOverPipe(options) : options); + return await launchWithChromeSandboxFallback(prepared.args ?? [], (args) => + launchOwnedBrowser(puppeteer, { ...prepared, args }), + ); } catch (error) { const unnamedSeatbelt = !sandboxed && diff --git a/packages/cli/src/browser/sandboxDenied.test.ts b/packages/cli/src/browser/sandboxDenied.test.ts index 2689eb96f4..bec5a228b0 100644 --- a/packages/cli/src/browser/sandboxDenied.test.ts +++ b/packages/cli/src/browser/sandboxDenied.test.ts @@ -124,9 +124,33 @@ describe("a browser inside an agent's macOS seatbelt", () => { "Protocol error (Target.setDiscoverTargets): Target closed", ); const { puppeteer, launches } = recordingPuppeteer([closed]); - await expect(launchManagedBrowser(puppeteer, { args: ["--headless"] })).rejects.toBe(closed); + await expect( + launchManagedBrowser(puppeteer, { args: ["--headless", "--no-sandbox"] }), + ).rejects.toBe(closed); expect(launches).toHaveLength(1); }); + + it("retries once without Chrome's sandbox when Linux closes a sandboxed launch at startup", async () => { + vi.stubEnv("CODEX_SANDBOX", ""); + Object.defineProperty(process, "platform", { value: "linux" }); + const warn = vi.spyOn(console, "warn").mockImplementation(() => {}); + const { puppeteer, launches } = recordingPuppeteer([ + new TargetCloseError("Protocol error (Target.setDiscoverTargets): Target closed"), + ]); + await expect(launchManagedBrowser(puppeteer, { args: ["--headless"] })).resolves.toBeDefined(); + expect(launches.map((options) => options.args)).toEqual([ + ["--headless", HYPERFRAMES_BROWSER_MARKER_ARG], + [ + "--headless", + "--no-sandbox", + "--disable-setuid-sandbox", + "--no-zygote", + HYPERFRAMES_BROWSER_MARKER_ARG, + ], + ]); + expect(warn).toHaveBeenCalledOnce(); + warn.mockRestore(); + }); }); describe("a browser launch that never settles", () => { diff --git a/packages/engine/src/index.ts b/packages/engine/src/index.ts index 39c69c7cad..47bec6f3e7 100644 --- a/packages/engine/src/index.ts +++ b/packages/engine/src/index.ts @@ -561,6 +561,7 @@ export { pipeLaunchClosedOnMac, sandboxDeniedRemediation, } from "./services/sandboxLaunch.js"; +export { launchWithChromeSandboxFallback } from "./services/chromeSandbox.js"; export { VariablesInvalidError, type VariablesIssue, diff --git a/packages/engine/src/services/browserManager.ts b/packages/engine/src/services/browserManager.ts index 783dd981aa..9f27577d8e 100644 --- a/packages/engine/src/services/browserManager.ts +++ b/packages/engine/src/services/browserManager.ts @@ -31,6 +31,7 @@ import { } from "./chromeHostCeiling.js"; import { DEFAULT_CONFIG, type EngineConfig } from "../config.js"; import { getSystemTotalMb, LOW_MEMORY_TOTAL_MB_THRESHOLD } from "./systemMemory.js"; +import { chromeSandboxArgs, launchWithChromeSandboxFallback } from "./chromeSandbox.js"; import { describeWslGpuAccess, detectWsl, type WslHostFacts } from "./wslHost.js"; import { BrowserLeasePool, @@ -923,16 +924,18 @@ async function launchBrowser( let captureMode = fingerprint.requestedCaptureMode; let browser: Browser | undefined; const launch = (args: readonly string[]) => - launchInsideSandbox(ppt, { - ...ownedLaunchOptions(), - headless: true, - args: withHyperframesBrowserMarker(args), - defaultViewport: null, - executablePath: fingerprint.executablePath, - env, - timeout: fingerprint.browserTimeoutMs, - protocolTimeout: fingerprint.protocolTimeoutMs, - }).catch(async (error: unknown) => { + launchWithChromeSandboxFallback(args, (launchArgs) => + launchInsideSandbox(ppt, { + ...ownedLaunchOptions(), + headless: true, + args: withHyperframesBrowserMarker(launchArgs), + defaultViewport: null, + executablePath: fingerprint.executablePath, + env, + timeout: fingerprint.browserTimeoutMs, + protocolTimeout: fingerprint.protocolTimeoutMs, + }), + ).catch(async (error: unknown) => { const remediation = process.platform === "darwin" && sandboxDeniedRemediation(error instanceof Error ? error.message : ""); @@ -1105,6 +1108,7 @@ export interface BuildChromeArgsOptions { platform?: NodeJS.Platform; /** The composition declares `data-requires-webgpu`; forces the WebGPU flag even in "software" mode. */ requiresWebGpu?: boolean; + sandbox?: boolean; } const CANVAS_DRAW_ELEMENT_FEATURE_FLAG = "--enable-features=CanvasDrawElement"; @@ -1143,8 +1147,7 @@ export function buildChromeArgs(options: BuildChromeArgsOptions, config?: GpuCon // appear in Puppeteer's defaults, Playwright, Remotion, and Chrome's own // headless-shell guidance. const chromeArgs = [ - "--no-sandbox", - "--disable-setuid-sandbox", + ...chromeSandboxArgs(options.sandbox), "--disable-dev-shm-usage", softwareWebGpu ? `${CANVAS_DRAW_ELEMENT_FEATURE_FLAG},Vulkan` @@ -1175,7 +1178,6 @@ export function buildChromeArgs(options: BuildChromeArgsOptions, config?: GpuCon "--disable-domain-reliability", "--disable-print-preview", "--no-pings", - "--no-zygote", // Memory — scale GPU budget to available system RAM `--force-gpu-mem-available-mb=${getGpuMemBudgetMb()}`, "--disk-cache-size=268435456", diff --git a/packages/engine/src/services/chromeSandbox.test.ts b/packages/engine/src/services/chromeSandbox.test.ts new file mode 100644 index 0000000000..b67234a0ef --- /dev/null +++ b/packages/engine/src/services/chromeSandbox.test.ts @@ -0,0 +1,166 @@ +import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; +import { + _resetChromeSandboxFallbackForTests, + chromeSandboxEnabled, + launchWithChromeSandboxFallback, + NO_SANDBOX_ARGS, + type ChromeSandboxHost, +} from "./chromeSandbox.js"; +import { buildChromeArgs } from "./browserManager.js"; + +function host(overrides: Partial = {}): ChromeSandboxHost { + return { + platform: "linux", + env: {}, + uid: 1000, + exists: () => false, + read: () => undefined, + ...overrides, + }; +} + +const withFile = (path: string) => (candidate: string) => candidate === path; +const withSysctl = (path: string, value: string) => (candidate: string) => + candidate === path ? value : undefined; + +describe("chromeSandboxEnabled", () => { + it.each(["darwin", "win32"] as const)("keeps the sandbox on %s", (platform) => { + expect(chromeSandboxEnabled(host({ platform }))).toBe(true); + expect(chromeSandboxEnabled(host({ platform, uid: 0 }))).toBe(true); + }); + + it("keeps the sandbox for a non-root Linux user outside containers", () => { + expect(chromeSandboxEnabled(host())).toBe(true); + expect( + chromeSandboxEnabled( + host({ read: withSysctl("/proc/sys/kernel/apparmor_restrict_unprivileged_userns", "0") }), + ), + ).toBe(true); + }); + + it.each<[string, Partial]>([ + ["root", { uid: 0 }], + ["AWS Lambda", { env: { AWS_LAMBDA_FUNCTION_NAME: "render" } }], + ["the Lambda runtime", { env: { AWS_EXECUTION_ENV: "AWS_Lambda_nodejs22.x" } }], + ["a Cloud Run service", { env: { K_SERVICE: "renderer" } }], + ["a Cloud Run job", { env: { CLOUD_RUN_JOB: "renderer" } }], + ["Kubernetes", { env: { KUBERNETES_SERVICE_HOST: "10.0.0.1" } }], + ["Docker", { exists: withFile("/.dockerenv") }], + ["Podman", { exists: withFile("/run/.containerenv") }], + [ + "AppArmor-restricted user namespaces", + { read: withSysctl("/proc/sys/kernel/apparmor_restrict_unprivileged_userns", "1") }, + ], + [ + "disabled user namespaces", + { read: withSysctl("/proc/sys/kernel/unprivileged_userns_clone", "0") }, + ], + ])("drops the sandbox on Linux under %s", (_, overrides) => { + expect(chromeSandboxEnabled(host(overrides))).toBe(false); + }); + + it("lets HYPERFRAMES_CHROME_SANDBOX override detection", () => { + expect(chromeSandboxEnabled(host({ uid: 0, env: { HYPERFRAMES_CHROME_SANDBOX: "1" } }))).toBe( + true, + ); + expect( + chromeSandboxEnabled(host({ platform: "darwin", env: { HYPERFRAMES_CHROME_SANDBOX: "0" } })), + ).toBe(false); + expect(chromeSandboxEnabled(host({ env: { HYPERFRAMES_CHROME_SANDBOX: "yes" } }))).toBe(true); + }); +}); + +describe("buildChromeArgs sandbox flags", () => { + const base = { width: 320, height: 240, platform: "linux" as const }; + + it("omits every no-sandbox flag, --no-zygote included, when the sandbox is on", () => { + const args = buildChromeArgs({ ...base, sandbox: true }); + for (const flag of NO_SANDBOX_ARGS) expect(args).not.toContain(flag); + }); + + it("keeps --no-zygote alongside --no-sandbox when the sandbox is off", () => { + expect(buildChromeArgs({ ...base, sandbox: false })).toEqual( + expect.arrayContaining(["--no-sandbox", "--disable-setuid-sandbox", "--no-zygote"]), + ); + }); +}); + +describe("launchWithChromeSandboxFallback", () => { + const closedTarget = Object.assign(new Error("Protocol error: Target closed"), { + name: "TargetCloseError", + }); + let warn: ReturnType; + + beforeEach(() => { + _resetChromeSandboxFallbackForTests(); + warn = vi.spyOn(console, "warn").mockImplementation(() => {}); + }); + afterEach(() => warn.mockRestore()); + + it("retries a bare startup exit once without the sandbox, warns once, and latches on the second", async () => { + const launch = vi + .fn<(args: string[]) => Promise>() + .mockRejectedValueOnce(closedTarget) + .mockResolvedValueOnce("browser") + .mockRejectedValueOnce(closedTarget) + .mockResolvedValue("browser"); + + await expect(launchWithChromeSandboxFallback(["--a"], launch, "linux", {})).resolves.toBe( + "browser", + ); + await launchWithChromeSandboxFallback(["--a"], launch, "linux", {}); + await launchWithChromeSandboxFallback(["--a"], launch, "linux", {}); + + expect(launch.mock.calls.map(([args]) => args)).toEqual([ + ["--a"], + ["--a", ...NO_SANDBOX_ARGS], + ["--a"], + ["--a", ...NO_SANDBOX_ARGS], + ["--a", ...NO_SANDBOX_ARGS], + ]); + expect(warn).toHaveBeenCalledTimes(1); + }); + + it("latches at once when Chrome names the sandbox in its error", async () => { + const launch = vi + .fn<(args: string[]) => Promise>() + .mockRejectedValueOnce(new Error("Failed to launch the browser process: No usable sandbox!")) + .mockResolvedValue("browser"); + await expect(launchWithChromeSandboxFallback([], launch, "linux", {})).resolves.toBe("browser"); + await launchWithChromeSandboxFallback([], launch, "linux", {}); + expect(launch.mock.calls.map(([args]) => args)).toEqual([[], NO_SANDBOX_ARGS, NO_SANDBOX_ARGS]); + }); + + it("never drops the sandbox when HYPERFRAMES_CHROME_SANDBOX=1 requires it", async () => { + const refused = new Error("Running as root without --no-sandbox is not supported"); + const launch = vi.fn<(args: string[]) => Promise>().mockRejectedValue(refused); + const env = { HYPERFRAMES_CHROME_SANDBOX: "1" }; + await expect(launchWithChromeSandboxFallback([], launch, "linux", env)).rejects.toBe(refused); + expect(launch).toHaveBeenCalledTimes(1); + expect(warn).not.toHaveBeenCalled(); + }); + + it.each<[string, string[], NodeJS.Platform, Error]>([ + ["an unrelated failure", [], "linux", new Error("spawn ENOENT")], + ["an unsandboxed launch", ["--no-sandbox"], "linux", closedTarget], + ["macOS", [], "darwin", closedTarget], + ["Windows", [], "win32", closedTarget], + ])("does not retry %s", async (_, args, platform, error) => { + const launch = vi.fn<(args: string[]) => Promise>().mockRejectedValue(error); + await expect(launchWithChromeSandboxFallback(args, launch, platform, {})).rejects.toBe(error); + expect(launch).toHaveBeenCalledTimes(1); + expect(warn).not.toHaveBeenCalled(); + }); + + it("surfaces the retry's own failure and tries the sandbox again next time", async () => { + const retryError = new Error("retry failed"); + const launch = vi + .fn<(args: string[]) => Promise>() + .mockRejectedValueOnce(closedTarget) + .mockRejectedValueOnce(retryError) + .mockResolvedValue("browser"); + await expect(launchWithChromeSandboxFallback([], launch, "linux", {})).rejects.toBe(retryError); + await launchWithChromeSandboxFallback([], launch, "linux", {}); + expect(launch).toHaveBeenLastCalledWith([]); + }); +}); diff --git a/packages/engine/src/services/chromeSandbox.ts b/packages/engine/src/services/chromeSandbox.ts new file mode 100644 index 0000000000..52868227ba --- /dev/null +++ b/packages/engine/src/services/chromeSandbox.ts @@ -0,0 +1,119 @@ +import { existsSync, readFileSync } from "node:fs"; + +export const NO_SANDBOX_ARGS = ["--no-sandbox", "--disable-setuid-sandbox", "--no-zygote"]; + +const SANDBOX_START_FAILURE = + /No usable sandbox|Running as root without --no-sandbox|Failed to move to new namespace/; + +export interface ChromeSandboxHost { + platform: NodeJS.Platform; + env: NodeJS.ProcessEnv; + uid: number | undefined; + exists: (path: string) => boolean; + read: (path: string) => string | undefined; +} + +function readTrimmed(path: string): string | undefined { + try { + return readFileSync(path, "utf8").trim(); + } catch { + return undefined; + } +} + +const currentHost = (): ChromeSandboxHost => ({ + platform: process.platform, + env: process.env, + uid: process.getuid?.(), + exists: existsSync, + read: readTrimmed, +}); + +const SERVERLESS_OR_ORCHESTRATED_ENV = [ + "AWS_LAMBDA_FUNCTION_NAME", + "AWS_EXECUTION_ENV", + "K_SERVICE", + "CLOUD_RUN_JOB", + "KUBERNETES_SERVICE_HOST", +]; +const CONTAINER_MARKERS = ["/.dockerenv", "/run/.containerenv"]; +const USERNS_BLOCKED_SYSCTLS: ReadonlyArray = [ + ["/proc/sys/kernel/apparmor_restrict_unprivileged_userns", "1"], + ["/proc/sys/kernel/unprivileged_userns_clone", "0"], +]; + +function linuxCannotSandbox({ env, uid, exists, read }: ChromeSandboxHost): boolean { + return ( + uid === 0 || + SERVERLESS_OR_ORCHESTRATED_ENV.some((name) => !!env[name]) || + CONTAINER_MARKERS.some((path) => exists(path)) || + USERNS_BLOCKED_SYSCTLS.some(([path, blocked]) => read(path) === blocked) + ); +} + +/** Chrome keeps its sandbox unless the host can't give it one: root, serverless, a container, or blocked user namespaces. */ +export function chromeSandboxEnabled(host: ChromeSandboxHost = currentHost()): boolean { + const override = host.env.HYPERFRAMES_CHROME_SANDBOX; + if (override === "0" || override === "1") return override === "1"; + return host.platform !== "linux" || !linuxCannotSandbox(host); +} + +export function chromeSandboxArgs(sandbox = chromeSandboxEnabled()): string[] { + return sandbox ? [] : NO_SANDBOX_ARGS; +} + +function withoutChromeSandbox(args: readonly string[]): string[] { + return [...args.filter((arg) => !NO_SANDBOX_ARGS.includes(arg)), ...NO_SANDBOX_ARGS]; +} + +let sandboxRefused = false; +let closedTargetRetries = 0; +let warned = false; + +export function _resetChromeSandboxFallbackForTests(): void { + sandboxRefused = false; + closedTargetRetries = 0; + warned = false; +} + +function chromeNamedSandboxFailure(error: unknown): boolean { + return error instanceof Error && SANDBOX_START_FAILURE.test(error.message); +} + +function sandboxStartFailure(error: unknown): boolean { + // Over a pipe, a Chrome that exits at startup surfaces only as a closed target. + return ( + chromeNamedSandboxFailure(error) || + (error instanceof Error && error.name === "TargetCloseError") + ); +} + +/** + * A sandboxed Linux launch Chrome refuses at startup is retried once without the sandbox. + * The sandbox stays off for the process once Chrome names it as the cause, or after a second + * bare startup exit (a single one may be an unrelated kill). HYPERFRAMES_CHROME_SANDBOX=1 never retries. + */ +export async function launchWithChromeSandboxFallback( + args: readonly string[], + launch: (args: string[]) => Promise, + platform: NodeJS.Platform = process.platform, + env: NodeJS.ProcessEnv = process.env, +): Promise { + const sandboxed = platform === "linux" && !args.includes("--no-sandbox"); + const sandboxRequired = env.HYPERFRAMES_CHROME_SANDBOX === "1"; + if (sandboxed && sandboxRefused && !sandboxRequired) return launch(withoutChromeSandbox(args)); + try { + return await launch([...args]); + } catch (error) { + if (!sandboxed || sandboxRequired || !sandboxStartFailure(error)) throw error; + const browser = await launch(withoutChromeSandbox(args)); + if (chromeNamedSandboxFailure(error) || ++closedTargetRetries >= 2) sandboxRefused = true; + if (!warned) { + warned = true; + console.warn( + "[hyperframes] Chrome could not start with its sandbox on this host (user namespaces are likely restricted), so it runs without it. Set HYPERFRAMES_CHROME_SANDBOX=0 to skip the sandboxed attempt.", + ); + } + return browser; + } +}