diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index dc783fb9b..54cc5cd97 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -35,6 +35,7 @@ jobs: cli: ${{ steps.filter.outputs.cli }} skills: ${{ steps.filter.outputs.skills }} codex_plugin: ${{ steps.filter.outputs.codex_plugin }} + gcp_beginframe: ${{ steps.filter.outputs.gcp_beginframe }} steps: # Force git-based change detection instead of the pull_request REST API. # The API path can fail the whole workflow on transient listFiles @@ -76,6 +77,33 @@ jobs: - "scripts/package-codex-plugin.mjs" - "package.json" - ".github/workflows/ci.yml" + gcp_beginframe: + - "packages/gcp-cloud-run/Dockerfile" + - "packages/aws-lambda/scripts/probe-beginframe.ts" + - "packages/engine/src/services/browserManager.ts" + - "package.json" + - "bun.lock" + - ".github/workflows/ci.yml" + + gcp-beginframe-contract: + name: GCP BeginFrame image contract + needs: changes + if: needs.changes.outputs.gcp_beginframe == 'true' + runs-on: ubuntu-latest + timeout-minutes: 15 + steps: + - uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4 + with: + lfs: true + - uses: docker/setup-buildx-action@e468171a9de216ec08956ac3ada2f0791b6bd435 # v3 + - uses: docker/build-push-action@263435318d21b8e681c14492fe198d362a7d2c83 # v6 + with: + context: . + file: packages/gcp-cloud-run/Dockerfile + target: beginframe-contract + push: false + cache-from: type=gha,scope=gcp-beginframe-contract + cache-to: type=gha,mode=max,scope=gcp-beginframe-contract build: name: Build diff --git a/examples/gcp-cloud-run/scripts/smoke.sh b/examples/gcp-cloud-run/scripts/smoke.sh index ff14309eb..91a4d4d58 100755 --- a/examples/gcp-cloud-run/scripts/smoke.sh +++ b/examples/gcp-cloud-run/scripts/smoke.sh @@ -408,6 +408,8 @@ else !packages/shader-transitions/package.json !packages/aws-lambda/ !packages/aws-lambda/package.json +!packages/aws-lambda/scripts/ +!packages/aws-lambda/scripts/probe-beginframe.ts packages/**/node_modules/** packages/**/dist/** packages/**/coverage/** @@ -548,6 +550,19 @@ for protocol in "${PROTOCOL_LIST[@]}"; do continue fi + CAPTURE_MODES="$(jq -r \ + '.result | fromjson | [.Chunks[]?.CaptureMode // ""] | unique | join(",")' \ + "$EXECUTION_JSON")" + if jq -e \ + '.result | fromjson | .Chunks as $chunks | + (($chunks | length) > 0 and all($chunks[]; .CaptureMode == "beginframe"))' \ + "$EXECUTION_JSON" >/dev/null; then + echo " ✓ effective capture mode=beginframe" + else + echo " ✗ expected every chunk to use beginframe; observed=${CAPTURE_MODES:-}" + [ "$OVERALL_RC" -ne 0 ] || OVERALL_RC=6 + fi + OUTPUT_LOCAL="$RENDER_DIR/$protocol-c$chunk_size-output.mp4" gcloud storage cp "$OUTPUT_GCS" "$OUTPUT_LOCAL" --project "$PROJECT" >/dev/null diff --git a/packages/aws-lambda/scripts/probe-beginframe.test.ts b/packages/aws-lambda/scripts/probe-beginframe.test.ts new file mode 100644 index 000000000..b835e0eb0 --- /dev/null +++ b/packages/aws-lambda/scripts/probe-beginframe.test.ts @@ -0,0 +1,80 @@ +import { describe, expect, it } from "bun:test"; +import { mkdtempSync, rmSync, writeFileSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join, resolve } from "node:path"; +import { + _awaitBeforeDeadlineForTests, + _closeBrowserForProbeTests, + parseProbeArgs, +} from "./probe-beginframe.js"; + +describe("parseProbeArgs", () => { + it("defaults to the @sparticuz/chromium source", () => { + expect(parseProbeArgs([])).toEqual({}); + }); + + it("accepts a standalone executable path", () => { + expect(parseProbeArgs(["--executable-path", "/opt/chrome/chrome-headless-shell"])).toEqual({ + executablePath: "/opt/chrome/chrome-headless-shell", + }); + }); + + it("accepts the equals form and resolves relative paths", () => { + expect(parseProbeArgs(["--executable-path=./chrome"])).toEqual({ + executablePath: resolve("./chrome"), + }); + }); + + it("loads exact production launch arguments from JSON", () => { + const dir = mkdtempSync(join(tmpdir(), "hf-probe-args-")); + const argsPath = join(dir, "args.json"); + writeFileSync(argsPath, JSON.stringify(["--enable-begin-frame-control", "--no-sandbox"])); + try { + expect(parseProbeArgs(["--launch-args-json", argsPath])).toEqual({ + launchArgs: ["--enable-begin-frame-control", "--no-sandbox"], + }); + } finally { + rmSync(dir, { recursive: true, force: true }); + } + }); + + it("rejects missing values and unknown arguments", () => { + expect(() => parseProbeArgs(["--executable-path"])).toThrow( + "--executable-path requires a path", + ); + expect(() => parseProbeArgs(["--source", "chrome"])).toThrow("Unknown argument: --source"); + expect(() => parseProbeArgs(["--launch-args-json"])).toThrow( + "--launch-args-json requires a path", + ); + }); + + it("bounds a CDP operation that never resolves", async () => { + const never = new Promise(() => {}); + await expect( + _awaitBeforeDeadlineForTests(never, Date.now() + 25, "screenshot beginFrame"), + ).rejects.toThrow("timeout during screenshot beginFrame"); + }); + + it("force-kills and disconnects when graceful cleanup never resolves", async () => { + let killedWith: NodeJS.Signals | number | undefined; + let disconnected = false; + await _closeBrowserForProbeTests( + { + close: () => new Promise(() => {}), + process: () => ({ + kill: (signal) => { + killedWith = signal; + return true; + }, + }), + disconnect: async () => { + disconnected = true; + }, + }, + 25, + ); + + expect(killedWith).toBe("SIGKILL"); + expect(disconnected).toBe(true); + }); +}); diff --git a/packages/aws-lambda/scripts/probe-beginframe.ts b/packages/aws-lambda/scripts/probe-beginframe.ts index 0ac545ed5..9a332f195 100644 --- a/packages/aws-lambda/scripts/probe-beginframe.ts +++ b/packages/aws-lambda/scripts/probe-beginframe.ts @@ -1,14 +1,13 @@ #!/usr/bin/env tsx +// fallow-ignore-file code-duplication /** - * BeginFrame regression guard for `@sparticuz/chromium`. + * BeginFrame regression guard for a Chromium executable. * - * The load-bearing assumption of `@hyperframes/aws-lambda` is that the - * Chromium build shipped by `@sparticuz/chromium` honours CDP - * `HeadlessExperimental.beginFrame` with `screenshot: true`. This script - * boots that Chromium build (decompressing into `/tmp` per the library's - * runtime contract), navigates to a tiny static page, issues one - * `beginFrame` with a screenshot request, and asserts the response - * carries a PNG buffer. + * With no arguments, this boots the build shipped by `@sparticuz/chromium` + * (decompressing into `/tmp` per the library's runtime contract). Passing + * `--executable-path /path/to/chrome-headless-shell` probes an arbitrary + * executable instead; the GCP image build uses that form against the exact + * binary copied into the image. * * The script is the contract test, not a one-shot verification — every * release should run it inside the Docker container at @@ -17,15 +16,18 @@ * * Exits 0 on pass, 1 on fail. Run via: * - * bun run --cwd packages/aws-lambda probe:beginframe # host - * bun run --cwd packages/aws-lambda probe:beginframe:docker # Lambda-like + * bun run --cwd packages/aws-lambda probe:beginframe + * bun run --cwd packages/aws-lambda probe:beginframe -- \ + * --executable-path /opt/chrome/chrome-headless-shell + * bun run --cwd packages/aws-lambda probe:beginframe:docker */ -import { mkdtempSync, promises as fs } from "node:fs"; +import { mkdtempSync, promises as fs, readFileSync } from "node:fs"; import { tmpdir } from "node:os"; -import { join } from "node:path"; +import { join, resolve } from "node:path"; +import { fileURLToPath } from "node:url"; -interface ProbeResult { +export interface ProbeResult { passed: boolean; durationMs: number; chromiumPath: string; @@ -38,10 +40,140 @@ const PROBE_HTML = ` hf-beginframe-probe
hf-beginframe-probe
`; +const SCREENSHOT_ATTEMPTS = 10; +const PROBE_OPERATION_TIMEOUT_MS = 5000; +const PROBE_CLEANUP_TIMEOUT_MS = 250; + +export interface ProbeOptions { + executablePath?: string; + /** Exact launch arguments to probe instead of the standalone default profile. */ + launchArgs?: string[]; + /** Test override for the renderer/CDP operation deadline. */ + timeoutMs?: number; +} + +// The CLI accepts paired and equals forms for two independent path options. +// fallow-ignore-next-line complexity +export function parseProbeArgs(args: string[]): ProbeOptions { + let executablePath: string | undefined; + let launchArgs: string[] | undefined; + for (let i = 0; i < args.length; i += 1) { + const arg = args[i]; + if (arg === "--executable-path") { + const value = args[i + 1]; + if (!value || value.startsWith("--")) { + throw new Error("--executable-path requires a path"); + } + executablePath = resolve(value); + i += 1; + continue; + } + if (arg.startsWith("--executable-path=")) { + const value = arg.slice("--executable-path=".length); + if (!value) throw new Error("--executable-path requires a path"); + executablePath = resolve(value); + continue; + } + if (arg === "--launch-args-json") { + const value = args[i + 1]; + if (!value || value.startsWith("--")) { + throw new Error("--launch-args-json requires a path"); + } + launchArgs = readLaunchArgs(value); + i += 1; + continue; + } + if (arg.startsWith("--launch-args-json=")) { + const value = arg.slice("--launch-args-json=".length); + if (!value) throw new Error("--launch-args-json requires a path"); + launchArgs = readLaunchArgs(value); + continue; + } + throw new Error(`Unknown argument: ${arg}`); + } + return { + ...(executablePath ? { executablePath } : {}), + ...(launchArgs ? { launchArgs } : {}), + }; +} + +function readLaunchArgs(path: string): string[] { + const resolved = resolve(path); + const value: unknown = JSON.parse(readFileSync(resolved, "utf-8")); + if (!Array.isArray(value) || !value.every((item) => typeof item === "string")) { + throw new Error(`--launch-args-json must contain a JSON string array: ${resolved}`); + } + return value; +} + +async function awaitBeforeDeadline( + operation: Promise, + deadline: number, + label: string, +): Promise { + const remainingMs = deadline - Date.now(); + if (remainingMs <= 0) throw new Error(`BeginFrame probe timeout before ${label}`); + let timeout: ReturnType | undefined; + try { + return await Promise.race([ + operation, + new Promise((_, reject) => { + timeout = setTimeout( + () => reject(new Error(`BeginFrame probe timeout during ${label}`)), + remainingMs, + ); + }), + ]); + } finally { + if (timeout) clearTimeout(timeout); + } +} + +/** Test-only export for the standalone probe's bounded-operation contract. */ +export const _awaitBeforeDeadlineForTests = awaitBeforeDeadline; + +interface ProbeBrowserCleanup { + close(): Promise; + disconnect(): Promise; + process(): { kill(signal?: NodeJS.Signals | number): boolean } | null; +} + +async function settleWithin(operation: Promise, timeoutMs: number): Promise { + let timeout: ReturnType | undefined; + try { + return await Promise.race([ + operation.then( + () => true, + () => false, + ), + new Promise((resolveTimeout) => { + timeout = setTimeout(() => resolveTimeout(false), timeoutMs); + }), + ]); + } finally { + if (timeout) clearTimeout(timeout); + } +} + +async function closeBrowserForProbe( + browser: ProbeBrowserCleanup, + timeoutMs = PROBE_CLEANUP_TIMEOUT_MS, +): Promise { + if (await settleWithin(browser.close(), timeoutMs)) return; + try { + browser.process()?.kill("SIGKILL"); + } catch { + // Best effort; disconnect below still releases Puppeteer's transport. + } + await settleWithin(browser.disconnect(), timeoutMs); +} + +/** Test-only export for bounded standalone-probe cleanup. */ +export const _closeBrowserForProbeTests = closeBrowserForProbe; async function main(): Promise { const start = Date.now(); - const result = await probe(); + const result = await probe(parseProbeArgs(process.argv.slice(2))); result.durationMs = Date.now() - start; console.log(JSON.stringify(result, null, 2)); if (!result.passed) { @@ -49,12 +181,21 @@ async function main(): Promise { } } -async function probe(): Promise { +// This intentionally linear contract owns launch, renderer setup, CDP +// validation, diagnostics, and cleanup in one fail-closed lifecycle. +// fallow-ignore-next-line complexity +export async function probe(options: ProbeOptions = {}): Promise { let chromiumPath = ""; + let tmpHtmlDir = ""; try { - const { default: chromium } = await import("@sparticuz/chromium"); - chromiumPath = await chromium.executablePath(); - const args = chromium.args; + let sourceArgs: string[] = []; + if (options.executablePath) { + chromiumPath = options.executablePath; + } else { + const { default: chromium } = await import("@sparticuz/chromium"); + chromiumPath = await chromium.executablePath(); + sourceArgs = chromium.args; + } const puppeteer = await import("puppeteer-core"); @@ -63,7 +204,7 @@ async function probe(): Promise { // Chrome-side issue. `mkdtempSync` (vs `tmpdir() + Date.now()`) gives // an unguessable directory name so two concurrent probes on the same // host don't collide and CodeQL's insecure-tempfile rule clears. - const tmpHtmlDir = mkdtempSync(join(tmpdir(), "hf-beginframe-")); + tmpHtmlDir = mkdtempSync(join(tmpdir(), "hf-beginframe-")); const htmlPath = join(tmpHtmlDir, "probe.html"); await fs.writeFile(htmlPath, PROBE_HTML, "utf-8"); @@ -75,6 +216,11 @@ async function probe(): Promise { // ("Chrome's beginFrame with `screenshot` param always reports // hasDamage=true"). const beginFrameFlags = [ + "--no-sandbox", + "--disable-setuid-sandbox", + "--disable-dev-shm-usage", + "--enable-webgl", + "--ignore-gpu-blocklist", "--deterministic-mode", "--enable-begin-frame-control", "--disable-new-content-rendering-timeout", @@ -89,55 +235,97 @@ async function probe(): Promise { "--use-gl=angle", "--use-angle=swiftshader", "--enable-unsafe-swiftshader", + // Distributed Linux rendering explicitly uses software compositing to + // avoid stale transformed layers in SwiftShader (see browserManager). + "--disable-gpu-compositing", ]; const browser = await puppeteer.launch({ executablePath: chromiumPath, headless: "shell", - args: [...args, ...beginFrameFlags], + args: options.launchArgs ?? [...sourceArgs, ...beginFrameFlags], defaultViewport: { width: 800, height: 600 }, }); try { - const page = await browser.newPage(); - await page.goto(`file://${htmlPath}`, { waitUntil: "domcontentloaded", timeout: 30_000 }); - const session = await page.createCDPSession(); - await session.send("HeadlessExperimental.enable"); + const timeoutMs = options.timeoutMs ?? PROBE_OPERATION_TIMEOUT_MS; + const deadline = Date.now() + timeoutMs; + const page = await awaitBeforeDeadline(browser.newPage(), deadline, "newPage"); + await awaitBeforeDeadline( + page.goto(`file://${htmlPath}`, { waitUntil: "domcontentloaded", timeout: timeoutMs }), + deadline, + "navigation", + ); + const session = await awaitBeforeDeadline( + page.createCDPSession(), + deadline, + "CDP session creation", + ); + await awaitBeforeDeadline( + session.send("HeadlessExperimental.enable"), + deadline, + "HeadlessExperimental.enable", + ); // Warm-up beginFrame with noDisplayUpdates: true — drives the // compositor without producing a screenshot, matching how the engine // primes a capture loop. - await session.send("HeadlessExperimental.beginFrame", { - frameTimeTicks: 0, - interval: 33, - noDisplayUpdates: true, - }); - const response = await session.send("HeadlessExperimental.beginFrame", { - frameTimeTicks: 1000, - interval: 33, - screenshot: { format: "png" }, - }); - await fs.rm(tmpHtmlDir, { recursive: true, force: true }).catch(() => {}); - const screenshot = response.screenshotData ?? ""; - const bytes = screenshot ? Buffer.from(screenshot, "base64") : Buffer.alloc(0); - const isPng = - bytes.length >= 8 && - bytes[0] === 0x89 && - bytes[1] === 0x50 && - bytes[2] === 0x4e && - bytes[3] === 0x47; + await awaitBeforeDeadline( + session.send("HeadlessExperimental.beginFrame", { + frameTimeTicks: 0, + interval: 33, + noDisplayUpdates: true, + }), + deadline, + "warm-up beginFrame", + ); + let hasDamage = false; + let bytes = Buffer.alloc(0); + let isPng = false; + let attempts = 0; + // A renderer-ready document can still need more than one controlled + // frame before it submits a screenshot surface. Chromium explicitly + // permits screenshotData to be absent during renderer initialization, + // so retry a small bounded sequence with monotonically increasing ticks. + for (attempts = 1; attempts <= SCREENSHOT_ATTEMPTS; attempts += 1) { + const response = await awaitBeforeDeadline( + session.send("HeadlessExperimental.beginFrame", { + frameTimeTicks: 1000 + (attempts - 1) * 33, + interval: 33, + screenshot: { format: "png" }, + }), + deadline, + `screenshot beginFrame attempt ${attempts}`, + ); + hasDamage = response.hasDamage; + const screenshot = response.screenshotData ?? ""; + bytes = screenshot ? Buffer.from(screenshot, "base64") : Buffer.alloc(0); + isPng = + bytes.length >= 8 && + bytes[0] === 0x89 && + bytes[1] === 0x50 && + bytes[2] === 0x4e && + bytes[3] === 0x47; + if (isPng) break; + await awaitBeforeDeadline( + new Promise((resolveDelay) => setTimeout(resolveDelay, 10)), + deadline, + `screenshot retry delay ${attempts}`, + ); + } return { passed: isPng && bytes.length > 0, durationMs: 0, chromiumPath, screenshotBytes: bytes.length, - hasDamage: response.hasDamage, + hasDamage, detail: isPng - ? "OK — BeginFrame returned a PNG buffer." - : `FAIL — BeginFrame returned ${bytes.length} bytes, PNG signature ${ + ? `OK — BeginFrame returned a PNG buffer after ${attempts} attempt(s).` + : `FAIL — BeginFrame returned ${bytes.length} bytes after ${SCREENSHOT_ATTEMPTS} ` + + `attempts, PNG signature ${ bytes.length >= 4 ? bytes.subarray(0, 4).toString("hex") : "" }`, }; } finally { - await browser.close().catch(() => {}); + await closeBrowserForProbe(browser); } } catch (err) { return { @@ -148,10 +336,17 @@ async function probe(): Promise { hasDamage: false, detail: `FAIL — ${err instanceof Error ? err.message : String(err)}`, }; + } finally { + if (tmpHtmlDir) { + await fs.rm(tmpHtmlDir, { recursive: true, force: true }).catch(() => {}); + } } } -void main().catch((err) => { - console.error("[probe-beginframe] unexpected:", err); - process.exit(2); -}); +const invokedPath = process.argv[1] ? resolve(process.argv[1]) : ""; +if (invokedPath === fileURLToPath(import.meta.url)) { + void main().catch((err) => { + console.error("[probe-beginframe] unexpected:", err); + process.exit(2); + }); +} diff --git a/packages/aws-lambda/src/events.ts b/packages/aws-lambda/src/events.ts index e49da1d80..0756575f9 100644 --- a/packages/aws-lambda/src/events.ts +++ b/packages/aws-lambda/src/events.ts @@ -184,6 +184,8 @@ export interface RenderChunkLambdaResult { ChunkIndex: number; Sha256: string; FramesEncoded: number; + /** Effective engine mode after browser probing. Emitted by current handlers. */ + CaptureMode?: "beginframe" | "screenshot" | "drawelement"; DurationMs: number; } diff --git a/packages/aws-lambda/src/handler.test.ts b/packages/aws-lambda/src/handler.test.ts index 0b1787a11..5aa1adb94 100644 --- a/packages/aws-lambda/src/handler.test.ts +++ b/packages/aws-lambda/src/handler.test.ts @@ -377,6 +377,12 @@ describe("handler dispatch", () => { framesEncoded: 240, sha256: "0".repeat(64), durationMs: 12345, + planHashMs: 1, + sessionBootMs: 2, + captureStageMs: 3, + encodeStageMs: 4, + workers: 1, + captureMode: "beginframe", perfPath: outputChunkPath + ".perf.json", }; }, @@ -416,6 +422,7 @@ describe("handler dispatch", () => { expect(result.ChunkIndex).toBe(2); expect(result.Sha256).toBe("0".repeat(64)); expect(result.FramesEncoded).toBe(240); + expect(result.CaptureMode).toBe("beginframe"); expect(renderChunkMock).toHaveBeenCalledTimes(1); }); @@ -555,6 +562,12 @@ describe("handler dispatch", () => { framesEncoded: 30, sha256: "b".repeat(64), durationMs: 1, + planHashMs: 0, + sessionBootMs: 0, + captureStageMs: 1, + encodeStageMs: 0, + workers: 1, + captureMode: "beginframe", perfPath: `${outputPath}.perf.json`, }; }, @@ -631,6 +644,7 @@ describe("handler dispatch", () => { deps, ); if (chunk.Action !== "renderChunk") throw new Error("expected chunk result"); + expect(chunk.CaptureMode).toBe("beginframe"); const audioDigest = createHash("sha256").update("AAC").digest("hex"); const audioUri = `${planned.PlanV2ArtifactS3Prefix}/${audioDigest.slice(0, 2)}/${audioDigest}`; expect(s3.ops.slice(beforeChunk).some((operation) => operation.uri === audioUri)).toBe(false); diff --git a/packages/aws-lambda/src/handler.ts b/packages/aws-lambda/src/handler.ts index 9f2d6b97b..3a41b58fb 100644 --- a/packages/aws-lambda/src/handler.ts +++ b/packages/aws-lambda/src/handler.ts @@ -457,6 +457,7 @@ async function handleRenderChunk( ChunkIndex: event.ChunkIndex, Sha256: result.sha256, FramesEncoded: result.framesEncoded, + CaptureMode: result.captureMode, DurationMs: Date.now() - started, }; } finally { @@ -504,6 +505,7 @@ async function handleRenderChunkV2( ChunkIndex: event.ChunkIndex, Sha256: result.sha256, FramesEncoded: result.framesEncoded, + CaptureMode: result.captureMode, DurationMs: Date.now() - started, }; } finally { diff --git a/packages/engine/src/services/browserManager.test.ts b/packages/engine/src/services/browserManager.test.ts index 1e0d26c28..3ef277bae 100644 --- a/packages/engine/src/services/browserManager.test.ts +++ b/packages/engine/src/services/browserManager.test.ts @@ -1,3 +1,4 @@ +// fallow-ignore-file code-duplication import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; import { execFileSync } from "node:child_process"; import { mkdirSync, mkdtempSync, rmSync, writeFileSync } from "node:fs"; @@ -9,6 +10,9 @@ import type { Browser, PuppeteerNode } from "puppeteer-core"; import { _resetAutoBrowserGpuModeCacheForTests, _resetBrowserPoolForTests, + _closeBrowserAfterFailedProbeForTests, + _createBrowserLaunchFingerprintForTests, + _probeBeginFrameSupportForTests, _setPuppeteerForTests, acquireBrowser, buildChromeArgs, @@ -19,6 +23,102 @@ import { resolveBrowserGpuMode, } from "./browserManager.js"; +describe("BeginFrame capability probe", () => { + it("waits for a document and validates a PNG-returning frame", async () => { + const png = Buffer.from([0x89, 0x50, 0x4e, 0x47, 0x0d, 0x0a, 0x1a, 0x0a]); + const send = vi + .fn() + .mockResolvedValueOnce({}) + .mockResolvedValueOnce({}) + .mockResolvedValueOnce({ hasDamage: true, screenshotData: png.toString("base64") }); + const detach = vi.fn().mockResolvedValue(undefined); + const goto = vi.fn().mockResolvedValue(null); + const close = vi.fn().mockResolvedValue(undefined); + const browser = { + newPage: vi.fn().mockResolvedValue({ + goto, + createCDPSession: vi.fn().mockResolvedValue({ send, detach }), + close, + }), + } as unknown as Browser; + + const result = await _probeBeginFrameSupportForTests(browser); + + expect(result.supported).toBe(true); + expect(goto).toHaveBeenCalledWith(expect.stringContaining("hf-beginframe-probe"), { + waitUntil: "domcontentloaded", + timeout: 2000, + }); + expect(send.mock.calls.map(([method]) => method)).toEqual([ + "HeadlessExperimental.enable", + "HeadlessExperimental.beginFrame", + "HeadlessExperimental.beginFrame", + ]); + expect(detach).toHaveBeenCalledOnce(); + expect(close).toHaveBeenCalledOnce(); + }); + + it("reports an empty screenshot as unsupported", async () => { + const send = vi.fn().mockResolvedValue({ hasDamage: false }); + const browser = { + newPage: vi.fn().mockResolvedValue({ + goto: vi.fn().mockResolvedValue(null), + createCDPSession: vi.fn().mockResolvedValue({ + send, + detach: vi.fn().mockResolvedValue(undefined), + }), + close: vi.fn().mockResolvedValue(undefined), + }), + } as unknown as Browser; + + const result = await _probeBeginFrameSupportForTests(browser); + + expect(result.supported).toBe(false); + expect(result.detail).toContain("returned 0 bytes after 10 attempts"); + }); + + it("bounds a screenshot-bearing CDP call that never resolves", async () => { + const never = new Promise(() => {}); + const send = vi + .fn() + .mockResolvedValueOnce({}) + .mockResolvedValueOnce({}) + .mockReturnValueOnce(never); + const close = vi.fn().mockResolvedValue(undefined); + const browser = { + newPage: vi.fn().mockResolvedValue({ + goto: vi.fn().mockResolvedValue(null), + createCDPSession: vi.fn().mockResolvedValue({ + send, + detach: vi.fn().mockResolvedValue(undefined), + }), + close, + }), + } as unknown as Browser; + + const result = await _probeBeginFrameSupportForTests(browser, 25); + + expect(result.supported).toBe(false); + expect(result.detail).toContain("timeout during screenshot beginFrame attempt 1"); + expect(close).toHaveBeenCalledOnce(); + }); + + it("force-kills and disconnects when graceful browser cleanup never resolves", async () => { + const kill = vi.fn(); + const disconnect = vi.fn().mockResolvedValue(undefined); + const browser = { + close: vi.fn().mockReturnValue(new Promise(() => {})), + process: vi.fn().mockReturnValue({ kill }), + disconnect, + } as unknown as Browser; + + await _closeBrowserAfterFailedProbeForTests(browser, 25); + + expect(kill).toHaveBeenCalledWith("SIGKILL"); + expect(disconnect).toHaveBeenCalledOnce(); + }); +}); + describe("buildChromeArgs browser GPU mode", () => { const base = { width: 1920, height: 1080 }; @@ -89,6 +189,31 @@ describe("buildChromeArgs browser GPU mode", () => { }); }); +describe("browser launch capture-mode contract", () => { + it("derives BeginFrame from the actual launch flags, not forceScreenshot alone", () => { + const dir = mkdtempSync(join(tmpdir(), "hf-browser-fingerprint-")); + const chromePath = join(dir, "chrome-headless-shell"); + writeFileSync(chromePath, ""); + try { + const withoutControl = _createBrowserLaunchFingerprintForTests([], { + chromePath, + forceScreenshot: false, + }); + const withControl = _createBrowserLaunchFingerprintForTests( + ["--enable-begin-frame-control"], + { chromePath, forceScreenshot: false }, + ); + + expect(withoutControl.requestedCaptureMode).toBe("screenshot"); + expect(withControl.requestedCaptureMode).toBe( + process.platform === "linux" ? "beginframe" : "screenshot", + ); + } finally { + rmSync(dir, { recursive: true, force: true }); + } + }); +}); + describe("resolveBrowserGpuMode", () => { const setMockWebGlProbe = (info: { hasWebGL: boolean; vendor: string; renderer: string }) => { const close = vi.fn().mockResolvedValue(undefined); diff --git a/packages/engine/src/services/browserManager.ts b/packages/engine/src/services/browserManager.ts index 2351f4ede..1f6398b95 100644 --- a/packages/engine/src/services/browserManager.ts +++ b/packages/engine/src/services/browserManager.ts @@ -217,44 +217,192 @@ function stripBeginFrameFlags(args: string[]): string[] { } /** - * Probe whether the browser still speaks HeadlessExperimental.beginFrame. + * Probe the complete HeadlessExperimental.beginFrame runtime contract. * - * Recent chrome-headless-shell builds (observed on 147) expose the domain - * well enough that HeadlessExperimental.enable succeeds but drop the - * beginFrame method itself — the capture loop then dies on first frame with - * `'HeadlessExperimental.beginFrame' wasn't found`. So we probe BOTH: enable - * + one cheap beginFrame raced against a 2s timeout. In beginframe-control - * mode the command completes as soon as the compositor acks, so a real - * supported browser returns well under the timeout. - * - * Any failure (method missing, timeout, protocol error) is treated as - * unsupported. Real errors after launch would surface in the warmup loop and - * fall out through the caller's try/catch. + * Domain registration alone is insufficient: BeginFrame control must be + * enabled at launch and a renderer-ready target must return a real PNG. + * Every operation shares one short deadline so a wedged CDP call cannot hold + * a serverless cold start until Puppeteer's much longer protocol timeout. */ -async function probeBeginFrameSupport(browser: Browser): Promise { - let page; +interface BeginFrameProbeResult { + supported: boolean; + detail: string; + durationMs: number; +} + +const BEGINFRAME_SCREENSHOT_PROBE_ATTEMPTS = 10; +const BEGINFRAME_PROBE_TIMEOUT_MS = 2000; +const BEGINFRAME_PROBE_CLEANUP_TIMEOUT_MS = 250; + +async function awaitBeforeDeadline( + operation: Promise, + deadline: number, + label: string, +): Promise { + const remainingMs = deadline - Date.now(); + if (remainingMs <= 0) { + throw new Error(`beginFrame probe timeout before ${label}`); + } + let timeout: ReturnType | undefined; try { - page = await browser.newPage(); - const client = await page.createCDPSession(); - await client.send("HeadlessExperimental.enable"); - const beginFrame = client.send("HeadlessExperimental.beginFrame", { - frameTimeTicks: 0, - interval: 33, - noDisplayUpdates: true, - }); - const timeout = new Promise((_, reject) => - setTimeout(() => reject(new Error("beginFrame probe timeout")), 2000), - ); - await Promise.race([beginFrame, timeout]); - await client.detach().catch(() => {}); - return true; - } catch { - return false; + return await Promise.race([ + operation, + new Promise((_, reject) => { + timeout = setTimeout( + () => reject(new Error(`beginFrame probe timeout during ${label}`)), + remainingMs, + ); + }), + ]); } finally { - await page?.close().catch(() => {}); + if (timeout) clearTimeout(timeout); } } +async function settleWithin(operation: Promise, timeoutMs: number): Promise { + let timeout: ReturnType | undefined; + try { + return await Promise.race([ + operation.then( + () => true, + () => false, + ), + new Promise((resolveTimeout) => { + timeout = setTimeout(() => resolveTimeout(false), timeoutMs); + }), + ]); + } finally { + if (timeout) clearTimeout(timeout); + } +} + +async function closeBrowserAfterFailedProbe( + browser: Browser, + timeoutMs = BEGINFRAME_PROBE_CLEANUP_TIMEOUT_MS, +): Promise { + if (await settleWithin(browser.close(), timeoutMs)) return; + // A wedged CDP transport can make graceful close inherit Puppeteer's + // multi-minute protocol timeout. Kill the owned process and disconnect so + // screenshot fallback can launch promptly. + try { + browser.process()?.kill("SIGKILL"); + } catch { + // Best effort; disconnect below still releases Puppeteer's transport. + } + await settleWithin(browser.disconnect(), timeoutMs); +} + +// The probe keeps its renderer setup, bounded CDP sequence, PNG validation, +// diagnostics, and cleanup together so every failure uses one contract. +// fallow-ignore-next-line complexity +async function probeBeginFrameSupport( + browser: Browser, + timeoutMs = BEGINFRAME_PROBE_TIMEOUT_MS, +): Promise { + const started = Date.now(); + const deadline = started + timeoutMs; + let page; + let result: BeginFrameProbeResult; + try { + page = await awaitBeforeDeadline(browser.newPage(), deadline, "newPage"); + // `browser.newPage()` resolves before a cold renderer has necessarily + // submitted its first surface. Cloud Run exposed this as a false + // "unsupported Chromium" result: probing the untouched about:blank + // target raced renderer initialization, while the same binary passed + // once a real document was ready. Navigate first so this tests protocol + // capability rather than target-startup timing. + await awaitBeforeDeadline( + page.goto( + "data:text/html,
hf-beginframe-probe
", + { waitUntil: "domcontentloaded", timeout: timeoutMs }, + ), + deadline, + "navigation", + ); + const client = await awaitBeforeDeadline( + page.createCDPSession(), + deadline, + "CDP session creation", + ); + await awaitBeforeDeadline( + client.send("HeadlessExperimental.enable"), + deadline, + "HeadlessExperimental.enable", + ); + await awaitBeforeDeadline( + client.send("HeadlessExperimental.beginFrame", { + frameTimeTicks: 0, + interval: 33, + noDisplayUpdates: true, + }), + deadline, + "warm-up beginFrame", + ); + let bytes = Buffer.alloc(0); + let isPng = false; + let attempts = 0; + for (attempts = 1; attempts <= BEGINFRAME_SCREENSHOT_PROBE_ATTEMPTS; attempts += 1) { + const response = await awaitBeforeDeadline( + client.send("HeadlessExperimental.beginFrame", { + frameTimeTicks: 1000 + (attempts - 1) * 33, + interval: 33, + screenshot: { format: "png" }, + }), + deadline, + `screenshot beginFrame attempt ${attempts}`, + ); + const screenshot = response.screenshotData ?? ""; + bytes = screenshot ? Buffer.from(screenshot, "base64") : Buffer.alloc(0); + isPng = + bytes.length >= 8 && + bytes[0] === 0x89 && + bytes[1] === 0x50 && + bytes[2] === 0x4e && + bytes[3] === 0x47; + if (isPng) break; + await awaitBeforeDeadline( + new Promise((resolveDelay) => setTimeout(resolveDelay, 10)), + deadline, + `screenshot retry delay ${attempts}`, + ); + } + if (!isPng) { + throw new Error( + `beginFrame screenshot returned ${bytes.length} bytes after ` + + `${BEGINFRAME_SCREENSHOT_PROBE_ATTEMPTS} attempts with signature ` + + `${bytes.length >= 4 ? bytes.subarray(0, 4).toString("hex") : ""}`, + ); + } + await awaitBeforeDeadline(client.detach(), deadline, "CDP detach").catch(() => {}); + result = { + supported: true, + detail: + `enable + warm-up + ${bytes.length}-byte PNG beginFrame succeeded ` + + `after ${attempts} screenshot attempt(s)`, + durationMs: Date.now() - started, + }; + } catch (error) { + result = { + supported: false, + detail: error instanceof Error ? error.message : String(error), + durationMs: Date.now() - started, + }; + } + if (page && !(await settleWithin(page.close(), BEGINFRAME_PROBE_CLEANUP_TIMEOUT_MS))) { + return { + supported: false, + detail: `${result.detail}; probe page cleanup timed out`, + durationMs: Date.now() - started, + }; + } + return result; +} + +/** Test-only export for the renderer-readiness + PNG capability contract. */ +export const _probeBeginFrameSupportForTests = probeBeginFrameSupport; +/** Test-only export for the forced browser-cleanup fallback. */ +export const _closeBrowserAfterFailedProbeForTests = closeBrowserAfterFailedProbe; + /** * Cached *in-flight or resolved* probe Promise for `resolveBrowserGpuMode("auto", ...)`. * @@ -391,8 +539,17 @@ function createBrowserLaunchFingerprint( ...config, }; const headlessShell = resolveHeadlessShellPath(launchConfig); + // The launch arguments are the authoritative capture-mode contract. + // A caller can pass `forceScreenshot:false` while a later safety clamp + // deliberately omits BeginFrame control flags. Inferring solely from the + // raw config in that case makes BrowserManager probe a capability it never + // enabled, then mislabel the result as an unsupported Chromium build. + const beginFrameControlEnabled = chromeArgs.includes("--enable-begin-frame-control"); const requestedCaptureMode: CaptureMode = - headlessShell && process.platform === "linux" && !launchConfig.forceScreenshot + headlessShell && + process.platform === "linux" && + !launchConfig.forceScreenshot && + beginFrameControlEnabled ? "beginframe" : "screenshot"; return { @@ -404,6 +561,9 @@ function createBrowserLaunchFingerprint( }; } +/** Test-only export for launch-argument/capture-mode agreement. */ +export const _createBrowserLaunchFingerprintForTests = createBrowserLaunchFingerprint; + export async function acquireBrowser( chromeArgs: string[], config?: Partial< @@ -443,12 +603,17 @@ async function launchBrowser( ); if (captureMode === "beginframe") { - const supported = await probeBeginFrameSupport(browser).catch(() => true); - if (!supported) { - await browser.close().catch(() => {}); + const probe = await probeBeginFrameSupport(browser).catch((error) => ({ + supported: true, + detail: `probe harness error ignored: ${error instanceof Error ? error.message : String(error)}`, + durationMs: 0, + })); + if (!probe.supported) { + await closeBrowserAfterFailedProbe(browser); browser = undefined; console.warn( - "[BrowserManager] HeadlessExperimental.beginFrame unavailable in this Chromium build; falling back to screenshot mode.", + `[BrowserManager] HeadlessExperimental.beginFrame probe failed after ${probe.durationMs}ms: ` + + `${probe.detail}; falling back to screenshot mode.`, ); captureMode = "screenshot"; browser = await ppt.launch({ diff --git a/packages/gcp-cloud-run/Dockerfile b/packages/gcp-cloud-run/Dockerfile index a084849e8..f0126bd8d 100644 --- a/packages/gcp-cloud-run/Dockerfile +++ b/packages/gcp-cloud-run/Dockerfile @@ -21,7 +21,7 @@ # Unlike the AWS Lambda adapter there is no ZIP-size ceiling and no runtime # Chrome decompression step — the binary lives in the image at a fixed path. -FROM node:22-bookworm-slim +FROM node:22-bookworm-slim AS beginframe-contract # ── System dependencies (identical set to the producer's render image) ─────── RUN apt-get update && apt-get install -y --no-install-recommends \ @@ -55,9 +55,9 @@ RUN apt-get update && apt-get install -y --no-install-recommends \ && fc-cache -fv # ── chrome-headless-shell (deterministic BeginFrame capture) ───────────────── -# Pinned to the SAME build the producer regression baselines were generated -# against so distributed renders are pixel-comparable. Bump together with the -# producer's Dockerfile.test pin and a baseline regen. +# Pinned deliberately. The build-stage contract probe below launches this +# exact executable and requires a renderer-ready warm-up plus a PNG-returning +# HeadlessExperimental.beginFrame before the image can build. RUN npx --yes @puppeteer/browsers install chrome-headless-shell@148.0.7778.167 \ --path /opt/puppeteer \ && CHS="$(find /opt/puppeteer/chrome-headless-shell -name chrome-headless-shell -type f | head -n1)" \ @@ -100,6 +100,23 @@ COPY packages/studio-server/package.json packages/studio-server/package.json COPY scripts/package-subpaths.mjs scripts/package-subpaths.mjs RUN bun install --frozen-lockfile +# Fail the image build closed if the packaged executable cannot perform the +# same BeginFrame operation the distributed renderer relies on. Generate the +# argument list with the production engine helper so a launcher-flag regression +# invalidates this layer and fails the contract instead of testing a duplicate +# hard-coded profile. +COPY packages/engine/ packages/engine/ +COPY packages/aws-lambda/scripts/probe-beginframe.ts packages/aws-lambda/scripts/probe-beginframe.ts +RUN bun -e 'import { buildChromeArgs } from "./packages/engine/src/services/browserManager.ts"; await Bun.write("/tmp/hf-gcp-chrome-args.json", JSON.stringify(buildChromeArgs({ width: 800, height: 600, captureMode: "beginframe", platform: "linux" }, { browserGpuMode: "software" })))' \ + && bun packages/aws-lambda/scripts/probe-beginframe.ts \ + --executable-path /opt/chrome/chrome-headless-shell \ + --launch-args-json /tmp/hf-gcp-chrome-args.json + +# CI targets `beginframe-contract` directly, so pin/probe changes do not need +# the full monorepo build merely to verify browser capability. The deployable +# image continues from the already-proven binary and installed workspaces. +FROM beginframe-contract AS runtime + # Copy source for the packages the render path needs. COPY packages/core/ packages/core/ COPY packages/engine/ packages/engine/ diff --git a/packages/gcp-cloud-run/README.md b/packages/gcp-cloud-run/README.md index ec6faf985..c2aa7fb8c 100644 --- a/packages/gcp-cloud-run/README.md +++ b/packages/gcp-cloud-run/README.md @@ -50,7 +50,11 @@ decompresses `@sparticuz/chromium` into `/tmp` at runtime — Cloud Run runs a container image. The `Dockerfile` installs the same pinned `chrome-headless-shell` build and font set the production renderer uses, at a fixed path, and exports `HYPERFRAMES_CHROME_PATH`. CDP-level `BeginFrame` -works because the command lives in the protocol, not the binary. There is no +support is a binary/runtime capability, so the image build launches that +exact executable and requires an enable + warm-up + PNG-returning +`HeadlessExperimental.beginFrame` probe to pass. The end-to-end smoke also +requires every chunk to report effective `CaptureMode: "beginframe"`, which +catches runtime fallback separately from build-time packaging. There is no runtime decompression step and no packaging ceiling. ## Deploying diff --git a/packages/gcp-cloud-run/src/chromium.ts b/packages/gcp-cloud-run/src/chromium.ts index 5e18b6859..a5d0ff2d7 100644 --- a/packages/gcp-cloud-run/src/chromium.ts +++ b/packages/gcp-cloud-run/src/chromium.ts @@ -5,9 +5,10 @@ * via the engine's `BrowserManager`. Because Cloud Run runs a container * image rather than a size-capped ZIP, the Chrome story is far simpler than * the Lambda adapter's: the `Dockerfile` installs `chrome-headless-shell` - * (the same BeginFrame-capable build the K8s deploy uses) into the image at - * a known path and exports `HYPERFRAMES_CHROME_PATH`. There is no runtime - * decompression-into-/tmp step and no 250 MB packaging ceiling to fight. + * into the image at a known path and exports `HYPERFRAMES_CHROME_PATH`. + * The Dockerfile proves that exact executable's full BeginFrame screenshot + * contract during the image build. There is no runtime decompression-into- + * /tmp step and no 250 MB packaging ceiling to fight. * * Resolution order: * 1. `PRODUCER_HEADLESS_SHELL_PATH` — the engine's own override. If a diff --git a/packages/gcp-cloud-run/src/events.ts b/packages/gcp-cloud-run/src/events.ts index 98ab840aa..a65e0df76 100644 --- a/packages/gcp-cloud-run/src/events.ts +++ b/packages/gcp-cloud-run/src/events.ts @@ -203,6 +203,8 @@ export interface RenderChunkResultBody { ChunkIndex: number; Sha256: string; FramesEncoded: number; + /** Effective engine mode after browser probing. Emitted by current handlers. */ + CaptureMode?: "beginframe" | "screenshot" | "drawelement"; DurationMs: number; } diff --git a/packages/gcp-cloud-run/src/server.test.ts b/packages/gcp-cloud-run/src/server.test.ts index 0028e9d2d..a6a1596f6 100644 --- a/packages/gcp-cloud-run/src/server.test.ts +++ b/packages/gcp-cloud-run/src/server.test.ts @@ -121,6 +121,14 @@ function depsWith( outputKind: "file", framesEncoded: 30, sha256: `sha-${chunkIndex}`, + captureMode: "beginframe", + durationMs: 0, + planHashMs: 0, + sessionBootMs: 0, + captureStageMs: 0, + encodeStageMs: 0, + workers: 1, + perfPath: `${outputBase}.perf.json`, } satisfies ChunkResult; }; const assemble = async ( @@ -201,6 +209,7 @@ describe("dispatch", () => { const res = await dispatch(event, depsWith(gcs)); if (res.Action !== "renderChunk") throw new Error("unreachable"); expect(res.ChunkIndex).toBe(2); + expect(res.CaptureMode).toBe("beginframe"); expect(res.ChunkGcsUri).toBe("gs://b/renders/r1/chunks/0002.mp4"); expect(gcs.objects.has("gs://b/renders/r1/chunks/0002.mp4")).toBe(true); }); @@ -270,6 +279,14 @@ describe("dispatch", () => { outputKind: "file", framesEncoded: 30, sha256: "b".repeat(64), + captureMode: "beginframe", + durationMs: 0, + planHashMs: 0, + sessionBootMs: 0, + captureStageMs: 0, + encodeStageMs: 0, + workers: 1, + perfPath: `${outputBase}.perf.json`, }; }; const assemble = async ( diff --git a/packages/gcp-cloud-run/src/server.ts b/packages/gcp-cloud-run/src/server.ts index 02959b7c2..fd2ba6b7b 100644 --- a/packages/gcp-cloud-run/src/server.ts +++ b/packages/gcp-cloud-run/src/server.ts @@ -455,6 +455,7 @@ async function handleRenderChunk( ChunkIndex: event.ChunkIndex, Sha256: result.sha256, FramesEncoded: result.framesEncoded, + CaptureMode: result.captureMode, DurationMs: Date.now() - started, }; } finally { @@ -500,6 +501,7 @@ async function handleRenderChunkV2( ChunkIndex: event.ChunkIndex, Sha256: result.sha256, FramesEncoded: result.framesEncoded, + CaptureMode: result.captureMode, DurationMs: Date.now() - started, }; } finally { diff --git a/packages/gcp-cloud-run/terraform/smoke-safety.test.ts b/packages/gcp-cloud-run/terraform/smoke-safety.test.ts index 9b8bd0248..4ba0df321 100644 --- a/packages/gcp-cloud-run/terraform/smoke-safety.test.ts +++ b/packages/gcp-cloud-run/terraform/smoke-safety.test.ts @@ -14,6 +14,8 @@ describe("GCP smoke ownership and protocol safety", () => { expect(smoke).toContain("decodedFramesEqual"); expect(smoke).toContain("decodedAudioEqual"); expect(smoke).toContain("normalizedMetadataEqual"); + expect(smoke).toContain('.CaptureMode == "beginframe"'); + expect(smoke).toContain("expected every chunk to use beginframe"); }); it("derives a length-safe owner prefix and isolates Terraform state", () => { @@ -51,6 +53,7 @@ describe("GCP smoke ownership and protocol safety", () => { expect(smoke).toContain("--ignore-file"); expect(smoke).toContain("--gcs-source-staging-dir"); expect(smoke).toContain("!scripts/package-subpaths.mjs"); + expect(smoke).toContain("!packages/aws-lambda/scripts/probe-beginframe.ts"); expect(dockerfile).toContain("COPY scripts/package-subpaths.mjs scripts/package-subpaths.mjs"); expect(smoke).not.toContain("gcloud services enable"); expect(smoke).toContain("gcloud services list"); diff --git a/packages/producer/src/services/distributed/crossWorkerIdempotency.test.ts b/packages/producer/src/services/distributed/crossWorkerIdempotency.test.ts index 219e76c56..1de53db2b 100644 --- a/packages/producer/src/services/distributed/crossWorkerIdempotency.test.ts +++ b/packages/producer/src/services/distributed/crossWorkerIdempotency.test.ts @@ -238,6 +238,8 @@ describe("cross-worker idempotency", () => { expect(b.outputKind).toBe("file"); expect(a.framesEncoded).toBeGreaterThan(0); expect(b.framesEncoded).toBe(a.framesEncoded); + expect(a.captureMode).toBe("beginframe"); + expect(b.captureMode).toBe("beginframe"); expect(a.sha256).toBe(b.sha256); assertBytesEqual(outA, outB, "file", `mp4 chunk ${chunkIndex}`); diff --git a/packages/producer/src/services/distributed/plan.test.ts b/packages/producer/src/services/distributed/plan.test.ts index 5473e53f7..e8dd5ecb8 100644 --- a/packages/producer/src/services/distributed/plan.test.ts +++ b/packages/producer/src/services/distributed/plan.test.ts @@ -19,6 +19,7 @@ import { afterAll, beforeAll, describe, expect, it } from "bun:test"; import { existsSync, mkdirSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from "node:fs"; import { tmpdir } from "node:os"; import { join } from "node:path"; +import { applyConcreteGpuScreenshotClamp, buildChromeArgs } from "@hyperframes/engine"; import { recomputePlanHashFromPlanDir } from "../render/stages/freezePlan.js"; import { RenderQualityError } from "../renderOrchestrator.js"; import { CURRENT_PLAN_PROTOCOL } from "./planProtocol.js"; @@ -29,6 +30,7 @@ import { DEFAULT_MAX_PARALLEL_CHUNKS, MIN_CHUNK_SIZE, plan, + resolveDistributedEngineConfig, resolveChunkPlan, } from "./plan.js"; import { buildSyntheticRenderJob } from "./shared.js"; @@ -155,6 +157,26 @@ describe("distributed synthetic render job", () => { expect(job.config.variables).toEqual(variables); }); + + it("keeps the production software-GPU launch on BeginFrame control", () => { + const cfg = resolveDistributedEngineConfig({ + fps: 30, + width: 320, + height: 240, + format: "mp4", + }); + const forceScreenshot = applyConcreteGpuScreenshotClamp( + cfg.forceScreenshot, + "software", + cfg, + {}, + ); + const captureMode = forceScreenshot ? "screenshot" : "beginframe"; + const args = buildChromeArgs({ width: 320, height: 240, captureMode, platform: "linux" }, cfg); + + expect(forceScreenshot).toBe(false); + expect(args).toContain("--enable-begin-frame-control"); + }); }); describe("resolveChunkPlan", () => { diff --git a/packages/producer/src/services/distributed/plan.ts b/packages/producer/src/services/distributed/plan.ts index db414b7f9..2960251dc 100644 --- a/packages/producer/src/services/distributed/plan.ts +++ b/packages/producer/src/services/distributed/plan.ts @@ -778,6 +778,20 @@ function resolveNonMp4EncoderTriple( return { encoder: "png-sequence", pixelFormat: "rgba", preset: "lossless" }; } +/** Test-visible construction of the engine config used by distributed planning. */ +export function resolveDistributedEngineConfig(config: DistributedRenderConfig): EngineConfig { + return { + ...(config.producerConfig ?? config.engineConfig ?? resolveConfig()), + browserGpuMode: "software", + forceScreenshot: false, + // Distributed rendering deliberately opts into deterministic BeginFrame + // on Linux SwiftShader. Preserve the provenance bit consumed by the + // engine's software-GPU screenshot clamp; assigning the boolean alone + // loses the distinction between a default false and this explicit opt-out. + forceScreenshotExplicitlyOptedOut: true, + }; +} + /** * Activity A of the distributed render pipeline. Produces a self-contained * `/` from a project + config. See module docstring for the @@ -808,11 +822,7 @@ export async function plan( throw new Error("[plan] render_cancelled"); } }; - const cfg: EngineConfig = { - ...(config.producerConfig ?? config.engineConfig ?? resolveConfig()), - browserGpuMode: "software", - forceScreenshot: false, - }; + const cfg = resolveDistributedEngineConfig(config); const job = buildSyntheticRenderJob({ fps: { num: config.fps, den: 1 }, diff --git a/packages/producer/src/services/distributed/renderChunk.test.ts b/packages/producer/src/services/distributed/renderChunk.test.ts index ab0708d7f..f45ea6ef9 100644 --- a/packages/producer/src/services/distributed/renderChunk.test.ts +++ b/packages/producer/src/services/distributed/renderChunk.test.ts @@ -203,8 +203,11 @@ describe("renderChunk()", () => { expect(a.captureStageMs).toBeGreaterThan(0); expect(a.encodeStageMs).toBeGreaterThanOrEqual(0); expect(a.workers).toBeGreaterThanOrEqual(1); + expect(["beginframe", "screenshot", "drawelement"]).toContain(a.captureMode); + expect(b.captureMode).toBe(a.captureMode); expect(a.captureStageMs + a.encodeStageMs).toBeLessThanOrEqual(a.durationMs); const perf = JSON.parse(readFileSync(a.perfPath, "utf-8")); + expect(perf.captureMode).toBe(a.captureMode); for (const key of [ "planHashMs", "sessionBootMs", diff --git a/packages/producer/src/services/distributed/renderChunk.ts b/packages/producer/src/services/distributed/renderChunk.ts index f940b0379..daaa41cc8 100644 --- a/packages/producer/src/services/distributed/renderChunk.ts +++ b/packages/producer/src/services/distributed/renderChunk.ts @@ -43,6 +43,8 @@ import { BROWSER_GPU_NOT_SOFTWARE, calculateOptimalWorkers, type CaptureOptions, + type CaptureMode, + type CapturePerfSummary, type CaptureSession, closeCaptureSession, createCaptureSession, @@ -158,6 +160,8 @@ export interface ChunkResult { encodeStageMs: number; /** Capture workers used for this chunk (`calculateOptimalWorkers` result). */ workers: number; + /** Effective engine mode used by every worker, after any browser fallback. */ + captureMode: CaptureMode; /** * Path to a sidecar JSON containing per-chunk perf counters. Adapters * upload this alongside the chunk so per-chunk regressions are @@ -330,6 +334,7 @@ export function resolveLockedVp9CpuUsed( * outputs — the caller picks the right shape based on `meta/encoder.json`. * `renderChunk` enforces the same choice via `outputKind` on the result. */ +// fallow-ignore-next-line complexity export async function renderChunk( planDir: string, chunkIndex: number, @@ -485,6 +490,10 @@ export async function renderChunk( ...resolveConfig(), browserGpuMode: "software", forceScreenshot: encoder.forceScreenshot, + // `encoder.forceScreenshot=false` is a locked distributed-render + // decision, not the engine default. Carry that explicit opt-out through + // the software-GPU clamp so buildChromeArgs includes BeginFrameControl. + forceScreenshotExplicitlyOptedOut: !encoder.forceScreenshot, }; // Build the BeforeCaptureHook that injects pre-extracted video frames @@ -585,6 +594,8 @@ export async function renderChunk( let sessionBootMs = 0; let captureStageMs = 0; let encodeStageMs = 0; + let captureMode: CaptureMode | undefined; + const capturePerfs: CapturePerfSummary[] = []; try { if (chunkWorkerCount === 1) { // Sequential branch reuses the probe session for the actual capture. @@ -638,9 +649,10 @@ export async function renderChunk( log, probeSession: session, captureAttempts: [], - // Distributed chunks run on Linux (beginframe) where dedup never arms; - // a throwaway sink satisfies the type without per-chunk dedup reporting. - dedupPerfs: [], + // This sink also records each worker's effective capture mode. That + // makes a BeginFrame → screenshot fallback observable to adapters and + // end-to-end smoke tests instead of existing only in stderr. + dedupPerfs: capturePerfs, buildCaptureOptions: () => captureOptions, createRenderVideoFrameInjector: () => videoInjector, abortSignal: undefined, @@ -650,6 +662,20 @@ export async function renderChunk( // captureStage closes the session it consumed. captureStageMs = Date.now() - captureStarted; session = null; + const observedModes = new Set(capturePerfs.map((perf) => perf.captureMode)); + const validModes = new Set(["beginframe", "screenshot", "drawelement"]); + if ( + observedModes.size !== 1 || + ![...observedModes].every((mode): mode is CaptureMode => + validModes.has(mode as CaptureMode), + ) + ) { + throw new Error( + `[renderChunk] capture workers reported invalid or inconsistent modes: ` + + `${[...observedModes].join(",") || ""}`, + ); + } + captureMode = [...observedModes][0] as CaptureMode; framesEncoded = framesInChunk; // ── Encode the chunk ── @@ -737,6 +763,9 @@ export async function renderChunk( } // ── Hash the output + write the perf sidecar ── + if (!captureMode) { + throw new Error("[renderChunk] capture stage completed without reporting a capture mode"); + } const sha256 = hashChunkOutput(outputChunkPath, outputKind); const durationMs = Date.now() - start; const perfPath = `${outputChunkPath}.perf.json`; @@ -752,6 +781,7 @@ export async function renderChunk( captureStageMs, encodeStageMs, workers: chunkWorkerCount, + captureMode, sha256, outputKind, producerVersion: plan.producerVersion, @@ -781,6 +811,7 @@ export async function renderChunk( captureStageMs, encodeStageMs, workers: chunkWorkerCount, + captureMode, perfPath, }; } finally {