mirror of
https://github.com/heygen-com/hyperframes.git
synced 2026-09-01 19:42:03 +00:00
Merge pull request #2817 from heygen-com/fix/2810-gcp-beginframe-contract
fix(gcp): enforce effective BeginFrame capture
This commit is contained in:
@@ -35,6 +35,7 @@ jobs:
|
||||
cli: ${{ steps.filter.outputs.cli }}
|
||||
skills: ${{ steps.filter.outputs.skills }}
|
||||
codex_plugin: ${{ steps.filter.outputs.codex_plugin }}
|
||||
gcp_beginframe: ${{ steps.filter.outputs.gcp_beginframe }}
|
||||
steps:
|
||||
# Force git-based change detection instead of the pull_request REST API.
|
||||
# The API path can fail the whole workflow on transient listFiles
|
||||
@@ -76,6 +77,33 @@ jobs:
|
||||
- "scripts/package-codex-plugin.mjs"
|
||||
- "package.json"
|
||||
- ".github/workflows/ci.yml"
|
||||
gcp_beginframe:
|
||||
- "packages/gcp-cloud-run/Dockerfile"
|
||||
- "packages/aws-lambda/scripts/probe-beginframe.ts"
|
||||
- "packages/engine/src/services/browserManager.ts"
|
||||
- "package.json"
|
||||
- "bun.lock"
|
||||
- ".github/workflows/ci.yml"
|
||||
|
||||
gcp-beginframe-contract:
|
||||
name: GCP BeginFrame image contract
|
||||
needs: changes
|
||||
if: needs.changes.outputs.gcp_beginframe == 'true'
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 15
|
||||
steps:
|
||||
- uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4
|
||||
with:
|
||||
lfs: true
|
||||
- uses: docker/setup-buildx-action@e468171a9de216ec08956ac3ada2f0791b6bd435 # v3
|
||||
- uses: docker/build-push-action@263435318d21b8e681c14492fe198d362a7d2c83 # v6
|
||||
with:
|
||||
context: .
|
||||
file: packages/gcp-cloud-run/Dockerfile
|
||||
target: beginframe-contract
|
||||
push: false
|
||||
cache-from: type=gha,scope=gcp-beginframe-contract
|
||||
cache-to: type=gha,mode=max,scope=gcp-beginframe-contract
|
||||
|
||||
build:
|
||||
name: Build
|
||||
|
||||
@@ -408,6 +408,8 @@ else
|
||||
!packages/shader-transitions/package.json
|
||||
!packages/aws-lambda/
|
||||
!packages/aws-lambda/package.json
|
||||
!packages/aws-lambda/scripts/
|
||||
!packages/aws-lambda/scripts/probe-beginframe.ts
|
||||
packages/**/node_modules/**
|
||||
packages/**/dist/**
|
||||
packages/**/coverage/**
|
||||
@@ -548,6 +550,19 @@ for protocol in "${PROTOCOL_LIST[@]}"; do
|
||||
continue
|
||||
fi
|
||||
|
||||
CAPTURE_MODES="$(jq -r \
|
||||
'.result | fromjson | [.Chunks[]?.CaptureMode // "<missing>"] | unique | join(",")' \
|
||||
"$EXECUTION_JSON")"
|
||||
if jq -e \
|
||||
'.result | fromjson | .Chunks as $chunks |
|
||||
(($chunks | length) > 0 and all($chunks[]; .CaptureMode == "beginframe"))' \
|
||||
"$EXECUTION_JSON" >/dev/null; then
|
||||
echo " ✓ effective capture mode=beginframe"
|
||||
else
|
||||
echo " ✗ expected every chunk to use beginframe; observed=${CAPTURE_MODES:-<none>}"
|
||||
[ "$OVERALL_RC" -ne 0 ] || OVERALL_RC=6
|
||||
fi
|
||||
|
||||
OUTPUT_LOCAL="$RENDER_DIR/$protocol-c$chunk_size-output.mp4"
|
||||
gcloud storage cp "$OUTPUT_GCS" "$OUTPUT_LOCAL" --project "$PROJECT" >/dev/null
|
||||
|
||||
|
||||
@@ -0,0 +1,80 @@
|
||||
import { describe, expect, it } from "bun:test";
|
||||
import { mkdtempSync, rmSync, writeFileSync } from "node:fs";
|
||||
import { tmpdir } from "node:os";
|
||||
import { join, resolve } from "node:path";
|
||||
import {
|
||||
_awaitBeforeDeadlineForTests,
|
||||
_closeBrowserForProbeTests,
|
||||
parseProbeArgs,
|
||||
} from "./probe-beginframe.js";
|
||||
|
||||
describe("parseProbeArgs", () => {
|
||||
it("defaults to the @sparticuz/chromium source", () => {
|
||||
expect(parseProbeArgs([])).toEqual({});
|
||||
});
|
||||
|
||||
it("accepts a standalone executable path", () => {
|
||||
expect(parseProbeArgs(["--executable-path", "/opt/chrome/chrome-headless-shell"])).toEqual({
|
||||
executablePath: "/opt/chrome/chrome-headless-shell",
|
||||
});
|
||||
});
|
||||
|
||||
it("accepts the equals form and resolves relative paths", () => {
|
||||
expect(parseProbeArgs(["--executable-path=./chrome"])).toEqual({
|
||||
executablePath: resolve("./chrome"),
|
||||
});
|
||||
});
|
||||
|
||||
it("loads exact production launch arguments from JSON", () => {
|
||||
const dir = mkdtempSync(join(tmpdir(), "hf-probe-args-"));
|
||||
const argsPath = join(dir, "args.json");
|
||||
writeFileSync(argsPath, JSON.stringify(["--enable-begin-frame-control", "--no-sandbox"]));
|
||||
try {
|
||||
expect(parseProbeArgs(["--launch-args-json", argsPath])).toEqual({
|
||||
launchArgs: ["--enable-begin-frame-control", "--no-sandbox"],
|
||||
});
|
||||
} finally {
|
||||
rmSync(dir, { recursive: true, force: true });
|
||||
}
|
||||
});
|
||||
|
||||
it("rejects missing values and unknown arguments", () => {
|
||||
expect(() => parseProbeArgs(["--executable-path"])).toThrow(
|
||||
"--executable-path requires a path",
|
||||
);
|
||||
expect(() => parseProbeArgs(["--source", "chrome"])).toThrow("Unknown argument: --source");
|
||||
expect(() => parseProbeArgs(["--launch-args-json"])).toThrow(
|
||||
"--launch-args-json requires a path",
|
||||
);
|
||||
});
|
||||
|
||||
it("bounds a CDP operation that never resolves", async () => {
|
||||
const never = new Promise<never>(() => {});
|
||||
await expect(
|
||||
_awaitBeforeDeadlineForTests(never, Date.now() + 25, "screenshot beginFrame"),
|
||||
).rejects.toThrow("timeout during screenshot beginFrame");
|
||||
});
|
||||
|
||||
it("force-kills and disconnects when graceful cleanup never resolves", async () => {
|
||||
let killedWith: NodeJS.Signals | number | undefined;
|
||||
let disconnected = false;
|
||||
await _closeBrowserForProbeTests(
|
||||
{
|
||||
close: () => new Promise<never>(() => {}),
|
||||
process: () => ({
|
||||
kill: (signal) => {
|
||||
killedWith = signal;
|
||||
return true;
|
||||
},
|
||||
}),
|
||||
disconnect: async () => {
|
||||
disconnected = true;
|
||||
},
|
||||
},
|
||||
25,
|
||||
);
|
||||
|
||||
expect(killedWith).toBe("SIGKILL");
|
||||
expect(disconnected).toBe(true);
|
||||
});
|
||||
});
|
||||
@@ -1,14 +1,13 @@
|
||||
#!/usr/bin/env tsx
|
||||
// fallow-ignore-file code-duplication
|
||||
/**
|
||||
* BeginFrame regression guard for `@sparticuz/chromium`.
|
||||
* BeginFrame regression guard for a Chromium executable.
|
||||
*
|
||||
* The load-bearing assumption of `@hyperframes/aws-lambda` is that the
|
||||
* Chromium build shipped by `@sparticuz/chromium` honours CDP
|
||||
* `HeadlessExperimental.beginFrame` with `screenshot: true`. This script
|
||||
* boots that Chromium build (decompressing into `/tmp` per the library's
|
||||
* runtime contract), navigates to a tiny static page, issues one
|
||||
* `beginFrame` with a screenshot request, and asserts the response
|
||||
* carries a PNG buffer.
|
||||
* With no arguments, this boots the build shipped by `@sparticuz/chromium`
|
||||
* (decompressing into `/tmp` per the library's runtime contract). Passing
|
||||
* `--executable-path /path/to/chrome-headless-shell` probes an arbitrary
|
||||
* executable instead; the GCP image build uses that form against the exact
|
||||
* binary copied into the image.
|
||||
*
|
||||
* The script is the contract test, not a one-shot verification — every
|
||||
* release should run it inside the Docker container at
|
||||
@@ -17,15 +16,18 @@
|
||||
*
|
||||
* Exits 0 on pass, 1 on fail. Run via:
|
||||
*
|
||||
* bun run --cwd packages/aws-lambda probe:beginframe # host
|
||||
* bun run --cwd packages/aws-lambda probe:beginframe:docker # Lambda-like
|
||||
* bun run --cwd packages/aws-lambda probe:beginframe
|
||||
* bun run --cwd packages/aws-lambda probe:beginframe -- \
|
||||
* --executable-path /opt/chrome/chrome-headless-shell
|
||||
* bun run --cwd packages/aws-lambda probe:beginframe:docker
|
||||
*/
|
||||
|
||||
import { mkdtempSync, promises as fs } from "node:fs";
|
||||
import { mkdtempSync, promises as fs, readFileSync } from "node:fs";
|
||||
import { tmpdir } from "node:os";
|
||||
import { join } from "node:path";
|
||||
import { join, resolve } from "node:path";
|
||||
import { fileURLToPath } from "node:url";
|
||||
|
||||
interface ProbeResult {
|
||||
export interface ProbeResult {
|
||||
passed: boolean;
|
||||
durationMs: number;
|
||||
chromiumPath: string;
|
||||
@@ -38,10 +40,140 @@ const PROBE_HTML = `<!doctype html>
|
||||
<html><head><meta charset="utf-8"><title>hf-beginframe-probe</title>
|
||||
<style>html,body{margin:0;background:#173;color:#fff;font:48px/1 sans-serif;display:flex;align-items:center;justify-content:center;height:100vh}</style>
|
||||
</head><body><div id="x">hf-beginframe-probe</div></body></html>`;
|
||||
const SCREENSHOT_ATTEMPTS = 10;
|
||||
const PROBE_OPERATION_TIMEOUT_MS = 5000;
|
||||
const PROBE_CLEANUP_TIMEOUT_MS = 250;
|
||||
|
||||
export interface ProbeOptions {
|
||||
executablePath?: string;
|
||||
/** Exact launch arguments to probe instead of the standalone default profile. */
|
||||
launchArgs?: string[];
|
||||
/** Test override for the renderer/CDP operation deadline. */
|
||||
timeoutMs?: number;
|
||||
}
|
||||
|
||||
// The CLI accepts paired and equals forms for two independent path options.
|
||||
// fallow-ignore-next-line complexity
|
||||
export function parseProbeArgs(args: string[]): ProbeOptions {
|
||||
let executablePath: string | undefined;
|
||||
let launchArgs: string[] | undefined;
|
||||
for (let i = 0; i < args.length; i += 1) {
|
||||
const arg = args[i];
|
||||
if (arg === "--executable-path") {
|
||||
const value = args[i + 1];
|
||||
if (!value || value.startsWith("--")) {
|
||||
throw new Error("--executable-path requires a path");
|
||||
}
|
||||
executablePath = resolve(value);
|
||||
i += 1;
|
||||
continue;
|
||||
}
|
||||
if (arg.startsWith("--executable-path=")) {
|
||||
const value = arg.slice("--executable-path=".length);
|
||||
if (!value) throw new Error("--executable-path requires a path");
|
||||
executablePath = resolve(value);
|
||||
continue;
|
||||
}
|
||||
if (arg === "--launch-args-json") {
|
||||
const value = args[i + 1];
|
||||
if (!value || value.startsWith("--")) {
|
||||
throw new Error("--launch-args-json requires a path");
|
||||
}
|
||||
launchArgs = readLaunchArgs(value);
|
||||
i += 1;
|
||||
continue;
|
||||
}
|
||||
if (arg.startsWith("--launch-args-json=")) {
|
||||
const value = arg.slice("--launch-args-json=".length);
|
||||
if (!value) throw new Error("--launch-args-json requires a path");
|
||||
launchArgs = readLaunchArgs(value);
|
||||
continue;
|
||||
}
|
||||
throw new Error(`Unknown argument: ${arg}`);
|
||||
}
|
||||
return {
|
||||
...(executablePath ? { executablePath } : {}),
|
||||
...(launchArgs ? { launchArgs } : {}),
|
||||
};
|
||||
}
|
||||
|
||||
function readLaunchArgs(path: string): string[] {
|
||||
const resolved = resolve(path);
|
||||
const value: unknown = JSON.parse(readFileSync(resolved, "utf-8"));
|
||||
if (!Array.isArray(value) || !value.every((item) => typeof item === "string")) {
|
||||
throw new Error(`--launch-args-json must contain a JSON string array: ${resolved}`);
|
||||
}
|
||||
return value;
|
||||
}
|
||||
|
||||
async function awaitBeforeDeadline<T>(
|
||||
operation: Promise<T>,
|
||||
deadline: number,
|
||||
label: string,
|
||||
): Promise<T> {
|
||||
const remainingMs = deadline - Date.now();
|
||||
if (remainingMs <= 0) throw new Error(`BeginFrame probe timeout before ${label}`);
|
||||
let timeout: ReturnType<typeof setTimeout> | undefined;
|
||||
try {
|
||||
return await Promise.race([
|
||||
operation,
|
||||
new Promise<never>((_, reject) => {
|
||||
timeout = setTimeout(
|
||||
() => reject(new Error(`BeginFrame probe timeout during ${label}`)),
|
||||
remainingMs,
|
||||
);
|
||||
}),
|
||||
]);
|
||||
} finally {
|
||||
if (timeout) clearTimeout(timeout);
|
||||
}
|
||||
}
|
||||
|
||||
/** Test-only export for the standalone probe's bounded-operation contract. */
|
||||
export const _awaitBeforeDeadlineForTests = awaitBeforeDeadline;
|
||||
|
||||
interface ProbeBrowserCleanup {
|
||||
close(): Promise<void>;
|
||||
disconnect(): Promise<void>;
|
||||
process(): { kill(signal?: NodeJS.Signals | number): boolean } | null;
|
||||
}
|
||||
|
||||
async function settleWithin(operation: Promise<unknown>, timeoutMs: number): Promise<boolean> {
|
||||
let timeout: ReturnType<typeof setTimeout> | undefined;
|
||||
try {
|
||||
return await Promise.race([
|
||||
operation.then(
|
||||
() => true,
|
||||
() => false,
|
||||
),
|
||||
new Promise<false>((resolveTimeout) => {
|
||||
timeout = setTimeout(() => resolveTimeout(false), timeoutMs);
|
||||
}),
|
||||
]);
|
||||
} finally {
|
||||
if (timeout) clearTimeout(timeout);
|
||||
}
|
||||
}
|
||||
|
||||
async function closeBrowserForProbe(
|
||||
browser: ProbeBrowserCleanup,
|
||||
timeoutMs = PROBE_CLEANUP_TIMEOUT_MS,
|
||||
): Promise<void> {
|
||||
if (await settleWithin(browser.close(), timeoutMs)) return;
|
||||
try {
|
||||
browser.process()?.kill("SIGKILL");
|
||||
} catch {
|
||||
// Best effort; disconnect below still releases Puppeteer's transport.
|
||||
}
|
||||
await settleWithin(browser.disconnect(), timeoutMs);
|
||||
}
|
||||
|
||||
/** Test-only export for bounded standalone-probe cleanup. */
|
||||
export const _closeBrowserForProbeTests = closeBrowserForProbe;
|
||||
|
||||
async function main(): Promise<void> {
|
||||
const start = Date.now();
|
||||
const result = await probe();
|
||||
const result = await probe(parseProbeArgs(process.argv.slice(2)));
|
||||
result.durationMs = Date.now() - start;
|
||||
console.log(JSON.stringify(result, null, 2));
|
||||
if (!result.passed) {
|
||||
@@ -49,12 +181,21 @@ async function main(): Promise<void> {
|
||||
}
|
||||
}
|
||||
|
||||
async function probe(): Promise<ProbeResult> {
|
||||
// This intentionally linear contract owns launch, renderer setup, CDP
|
||||
// validation, diagnostics, and cleanup in one fail-closed lifecycle.
|
||||
// fallow-ignore-next-line complexity
|
||||
export async function probe(options: ProbeOptions = {}): Promise<ProbeResult> {
|
||||
let chromiumPath = "";
|
||||
let tmpHtmlDir = "";
|
||||
try {
|
||||
const { default: chromium } = await import("@sparticuz/chromium");
|
||||
chromiumPath = await chromium.executablePath();
|
||||
const args = chromium.args;
|
||||
let sourceArgs: string[] = [];
|
||||
if (options.executablePath) {
|
||||
chromiumPath = options.executablePath;
|
||||
} else {
|
||||
const { default: chromium } = await import("@sparticuz/chromium");
|
||||
chromiumPath = await chromium.executablePath();
|
||||
sourceArgs = chromium.args;
|
||||
}
|
||||
|
||||
const puppeteer = await import("puppeteer-core");
|
||||
|
||||
@@ -63,7 +204,7 @@ async function probe(): Promise<ProbeResult> {
|
||||
// Chrome-side issue. `mkdtempSync` (vs `tmpdir() + Date.now()`) gives
|
||||
// an unguessable directory name so two concurrent probes on the same
|
||||
// host don't collide and CodeQL's insecure-tempfile rule clears.
|
||||
const tmpHtmlDir = mkdtempSync(join(tmpdir(), "hf-beginframe-"));
|
||||
tmpHtmlDir = mkdtempSync(join(tmpdir(), "hf-beginframe-"));
|
||||
const htmlPath = join(tmpHtmlDir, "probe.html");
|
||||
await fs.writeFile(htmlPath, PROBE_HTML, "utf-8");
|
||||
|
||||
@@ -75,6 +216,11 @@ async function probe(): Promise<ProbeResult> {
|
||||
// ("Chrome's beginFrame with `screenshot` param always reports
|
||||
// hasDamage=true").
|
||||
const beginFrameFlags = [
|
||||
"--no-sandbox",
|
||||
"--disable-setuid-sandbox",
|
||||
"--disable-dev-shm-usage",
|
||||
"--enable-webgl",
|
||||
"--ignore-gpu-blocklist",
|
||||
"--deterministic-mode",
|
||||
"--enable-begin-frame-control",
|
||||
"--disable-new-content-rendering-timeout",
|
||||
@@ -89,55 +235,97 @@ async function probe(): Promise<ProbeResult> {
|
||||
"--use-gl=angle",
|
||||
"--use-angle=swiftshader",
|
||||
"--enable-unsafe-swiftshader",
|
||||
// Distributed Linux rendering explicitly uses software compositing to
|
||||
// avoid stale transformed layers in SwiftShader (see browserManager).
|
||||
"--disable-gpu-compositing",
|
||||
];
|
||||
|
||||
const browser = await puppeteer.launch({
|
||||
executablePath: chromiumPath,
|
||||
headless: "shell",
|
||||
args: [...args, ...beginFrameFlags],
|
||||
args: options.launchArgs ?? [...sourceArgs, ...beginFrameFlags],
|
||||
defaultViewport: { width: 800, height: 600 },
|
||||
});
|
||||
try {
|
||||
const page = await browser.newPage();
|
||||
await page.goto(`file://${htmlPath}`, { waitUntil: "domcontentloaded", timeout: 30_000 });
|
||||
const session = await page.createCDPSession();
|
||||
await session.send("HeadlessExperimental.enable");
|
||||
const timeoutMs = options.timeoutMs ?? PROBE_OPERATION_TIMEOUT_MS;
|
||||
const deadline = Date.now() + timeoutMs;
|
||||
const page = await awaitBeforeDeadline(browser.newPage(), deadline, "newPage");
|
||||
await awaitBeforeDeadline(
|
||||
page.goto(`file://${htmlPath}`, { waitUntil: "domcontentloaded", timeout: timeoutMs }),
|
||||
deadline,
|
||||
"navigation",
|
||||
);
|
||||
const session = await awaitBeforeDeadline(
|
||||
page.createCDPSession(),
|
||||
deadline,
|
||||
"CDP session creation",
|
||||
);
|
||||
await awaitBeforeDeadline(
|
||||
session.send("HeadlessExperimental.enable"),
|
||||
deadline,
|
||||
"HeadlessExperimental.enable",
|
||||
);
|
||||
// Warm-up beginFrame with noDisplayUpdates: true — drives the
|
||||
// compositor without producing a screenshot, matching how the engine
|
||||
// primes a capture loop.
|
||||
await session.send("HeadlessExperimental.beginFrame", {
|
||||
frameTimeTicks: 0,
|
||||
interval: 33,
|
||||
noDisplayUpdates: true,
|
||||
});
|
||||
const response = await session.send("HeadlessExperimental.beginFrame", {
|
||||
frameTimeTicks: 1000,
|
||||
interval: 33,
|
||||
screenshot: { format: "png" },
|
||||
});
|
||||
await fs.rm(tmpHtmlDir, { recursive: true, force: true }).catch(() => {});
|
||||
const screenshot = response.screenshotData ?? "";
|
||||
const bytes = screenshot ? Buffer.from(screenshot, "base64") : Buffer.alloc(0);
|
||||
const isPng =
|
||||
bytes.length >= 8 &&
|
||||
bytes[0] === 0x89 &&
|
||||
bytes[1] === 0x50 &&
|
||||
bytes[2] === 0x4e &&
|
||||
bytes[3] === 0x47;
|
||||
await awaitBeforeDeadline(
|
||||
session.send("HeadlessExperimental.beginFrame", {
|
||||
frameTimeTicks: 0,
|
||||
interval: 33,
|
||||
noDisplayUpdates: true,
|
||||
}),
|
||||
deadline,
|
||||
"warm-up beginFrame",
|
||||
);
|
||||
let hasDamage = false;
|
||||
let bytes = Buffer.alloc(0);
|
||||
let isPng = false;
|
||||
let attempts = 0;
|
||||
// A renderer-ready document can still need more than one controlled
|
||||
// frame before it submits a screenshot surface. Chromium explicitly
|
||||
// permits screenshotData to be absent during renderer initialization,
|
||||
// so retry a small bounded sequence with monotonically increasing ticks.
|
||||
for (attempts = 1; attempts <= SCREENSHOT_ATTEMPTS; attempts += 1) {
|
||||
const response = await awaitBeforeDeadline(
|
||||
session.send("HeadlessExperimental.beginFrame", {
|
||||
frameTimeTicks: 1000 + (attempts - 1) * 33,
|
||||
interval: 33,
|
||||
screenshot: { format: "png" },
|
||||
}),
|
||||
deadline,
|
||||
`screenshot beginFrame attempt ${attempts}`,
|
||||
);
|
||||
hasDamage = response.hasDamage;
|
||||
const screenshot = response.screenshotData ?? "";
|
||||
bytes = screenshot ? Buffer.from(screenshot, "base64") : Buffer.alloc(0);
|
||||
isPng =
|
||||
bytes.length >= 8 &&
|
||||
bytes[0] === 0x89 &&
|
||||
bytes[1] === 0x50 &&
|
||||
bytes[2] === 0x4e &&
|
||||
bytes[3] === 0x47;
|
||||
if (isPng) break;
|
||||
await awaitBeforeDeadline(
|
||||
new Promise((resolveDelay) => setTimeout(resolveDelay, 10)),
|
||||
deadline,
|
||||
`screenshot retry delay ${attempts}`,
|
||||
);
|
||||
}
|
||||
return {
|
||||
passed: isPng && bytes.length > 0,
|
||||
durationMs: 0,
|
||||
chromiumPath,
|
||||
screenshotBytes: bytes.length,
|
||||
hasDamage: response.hasDamage,
|
||||
hasDamage,
|
||||
detail: isPng
|
||||
? "OK — BeginFrame returned a PNG buffer."
|
||||
: `FAIL — BeginFrame returned ${bytes.length} bytes, PNG signature ${
|
||||
? `OK — BeginFrame returned a PNG buffer after ${attempts} attempt(s).`
|
||||
: `FAIL — BeginFrame returned ${bytes.length} bytes after ${SCREENSHOT_ATTEMPTS} ` +
|
||||
`attempts, PNG signature ${
|
||||
bytes.length >= 4 ? bytes.subarray(0, 4).toString("hex") : "<empty>"
|
||||
}`,
|
||||
};
|
||||
} finally {
|
||||
await browser.close().catch(() => {});
|
||||
await closeBrowserForProbe(browser);
|
||||
}
|
||||
} catch (err) {
|
||||
return {
|
||||
@@ -148,10 +336,17 @@ async function probe(): Promise<ProbeResult> {
|
||||
hasDamage: false,
|
||||
detail: `FAIL — ${err instanceof Error ? err.message : String(err)}`,
|
||||
};
|
||||
} finally {
|
||||
if (tmpHtmlDir) {
|
||||
await fs.rm(tmpHtmlDir, { recursive: true, force: true }).catch(() => {});
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
void main().catch((err) => {
|
||||
console.error("[probe-beginframe] unexpected:", err);
|
||||
process.exit(2);
|
||||
});
|
||||
const invokedPath = process.argv[1] ? resolve(process.argv[1]) : "";
|
||||
if (invokedPath === fileURLToPath(import.meta.url)) {
|
||||
void main().catch((err) => {
|
||||
console.error("[probe-beginframe] unexpected:", err);
|
||||
process.exit(2);
|
||||
});
|
||||
}
|
||||
|
||||
@@ -184,6 +184,8 @@ export interface RenderChunkLambdaResult {
|
||||
ChunkIndex: number;
|
||||
Sha256: string;
|
||||
FramesEncoded: number;
|
||||
/** Effective engine mode after browser probing. Emitted by current handlers. */
|
||||
CaptureMode?: "beginframe" | "screenshot" | "drawelement";
|
||||
DurationMs: number;
|
||||
}
|
||||
|
||||
|
||||
@@ -377,6 +377,12 @@ describe("handler dispatch", () => {
|
||||
framesEncoded: 240,
|
||||
sha256: "0".repeat(64),
|
||||
durationMs: 12345,
|
||||
planHashMs: 1,
|
||||
sessionBootMs: 2,
|
||||
captureStageMs: 3,
|
||||
encodeStageMs: 4,
|
||||
workers: 1,
|
||||
captureMode: "beginframe",
|
||||
perfPath: outputChunkPath + ".perf.json",
|
||||
};
|
||||
},
|
||||
@@ -416,6 +422,7 @@ describe("handler dispatch", () => {
|
||||
expect(result.ChunkIndex).toBe(2);
|
||||
expect(result.Sha256).toBe("0".repeat(64));
|
||||
expect(result.FramesEncoded).toBe(240);
|
||||
expect(result.CaptureMode).toBe("beginframe");
|
||||
expect(renderChunkMock).toHaveBeenCalledTimes(1);
|
||||
});
|
||||
|
||||
@@ -555,6 +562,12 @@ describe("handler dispatch", () => {
|
||||
framesEncoded: 30,
|
||||
sha256: "b".repeat(64),
|
||||
durationMs: 1,
|
||||
planHashMs: 0,
|
||||
sessionBootMs: 0,
|
||||
captureStageMs: 1,
|
||||
encodeStageMs: 0,
|
||||
workers: 1,
|
||||
captureMode: "beginframe",
|
||||
perfPath: `${outputPath}.perf.json`,
|
||||
};
|
||||
},
|
||||
@@ -631,6 +644,7 @@ describe("handler dispatch", () => {
|
||||
deps,
|
||||
);
|
||||
if (chunk.Action !== "renderChunk") throw new Error("expected chunk result");
|
||||
expect(chunk.CaptureMode).toBe("beginframe");
|
||||
const audioDigest = createHash("sha256").update("AAC").digest("hex");
|
||||
const audioUri = `${planned.PlanV2ArtifactS3Prefix}/${audioDigest.slice(0, 2)}/${audioDigest}`;
|
||||
expect(s3.ops.slice(beforeChunk).some((operation) => operation.uri === audioUri)).toBe(false);
|
||||
|
||||
@@ -457,6 +457,7 @@ async function handleRenderChunk(
|
||||
ChunkIndex: event.ChunkIndex,
|
||||
Sha256: result.sha256,
|
||||
FramesEncoded: result.framesEncoded,
|
||||
CaptureMode: result.captureMode,
|
||||
DurationMs: Date.now() - started,
|
||||
};
|
||||
} finally {
|
||||
@@ -504,6 +505,7 @@ async function handleRenderChunkV2(
|
||||
ChunkIndex: event.ChunkIndex,
|
||||
Sha256: result.sha256,
|
||||
FramesEncoded: result.framesEncoded,
|
||||
CaptureMode: result.captureMode,
|
||||
DurationMs: Date.now() - started,
|
||||
};
|
||||
} finally {
|
||||
|
||||
@@ -1,3 +1,4 @@
|
||||
// fallow-ignore-file code-duplication
|
||||
import { afterEach, beforeEach, describe, expect, it, vi } from "vitest";
|
||||
import { execFileSync } from "node:child_process";
|
||||
import { mkdirSync, mkdtempSync, rmSync, writeFileSync } from "node:fs";
|
||||
@@ -9,6 +10,9 @@ import type { Browser, PuppeteerNode } from "puppeteer-core";
|
||||
import {
|
||||
_resetAutoBrowserGpuModeCacheForTests,
|
||||
_resetBrowserPoolForTests,
|
||||
_closeBrowserAfterFailedProbeForTests,
|
||||
_createBrowserLaunchFingerprintForTests,
|
||||
_probeBeginFrameSupportForTests,
|
||||
_setPuppeteerForTests,
|
||||
acquireBrowser,
|
||||
buildChromeArgs,
|
||||
@@ -19,6 +23,102 @@ import {
|
||||
resolveBrowserGpuMode,
|
||||
} from "./browserManager.js";
|
||||
|
||||
describe("BeginFrame capability probe", () => {
|
||||
it("waits for a document and validates a PNG-returning frame", async () => {
|
||||
const png = Buffer.from([0x89, 0x50, 0x4e, 0x47, 0x0d, 0x0a, 0x1a, 0x0a]);
|
||||
const send = vi
|
||||
.fn()
|
||||
.mockResolvedValueOnce({})
|
||||
.mockResolvedValueOnce({})
|
||||
.mockResolvedValueOnce({ hasDamage: true, screenshotData: png.toString("base64") });
|
||||
const detach = vi.fn().mockResolvedValue(undefined);
|
||||
const goto = vi.fn().mockResolvedValue(null);
|
||||
const close = vi.fn().mockResolvedValue(undefined);
|
||||
const browser = {
|
||||
newPage: vi.fn().mockResolvedValue({
|
||||
goto,
|
||||
createCDPSession: vi.fn().mockResolvedValue({ send, detach }),
|
||||
close,
|
||||
}),
|
||||
} as unknown as Browser;
|
||||
|
||||
const result = await _probeBeginFrameSupportForTests(browser);
|
||||
|
||||
expect(result.supported).toBe(true);
|
||||
expect(goto).toHaveBeenCalledWith(expect.stringContaining("hf-beginframe-probe"), {
|
||||
waitUntil: "domcontentloaded",
|
||||
timeout: 2000,
|
||||
});
|
||||
expect(send.mock.calls.map(([method]) => method)).toEqual([
|
||||
"HeadlessExperimental.enable",
|
||||
"HeadlessExperimental.beginFrame",
|
||||
"HeadlessExperimental.beginFrame",
|
||||
]);
|
||||
expect(detach).toHaveBeenCalledOnce();
|
||||
expect(close).toHaveBeenCalledOnce();
|
||||
});
|
||||
|
||||
it("reports an empty screenshot as unsupported", async () => {
|
||||
const send = vi.fn().mockResolvedValue({ hasDamage: false });
|
||||
const browser = {
|
||||
newPage: vi.fn().mockResolvedValue({
|
||||
goto: vi.fn().mockResolvedValue(null),
|
||||
createCDPSession: vi.fn().mockResolvedValue({
|
||||
send,
|
||||
detach: vi.fn().mockResolvedValue(undefined),
|
||||
}),
|
||||
close: vi.fn().mockResolvedValue(undefined),
|
||||
}),
|
||||
} as unknown as Browser;
|
||||
|
||||
const result = await _probeBeginFrameSupportForTests(browser);
|
||||
|
||||
expect(result.supported).toBe(false);
|
||||
expect(result.detail).toContain("returned 0 bytes after 10 attempts");
|
||||
});
|
||||
|
||||
it("bounds a screenshot-bearing CDP call that never resolves", async () => {
|
||||
const never = new Promise<never>(() => {});
|
||||
const send = vi
|
||||
.fn()
|
||||
.mockResolvedValueOnce({})
|
||||
.mockResolvedValueOnce({})
|
||||
.mockReturnValueOnce(never);
|
||||
const close = vi.fn().mockResolvedValue(undefined);
|
||||
const browser = {
|
||||
newPage: vi.fn().mockResolvedValue({
|
||||
goto: vi.fn().mockResolvedValue(null),
|
||||
createCDPSession: vi.fn().mockResolvedValue({
|
||||
send,
|
||||
detach: vi.fn().mockResolvedValue(undefined),
|
||||
}),
|
||||
close,
|
||||
}),
|
||||
} as unknown as Browser;
|
||||
|
||||
const result = await _probeBeginFrameSupportForTests(browser, 25);
|
||||
|
||||
expect(result.supported).toBe(false);
|
||||
expect(result.detail).toContain("timeout during screenshot beginFrame attempt 1");
|
||||
expect(close).toHaveBeenCalledOnce();
|
||||
});
|
||||
|
||||
it("force-kills and disconnects when graceful browser cleanup never resolves", async () => {
|
||||
const kill = vi.fn();
|
||||
const disconnect = vi.fn().mockResolvedValue(undefined);
|
||||
const browser = {
|
||||
close: vi.fn().mockReturnValue(new Promise<never>(() => {})),
|
||||
process: vi.fn().mockReturnValue({ kill }),
|
||||
disconnect,
|
||||
} as unknown as Browser;
|
||||
|
||||
await _closeBrowserAfterFailedProbeForTests(browser, 25);
|
||||
|
||||
expect(kill).toHaveBeenCalledWith("SIGKILL");
|
||||
expect(disconnect).toHaveBeenCalledOnce();
|
||||
});
|
||||
});
|
||||
|
||||
describe("buildChromeArgs browser GPU mode", () => {
|
||||
const base = { width: 1920, height: 1080 };
|
||||
|
||||
@@ -89,6 +189,31 @@ describe("buildChromeArgs browser GPU mode", () => {
|
||||
});
|
||||
});
|
||||
|
||||
describe("browser launch capture-mode contract", () => {
|
||||
it("derives BeginFrame from the actual launch flags, not forceScreenshot alone", () => {
|
||||
const dir = mkdtempSync(join(tmpdir(), "hf-browser-fingerprint-"));
|
||||
const chromePath = join(dir, "chrome-headless-shell");
|
||||
writeFileSync(chromePath, "");
|
||||
try {
|
||||
const withoutControl = _createBrowserLaunchFingerprintForTests([], {
|
||||
chromePath,
|
||||
forceScreenshot: false,
|
||||
});
|
||||
const withControl = _createBrowserLaunchFingerprintForTests(
|
||||
["--enable-begin-frame-control"],
|
||||
{ chromePath, forceScreenshot: false },
|
||||
);
|
||||
|
||||
expect(withoutControl.requestedCaptureMode).toBe("screenshot");
|
||||
expect(withControl.requestedCaptureMode).toBe(
|
||||
process.platform === "linux" ? "beginframe" : "screenshot",
|
||||
);
|
||||
} finally {
|
||||
rmSync(dir, { recursive: true, force: true });
|
||||
}
|
||||
});
|
||||
});
|
||||
|
||||
describe("resolveBrowserGpuMode", () => {
|
||||
const setMockWebGlProbe = (info: { hasWebGL: boolean; vendor: string; renderer: string }) => {
|
||||
const close = vi.fn().mockResolvedValue(undefined);
|
||||
|
||||
@@ -217,44 +217,192 @@ function stripBeginFrameFlags(args: string[]): string[] {
|
||||
}
|
||||
|
||||
/**
|
||||
* Probe whether the browser still speaks HeadlessExperimental.beginFrame.
|
||||
* Probe the complete HeadlessExperimental.beginFrame runtime contract.
|
||||
*
|
||||
* Recent chrome-headless-shell builds (observed on 147) expose the domain
|
||||
* well enough that HeadlessExperimental.enable succeeds but drop the
|
||||
* beginFrame method itself — the capture loop then dies on first frame with
|
||||
* `'HeadlessExperimental.beginFrame' wasn't found`. So we probe BOTH: enable
|
||||
* + one cheap beginFrame raced against a 2s timeout. In beginframe-control
|
||||
* mode the command completes as soon as the compositor acks, so a real
|
||||
* supported browser returns well under the timeout.
|
||||
*
|
||||
* Any failure (method missing, timeout, protocol error) is treated as
|
||||
* unsupported. Real errors after launch would surface in the warmup loop and
|
||||
* fall out through the caller's try/catch.
|
||||
* Domain registration alone is insufficient: BeginFrame control must be
|
||||
* enabled at launch and a renderer-ready target must return a real PNG.
|
||||
* Every operation shares one short deadline so a wedged CDP call cannot hold
|
||||
* a serverless cold start until Puppeteer's much longer protocol timeout.
|
||||
*/
|
||||
async function probeBeginFrameSupport(browser: Browser): Promise<boolean> {
|
||||
let page;
|
||||
interface BeginFrameProbeResult {
|
||||
supported: boolean;
|
||||
detail: string;
|
||||
durationMs: number;
|
||||
}
|
||||
|
||||
const BEGINFRAME_SCREENSHOT_PROBE_ATTEMPTS = 10;
|
||||
const BEGINFRAME_PROBE_TIMEOUT_MS = 2000;
|
||||
const BEGINFRAME_PROBE_CLEANUP_TIMEOUT_MS = 250;
|
||||
|
||||
async function awaitBeforeDeadline<T>(
|
||||
operation: Promise<T>,
|
||||
deadline: number,
|
||||
label: string,
|
||||
): Promise<T> {
|
||||
const remainingMs = deadline - Date.now();
|
||||
if (remainingMs <= 0) {
|
||||
throw new Error(`beginFrame probe timeout before ${label}`);
|
||||
}
|
||||
let timeout: ReturnType<typeof setTimeout> | undefined;
|
||||
try {
|
||||
page = await browser.newPage();
|
||||
const client = await page.createCDPSession();
|
||||
await client.send("HeadlessExperimental.enable");
|
||||
const beginFrame = client.send("HeadlessExperimental.beginFrame", {
|
||||
frameTimeTicks: 0,
|
||||
interval: 33,
|
||||
noDisplayUpdates: true,
|
||||
});
|
||||
const timeout = new Promise<never>((_, reject) =>
|
||||
setTimeout(() => reject(new Error("beginFrame probe timeout")), 2000),
|
||||
);
|
||||
await Promise.race([beginFrame, timeout]);
|
||||
await client.detach().catch(() => {});
|
||||
return true;
|
||||
} catch {
|
||||
return false;
|
||||
return await Promise.race([
|
||||
operation,
|
||||
new Promise<never>((_, reject) => {
|
||||
timeout = setTimeout(
|
||||
() => reject(new Error(`beginFrame probe timeout during ${label}`)),
|
||||
remainingMs,
|
||||
);
|
||||
}),
|
||||
]);
|
||||
} finally {
|
||||
await page?.close().catch(() => {});
|
||||
if (timeout) clearTimeout(timeout);
|
||||
}
|
||||
}
|
||||
|
||||
async function settleWithin(operation: Promise<unknown>, timeoutMs: number): Promise<boolean> {
|
||||
let timeout: ReturnType<typeof setTimeout> | undefined;
|
||||
try {
|
||||
return await Promise.race([
|
||||
operation.then(
|
||||
() => true,
|
||||
() => false,
|
||||
),
|
||||
new Promise<false>((resolveTimeout) => {
|
||||
timeout = setTimeout(() => resolveTimeout(false), timeoutMs);
|
||||
}),
|
||||
]);
|
||||
} finally {
|
||||
if (timeout) clearTimeout(timeout);
|
||||
}
|
||||
}
|
||||
|
||||
async function closeBrowserAfterFailedProbe(
|
||||
browser: Browser,
|
||||
timeoutMs = BEGINFRAME_PROBE_CLEANUP_TIMEOUT_MS,
|
||||
): Promise<void> {
|
||||
if (await settleWithin(browser.close(), timeoutMs)) return;
|
||||
// A wedged CDP transport can make graceful close inherit Puppeteer's
|
||||
// multi-minute protocol timeout. Kill the owned process and disconnect so
|
||||
// screenshot fallback can launch promptly.
|
||||
try {
|
||||
browser.process()?.kill("SIGKILL");
|
||||
} catch {
|
||||
// Best effort; disconnect below still releases Puppeteer's transport.
|
||||
}
|
||||
await settleWithin(browser.disconnect(), timeoutMs);
|
||||
}
|
||||
|
||||
// The probe keeps its renderer setup, bounded CDP sequence, PNG validation,
|
||||
// diagnostics, and cleanup together so every failure uses one contract.
|
||||
// fallow-ignore-next-line complexity
|
||||
async function probeBeginFrameSupport(
|
||||
browser: Browser,
|
||||
timeoutMs = BEGINFRAME_PROBE_TIMEOUT_MS,
|
||||
): Promise<BeginFrameProbeResult> {
|
||||
const started = Date.now();
|
||||
const deadline = started + timeoutMs;
|
||||
let page;
|
||||
let result: BeginFrameProbeResult;
|
||||
try {
|
||||
page = await awaitBeforeDeadline(browser.newPage(), deadline, "newPage");
|
||||
// `browser.newPage()` resolves before a cold renderer has necessarily
|
||||
// submitted its first surface. Cloud Run exposed this as a false
|
||||
// "unsupported Chromium" result: probing the untouched about:blank
|
||||
// target raced renderer initialization, while the same binary passed
|
||||
// once a real document was ready. Navigate first so this tests protocol
|
||||
// capability rather than target-startup timing.
|
||||
await awaitBeforeDeadline(
|
||||
page.goto(
|
||||
"data:text/html,<style>html,body{margin:0;background:%23173}</style><div>hf-beginframe-probe</div>",
|
||||
{ waitUntil: "domcontentloaded", timeout: timeoutMs },
|
||||
),
|
||||
deadline,
|
||||
"navigation",
|
||||
);
|
||||
const client = await awaitBeforeDeadline(
|
||||
page.createCDPSession(),
|
||||
deadline,
|
||||
"CDP session creation",
|
||||
);
|
||||
await awaitBeforeDeadline(
|
||||
client.send("HeadlessExperimental.enable"),
|
||||
deadline,
|
||||
"HeadlessExperimental.enable",
|
||||
);
|
||||
await awaitBeforeDeadline(
|
||||
client.send("HeadlessExperimental.beginFrame", {
|
||||
frameTimeTicks: 0,
|
||||
interval: 33,
|
||||
noDisplayUpdates: true,
|
||||
}),
|
||||
deadline,
|
||||
"warm-up beginFrame",
|
||||
);
|
||||
let bytes = Buffer.alloc(0);
|
||||
let isPng = false;
|
||||
let attempts = 0;
|
||||
for (attempts = 1; attempts <= BEGINFRAME_SCREENSHOT_PROBE_ATTEMPTS; attempts += 1) {
|
||||
const response = await awaitBeforeDeadline(
|
||||
client.send("HeadlessExperimental.beginFrame", {
|
||||
frameTimeTicks: 1000 + (attempts - 1) * 33,
|
||||
interval: 33,
|
||||
screenshot: { format: "png" },
|
||||
}),
|
||||
deadline,
|
||||
`screenshot beginFrame attempt ${attempts}`,
|
||||
);
|
||||
const screenshot = response.screenshotData ?? "";
|
||||
bytes = screenshot ? Buffer.from(screenshot, "base64") : Buffer.alloc(0);
|
||||
isPng =
|
||||
bytes.length >= 8 &&
|
||||
bytes[0] === 0x89 &&
|
||||
bytes[1] === 0x50 &&
|
||||
bytes[2] === 0x4e &&
|
||||
bytes[3] === 0x47;
|
||||
if (isPng) break;
|
||||
await awaitBeforeDeadline(
|
||||
new Promise((resolveDelay) => setTimeout(resolveDelay, 10)),
|
||||
deadline,
|
||||
`screenshot retry delay ${attempts}`,
|
||||
);
|
||||
}
|
||||
if (!isPng) {
|
||||
throw new Error(
|
||||
`beginFrame screenshot returned ${bytes.length} bytes after ` +
|
||||
`${BEGINFRAME_SCREENSHOT_PROBE_ATTEMPTS} attempts with signature ` +
|
||||
`${bytes.length >= 4 ? bytes.subarray(0, 4).toString("hex") : "<empty>"}`,
|
||||
);
|
||||
}
|
||||
await awaitBeforeDeadline(client.detach(), deadline, "CDP detach").catch(() => {});
|
||||
result = {
|
||||
supported: true,
|
||||
detail:
|
||||
`enable + warm-up + ${bytes.length}-byte PNG beginFrame succeeded ` +
|
||||
`after ${attempts} screenshot attempt(s)`,
|
||||
durationMs: Date.now() - started,
|
||||
};
|
||||
} catch (error) {
|
||||
result = {
|
||||
supported: false,
|
||||
detail: error instanceof Error ? error.message : String(error),
|
||||
durationMs: Date.now() - started,
|
||||
};
|
||||
}
|
||||
if (page && !(await settleWithin(page.close(), BEGINFRAME_PROBE_CLEANUP_TIMEOUT_MS))) {
|
||||
return {
|
||||
supported: false,
|
||||
detail: `${result.detail}; probe page cleanup timed out`,
|
||||
durationMs: Date.now() - started,
|
||||
};
|
||||
}
|
||||
return result;
|
||||
}
|
||||
|
||||
/** Test-only export for the renderer-readiness + PNG capability contract. */
|
||||
export const _probeBeginFrameSupportForTests = probeBeginFrameSupport;
|
||||
/** Test-only export for the forced browser-cleanup fallback. */
|
||||
export const _closeBrowserAfterFailedProbeForTests = closeBrowserAfterFailedProbe;
|
||||
|
||||
/**
|
||||
* Cached *in-flight or resolved* probe Promise for `resolveBrowserGpuMode("auto", ...)`.
|
||||
*
|
||||
@@ -391,8 +539,17 @@ function createBrowserLaunchFingerprint(
|
||||
...config,
|
||||
};
|
||||
const headlessShell = resolveHeadlessShellPath(launchConfig);
|
||||
// The launch arguments are the authoritative capture-mode contract.
|
||||
// A caller can pass `forceScreenshot:false` while a later safety clamp
|
||||
// deliberately omits BeginFrame control flags. Inferring solely from the
|
||||
// raw config in that case makes BrowserManager probe a capability it never
|
||||
// enabled, then mislabel the result as an unsupported Chromium build.
|
||||
const beginFrameControlEnabled = chromeArgs.includes("--enable-begin-frame-control");
|
||||
const requestedCaptureMode: CaptureMode =
|
||||
headlessShell && process.platform === "linux" && !launchConfig.forceScreenshot
|
||||
headlessShell &&
|
||||
process.platform === "linux" &&
|
||||
!launchConfig.forceScreenshot &&
|
||||
beginFrameControlEnabled
|
||||
? "beginframe"
|
||||
: "screenshot";
|
||||
return {
|
||||
@@ -404,6 +561,9 @@ function createBrowserLaunchFingerprint(
|
||||
};
|
||||
}
|
||||
|
||||
/** Test-only export for launch-argument/capture-mode agreement. */
|
||||
export const _createBrowserLaunchFingerprintForTests = createBrowserLaunchFingerprint;
|
||||
|
||||
export async function acquireBrowser(
|
||||
chromeArgs: string[],
|
||||
config?: Partial<
|
||||
@@ -443,12 +603,17 @@ async function launchBrowser(
|
||||
);
|
||||
|
||||
if (captureMode === "beginframe") {
|
||||
const supported = await probeBeginFrameSupport(browser).catch(() => true);
|
||||
if (!supported) {
|
||||
await browser.close().catch(() => {});
|
||||
const probe = await probeBeginFrameSupport(browser).catch((error) => ({
|
||||
supported: true,
|
||||
detail: `probe harness error ignored: ${error instanceof Error ? error.message : String(error)}`,
|
||||
durationMs: 0,
|
||||
}));
|
||||
if (!probe.supported) {
|
||||
await closeBrowserAfterFailedProbe(browser);
|
||||
browser = undefined;
|
||||
console.warn(
|
||||
"[BrowserManager] HeadlessExperimental.beginFrame unavailable in this Chromium build; falling back to screenshot mode.",
|
||||
`[BrowserManager] HeadlessExperimental.beginFrame probe failed after ${probe.durationMs}ms: ` +
|
||||
`${probe.detail}; falling back to screenshot mode.`,
|
||||
);
|
||||
captureMode = "screenshot";
|
||||
browser = await ppt.launch({
|
||||
|
||||
@@ -21,7 +21,7 @@
|
||||
# Unlike the AWS Lambda adapter there is no ZIP-size ceiling and no runtime
|
||||
# Chrome decompression step — the binary lives in the image at a fixed path.
|
||||
|
||||
FROM node:22-bookworm-slim
|
||||
FROM node:22-bookworm-slim AS beginframe-contract
|
||||
|
||||
# ── System dependencies (identical set to the producer's render image) ───────
|
||||
RUN apt-get update && apt-get install -y --no-install-recommends \
|
||||
@@ -55,9 +55,9 @@ RUN apt-get update && apt-get install -y --no-install-recommends \
|
||||
&& fc-cache -fv
|
||||
|
||||
# ── chrome-headless-shell (deterministic BeginFrame capture) ─────────────────
|
||||
# Pinned to the SAME build the producer regression baselines were generated
|
||||
# against so distributed renders are pixel-comparable. Bump together with the
|
||||
# producer's Dockerfile.test pin and a baseline regen.
|
||||
# Pinned deliberately. The build-stage contract probe below launches this
|
||||
# exact executable and requires a renderer-ready warm-up plus a PNG-returning
|
||||
# HeadlessExperimental.beginFrame before the image can build.
|
||||
RUN npx --yes @puppeteer/browsers install chrome-headless-shell@148.0.7778.167 \
|
||||
--path /opt/puppeteer \
|
||||
&& CHS="$(find /opt/puppeteer/chrome-headless-shell -name chrome-headless-shell -type f | head -n1)" \
|
||||
@@ -100,6 +100,23 @@ COPY packages/studio-server/package.json packages/studio-server/package.json
|
||||
COPY scripts/package-subpaths.mjs scripts/package-subpaths.mjs
|
||||
RUN bun install --frozen-lockfile
|
||||
|
||||
# Fail the image build closed if the packaged executable cannot perform the
|
||||
# same BeginFrame operation the distributed renderer relies on. Generate the
|
||||
# argument list with the production engine helper so a launcher-flag regression
|
||||
# invalidates this layer and fails the contract instead of testing a duplicate
|
||||
# hard-coded profile.
|
||||
COPY packages/engine/ packages/engine/
|
||||
COPY packages/aws-lambda/scripts/probe-beginframe.ts packages/aws-lambda/scripts/probe-beginframe.ts
|
||||
RUN bun -e 'import { buildChromeArgs } from "./packages/engine/src/services/browserManager.ts"; await Bun.write("/tmp/hf-gcp-chrome-args.json", JSON.stringify(buildChromeArgs({ width: 800, height: 600, captureMode: "beginframe", platform: "linux" }, { browserGpuMode: "software" })))' \
|
||||
&& bun packages/aws-lambda/scripts/probe-beginframe.ts \
|
||||
--executable-path /opt/chrome/chrome-headless-shell \
|
||||
--launch-args-json /tmp/hf-gcp-chrome-args.json
|
||||
|
||||
# CI targets `beginframe-contract` directly, so pin/probe changes do not need
|
||||
# the full monorepo build merely to verify browser capability. The deployable
|
||||
# image continues from the already-proven binary and installed workspaces.
|
||||
FROM beginframe-contract AS runtime
|
||||
|
||||
# Copy source for the packages the render path needs.
|
||||
COPY packages/core/ packages/core/
|
||||
COPY packages/engine/ packages/engine/
|
||||
|
||||
@@ -50,7 +50,11 @@ decompresses `@sparticuz/chromium` into `/tmp` at runtime — Cloud Run runs a
|
||||
container image. The `Dockerfile` installs the same pinned
|
||||
`chrome-headless-shell` build and font set the production renderer uses, at a
|
||||
fixed path, and exports `HYPERFRAMES_CHROME_PATH`. CDP-level `BeginFrame`
|
||||
works because the command lives in the protocol, not the binary. There is no
|
||||
support is a binary/runtime capability, so the image build launches that
|
||||
exact executable and requires an enable + warm-up + PNG-returning
|
||||
`HeadlessExperimental.beginFrame` probe to pass. The end-to-end smoke also
|
||||
requires every chunk to report effective `CaptureMode: "beginframe"`, which
|
||||
catches runtime fallback separately from build-time packaging. There is no
|
||||
runtime decompression step and no packaging ceiling.
|
||||
|
||||
## Deploying
|
||||
|
||||
@@ -5,9 +5,10 @@
|
||||
* via the engine's `BrowserManager`. Because Cloud Run runs a container
|
||||
* image rather than a size-capped ZIP, the Chrome story is far simpler than
|
||||
* the Lambda adapter's: the `Dockerfile` installs `chrome-headless-shell`
|
||||
* (the same BeginFrame-capable build the K8s deploy uses) into the image at
|
||||
* a known path and exports `HYPERFRAMES_CHROME_PATH`. There is no runtime
|
||||
* decompression-into-/tmp step and no 250 MB packaging ceiling to fight.
|
||||
* into the image at a known path and exports `HYPERFRAMES_CHROME_PATH`.
|
||||
* The Dockerfile proves that exact executable's full BeginFrame screenshot
|
||||
* contract during the image build. There is no runtime decompression-into-
|
||||
* /tmp step and no 250 MB packaging ceiling to fight.
|
||||
*
|
||||
* Resolution order:
|
||||
* 1. `PRODUCER_HEADLESS_SHELL_PATH` — the engine's own override. If a
|
||||
|
||||
@@ -203,6 +203,8 @@ export interface RenderChunkResultBody {
|
||||
ChunkIndex: number;
|
||||
Sha256: string;
|
||||
FramesEncoded: number;
|
||||
/** Effective engine mode after browser probing. Emitted by current handlers. */
|
||||
CaptureMode?: "beginframe" | "screenshot" | "drawelement";
|
||||
DurationMs: number;
|
||||
}
|
||||
|
||||
|
||||
@@ -121,6 +121,14 @@ function depsWith(
|
||||
outputKind: "file",
|
||||
framesEncoded: 30,
|
||||
sha256: `sha-${chunkIndex}`,
|
||||
captureMode: "beginframe",
|
||||
durationMs: 0,
|
||||
planHashMs: 0,
|
||||
sessionBootMs: 0,
|
||||
captureStageMs: 0,
|
||||
encodeStageMs: 0,
|
||||
workers: 1,
|
||||
perfPath: `${outputBase}.perf.json`,
|
||||
} satisfies ChunkResult;
|
||||
};
|
||||
const assemble = async (
|
||||
@@ -201,6 +209,7 @@ describe("dispatch", () => {
|
||||
const res = await dispatch(event, depsWith(gcs));
|
||||
if (res.Action !== "renderChunk") throw new Error("unreachable");
|
||||
expect(res.ChunkIndex).toBe(2);
|
||||
expect(res.CaptureMode).toBe("beginframe");
|
||||
expect(res.ChunkGcsUri).toBe("gs://b/renders/r1/chunks/0002.mp4");
|
||||
expect(gcs.objects.has("gs://b/renders/r1/chunks/0002.mp4")).toBe(true);
|
||||
});
|
||||
@@ -270,6 +279,14 @@ describe("dispatch", () => {
|
||||
outputKind: "file",
|
||||
framesEncoded: 30,
|
||||
sha256: "b".repeat(64),
|
||||
captureMode: "beginframe",
|
||||
durationMs: 0,
|
||||
planHashMs: 0,
|
||||
sessionBootMs: 0,
|
||||
captureStageMs: 0,
|
||||
encodeStageMs: 0,
|
||||
workers: 1,
|
||||
perfPath: `${outputBase}.perf.json`,
|
||||
};
|
||||
};
|
||||
const assemble = async (
|
||||
|
||||
@@ -455,6 +455,7 @@ async function handleRenderChunk(
|
||||
ChunkIndex: event.ChunkIndex,
|
||||
Sha256: result.sha256,
|
||||
FramesEncoded: result.framesEncoded,
|
||||
CaptureMode: result.captureMode,
|
||||
DurationMs: Date.now() - started,
|
||||
};
|
||||
} finally {
|
||||
@@ -500,6 +501,7 @@ async function handleRenderChunkV2(
|
||||
ChunkIndex: event.ChunkIndex,
|
||||
Sha256: result.sha256,
|
||||
FramesEncoded: result.framesEncoded,
|
||||
CaptureMode: result.captureMode,
|
||||
DurationMs: Date.now() - started,
|
||||
};
|
||||
} finally {
|
||||
|
||||
@@ -14,6 +14,8 @@ describe("GCP smoke ownership and protocol safety", () => {
|
||||
expect(smoke).toContain("decodedFramesEqual");
|
||||
expect(smoke).toContain("decodedAudioEqual");
|
||||
expect(smoke).toContain("normalizedMetadataEqual");
|
||||
expect(smoke).toContain('.CaptureMode == "beginframe"');
|
||||
expect(smoke).toContain("expected every chunk to use beginframe");
|
||||
});
|
||||
|
||||
it("derives a length-safe owner prefix and isolates Terraform state", () => {
|
||||
@@ -51,6 +53,7 @@ describe("GCP smoke ownership and protocol safety", () => {
|
||||
expect(smoke).toContain("--ignore-file");
|
||||
expect(smoke).toContain("--gcs-source-staging-dir");
|
||||
expect(smoke).toContain("!scripts/package-subpaths.mjs");
|
||||
expect(smoke).toContain("!packages/aws-lambda/scripts/probe-beginframe.ts");
|
||||
expect(dockerfile).toContain("COPY scripts/package-subpaths.mjs scripts/package-subpaths.mjs");
|
||||
expect(smoke).not.toContain("gcloud services enable");
|
||||
expect(smoke).toContain("gcloud services list");
|
||||
|
||||
@@ -238,6 +238,8 @@ describe("cross-worker idempotency", () => {
|
||||
expect(b.outputKind).toBe("file");
|
||||
expect(a.framesEncoded).toBeGreaterThan(0);
|
||||
expect(b.framesEncoded).toBe(a.framesEncoded);
|
||||
expect(a.captureMode).toBe("beginframe");
|
||||
expect(b.captureMode).toBe("beginframe");
|
||||
|
||||
expect(a.sha256).toBe(b.sha256);
|
||||
assertBytesEqual(outA, outB, "file", `mp4 chunk ${chunkIndex}`);
|
||||
|
||||
@@ -19,6 +19,7 @@ import { afterAll, beforeAll, describe, expect, it } from "bun:test";
|
||||
import { existsSync, mkdirSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from "node:fs";
|
||||
import { tmpdir } from "node:os";
|
||||
import { join } from "node:path";
|
||||
import { applyConcreteGpuScreenshotClamp, buildChromeArgs } from "@hyperframes/engine";
|
||||
import { recomputePlanHashFromPlanDir } from "../render/stages/freezePlan.js";
|
||||
import { RenderQualityError } from "../renderOrchestrator.js";
|
||||
import { CURRENT_PLAN_PROTOCOL } from "./planProtocol.js";
|
||||
@@ -29,6 +30,7 @@ import {
|
||||
DEFAULT_MAX_PARALLEL_CHUNKS,
|
||||
MIN_CHUNK_SIZE,
|
||||
plan,
|
||||
resolveDistributedEngineConfig,
|
||||
resolveChunkPlan,
|
||||
} from "./plan.js";
|
||||
import { buildSyntheticRenderJob } from "./shared.js";
|
||||
@@ -155,6 +157,26 @@ describe("distributed synthetic render job", () => {
|
||||
|
||||
expect(job.config.variables).toEqual(variables);
|
||||
});
|
||||
|
||||
it("keeps the production software-GPU launch on BeginFrame control", () => {
|
||||
const cfg = resolveDistributedEngineConfig({
|
||||
fps: 30,
|
||||
width: 320,
|
||||
height: 240,
|
||||
format: "mp4",
|
||||
});
|
||||
const forceScreenshot = applyConcreteGpuScreenshotClamp(
|
||||
cfg.forceScreenshot,
|
||||
"software",
|
||||
cfg,
|
||||
{},
|
||||
);
|
||||
const captureMode = forceScreenshot ? "screenshot" : "beginframe";
|
||||
const args = buildChromeArgs({ width: 320, height: 240, captureMode, platform: "linux" }, cfg);
|
||||
|
||||
expect(forceScreenshot).toBe(false);
|
||||
expect(args).toContain("--enable-begin-frame-control");
|
||||
});
|
||||
});
|
||||
|
||||
describe("resolveChunkPlan", () => {
|
||||
|
||||
@@ -778,6 +778,20 @@ function resolveNonMp4EncoderTriple(
|
||||
return { encoder: "png-sequence", pixelFormat: "rgba", preset: "lossless" };
|
||||
}
|
||||
|
||||
/** Test-visible construction of the engine config used by distributed planning. */
|
||||
export function resolveDistributedEngineConfig(config: DistributedRenderConfig): EngineConfig {
|
||||
return {
|
||||
...(config.producerConfig ?? config.engineConfig ?? resolveConfig()),
|
||||
browserGpuMode: "software",
|
||||
forceScreenshot: false,
|
||||
// Distributed rendering deliberately opts into deterministic BeginFrame
|
||||
// on Linux SwiftShader. Preserve the provenance bit consumed by the
|
||||
// engine's software-GPU screenshot clamp; assigning the boolean alone
|
||||
// loses the distinction between a default false and this explicit opt-out.
|
||||
forceScreenshotExplicitlyOptedOut: true,
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Activity A of the distributed render pipeline. Produces a self-contained
|
||||
* `<planDir>/` from a project + config. See module docstring for the
|
||||
@@ -808,11 +822,7 @@ export async function plan(
|
||||
throw new Error("[plan] render_cancelled");
|
||||
}
|
||||
};
|
||||
const cfg: EngineConfig = {
|
||||
...(config.producerConfig ?? config.engineConfig ?? resolveConfig()),
|
||||
browserGpuMode: "software",
|
||||
forceScreenshot: false,
|
||||
};
|
||||
const cfg = resolveDistributedEngineConfig(config);
|
||||
|
||||
const job = buildSyntheticRenderJob({
|
||||
fps: { num: config.fps, den: 1 },
|
||||
|
||||
@@ -203,8 +203,11 @@ describe("renderChunk()", () => {
|
||||
expect(a.captureStageMs).toBeGreaterThan(0);
|
||||
expect(a.encodeStageMs).toBeGreaterThanOrEqual(0);
|
||||
expect(a.workers).toBeGreaterThanOrEqual(1);
|
||||
expect(["beginframe", "screenshot", "drawelement"]).toContain(a.captureMode);
|
||||
expect(b.captureMode).toBe(a.captureMode);
|
||||
expect(a.captureStageMs + a.encodeStageMs).toBeLessThanOrEqual(a.durationMs);
|
||||
const perf = JSON.parse(readFileSync(a.perfPath, "utf-8"));
|
||||
expect(perf.captureMode).toBe(a.captureMode);
|
||||
for (const key of [
|
||||
"planHashMs",
|
||||
"sessionBootMs",
|
||||
|
||||
@@ -43,6 +43,8 @@ import {
|
||||
BROWSER_GPU_NOT_SOFTWARE,
|
||||
calculateOptimalWorkers,
|
||||
type CaptureOptions,
|
||||
type CaptureMode,
|
||||
type CapturePerfSummary,
|
||||
type CaptureSession,
|
||||
closeCaptureSession,
|
||||
createCaptureSession,
|
||||
@@ -158,6 +160,8 @@ export interface ChunkResult {
|
||||
encodeStageMs: number;
|
||||
/** Capture workers used for this chunk (`calculateOptimalWorkers` result). */
|
||||
workers: number;
|
||||
/** Effective engine mode used by every worker, after any browser fallback. */
|
||||
captureMode: CaptureMode;
|
||||
/**
|
||||
* Path to a sidecar JSON containing per-chunk perf counters. Adapters
|
||||
* upload this alongside the chunk so per-chunk regressions are
|
||||
@@ -330,6 +334,7 @@ export function resolveLockedVp9CpuUsed(
|
||||
* outputs — the caller picks the right shape based on `meta/encoder.json`.
|
||||
* `renderChunk` enforces the same choice via `outputKind` on the result.
|
||||
*/
|
||||
// fallow-ignore-next-line complexity
|
||||
export async function renderChunk(
|
||||
planDir: string,
|
||||
chunkIndex: number,
|
||||
@@ -485,6 +490,10 @@ export async function renderChunk(
|
||||
...resolveConfig(),
|
||||
browserGpuMode: "software",
|
||||
forceScreenshot: encoder.forceScreenshot,
|
||||
// `encoder.forceScreenshot=false` is a locked distributed-render
|
||||
// decision, not the engine default. Carry that explicit opt-out through
|
||||
// the software-GPU clamp so buildChromeArgs includes BeginFrameControl.
|
||||
forceScreenshotExplicitlyOptedOut: !encoder.forceScreenshot,
|
||||
};
|
||||
|
||||
// Build the BeforeCaptureHook that injects pre-extracted video frames
|
||||
@@ -585,6 +594,8 @@ export async function renderChunk(
|
||||
let sessionBootMs = 0;
|
||||
let captureStageMs = 0;
|
||||
let encodeStageMs = 0;
|
||||
let captureMode: CaptureMode | undefined;
|
||||
const capturePerfs: CapturePerfSummary[] = [];
|
||||
try {
|
||||
if (chunkWorkerCount === 1) {
|
||||
// Sequential branch reuses the probe session for the actual capture.
|
||||
@@ -638,9 +649,10 @@ export async function renderChunk(
|
||||
log,
|
||||
probeSession: session,
|
||||
captureAttempts: [],
|
||||
// Distributed chunks run on Linux (beginframe) where dedup never arms;
|
||||
// a throwaway sink satisfies the type without per-chunk dedup reporting.
|
||||
dedupPerfs: [],
|
||||
// This sink also records each worker's effective capture mode. That
|
||||
// makes a BeginFrame → screenshot fallback observable to adapters and
|
||||
// end-to-end smoke tests instead of existing only in stderr.
|
||||
dedupPerfs: capturePerfs,
|
||||
buildCaptureOptions: () => captureOptions,
|
||||
createRenderVideoFrameInjector: () => videoInjector,
|
||||
abortSignal: undefined,
|
||||
@@ -650,6 +662,20 @@ export async function renderChunk(
|
||||
// captureStage closes the session it consumed.
|
||||
captureStageMs = Date.now() - captureStarted;
|
||||
session = null;
|
||||
const observedModes = new Set(capturePerfs.map((perf) => perf.captureMode));
|
||||
const validModes = new Set<CaptureMode>(["beginframe", "screenshot", "drawelement"]);
|
||||
if (
|
||||
observedModes.size !== 1 ||
|
||||
![...observedModes].every((mode): mode is CaptureMode =>
|
||||
validModes.has(mode as CaptureMode),
|
||||
)
|
||||
) {
|
||||
throw new Error(
|
||||
`[renderChunk] capture workers reported invalid or inconsistent modes: ` +
|
||||
`${[...observedModes].join(",") || "<none>"}`,
|
||||
);
|
||||
}
|
||||
captureMode = [...observedModes][0] as CaptureMode;
|
||||
framesEncoded = framesInChunk;
|
||||
|
||||
// ── Encode the chunk ──
|
||||
@@ -737,6 +763,9 @@ export async function renderChunk(
|
||||
}
|
||||
|
||||
// ── Hash the output + write the perf sidecar ──
|
||||
if (!captureMode) {
|
||||
throw new Error("[renderChunk] capture stage completed without reporting a capture mode");
|
||||
}
|
||||
const sha256 = hashChunkOutput(outputChunkPath, outputKind);
|
||||
const durationMs = Date.now() - start;
|
||||
const perfPath = `${outputChunkPath}.perf.json`;
|
||||
@@ -752,6 +781,7 @@ export async function renderChunk(
|
||||
captureStageMs,
|
||||
encodeStageMs,
|
||||
workers: chunkWorkerCount,
|
||||
captureMode,
|
||||
sha256,
|
||||
outputKind,
|
||||
producerVersion: plan.producerVersion,
|
||||
@@ -781,6 +811,7 @@ export async function renderChunk(
|
||||
captureStageMs,
|
||||
encodeStageMs,
|
||||
workers: chunkWorkerCount,
|
||||
captureMode,
|
||||
perfPath,
|
||||
};
|
||||
} finally {
|
||||
|
||||
Reference in New Issue
Block a user