diff --git a/packages/engine/src/services/frameCapture-verifyArming.browser.test.ts b/packages/engine/src/services/frameCapture-verifyArming.browser.test.ts
new file mode 100644
index 0000000000..844a8eb8ee
--- /dev/null
+++ b/packages/engine/src/services/frameCapture-verifyArming.browser.test.ts
@@ -0,0 +1,101 @@
+import { spawnSync } from "node:child_process";
+import { createServer, type Server } from "node:http";
+import { afterAll, afterEach, beforeAll, describe, expect, it } from "vitest";
+import type { Browser, Page } from "puppeteer-core";
+import { closeTestChrome, launchTestChrome, listenOnLoopback } from "./__fixtures__/loopback.js";
+import { resolveHeadlessShellPath } from "./browserManager.js";
+import { captureDeVerificationFrames, type CaptureSession } from "./frameCapture.js";
+
+const chromePath = resolveHeadlessShellPath();
+const hasBrowser = chromePath !== undefined && spawnSync(chromePath, ["--version"]).status === 0;
+
+const HF_PAGE = `
`;
+
+describe.skipIf(!hasBrowser)("drawElement self-verify arming", () => {
+ let browser: Browser;
+ let server: Server;
+ let serverUrl: string;
+ let served = HF_PAGE;
+ const pages: Page[] = [];
+ const priorSamples = process.env.HF_DE_VERIFY;
+
+ beforeAll(async () => {
+ browser = await launchTestChrome(chromePath);
+ server = createServer((_req, res) => {
+ res.setHeader("content-type", "text/html");
+ res.end(served);
+ });
+ serverUrl = await listenOnLoopback(server);
+ });
+
+ afterEach(async () => {
+ served = HF_PAGE;
+ if (priorSamples === undefined) delete process.env.HF_DE_VERIFY;
+ else process.env.HF_DE_VERIFY = priorSamples;
+ await Promise.all(pages.splice(0).map((page) => page.close()));
+ });
+
+ afterAll(async () => {
+ await closeTestChrome(browser);
+ await new Promise((resolve) => server.close(resolve));
+ });
+
+ const arm = async (html = HF_PAGE) => {
+ served = html;
+ const page = await browser.newPage();
+ pages.push(page);
+ await page.goto(`${serverUrl}/index.html`);
+ const session = {
+ browser,
+ page,
+ serverUrl,
+ onBeforeCapture: null,
+ scriptLoadFailures: [],
+ subTimelineWaitMemo: {},
+ options: {
+ width: 64,
+ height: 64,
+ fps: { num: 30, den: 1 },
+ format: "jpeg",
+ compositionDurationSeconds: 2,
+ },
+ } as unknown as CaptureSession;
+ await captureDeVerificationFrames(session, page, () => {});
+ return session.deVerifyFrames;
+ };
+
+ it("takes the ground truth at the fixed fractions, each showing its own frame", async () => {
+ const frames = await arm();
+ // 60 frames at fractions 1/4, 1/2, 3/4 and 0.95.
+ expect([...(frames?.keys() ?? [])]).toEqual([15, 30, 45, 57]);
+ expect(new Set([...(frames?.values() ?? [])].map((b) => b.toString("base64"))).size).toBe(4);
+ for (const truth of frames?.values() ?? []) {
+ expect(truth.subarray(0, 2).toString("hex")).toBe("ffd8");
+ }
+ });
+
+ it("honours the requested sample count", async () => {
+ process.env.HF_DE_VERIFY = "1";
+ expect([...((await arm())?.keys() ?? [])]).toEqual([57]);
+ });
+
+ it("arms nothing when self-verify is turned off", async () => {
+ process.env.HF_DE_VERIFY = "0";
+ expect(await arm()).toBeUndefined();
+ });
+
+ it("moves a sample off a clip cut", async () => {
+ const frames = await arm(`${HF_PAGE}`);
+ expect(frames?.has(30)).toBe(false);
+ expect(frames?.size).toBe(4);
+ });
+
+ it("arms nothing on a video composition with no frame injector, whose truth would be black", async () => {
+ expect(await arm(`${HF_PAGE}`)).toBeUndefined();
+ });
+});
diff --git a/packages/engine/src/services/frameCapture.ts b/packages/engine/src/services/frameCapture.ts
index a145594a90..2bb1299588 100644
--- a/packages/engine/src/services/frameCapture.ts
+++ b/packages/engine/src/services/frameCapture.ts
@@ -5712,7 +5712,7 @@ export async function seekDeVerifyTruth(
);
}
-async function captureDeVerificationFrames(
+export async function captureDeVerificationFrames(
session: CaptureSession,
page: Page,
logInitPhase: (phase: string) => void,
diff --git a/packages/engine/src/services/parallelCoordinator.ts b/packages/engine/src/services/parallelCoordinator.ts
index e91ecde05f..3d5efefc61 100644
--- a/packages/engine/src/services/parallelCoordinator.ts
+++ b/packages/engine/src/services/parallelCoordinator.ts
@@ -814,8 +814,22 @@ async function psnrForDiskSample(
workerId: number,
idx: number,
): Promise {
+ const skipSample = (err: unknown): null => {
+ console.warn(
+ `[Parallel] drawElement disk self-verify sample skipped (worker ${workerId}, ` +
+ `frame ${idx}): ${err instanceof Error ? err.message : String(err)}`,
+ );
+ return null;
+ };
+ // Read apart from the compare: a frame file's ENOENT says nothing about ffmpeg.
+ let frame: Buffer;
try {
- return await psnrDb(await readFile(framePath), truth);
+ frame = await readFile(framePath);
+ } catch (err) {
+ return skipSample(err);
+ }
+ try {
+ return await psnrDb(frame, truth);
} catch (err) {
if (isFfmpegInfrastructureFailure(err)) {
const detail = err instanceof Error ? err.message : String(err);
@@ -828,11 +842,7 @@ async function psnrForDiskSample(
{ cause: err },
);
}
- console.warn(
- `[Parallel] drawElement disk self-verify sample skipped (worker ${workerId}, ` +
- `frame ${idx}): ${err instanceof Error ? err.message : String(err)}`,
- );
- return null;
+ return skipSample(err);
}
}
diff --git a/packages/engine/src/services/parallelCoordinator.verifyDisk.test.ts b/packages/engine/src/services/parallelCoordinator.verifyDisk.test.ts
new file mode 100644
index 0000000000..c207c8c710
--- /dev/null
+++ b/packages/engine/src/services/parallelCoordinator.verifyDisk.test.ts
@@ -0,0 +1,157 @@
+import { mkdtempSync, rmSync, writeFileSync } from "node:fs";
+import { tmpdir } from "node:os";
+import { join } from "node:path";
+import { afterEach, beforeEach, describe, expect, it, vi } from "vitest";
+import { psnrDb } from "../utils/psnr.js";
+import { isDrawElementVerificationError, type CaptureSession } from "./frameCapture.js";
+import { verifyDiskDrawElementSamples, type WorkerTask } from "./parallelCoordinator.js";
+
+vi.mock("../utils/psnr.js", async (importOriginal) => ({
+ ...(await importOriginal()),
+ psnrDb: vi.fn(),
+}));
+
+const psnr = vi.mocked(psnrDb);
+const FLOOR_DB = 32;
+
+describe("verifyDiskDrawElementSamples", () => {
+ let outputDir: string;
+
+ beforeEach(() => {
+ outputDir = mkdtempSync(join(tmpdir(), "hf-disk-verify-"));
+ psnr.mockReset();
+ vi.spyOn(console, "error").mockImplementation(() => {});
+ vi.spyOn(console, "warn").mockImplementation(() => {});
+ });
+
+ afterEach(() => {
+ rmSync(outputDir, { recursive: true, force: true });
+ vi.restoreAllMocks();
+ });
+
+ const truthFor = (frame: number) => Buffer.from(`truth-${frame}`);
+ const sessionWith = (
+ frames: number[],
+ overrides: { captureMode?: CaptureSession["captureMode"]; format?: "jpeg" | "png" } = {},
+ ) =>
+ ({
+ captureMode: overrides.captureMode ?? "drawelement",
+ options: { format: overrides.format ?? "jpeg" },
+ deVerifyFrames: new Map(frames.map((frame) => [frame, truthFor(frame)])),
+ }) as unknown as CaptureSession;
+ const taskWith = (overrides: Partial = {}) =>
+ ({ workerId: 1, startFrame: 0, endFrame: 10, outputDir, ...overrides }) as WorkerTask;
+ const writeFrame = (fileIndex: number, ext = "jpg") => {
+ const bytes = Buffer.from(`captured-${fileIndex}`);
+ writeFileSync(join(outputDir, `frame_${String(fileIndex).padStart(6, "0")}.${ext}`), bytes);
+ return bytes;
+ };
+
+ it("fails the worker on a sample below the floor, naming the frame and its score", async () => {
+ writeFrame(2);
+ writeFrame(4);
+ psnr.mockResolvedValueOnce(50).mockResolvedValueOnce(FLOOR_DB - 0.1);
+
+ const error = await verifyDiskDrawElementSamples(sessionWith([2, 4]), taskWith(), false).catch(
+ (e: unknown) => e,
+ );
+
+ expect(isDrawElementVerificationError(error)).toBe(true);
+ expect(error).toMatchObject({
+ kind: "psnr",
+ frameIndex: 4,
+ failedDb: FLOOR_DB - 0.1,
+ verifyThresholdDb: FLOOR_DB,
+ });
+ });
+
+ it("passes a sample exactly on the floor", async () => {
+ writeFrame(2);
+ psnr.mockResolvedValue(FLOOR_DB);
+ await expect(
+ verifyDiskDrawElementSamples(sessionWith([2]), taskWith(), false),
+ ).resolves.toBeUndefined();
+ expect(psnr).toHaveBeenCalledTimes(1);
+ });
+
+ it("compares each captured file against its own ground truth", async () => {
+ const two = writeFrame(2);
+ const four = writeFrame(4);
+ psnr.mockResolvedValue(50);
+ await verifyDiskDrawElementSamples(sessionWith([4, 2]), taskWith(), false);
+ expect(psnr.mock.calls).toEqual([
+ [two, truthFor(2)],
+ [four, truthFor(4)],
+ ]);
+ });
+
+ it("finds a chunk's frames through the output offset", async () => {
+ const onDisk = writeFrame(2);
+ psnr.mockResolvedValue(50);
+ await verifyDiskDrawElementSamples(
+ sessionWith([12]),
+ taskWith({ startFrame: 10, endFrame: 20, outputFrameOffset: 10 }),
+ false,
+ );
+ expect(psnr.mock.calls).toEqual([[onDisk, truthFor(12)]]);
+ });
+
+ it("reads png frames from a png capture", async () => {
+ const onDisk = writeFrame(2, "png");
+ psnr.mockResolvedValue(50);
+ await verifyDiskDrawElementSamples(sessionWith([2], { format: "png" }), taskWith(), false);
+ expect(psnr.mock.calls).toEqual([[onDisk, truthFor(2)]]);
+ });
+
+ it("checks only the samples this worker captured", async () => {
+ const mine = writeFrame(4);
+ psnr.mockResolvedValue(50);
+ await verifyDiskDrawElementSamples(
+ sessionWith([1, 3, 4, 12]),
+ taskWith({ startFrame: 0, endFrame: 10, frameStride: 2 }),
+ false,
+ );
+ expect(psnr.mock.calls).toEqual([[mine, truthFor(4)]]);
+ });
+
+ it.each([
+ ["a streaming capture, whose drain already checks", true, "drawelement"],
+ ["a session that fell back to screenshots", false, "screenshot"],
+ ] as const)("does not compare on %s", async (_name, streaming, captureMode) => {
+ writeFrame(2);
+ await verifyDiskDrawElementSamples(sessionWith([2], { captureMode }), taskWith(), streaming);
+ expect(psnr).not.toHaveBeenCalled();
+ });
+
+ it("skips a sample whose file cannot be read, and still checks the next", async () => {
+ writeFrame(4);
+ psnr.mockResolvedValue(FLOOR_DB - 1);
+ const error = await verifyDiskDrawElementSamples(sessionWith([2, 4]), taskWith(), false).catch(
+ (e: unknown) => e,
+ );
+ expect(error).toMatchObject({ frameIndex: 4 });
+ });
+
+ it("skips a sample the compare could not score, and still checks the next", async () => {
+ writeFrame(2);
+ writeFrame(4);
+ psnr.mockRejectedValueOnce(new Error("could not parse psnr")).mockResolvedValue(FLOOR_DB - 1);
+ const error = await verifyDiskDrawElementSamples(sessionWith([2, 4]), taskWith(), false).catch(
+ (e: unknown) => e,
+ );
+ expect(error).toMatchObject({ frameIndex: 4 });
+ });
+
+ it.each([
+ ["a missing filter", new Error("No such filter: 'psnr'")],
+ ["a missing binary", Object.assign(new Error("spawn ffmpeg ENOENT"), { code: "ENOENT" })],
+ ])("aborts rather than ship unverified frames on %s", async (_name, failure) => {
+ writeFrame(2);
+ psnr.mockRejectedValue(failure);
+ const error = await verifyDiskDrawElementSamples(sessionWith([2]), taskWith(), false).catch(
+ (e: unknown) => e,
+ );
+ expect(isDrawElementVerificationError(error)).toBe(false);
+ expect((error as Error).message).toContain("disk self-verify aborted");
+ });
+});