From cfa0d0cef9f3dcde7057ac42cf268c78ed3dad66 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Miguel=20=C3=81ngel?= Date: Fri, 6 Nov 2026 20:45:36 +0000 Subject: [PATCH] fix(engine): a missing frame file skips its self-verify sample instead of aborting the render (#7334) --- .../frameCapture-verifyArming.browser.test.ts | 101 +++++++++++ packages/engine/src/services/frameCapture.ts | 2 +- .../src/services/parallelCoordinator.ts | 22 ++- .../parallelCoordinator.verifyDisk.test.ts | 157 ++++++++++++++++++ 4 files changed, 275 insertions(+), 7 deletions(-) create mode 100644 packages/engine/src/services/frameCapture-verifyArming.browser.test.ts create mode 100644 packages/engine/src/services/parallelCoordinator.verifyDisk.test.ts diff --git a/packages/engine/src/services/frameCapture-verifyArming.browser.test.ts b/packages/engine/src/services/frameCapture-verifyArming.browser.test.ts new file mode 100644 index 0000000000..844a8eb8ee --- /dev/null +++ b/packages/engine/src/services/frameCapture-verifyArming.browser.test.ts @@ -0,0 +1,101 @@ +import { spawnSync } from "node:child_process"; +import { createServer, type Server } from "node:http"; +import { afterAll, afterEach, beforeAll, describe, expect, it } from "vitest"; +import type { Browser, Page } from "puppeteer-core"; +import { closeTestChrome, launchTestChrome, listenOnLoopback } from "./__fixtures__/loopback.js"; +import { resolveHeadlessShellPath } from "./browserManager.js"; +import { captureDeVerificationFrames, type CaptureSession } from "./frameCapture.js"; + +const chromePath = resolveHeadlessShellPath(); +const hasBrowser = chromePath !== undefined && spawnSync(chromePath, ["--version"]).status === 0; + +const HF_PAGE = ``; + +describe.skipIf(!hasBrowser)("drawElement self-verify arming", () => { + let browser: Browser; + let server: Server; + let serverUrl: string; + let served = HF_PAGE; + const pages: Page[] = []; + const priorSamples = process.env.HF_DE_VERIFY; + + beforeAll(async () => { + browser = await launchTestChrome(chromePath); + server = createServer((_req, res) => { + res.setHeader("content-type", "text/html"); + res.end(served); + }); + serverUrl = await listenOnLoopback(server); + }); + + afterEach(async () => { + served = HF_PAGE; + if (priorSamples === undefined) delete process.env.HF_DE_VERIFY; + else process.env.HF_DE_VERIFY = priorSamples; + await Promise.all(pages.splice(0).map((page) => page.close())); + }); + + afterAll(async () => { + await closeTestChrome(browser); + await new Promise((resolve) => server.close(resolve)); + }); + + const arm = async (html = HF_PAGE) => { + served = html; + const page = await browser.newPage(); + pages.push(page); + await page.goto(`${serverUrl}/index.html`); + const session = { + browser, + page, + serverUrl, + onBeforeCapture: null, + scriptLoadFailures: [], + subTimelineWaitMemo: {}, + options: { + width: 64, + height: 64, + fps: { num: 30, den: 1 }, + format: "jpeg", + compositionDurationSeconds: 2, + }, + } as unknown as CaptureSession; + await captureDeVerificationFrames(session, page, () => {}); + return session.deVerifyFrames; + }; + + it("takes the ground truth at the fixed fractions, each showing its own frame", async () => { + const frames = await arm(); + // 60 frames at fractions 1/4, 1/2, 3/4 and 0.95. + expect([...(frames?.keys() ?? [])]).toEqual([15, 30, 45, 57]); + expect(new Set([...(frames?.values() ?? [])].map((b) => b.toString("base64"))).size).toBe(4); + for (const truth of frames?.values() ?? []) { + expect(truth.subarray(0, 2).toString("hex")).toBe("ffd8"); + } + }); + + it("honours the requested sample count", async () => { + process.env.HF_DE_VERIFY = "1"; + expect([...((await arm())?.keys() ?? [])]).toEqual([57]); + }); + + it("arms nothing when self-verify is turned off", async () => { + process.env.HF_DE_VERIFY = "0"; + expect(await arm()).toBeUndefined(); + }); + + it("moves a sample off a clip cut", async () => { + const frames = await arm(`${HF_PAGE}
`); + expect(frames?.has(30)).toBe(false); + expect(frames?.size).toBe(4); + }); + + it("arms nothing on a video composition with no frame injector, whose truth would be black", async () => { + expect(await arm(`${HF_PAGE}`)).toBeUndefined(); + }); +}); diff --git a/packages/engine/src/services/frameCapture.ts b/packages/engine/src/services/frameCapture.ts index a145594a90..2bb1299588 100644 --- a/packages/engine/src/services/frameCapture.ts +++ b/packages/engine/src/services/frameCapture.ts @@ -5712,7 +5712,7 @@ export async function seekDeVerifyTruth( ); } -async function captureDeVerificationFrames( +export async function captureDeVerificationFrames( session: CaptureSession, page: Page, logInitPhase: (phase: string) => void, diff --git a/packages/engine/src/services/parallelCoordinator.ts b/packages/engine/src/services/parallelCoordinator.ts index e91ecde05f..3d5efefc61 100644 --- a/packages/engine/src/services/parallelCoordinator.ts +++ b/packages/engine/src/services/parallelCoordinator.ts @@ -814,8 +814,22 @@ async function psnrForDiskSample( workerId: number, idx: number, ): Promise { + const skipSample = (err: unknown): null => { + console.warn( + `[Parallel] drawElement disk self-verify sample skipped (worker ${workerId}, ` + + `frame ${idx}): ${err instanceof Error ? err.message : String(err)}`, + ); + return null; + }; + // Read apart from the compare: a frame file's ENOENT says nothing about ffmpeg. + let frame: Buffer; try { - return await psnrDb(await readFile(framePath), truth); + frame = await readFile(framePath); + } catch (err) { + return skipSample(err); + } + try { + return await psnrDb(frame, truth); } catch (err) { if (isFfmpegInfrastructureFailure(err)) { const detail = err instanceof Error ? err.message : String(err); @@ -828,11 +842,7 @@ async function psnrForDiskSample( { cause: err }, ); } - console.warn( - `[Parallel] drawElement disk self-verify sample skipped (worker ${workerId}, ` + - `frame ${idx}): ${err instanceof Error ? err.message : String(err)}`, - ); - return null; + return skipSample(err); } } diff --git a/packages/engine/src/services/parallelCoordinator.verifyDisk.test.ts b/packages/engine/src/services/parallelCoordinator.verifyDisk.test.ts new file mode 100644 index 0000000000..c207c8c710 --- /dev/null +++ b/packages/engine/src/services/parallelCoordinator.verifyDisk.test.ts @@ -0,0 +1,157 @@ +import { mkdtempSync, rmSync, writeFileSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; +import { psnrDb } from "../utils/psnr.js"; +import { isDrawElementVerificationError, type CaptureSession } from "./frameCapture.js"; +import { verifyDiskDrawElementSamples, type WorkerTask } from "./parallelCoordinator.js"; + +vi.mock("../utils/psnr.js", async (importOriginal) => ({ + ...(await importOriginal()), + psnrDb: vi.fn(), +})); + +const psnr = vi.mocked(psnrDb); +const FLOOR_DB = 32; + +describe("verifyDiskDrawElementSamples", () => { + let outputDir: string; + + beforeEach(() => { + outputDir = mkdtempSync(join(tmpdir(), "hf-disk-verify-")); + psnr.mockReset(); + vi.spyOn(console, "error").mockImplementation(() => {}); + vi.spyOn(console, "warn").mockImplementation(() => {}); + }); + + afterEach(() => { + rmSync(outputDir, { recursive: true, force: true }); + vi.restoreAllMocks(); + }); + + const truthFor = (frame: number) => Buffer.from(`truth-${frame}`); + const sessionWith = ( + frames: number[], + overrides: { captureMode?: CaptureSession["captureMode"]; format?: "jpeg" | "png" } = {}, + ) => + ({ + captureMode: overrides.captureMode ?? "drawelement", + options: { format: overrides.format ?? "jpeg" }, + deVerifyFrames: new Map(frames.map((frame) => [frame, truthFor(frame)])), + }) as unknown as CaptureSession; + const taskWith = (overrides: Partial = {}) => + ({ workerId: 1, startFrame: 0, endFrame: 10, outputDir, ...overrides }) as WorkerTask; + const writeFrame = (fileIndex: number, ext = "jpg") => { + const bytes = Buffer.from(`captured-${fileIndex}`); + writeFileSync(join(outputDir, `frame_${String(fileIndex).padStart(6, "0")}.${ext}`), bytes); + return bytes; + }; + + it("fails the worker on a sample below the floor, naming the frame and its score", async () => { + writeFrame(2); + writeFrame(4); + psnr.mockResolvedValueOnce(50).mockResolvedValueOnce(FLOOR_DB - 0.1); + + const error = await verifyDiskDrawElementSamples(sessionWith([2, 4]), taskWith(), false).catch( + (e: unknown) => e, + ); + + expect(isDrawElementVerificationError(error)).toBe(true); + expect(error).toMatchObject({ + kind: "psnr", + frameIndex: 4, + failedDb: FLOOR_DB - 0.1, + verifyThresholdDb: FLOOR_DB, + }); + }); + + it("passes a sample exactly on the floor", async () => { + writeFrame(2); + psnr.mockResolvedValue(FLOOR_DB); + await expect( + verifyDiskDrawElementSamples(sessionWith([2]), taskWith(), false), + ).resolves.toBeUndefined(); + expect(psnr).toHaveBeenCalledTimes(1); + }); + + it("compares each captured file against its own ground truth", async () => { + const two = writeFrame(2); + const four = writeFrame(4); + psnr.mockResolvedValue(50); + await verifyDiskDrawElementSamples(sessionWith([4, 2]), taskWith(), false); + expect(psnr.mock.calls).toEqual([ + [two, truthFor(2)], + [four, truthFor(4)], + ]); + }); + + it("finds a chunk's frames through the output offset", async () => { + const onDisk = writeFrame(2); + psnr.mockResolvedValue(50); + await verifyDiskDrawElementSamples( + sessionWith([12]), + taskWith({ startFrame: 10, endFrame: 20, outputFrameOffset: 10 }), + false, + ); + expect(psnr.mock.calls).toEqual([[onDisk, truthFor(12)]]); + }); + + it("reads png frames from a png capture", async () => { + const onDisk = writeFrame(2, "png"); + psnr.mockResolvedValue(50); + await verifyDiskDrawElementSamples(sessionWith([2], { format: "png" }), taskWith(), false); + expect(psnr.mock.calls).toEqual([[onDisk, truthFor(2)]]); + }); + + it("checks only the samples this worker captured", async () => { + const mine = writeFrame(4); + psnr.mockResolvedValue(50); + await verifyDiskDrawElementSamples( + sessionWith([1, 3, 4, 12]), + taskWith({ startFrame: 0, endFrame: 10, frameStride: 2 }), + false, + ); + expect(psnr.mock.calls).toEqual([[mine, truthFor(4)]]); + }); + + it.each([ + ["a streaming capture, whose drain already checks", true, "drawelement"], + ["a session that fell back to screenshots", false, "screenshot"], + ] as const)("does not compare on %s", async (_name, streaming, captureMode) => { + writeFrame(2); + await verifyDiskDrawElementSamples(sessionWith([2], { captureMode }), taskWith(), streaming); + expect(psnr).not.toHaveBeenCalled(); + }); + + it("skips a sample whose file cannot be read, and still checks the next", async () => { + writeFrame(4); + psnr.mockResolvedValue(FLOOR_DB - 1); + const error = await verifyDiskDrawElementSamples(sessionWith([2, 4]), taskWith(), false).catch( + (e: unknown) => e, + ); + expect(error).toMatchObject({ frameIndex: 4 }); + }); + + it("skips a sample the compare could not score, and still checks the next", async () => { + writeFrame(2); + writeFrame(4); + psnr.mockRejectedValueOnce(new Error("could not parse psnr")).mockResolvedValue(FLOOR_DB - 1); + const error = await verifyDiskDrawElementSamples(sessionWith([2, 4]), taskWith(), false).catch( + (e: unknown) => e, + ); + expect(error).toMatchObject({ frameIndex: 4 }); + }); + + it.each([ + ["a missing filter", new Error("No such filter: 'psnr'")], + ["a missing binary", Object.assign(new Error("spawn ffmpeg ENOENT"), { code: "ENOENT" })], + ])("aborts rather than ship unverified frames on %s", async (_name, failure) => { + writeFrame(2); + psnr.mockRejectedValue(failure); + const error = await verifyDiskDrawElementSamples(sessionWith([2]), taskWith(), false).catch( + (e: unknown) => e, + ); + expect(isDrawElementVerificationError(error)).toBe(false); + expect((error as Error).message).toContain("disk self-verify aborted"); + }); +});