From 8df4f21c20679fc92ae338ad647da5f5d9a69d76 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Miguel=20=C3=81ngel?= Date: Thu, 5 Nov 2026 18:43:59 +0000 Subject: [PATCH] fix(producer): segmented and sequential capture check drawElement samples (#7255) * fix(producer): segmented capture checks drawElement frames against their screenshots * fix(producer): streaming sequential loop checks drawElement samples too --- .../stages/captureSegmentedStage.test.ts | 70 +++++++++++++++ .../render/stages/captureSegmentedStage.ts | 10 +++ .../stages/captureStreamingStage.test.ts | 41 +++++++++ .../render/stages/captureStreamingStage.ts | 89 +++++++++++-------- 4 files changed, 175 insertions(+), 35 deletions(-) diff --git a/packages/producer/src/services/render/stages/captureSegmentedStage.test.ts b/packages/producer/src/services/render/stages/captureSegmentedStage.test.ts index 6bc6bf9ab2..6947a2319b 100644 --- a/packages/producer/src/services/render/stages/captureSegmentedStage.test.ts +++ b/packages/producer/src/services/render/stages/captureSegmentedStage.test.ts @@ -32,6 +32,9 @@ const createdSessionConfigs: Array<{ enableBrowserPool?: boolean }> = []; const createdSessionHooks: unknown[] = []; const reusedSessionHooks: unknown[] = []; class DrawElementVerificationError extends Error {} +/** PSNR the fake comparison reports for a frame; every call is recorded. */ +let samplePsnr = (_frame: Buffer) => 50; +const psnrCalls: Array<{ frame: Buffer; truth: Buffer }> = []; mock.module("@hyperframes/engine", () => ({ calculateOptimalWorkers: () => 1, @@ -137,7 +140,12 @@ mock.module("@hyperframes/engine", () => ({ throw new Error("prepare reuse failed: ENOSPC"); } }, + psnrDb: async (frame: Buffer, truth: Buffer) => { + psnrCalls.push({ frame, truth }); + return samplePsnr(frame); + }, recaptureDrawElementFrameForVerify: async () => Buffer.from("frame"), + resolveDeVerifyMinDb: () => 32, resolveHeadlessShellPath: () => "/headless-shell", shouldDisableBrowserPoolForParallelWorker: ({ parallel }: { parallel?: boolean }) => Boolean(parallel), @@ -931,6 +939,68 @@ describe("runCaptureSegmentedStage", () => { await new Promise((r) => setTimeout(r, 40)); expect(capturedAfterClose).toBe(0); }); + + describe("drawElement self-verify", () => { + const truths = new Map([ + [2, Buffer.from("truth-2")], + [7, Buffer.from("truth-7")], + ]); + const frameBuffer = (i: number) => Buffer.from(`frame-${i}`); + + function verifiedRun(dir: string, written: number[]) { + return runCaptureSegmentedStage({ + ...fakeStageInput({ totalFrames: 9 }), + probeSession: null, + segmentFrames: 3, + segmentDir: join(fixtureRoot, dir), + workerCount: 2, + sessionFactoryForWorker: (id) => ({ + create: async () => ({ + ...fakeSession(id), + captureMode: "drawelement" as const, + deVerifyFrames: truths, + }), + }), + deps: { + spawnEncoder: mock(async () => ({ + ...okEncoder(), + writeFrame: async (buf: Buffer) => { + written.push(Number(buf.toString().split("-")[1])); + return true; + }, + })), + captureFrame: mock(async (_s: unknown, i: number) => ({ buffer: frameBuffer(i) })), + concat: mock(async () => ({ success: true as const })), + closeSession: mock(async () => {}), + removeFile: () => {}, + }, + }); + } + + it("compares every armed sample against its ground truth", async () => { + psnrCalls.length = 0; + samplePsnr = () => 50; + const result = await verifiedRun("verify-pass", []); + expect(result.success).toBe(true); + const checked = psnrCalls + .map((c) => ({ frame: c.frame.toString(), truth: c.truth.toString() })) + .sort((a, b) => a.frame.localeCompare(b.frame)); + expect(checked).toEqual([ + { frame: "frame-2", truth: "truth-2" }, + { frame: "frame-7", truth: "truth-7" }, + ]); + }); + + it("fails the render before a sample below the floor is encoded", async () => { + psnrCalls.length = 0; + samplePsnr = (frame) => (frame.toString() === "frame-7" ? 20 : 50); + const written: number[] = []; + const error = await verifiedRun("verify-breach", written).catch((e: unknown) => e); + expect((error as { cause?: unknown }).cause).toBeInstanceOf(DrawElementVerificationError); + expect(written).not.toContain(7); + samplePsnr = () => 50; + }); + }); }); describe("runCaptureSegmentedStage with segmentVideoFrames", () => { diff --git a/packages/producer/src/services/render/stages/captureSegmentedStage.ts b/packages/producer/src/services/render/stages/captureSegmentedStage.ts index 5c4d62f540..c5d4de6fb0 100644 --- a/packages/producer/src/services/render/stages/captureSegmentedStage.ts +++ b/packages/producer/src/services/render/stages/captureSegmentedStage.ts @@ -23,6 +23,7 @@ import { getCapturePerfSummary, initializeSession, prepareCaptureSessionForReuse, + resolveDeVerifyMinDb, shouldDisableBrowserPoolForParallelWorker, spawnStreamingEncoder, type BeforeCaptureHook, @@ -48,6 +49,7 @@ import { isTargetLossError } from "../segmentRecycle.js"; import { raceAgainstStall, resolveCaptureStallTimeoutMs, + verifyDrawElementSample, type StreamingEncoderOptions, } from "./captureStreamingStage.js"; @@ -188,6 +190,8 @@ interface SegmentCaptureContext { spawnEncoder: SegmentedStageDeps["spawnEncoder"]; captureFrame: SegmentedStageDeps["captureFrame"]; stallTimeoutMs: number; + log: ProducerLogger; + verifyMinDb: number; abortSignal: AbortSignal | undefined; assertNotAborted: () => void; onProgress?: ProgressCallback; @@ -215,6 +219,10 @@ async function captureSegmentFrames( }, ctx.abortSignal, ); + await verifyDrawElementSample(ctx.session, i, buffer, { + log: ctx.log, + verifyMinDb: ctx.verifyMinDb, + }); ensureFrameWritten(await encoder.writeFrame(buffer), i, encoder); ctx.job.framesRendered = i + 1; lastProgressAt = Date.now(); @@ -451,6 +459,8 @@ async function createSegmentWorker(run: SegmentRun, id: number): Promise {}); class DrawElementVerificationError extends Error {} +// Ground truth the drawElement session arms, and the PSNR the fake comparison reports per frame. +let sessionVerifyFrames: Map | undefined; +let samplePsnr = (_frame: Buffer) => 50; +const psnrCalls: number[] = []; async function replaySlowCapture( signal: AbortSignal | undefined, @@ -165,6 +169,7 @@ function fakeSession(screenshot: boolean) { isScreenshotFallback: screenshot, onBeforeCapture: sessionHasBeforeCapture ? async () => {} : undefined, deRiskFrames: screenshot ? undefined : sessionRiskFrames, + deVerifyFrames: screenshot ? undefined : sessionVerifyFrames, isInitialized: false, browserConsoleBuffer, options: { captureBeyondViewport: false }, @@ -181,6 +186,11 @@ function fakeSession(screenshot: boolean) { mock.module("@hyperframes/engine", () => ({ calculateOptimalWorkers: () => 1, + psnrDb: async (frame: Buffer, truth: Buffer) => { + psnrCalls.push(Number(truth.toString().split("-")[1])); + return samplePsnr(frame); + }, + resolveDeVerifyMinDb: () => 32, convertTransfer: () => {}, captureFrame: async () => {}, captureFrameToBufferPipelined: async (_session: unknown, frameIndex: number) => { @@ -473,6 +483,37 @@ describe("runCaptureStreamingStage", () => { expect(spawnStreamingEncoder.mock.calls[0]?.[3]).toBe(cfg); }); + describe("drawElement self-verify on the sequential loop", () => { + const run = async (psnr: (frame: Buffer) => number) => { + const { runCaptureStreamingStage } = await import("./captureStreamingStage.js"); + captureSessionMode = "drawelement"; + sessionWorkerEncodeEnabled = false; + sessionVerifyFrames = new Map([[3, Buffer.from("truth-3")]]); + samplePsnr = psnr; + psnrCalls.length = 0; + const cfg = { forceScreenshot: false, ffmpegStreamingTimeout: 3_600_000 }; + try { + return await runCaptureStreamingStage({ ...createInput(cfg), totalFrames: 6 }).catch( + (e: unknown) => e, + ); + } finally { + sessionVerifyFrames = undefined; + samplePsnr = () => 50; + } + }; + + it("compares the armed sample against its ground truth", async () => { + const result = await run(() => 50); + expect(result).toMatchObject({ success: true }); + expect(psnrCalls).toEqual([3]); + }); + + it("fails the render on a sample below the floor", async () => { + const error = await run(() => 20); + expect((error as { cause?: unknown }).cause).toBeInstanceOf(DrawElementVerificationError); + }); + }); + it("wraps sequential capture failures with the browser console buffer", async () => { failCaptureFrameToBuffer = true; failInitializeSession = false; diff --git a/packages/producer/src/services/render/stages/captureStreamingStage.ts b/packages/producer/src/services/render/stages/captureStreamingStage.ts index 68cda7e7e0..2d882afd2c 100644 --- a/packages/producer/src/services/render/stages/captureStreamingStage.ts +++ b/packages/producer/src/services/render/stages/captureStreamingStage.ts @@ -462,45 +462,62 @@ function createDrainFrameGuard(args: { sizes.push(buf.length); if (sizes.length > 60) sizes.shift(); } - const truth = session.deVerifyFrames?.get(idx); - if (truth) { - let db: number; - try { - db = await psnrDb(buf, truth); - } catch (err) { - // Infrastructure failure (ffmpeg spawn/parse/tmpdir), not evidence of - // damage — skip this sample rather than failing or falling back. - log.warn("[Render] drawElement self-verify sample skipped (psnr infrastructure)", { - frame: idx, - error: err instanceof Error ? err.message : String(err), - }); - return buf; - } - if (db < verifyMinDb) { - // Keep the mismatched pair for diagnosis (tmpdir; OS-reaped). - const dumpDir = await mkdtemp(join(tmpdir(), "hf-de-verify-fail-")).catch(() => null); - if (dumpDir) { - await Promise.all([ - writeFile(join(dumpDir, `frame-${idx}-de.jpg`), buf), - writeFile(join(dumpDir, `frame-${idx}-truth.jpg`), truth), - ]).catch(() => {}); - } - throw new DrawElementVerificationError( - `drawElement self-verify failed at frame ${idx}: ${db.toFixed(1)}dB < ${verifyMinDb}dB vs pre-injection screenshot${dumpDir ? ` (pair: ${dumpDir})` : ""}`, - { kind: "psnr", frameIndex: idx, failedDb: db, verifyThresholdDb: verifyMinDb }, - ); - } - stats.verifyChecked += 1; - stats.verifyMinDb = stats.verifyMinDb === undefined ? db : Math.min(stats.verifyMinDb, db); - log.info("[Render] drawElement self-verify passed", { - frame: idx, - psnrDb: db === Infinity ? "inf" : Number(db.toFixed(1)), - }); - } + await verifyDrawElementSample(session, idx, buf, { log, stats, verifyMinDb }); return buf; }; } +/** At an armed sample index, a frame below the PSNR floor vs its pre-injection screenshot throws. */ +export async function verifyDrawElementSample( + session: CaptureSession, + idx: number, + buf: Buffer, + ctx: { + log: CaptureStreamingStageInput["log"]; + /** Telemetry counters; the stages that report none pass nothing. */ + stats?: Pick; + verifyMinDb: number; + }, +): Promise { + const { log, stats, verifyMinDb } = ctx; + const truth = session.deVerifyFrames?.get(idx); + if (!truth) return; + let db: number; + try { + db = await psnrDb(buf, truth); + } catch (err) { + // Infrastructure failure (ffmpeg spawn/parse/tmpdir), not evidence of + // damage — skip this sample rather than failing or falling back. + log.warn("[Render] drawElement self-verify sample skipped (psnr infrastructure)", { + frame: idx, + error: err instanceof Error ? err.message : String(err), + }); + return; + } + if (db < verifyMinDb) { + // Keep the mismatched pair for diagnosis (tmpdir; OS-reaped). + const dumpDir = await mkdtemp(join(tmpdir(), "hf-de-verify-fail-")).catch(() => null); + if (dumpDir) { + await Promise.all([ + writeFile(join(dumpDir, `frame-${idx}-de.jpg`), buf), + writeFile(join(dumpDir, `frame-${idx}-truth.jpg`), truth), + ]).catch(() => {}); + } + throw new DrawElementVerificationError( + `drawElement self-verify failed at frame ${idx}: ${db.toFixed(1)}dB < ${verifyMinDb}dB vs pre-injection screenshot${dumpDir ? ` (pair: ${dumpDir})` : ""}`, + { kind: "psnr", frameIndex: idx, failedDb: db, verifyThresholdDb: verifyMinDb }, + ); + } + if (stats) { + stats.verifyChecked += 1; + stats.verifyMinDb = stats.verifyMinDb === undefined ? db : Math.min(stats.verifyMinDb, db); + } + log.info("[Render] drawElement self-verify passed", { + frame: idx, + psnrDb: db === Infinity ? "inf" : Number(db.toFixed(1)), + }); +} + /** Frames drawElement cannot reproduce, and how to capture one by screenshot. */ interface RiskFrameCapture { frames: ReadonlySet; @@ -1126,6 +1143,7 @@ export async function runCaptureStreamingStage( ); } else { const stallTimeoutMs = resolveCaptureStallTimeoutMs(); + const verifyMinDb = resolveDeVerifyMinDb(); let lastProgressAt = Date.now(); for (let i = resume.from; i < totalFrames; i++) { assertNotAborted(); @@ -1142,6 +1160,7 @@ export async function runCaptureStreamingStage( { session, frameIndex: i, totalFrames, stallTimeoutMs: budget }, abortSignal, ); + await verifyDrawElementSample(session, i, buffer, { log, verifyMinDb }); await reorderBuffer.waitForFrame(i); ensureFrameWritten(await currentEncoder.writeFrame(buffer), i, currentEncoder); reorderBuffer.advanceTo(i + 1);