fix(producer): segmented and sequential capture check drawElement samples (#7255)

* fix(producer): segmented capture checks drawElement frames against their screenshots

* fix(producer): streaming sequential loop checks drawElement samples too
This commit is contained in:
Miguel Ángel
2026-11-05 18:43:59 +00:00
committed by GitHub
parent 599eaad2ba
commit 8df4f21c20
4 changed files with 175 additions and 35 deletions
@@ -32,6 +32,9 @@ const createdSessionConfigs: Array<{ enableBrowserPool?: boolean }> = [];
const createdSessionHooks: unknown[] = [];
const reusedSessionHooks: unknown[] = [];
class DrawElementVerificationError extends Error {}
/** PSNR the fake comparison reports for a frame; every call is recorded. */
let samplePsnr = (_frame: Buffer) => 50;
const psnrCalls: Array<{ frame: Buffer; truth: Buffer }> = [];
mock.module("@hyperframes/engine", () => ({
calculateOptimalWorkers: () => 1,
@@ -137,7 +140,12 @@ mock.module("@hyperframes/engine", () => ({
throw new Error("prepare reuse failed: ENOSPC");
}
},
psnrDb: async (frame: Buffer, truth: Buffer) => {
psnrCalls.push({ frame, truth });
return samplePsnr(frame);
},
recaptureDrawElementFrameForVerify: async () => Buffer.from("frame"),
resolveDeVerifyMinDb: () => 32,
resolveHeadlessShellPath: () => "/headless-shell",
shouldDisableBrowserPoolForParallelWorker: ({ parallel }: { parallel?: boolean }) =>
Boolean(parallel),
@@ -931,6 +939,68 @@ describe("runCaptureSegmentedStage", () => {
await new Promise((r) => setTimeout(r, 40));
expect(capturedAfterClose).toBe(0);
});
describe("drawElement self-verify", () => {
const truths = new Map([
[2, Buffer.from("truth-2")],
[7, Buffer.from("truth-7")],
]);
const frameBuffer = (i: number) => Buffer.from(`frame-${i}`);
function verifiedRun(dir: string, written: number[]) {
return runCaptureSegmentedStage({
...fakeStageInput({ totalFrames: 9 }),
probeSession: null,
segmentFrames: 3,
segmentDir: join(fixtureRoot, dir),
workerCount: 2,
sessionFactoryForWorker: (id) => ({
create: async () => ({
...fakeSession(id),
captureMode: "drawelement" as const,
deVerifyFrames: truths,
}),
}),
deps: {
spawnEncoder: mock(async () => ({
...okEncoder(),
writeFrame: async (buf: Buffer) => {
written.push(Number(buf.toString().split("-")[1]));
return true;
},
})),
captureFrame: mock(async (_s: unknown, i: number) => ({ buffer: frameBuffer(i) })),
concat: mock(async () => ({ success: true as const })),
closeSession: mock(async () => {}),
removeFile: () => {},
},
});
}
it("compares every armed sample against its ground truth", async () => {
psnrCalls.length = 0;
samplePsnr = () => 50;
const result = await verifiedRun("verify-pass", []);
expect(result.success).toBe(true);
const checked = psnrCalls
.map((c) => ({ frame: c.frame.toString(), truth: c.truth.toString() }))
.sort((a, b) => a.frame.localeCompare(b.frame));
expect(checked).toEqual([
{ frame: "frame-2", truth: "truth-2" },
{ frame: "frame-7", truth: "truth-7" },
]);
});
it("fails the render before a sample below the floor is encoded", async () => {
psnrCalls.length = 0;
samplePsnr = (frame) => (frame.toString() === "frame-7" ? 20 : 50);
const written: number[] = [];
const error = await verifiedRun("verify-breach", written).catch((e: unknown) => e);
expect((error as { cause?: unknown }).cause).toBeInstanceOf(DrawElementVerificationError);
expect(written).not.toContain(7);
samplePsnr = () => 50;
});
});
});
describe("runCaptureSegmentedStage with segmentVideoFrames", () => {
@@ -23,6 +23,7 @@ import {
getCapturePerfSummary,
initializeSession,
prepareCaptureSessionForReuse,
resolveDeVerifyMinDb,
shouldDisableBrowserPoolForParallelWorker,
spawnStreamingEncoder,
type BeforeCaptureHook,
@@ -48,6 +49,7 @@ import { isTargetLossError } from "../segmentRecycle.js";
import {
raceAgainstStall,
resolveCaptureStallTimeoutMs,
verifyDrawElementSample,
type StreamingEncoderOptions,
} from "./captureStreamingStage.js";
@@ -188,6 +190,8 @@ interface SegmentCaptureContext {
spawnEncoder: SegmentedStageDeps["spawnEncoder"];
captureFrame: SegmentedStageDeps["captureFrame"];
stallTimeoutMs: number;
log: ProducerLogger;
verifyMinDb: number;
abortSignal: AbortSignal | undefined;
assertNotAborted: () => void;
onProgress?: ProgressCallback;
@@ -215,6 +219,10 @@ async function captureSegmentFrames(
},
ctx.abortSignal,
);
await verifyDrawElementSample(ctx.session, i, buffer, {
log: ctx.log,
verifyMinDb: ctx.verifyMinDb,
});
ensureFrameWritten(await encoder.writeFrame(buffer), i, encoder);
ctx.job.framesRendered = i + 1;
lastProgressAt = Date.now();
@@ -451,6 +459,8 @@ async function createSegmentWorker(run: SegmentRun, id: number): Promise<Segment
spawnEncoder: deps.spawnEncoder,
captureFrame: deps.captureFrame,
stallTimeoutMs: resolveCaptureStallTimeoutMs(),
log: input.log,
verifyMinDb: resolveDeVerifyMinDb(),
abortSignal: input.abortSignal,
assertNotAborted: input.assertNotAborted,
onProgress: input.onProgress,
@@ -87,6 +87,10 @@ let initializeSessionErrorMessage = "initialize failed";
const browserConsoleBuffer = ["[FrameCapture:ERROR] page.goto failed"];
const closeCaptureSession = mock(async () => {});
class DrawElementVerificationError extends Error {}
// Ground truth the drawElement session arms, and the PSNR the fake comparison reports per frame.
let sessionVerifyFrames: Map<number, Buffer> | undefined;
let samplePsnr = (_frame: Buffer) => 50;
const psnrCalls: number[] = [];
async function replaySlowCapture(
signal: AbortSignal | undefined,
@@ -165,6 +169,7 @@ function fakeSession(screenshot: boolean) {
isScreenshotFallback: screenshot,
onBeforeCapture: sessionHasBeforeCapture ? async () => {} : undefined,
deRiskFrames: screenshot ? undefined : sessionRiskFrames,
deVerifyFrames: screenshot ? undefined : sessionVerifyFrames,
isInitialized: false,
browserConsoleBuffer,
options: { captureBeyondViewport: false },
@@ -181,6 +186,11 @@ function fakeSession(screenshot: boolean) {
mock.module("@hyperframes/engine", () => ({
calculateOptimalWorkers: () => 1,
psnrDb: async (frame: Buffer, truth: Buffer) => {
psnrCalls.push(Number(truth.toString().split("-")[1]));
return samplePsnr(frame);
},
resolveDeVerifyMinDb: () => 32,
convertTransfer: () => {},
captureFrame: async () => {},
captureFrameToBufferPipelined: async (_session: unknown, frameIndex: number) => {
@@ -473,6 +483,37 @@ describe("runCaptureStreamingStage", () => {
expect(spawnStreamingEncoder.mock.calls[0]?.[3]).toBe(cfg);
});
describe("drawElement self-verify on the sequential loop", () => {
const run = async (psnr: (frame: Buffer) => number) => {
const { runCaptureStreamingStage } = await import("./captureStreamingStage.js");
captureSessionMode = "drawelement";
sessionWorkerEncodeEnabled = false;
sessionVerifyFrames = new Map([[3, Buffer.from("truth-3")]]);
samplePsnr = psnr;
psnrCalls.length = 0;
const cfg = { forceScreenshot: false, ffmpegStreamingTimeout: 3_600_000 };
try {
return await runCaptureStreamingStage({ ...createInput(cfg), totalFrames: 6 }).catch(
(e: unknown) => e,
);
} finally {
sessionVerifyFrames = undefined;
samplePsnr = () => 50;
}
};
it("compares the armed sample against its ground truth", async () => {
const result = await run(() => 50);
expect(result).toMatchObject({ success: true });
expect(psnrCalls).toEqual([3]);
});
it("fails the render on a sample below the floor", async () => {
const error = await run(() => 20);
expect((error as { cause?: unknown }).cause).toBeInstanceOf(DrawElementVerificationError);
});
});
it("wraps sequential capture failures with the browser console buffer", async () => {
failCaptureFrameToBuffer = true;
failInitializeSession = false;
@@ -462,45 +462,62 @@ function createDrainFrameGuard(args: {
sizes.push(buf.length);
if (sizes.length > 60) sizes.shift();
}
const truth = session.deVerifyFrames?.get(idx);
if (truth) {
let db: number;
try {
db = await psnrDb(buf, truth);
} catch (err) {
// Infrastructure failure (ffmpeg spawn/parse/tmpdir), not evidence of
// damage — skip this sample rather than failing or falling back.
log.warn("[Render] drawElement self-verify sample skipped (psnr infrastructure)", {
frame: idx,
error: err instanceof Error ? err.message : String(err),
});
return buf;
}
if (db < verifyMinDb) {
// Keep the mismatched pair for diagnosis (tmpdir; OS-reaped).
const dumpDir = await mkdtemp(join(tmpdir(), "hf-de-verify-fail-")).catch(() => null);
if (dumpDir) {
await Promise.all([
writeFile(join(dumpDir, `frame-${idx}-de.jpg`), buf),
writeFile(join(dumpDir, `frame-${idx}-truth.jpg`), truth),
]).catch(() => {});
}
throw new DrawElementVerificationError(
`drawElement self-verify failed at frame ${idx}: ${db.toFixed(1)}dB < ${verifyMinDb}dB vs pre-injection screenshot${dumpDir ? ` (pair: ${dumpDir})` : ""}`,
{ kind: "psnr", frameIndex: idx, failedDb: db, verifyThresholdDb: verifyMinDb },
);
}
stats.verifyChecked += 1;
stats.verifyMinDb = stats.verifyMinDb === undefined ? db : Math.min(stats.verifyMinDb, db);
log.info("[Render] drawElement self-verify passed", {
frame: idx,
psnrDb: db === Infinity ? "inf" : Number(db.toFixed(1)),
});
}
await verifyDrawElementSample(session, idx, buf, { log, stats, verifyMinDb });
return buf;
};
}
/** At an armed sample index, a frame below the PSNR floor vs its pre-injection screenshot throws. */
export async function verifyDrawElementSample(
session: CaptureSession,
idx: number,
buf: Buffer,
ctx: {
log: CaptureStreamingStageInput["log"];
/** Telemetry counters; the stages that report none pass nothing. */
stats?: Pick<DeDrainStats, "verifyChecked" | "verifyMinDb">;
verifyMinDb: number;
},
): Promise<void> {
const { log, stats, verifyMinDb } = ctx;
const truth = session.deVerifyFrames?.get(idx);
if (!truth) return;
let db: number;
try {
db = await psnrDb(buf, truth);
} catch (err) {
// Infrastructure failure (ffmpeg spawn/parse/tmpdir), not evidence of
// damage — skip this sample rather than failing or falling back.
log.warn("[Render] drawElement self-verify sample skipped (psnr infrastructure)", {
frame: idx,
error: err instanceof Error ? err.message : String(err),
});
return;
}
if (db < verifyMinDb) {
// Keep the mismatched pair for diagnosis (tmpdir; OS-reaped).
const dumpDir = await mkdtemp(join(tmpdir(), "hf-de-verify-fail-")).catch(() => null);
if (dumpDir) {
await Promise.all([
writeFile(join(dumpDir, `frame-${idx}-de.jpg`), buf),
writeFile(join(dumpDir, `frame-${idx}-truth.jpg`), truth),
]).catch(() => {});
}
throw new DrawElementVerificationError(
`drawElement self-verify failed at frame ${idx}: ${db.toFixed(1)}dB < ${verifyMinDb}dB vs pre-injection screenshot${dumpDir ? ` (pair: ${dumpDir})` : ""}`,
{ kind: "psnr", frameIndex: idx, failedDb: db, verifyThresholdDb: verifyMinDb },
);
}
if (stats) {
stats.verifyChecked += 1;
stats.verifyMinDb = stats.verifyMinDb === undefined ? db : Math.min(stats.verifyMinDb, db);
}
log.info("[Render] drawElement self-verify passed", {
frame: idx,
psnrDb: db === Infinity ? "inf" : Number(db.toFixed(1)),
});
}
/** Frames drawElement cannot reproduce, and how to capture one by screenshot. */
interface RiskFrameCapture {
frames: ReadonlySet<number>;
@@ -1126,6 +1143,7 @@ export async function runCaptureStreamingStage(
);
} else {
const stallTimeoutMs = resolveCaptureStallTimeoutMs();
const verifyMinDb = resolveDeVerifyMinDb();
let lastProgressAt = Date.now();
for (let i = resume.from; i < totalFrames; i++) {
assertNotAborted();
@@ -1142,6 +1160,7 @@ export async function runCaptureStreamingStage(
{ session, frameIndex: i, totalFrames, stallTimeoutMs: budget },
abortSignal,
);
await verifyDrawElementSample(session, i, buffer, { log, verifyMinDb });
await reorderBuffer.waitForFrame(i);
ensureFrameWritten(await currentEncoder.writeFrame(buffer), i, currentEncoder);
reorderBuffer.advanceTo(i + 1);