mirror of
https://github.com/heygen-com/hyperframes.git
synced 2026-11-18 07:50:13 +00:00
fix(producer): segmented and sequential capture check drawElement samples (#7255)
* fix(producer): segmented capture checks drawElement frames against their screenshots * fix(producer): streaming sequential loop checks drawElement samples too
This commit is contained in:
@@ -32,6 +32,9 @@ const createdSessionConfigs: Array<{ enableBrowserPool?: boolean }> = [];
|
||||
const createdSessionHooks: unknown[] = [];
|
||||
const reusedSessionHooks: unknown[] = [];
|
||||
class DrawElementVerificationError extends Error {}
|
||||
/** PSNR the fake comparison reports for a frame; every call is recorded. */
|
||||
let samplePsnr = (_frame: Buffer) => 50;
|
||||
const psnrCalls: Array<{ frame: Buffer; truth: Buffer }> = [];
|
||||
|
||||
mock.module("@hyperframes/engine", () => ({
|
||||
calculateOptimalWorkers: () => 1,
|
||||
@@ -137,7 +140,12 @@ mock.module("@hyperframes/engine", () => ({
|
||||
throw new Error("prepare reuse failed: ENOSPC");
|
||||
}
|
||||
},
|
||||
psnrDb: async (frame: Buffer, truth: Buffer) => {
|
||||
psnrCalls.push({ frame, truth });
|
||||
return samplePsnr(frame);
|
||||
},
|
||||
recaptureDrawElementFrameForVerify: async () => Buffer.from("frame"),
|
||||
resolveDeVerifyMinDb: () => 32,
|
||||
resolveHeadlessShellPath: () => "/headless-shell",
|
||||
shouldDisableBrowserPoolForParallelWorker: ({ parallel }: { parallel?: boolean }) =>
|
||||
Boolean(parallel),
|
||||
@@ -931,6 +939,68 @@ describe("runCaptureSegmentedStage", () => {
|
||||
await new Promise((r) => setTimeout(r, 40));
|
||||
expect(capturedAfterClose).toBe(0);
|
||||
});
|
||||
|
||||
describe("drawElement self-verify", () => {
|
||||
const truths = new Map([
|
||||
[2, Buffer.from("truth-2")],
|
||||
[7, Buffer.from("truth-7")],
|
||||
]);
|
||||
const frameBuffer = (i: number) => Buffer.from(`frame-${i}`);
|
||||
|
||||
function verifiedRun(dir: string, written: number[]) {
|
||||
return runCaptureSegmentedStage({
|
||||
...fakeStageInput({ totalFrames: 9 }),
|
||||
probeSession: null,
|
||||
segmentFrames: 3,
|
||||
segmentDir: join(fixtureRoot, dir),
|
||||
workerCount: 2,
|
||||
sessionFactoryForWorker: (id) => ({
|
||||
create: async () => ({
|
||||
...fakeSession(id),
|
||||
captureMode: "drawelement" as const,
|
||||
deVerifyFrames: truths,
|
||||
}),
|
||||
}),
|
||||
deps: {
|
||||
spawnEncoder: mock(async () => ({
|
||||
...okEncoder(),
|
||||
writeFrame: async (buf: Buffer) => {
|
||||
written.push(Number(buf.toString().split("-")[1]));
|
||||
return true;
|
||||
},
|
||||
})),
|
||||
captureFrame: mock(async (_s: unknown, i: number) => ({ buffer: frameBuffer(i) })),
|
||||
concat: mock(async () => ({ success: true as const })),
|
||||
closeSession: mock(async () => {}),
|
||||
removeFile: () => {},
|
||||
},
|
||||
});
|
||||
}
|
||||
|
||||
it("compares every armed sample against its ground truth", async () => {
|
||||
psnrCalls.length = 0;
|
||||
samplePsnr = () => 50;
|
||||
const result = await verifiedRun("verify-pass", []);
|
||||
expect(result.success).toBe(true);
|
||||
const checked = psnrCalls
|
||||
.map((c) => ({ frame: c.frame.toString(), truth: c.truth.toString() }))
|
||||
.sort((a, b) => a.frame.localeCompare(b.frame));
|
||||
expect(checked).toEqual([
|
||||
{ frame: "frame-2", truth: "truth-2" },
|
||||
{ frame: "frame-7", truth: "truth-7" },
|
||||
]);
|
||||
});
|
||||
|
||||
it("fails the render before a sample below the floor is encoded", async () => {
|
||||
psnrCalls.length = 0;
|
||||
samplePsnr = (frame) => (frame.toString() === "frame-7" ? 20 : 50);
|
||||
const written: number[] = [];
|
||||
const error = await verifiedRun("verify-breach", written).catch((e: unknown) => e);
|
||||
expect((error as { cause?: unknown }).cause).toBeInstanceOf(DrawElementVerificationError);
|
||||
expect(written).not.toContain(7);
|
||||
samplePsnr = () => 50;
|
||||
});
|
||||
});
|
||||
});
|
||||
|
||||
describe("runCaptureSegmentedStage with segmentVideoFrames", () => {
|
||||
|
||||
@@ -23,6 +23,7 @@ import {
|
||||
getCapturePerfSummary,
|
||||
initializeSession,
|
||||
prepareCaptureSessionForReuse,
|
||||
resolveDeVerifyMinDb,
|
||||
shouldDisableBrowserPoolForParallelWorker,
|
||||
spawnStreamingEncoder,
|
||||
type BeforeCaptureHook,
|
||||
@@ -48,6 +49,7 @@ import { isTargetLossError } from "../segmentRecycle.js";
|
||||
import {
|
||||
raceAgainstStall,
|
||||
resolveCaptureStallTimeoutMs,
|
||||
verifyDrawElementSample,
|
||||
type StreamingEncoderOptions,
|
||||
} from "./captureStreamingStage.js";
|
||||
|
||||
@@ -188,6 +190,8 @@ interface SegmentCaptureContext {
|
||||
spawnEncoder: SegmentedStageDeps["spawnEncoder"];
|
||||
captureFrame: SegmentedStageDeps["captureFrame"];
|
||||
stallTimeoutMs: number;
|
||||
log: ProducerLogger;
|
||||
verifyMinDb: number;
|
||||
abortSignal: AbortSignal | undefined;
|
||||
assertNotAborted: () => void;
|
||||
onProgress?: ProgressCallback;
|
||||
@@ -215,6 +219,10 @@ async function captureSegmentFrames(
|
||||
},
|
||||
ctx.abortSignal,
|
||||
);
|
||||
await verifyDrawElementSample(ctx.session, i, buffer, {
|
||||
log: ctx.log,
|
||||
verifyMinDb: ctx.verifyMinDb,
|
||||
});
|
||||
ensureFrameWritten(await encoder.writeFrame(buffer), i, encoder);
|
||||
ctx.job.framesRendered = i + 1;
|
||||
lastProgressAt = Date.now();
|
||||
@@ -451,6 +459,8 @@ async function createSegmentWorker(run: SegmentRun, id: number): Promise<Segment
|
||||
spawnEncoder: deps.spawnEncoder,
|
||||
captureFrame: deps.captureFrame,
|
||||
stallTimeoutMs: resolveCaptureStallTimeoutMs(),
|
||||
log: input.log,
|
||||
verifyMinDb: resolveDeVerifyMinDb(),
|
||||
abortSignal: input.abortSignal,
|
||||
assertNotAborted: input.assertNotAborted,
|
||||
onProgress: input.onProgress,
|
||||
|
||||
@@ -87,6 +87,10 @@ let initializeSessionErrorMessage = "initialize failed";
|
||||
const browserConsoleBuffer = ["[FrameCapture:ERROR] page.goto failed"];
|
||||
const closeCaptureSession = mock(async () => {});
|
||||
class DrawElementVerificationError extends Error {}
|
||||
// Ground truth the drawElement session arms, and the PSNR the fake comparison reports per frame.
|
||||
let sessionVerifyFrames: Map<number, Buffer> | undefined;
|
||||
let samplePsnr = (_frame: Buffer) => 50;
|
||||
const psnrCalls: number[] = [];
|
||||
|
||||
async function replaySlowCapture(
|
||||
signal: AbortSignal | undefined,
|
||||
@@ -165,6 +169,7 @@ function fakeSession(screenshot: boolean) {
|
||||
isScreenshotFallback: screenshot,
|
||||
onBeforeCapture: sessionHasBeforeCapture ? async () => {} : undefined,
|
||||
deRiskFrames: screenshot ? undefined : sessionRiskFrames,
|
||||
deVerifyFrames: screenshot ? undefined : sessionVerifyFrames,
|
||||
isInitialized: false,
|
||||
browserConsoleBuffer,
|
||||
options: { captureBeyondViewport: false },
|
||||
@@ -181,6 +186,11 @@ function fakeSession(screenshot: boolean) {
|
||||
|
||||
mock.module("@hyperframes/engine", () => ({
|
||||
calculateOptimalWorkers: () => 1,
|
||||
psnrDb: async (frame: Buffer, truth: Buffer) => {
|
||||
psnrCalls.push(Number(truth.toString().split("-")[1]));
|
||||
return samplePsnr(frame);
|
||||
},
|
||||
resolveDeVerifyMinDb: () => 32,
|
||||
convertTransfer: () => {},
|
||||
captureFrame: async () => {},
|
||||
captureFrameToBufferPipelined: async (_session: unknown, frameIndex: number) => {
|
||||
@@ -473,6 +483,37 @@ describe("runCaptureStreamingStage", () => {
|
||||
expect(spawnStreamingEncoder.mock.calls[0]?.[3]).toBe(cfg);
|
||||
});
|
||||
|
||||
describe("drawElement self-verify on the sequential loop", () => {
|
||||
const run = async (psnr: (frame: Buffer) => number) => {
|
||||
const { runCaptureStreamingStage } = await import("./captureStreamingStage.js");
|
||||
captureSessionMode = "drawelement";
|
||||
sessionWorkerEncodeEnabled = false;
|
||||
sessionVerifyFrames = new Map([[3, Buffer.from("truth-3")]]);
|
||||
samplePsnr = psnr;
|
||||
psnrCalls.length = 0;
|
||||
const cfg = { forceScreenshot: false, ffmpegStreamingTimeout: 3_600_000 };
|
||||
try {
|
||||
return await runCaptureStreamingStage({ ...createInput(cfg), totalFrames: 6 }).catch(
|
||||
(e: unknown) => e,
|
||||
);
|
||||
} finally {
|
||||
sessionVerifyFrames = undefined;
|
||||
samplePsnr = () => 50;
|
||||
}
|
||||
};
|
||||
|
||||
it("compares the armed sample against its ground truth", async () => {
|
||||
const result = await run(() => 50);
|
||||
expect(result).toMatchObject({ success: true });
|
||||
expect(psnrCalls).toEqual([3]);
|
||||
});
|
||||
|
||||
it("fails the render on a sample below the floor", async () => {
|
||||
const error = await run(() => 20);
|
||||
expect((error as { cause?: unknown }).cause).toBeInstanceOf(DrawElementVerificationError);
|
||||
});
|
||||
});
|
||||
|
||||
it("wraps sequential capture failures with the browser console buffer", async () => {
|
||||
failCaptureFrameToBuffer = true;
|
||||
failInitializeSession = false;
|
||||
|
||||
@@ -462,45 +462,62 @@ function createDrainFrameGuard(args: {
|
||||
sizes.push(buf.length);
|
||||
if (sizes.length > 60) sizes.shift();
|
||||
}
|
||||
const truth = session.deVerifyFrames?.get(idx);
|
||||
if (truth) {
|
||||
let db: number;
|
||||
try {
|
||||
db = await psnrDb(buf, truth);
|
||||
} catch (err) {
|
||||
// Infrastructure failure (ffmpeg spawn/parse/tmpdir), not evidence of
|
||||
// damage — skip this sample rather than failing or falling back.
|
||||
log.warn("[Render] drawElement self-verify sample skipped (psnr infrastructure)", {
|
||||
frame: idx,
|
||||
error: err instanceof Error ? err.message : String(err),
|
||||
});
|
||||
return buf;
|
||||
}
|
||||
if (db < verifyMinDb) {
|
||||
// Keep the mismatched pair for diagnosis (tmpdir; OS-reaped).
|
||||
const dumpDir = await mkdtemp(join(tmpdir(), "hf-de-verify-fail-")).catch(() => null);
|
||||
if (dumpDir) {
|
||||
await Promise.all([
|
||||
writeFile(join(dumpDir, `frame-${idx}-de.jpg`), buf),
|
||||
writeFile(join(dumpDir, `frame-${idx}-truth.jpg`), truth),
|
||||
]).catch(() => {});
|
||||
}
|
||||
throw new DrawElementVerificationError(
|
||||
`drawElement self-verify failed at frame ${idx}: ${db.toFixed(1)}dB < ${verifyMinDb}dB vs pre-injection screenshot${dumpDir ? ` (pair: ${dumpDir})` : ""}`,
|
||||
{ kind: "psnr", frameIndex: idx, failedDb: db, verifyThresholdDb: verifyMinDb },
|
||||
);
|
||||
}
|
||||
stats.verifyChecked += 1;
|
||||
stats.verifyMinDb = stats.verifyMinDb === undefined ? db : Math.min(stats.verifyMinDb, db);
|
||||
log.info("[Render] drawElement self-verify passed", {
|
||||
frame: idx,
|
||||
psnrDb: db === Infinity ? "inf" : Number(db.toFixed(1)),
|
||||
});
|
||||
}
|
||||
await verifyDrawElementSample(session, idx, buf, { log, stats, verifyMinDb });
|
||||
return buf;
|
||||
};
|
||||
}
|
||||
|
||||
/** At an armed sample index, a frame below the PSNR floor vs its pre-injection screenshot throws. */
|
||||
export async function verifyDrawElementSample(
|
||||
session: CaptureSession,
|
||||
idx: number,
|
||||
buf: Buffer,
|
||||
ctx: {
|
||||
log: CaptureStreamingStageInput["log"];
|
||||
/** Telemetry counters; the stages that report none pass nothing. */
|
||||
stats?: Pick<DeDrainStats, "verifyChecked" | "verifyMinDb">;
|
||||
verifyMinDb: number;
|
||||
},
|
||||
): Promise<void> {
|
||||
const { log, stats, verifyMinDb } = ctx;
|
||||
const truth = session.deVerifyFrames?.get(idx);
|
||||
if (!truth) return;
|
||||
let db: number;
|
||||
try {
|
||||
db = await psnrDb(buf, truth);
|
||||
} catch (err) {
|
||||
// Infrastructure failure (ffmpeg spawn/parse/tmpdir), not evidence of
|
||||
// damage — skip this sample rather than failing or falling back.
|
||||
log.warn("[Render] drawElement self-verify sample skipped (psnr infrastructure)", {
|
||||
frame: idx,
|
||||
error: err instanceof Error ? err.message : String(err),
|
||||
});
|
||||
return;
|
||||
}
|
||||
if (db < verifyMinDb) {
|
||||
// Keep the mismatched pair for diagnosis (tmpdir; OS-reaped).
|
||||
const dumpDir = await mkdtemp(join(tmpdir(), "hf-de-verify-fail-")).catch(() => null);
|
||||
if (dumpDir) {
|
||||
await Promise.all([
|
||||
writeFile(join(dumpDir, `frame-${idx}-de.jpg`), buf),
|
||||
writeFile(join(dumpDir, `frame-${idx}-truth.jpg`), truth),
|
||||
]).catch(() => {});
|
||||
}
|
||||
throw new DrawElementVerificationError(
|
||||
`drawElement self-verify failed at frame ${idx}: ${db.toFixed(1)}dB < ${verifyMinDb}dB vs pre-injection screenshot${dumpDir ? ` (pair: ${dumpDir})` : ""}`,
|
||||
{ kind: "psnr", frameIndex: idx, failedDb: db, verifyThresholdDb: verifyMinDb },
|
||||
);
|
||||
}
|
||||
if (stats) {
|
||||
stats.verifyChecked += 1;
|
||||
stats.verifyMinDb = stats.verifyMinDb === undefined ? db : Math.min(stats.verifyMinDb, db);
|
||||
}
|
||||
log.info("[Render] drawElement self-verify passed", {
|
||||
frame: idx,
|
||||
psnrDb: db === Infinity ? "inf" : Number(db.toFixed(1)),
|
||||
});
|
||||
}
|
||||
|
||||
/** Frames drawElement cannot reproduce, and how to capture one by screenshot. */
|
||||
interface RiskFrameCapture {
|
||||
frames: ReadonlySet<number>;
|
||||
@@ -1126,6 +1143,7 @@ export async function runCaptureStreamingStage(
|
||||
);
|
||||
} else {
|
||||
const stallTimeoutMs = resolveCaptureStallTimeoutMs();
|
||||
const verifyMinDb = resolveDeVerifyMinDb();
|
||||
let lastProgressAt = Date.now();
|
||||
for (let i = resume.from; i < totalFrames; i++) {
|
||||
assertNotAborted();
|
||||
@@ -1142,6 +1160,7 @@ export async function runCaptureStreamingStage(
|
||||
{ session, frameIndex: i, totalFrames, stallTimeoutMs: budget },
|
||||
abortSignal,
|
||||
);
|
||||
await verifyDrawElementSample(session, i, buffer, { log, verifyMinDb });
|
||||
await reorderBuffer.waitForFrame(i);
|
||||
ensureFrameWritten(await currentEncoder.writeFrame(buffer), i, currentEncoder);
|
||||
reorderBuffer.advanceTo(i + 1);
|
||||
|
||||
Reference in New Issue
Block a user