mirror of
https://github.com/heygen-com/hyperframes.git
synced 2026-11-18 07:50:13 +00:00
fix(engine): a missing frame file skips its self-verify sample instead of aborting the render (#7334)
This commit is contained in:
@@ -0,0 +1,101 @@
|
||||
import { spawnSync } from "node:child_process";
|
||||
import { createServer, type Server } from "node:http";
|
||||
import { afterAll, afterEach, beforeAll, describe, expect, it } from "vitest";
|
||||
import type { Browser, Page } from "puppeteer-core";
|
||||
import { closeTestChrome, launchTestChrome, listenOnLoopback } from "./__fixtures__/loopback.js";
|
||||
import { resolveHeadlessShellPath } from "./browserManager.js";
|
||||
import { captureDeVerificationFrames, type CaptureSession } from "./frameCapture.js";
|
||||
|
||||
const chromePath = resolveHeadlessShellPath();
|
||||
const hasBrowser = chromePath !== undefined && spawnSync(chromePath, ["--version"]).status === 0;
|
||||
|
||||
const HF_PAGE = `<!doctype html><html><body style="margin:0"><script>
|
||||
window.__hf = {
|
||||
duration: 2,
|
||||
seek(t) { document.body.style.background = "rgb(" + Math.round(t * 100) + ",0,0)"; },
|
||||
};
|
||||
</script>`;
|
||||
|
||||
describe.skipIf(!hasBrowser)("drawElement self-verify arming", () => {
|
||||
let browser: Browser;
|
||||
let server: Server;
|
||||
let serverUrl: string;
|
||||
let served = HF_PAGE;
|
||||
const pages: Page[] = [];
|
||||
const priorSamples = process.env.HF_DE_VERIFY;
|
||||
|
||||
beforeAll(async () => {
|
||||
browser = await launchTestChrome(chromePath);
|
||||
server = createServer((_req, res) => {
|
||||
res.setHeader("content-type", "text/html");
|
||||
res.end(served);
|
||||
});
|
||||
serverUrl = await listenOnLoopback(server);
|
||||
});
|
||||
|
||||
afterEach(async () => {
|
||||
served = HF_PAGE;
|
||||
if (priorSamples === undefined) delete process.env.HF_DE_VERIFY;
|
||||
else process.env.HF_DE_VERIFY = priorSamples;
|
||||
await Promise.all(pages.splice(0).map((page) => page.close()));
|
||||
});
|
||||
|
||||
afterAll(async () => {
|
||||
await closeTestChrome(browser);
|
||||
await new Promise((resolve) => server.close(resolve));
|
||||
});
|
||||
|
||||
const arm = async (html = HF_PAGE) => {
|
||||
served = html;
|
||||
const page = await browser.newPage();
|
||||
pages.push(page);
|
||||
await page.goto(`${serverUrl}/index.html`);
|
||||
const session = {
|
||||
browser,
|
||||
page,
|
||||
serverUrl,
|
||||
onBeforeCapture: null,
|
||||
scriptLoadFailures: [],
|
||||
subTimelineWaitMemo: {},
|
||||
options: {
|
||||
width: 64,
|
||||
height: 64,
|
||||
fps: { num: 30, den: 1 },
|
||||
format: "jpeg",
|
||||
compositionDurationSeconds: 2,
|
||||
},
|
||||
} as unknown as CaptureSession;
|
||||
await captureDeVerificationFrames(session, page, () => {});
|
||||
return session.deVerifyFrames;
|
||||
};
|
||||
|
||||
it("takes the ground truth at the fixed fractions, each showing its own frame", async () => {
|
||||
const frames = await arm();
|
||||
// 60 frames at fractions 1/4, 1/2, 3/4 and 0.95.
|
||||
expect([...(frames?.keys() ?? [])]).toEqual([15, 30, 45, 57]);
|
||||
expect(new Set([...(frames?.values() ?? [])].map((b) => b.toString("base64"))).size).toBe(4);
|
||||
for (const truth of frames?.values() ?? []) {
|
||||
expect(truth.subarray(0, 2).toString("hex")).toBe("ffd8");
|
||||
}
|
||||
});
|
||||
|
||||
it("honours the requested sample count", async () => {
|
||||
process.env.HF_DE_VERIFY = "1";
|
||||
expect([...((await arm())?.keys() ?? [])]).toEqual([57]);
|
||||
});
|
||||
|
||||
it("arms nothing when self-verify is turned off", async () => {
|
||||
process.env.HF_DE_VERIFY = "0";
|
||||
expect(await arm()).toBeUndefined();
|
||||
});
|
||||
|
||||
it("moves a sample off a clip cut", async () => {
|
||||
const frames = await arm(`${HF_PAGE}<div data-start="0" data-duration="1"></div>`);
|
||||
expect(frames?.has(30)).toBe(false);
|
||||
expect(frames?.size).toBe(4);
|
||||
});
|
||||
|
||||
it("arms nothing on a video composition with no frame injector, whose truth would be black", async () => {
|
||||
expect(await arm(`${HF_PAGE}<video></video>`)).toBeUndefined();
|
||||
});
|
||||
});
|
||||
@@ -5712,7 +5712,7 @@ export async function seekDeVerifyTruth(
|
||||
);
|
||||
}
|
||||
|
||||
async function captureDeVerificationFrames(
|
||||
export async function captureDeVerificationFrames(
|
||||
session: CaptureSession,
|
||||
page: Page,
|
||||
logInitPhase: (phase: string) => void,
|
||||
|
||||
@@ -814,8 +814,22 @@ async function psnrForDiskSample(
|
||||
workerId: number,
|
||||
idx: number,
|
||||
): Promise<number | null> {
|
||||
const skipSample = (err: unknown): null => {
|
||||
console.warn(
|
||||
`[Parallel] drawElement disk self-verify sample skipped (worker ${workerId}, ` +
|
||||
`frame ${idx}): ${err instanceof Error ? err.message : String(err)}`,
|
||||
);
|
||||
return null;
|
||||
};
|
||||
// Read apart from the compare: a frame file's ENOENT says nothing about ffmpeg.
|
||||
let frame: Buffer;
|
||||
try {
|
||||
return await psnrDb(await readFile(framePath), truth);
|
||||
frame = await readFile(framePath);
|
||||
} catch (err) {
|
||||
return skipSample(err);
|
||||
}
|
||||
try {
|
||||
return await psnrDb(frame, truth);
|
||||
} catch (err) {
|
||||
if (isFfmpegInfrastructureFailure(err)) {
|
||||
const detail = err instanceof Error ? err.message : String(err);
|
||||
@@ -828,11 +842,7 @@ async function psnrForDiskSample(
|
||||
{ cause: err },
|
||||
);
|
||||
}
|
||||
console.warn(
|
||||
`[Parallel] drawElement disk self-verify sample skipped (worker ${workerId}, ` +
|
||||
`frame ${idx}): ${err instanceof Error ? err.message : String(err)}`,
|
||||
);
|
||||
return null;
|
||||
return skipSample(err);
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -0,0 +1,157 @@
|
||||
import { mkdtempSync, rmSync, writeFileSync } from "node:fs";
|
||||
import { tmpdir } from "node:os";
|
||||
import { join } from "node:path";
|
||||
import { afterEach, beforeEach, describe, expect, it, vi } from "vitest";
|
||||
import { psnrDb } from "../utils/psnr.js";
|
||||
import { isDrawElementVerificationError, type CaptureSession } from "./frameCapture.js";
|
||||
import { verifyDiskDrawElementSamples, type WorkerTask } from "./parallelCoordinator.js";
|
||||
|
||||
vi.mock("../utils/psnr.js", async (importOriginal) => ({
|
||||
...(await importOriginal<typeof import("../utils/psnr.js")>()),
|
||||
psnrDb: vi.fn(),
|
||||
}));
|
||||
|
||||
const psnr = vi.mocked(psnrDb);
|
||||
const FLOOR_DB = 32;
|
||||
|
||||
describe("verifyDiskDrawElementSamples", () => {
|
||||
let outputDir: string;
|
||||
|
||||
beforeEach(() => {
|
||||
outputDir = mkdtempSync(join(tmpdir(), "hf-disk-verify-"));
|
||||
psnr.mockReset();
|
||||
vi.spyOn(console, "error").mockImplementation(() => {});
|
||||
vi.spyOn(console, "warn").mockImplementation(() => {});
|
||||
});
|
||||
|
||||
afterEach(() => {
|
||||
rmSync(outputDir, { recursive: true, force: true });
|
||||
vi.restoreAllMocks();
|
||||
});
|
||||
|
||||
const truthFor = (frame: number) => Buffer.from(`truth-${frame}`);
|
||||
const sessionWith = (
|
||||
frames: number[],
|
||||
overrides: { captureMode?: CaptureSession["captureMode"]; format?: "jpeg" | "png" } = {},
|
||||
) =>
|
||||
({
|
||||
captureMode: overrides.captureMode ?? "drawelement",
|
||||
options: { format: overrides.format ?? "jpeg" },
|
||||
deVerifyFrames: new Map(frames.map((frame) => [frame, truthFor(frame)])),
|
||||
}) as unknown as CaptureSession;
|
||||
const taskWith = (overrides: Partial<WorkerTask> = {}) =>
|
||||
({ workerId: 1, startFrame: 0, endFrame: 10, outputDir, ...overrides }) as WorkerTask;
|
||||
const writeFrame = (fileIndex: number, ext = "jpg") => {
|
||||
const bytes = Buffer.from(`captured-${fileIndex}`);
|
||||
writeFileSync(join(outputDir, `frame_${String(fileIndex).padStart(6, "0")}.${ext}`), bytes);
|
||||
return bytes;
|
||||
};
|
||||
|
||||
it("fails the worker on a sample below the floor, naming the frame and its score", async () => {
|
||||
writeFrame(2);
|
||||
writeFrame(4);
|
||||
psnr.mockResolvedValueOnce(50).mockResolvedValueOnce(FLOOR_DB - 0.1);
|
||||
|
||||
const error = await verifyDiskDrawElementSamples(sessionWith([2, 4]), taskWith(), false).catch(
|
||||
(e: unknown) => e,
|
||||
);
|
||||
|
||||
expect(isDrawElementVerificationError(error)).toBe(true);
|
||||
expect(error).toMatchObject({
|
||||
kind: "psnr",
|
||||
frameIndex: 4,
|
||||
failedDb: FLOOR_DB - 0.1,
|
||||
verifyThresholdDb: FLOOR_DB,
|
||||
});
|
||||
});
|
||||
|
||||
it("passes a sample exactly on the floor", async () => {
|
||||
writeFrame(2);
|
||||
psnr.mockResolvedValue(FLOOR_DB);
|
||||
await expect(
|
||||
verifyDiskDrawElementSamples(sessionWith([2]), taskWith(), false),
|
||||
).resolves.toBeUndefined();
|
||||
expect(psnr).toHaveBeenCalledTimes(1);
|
||||
});
|
||||
|
||||
it("compares each captured file against its own ground truth", async () => {
|
||||
const two = writeFrame(2);
|
||||
const four = writeFrame(4);
|
||||
psnr.mockResolvedValue(50);
|
||||
await verifyDiskDrawElementSamples(sessionWith([4, 2]), taskWith(), false);
|
||||
expect(psnr.mock.calls).toEqual([
|
||||
[two, truthFor(2)],
|
||||
[four, truthFor(4)],
|
||||
]);
|
||||
});
|
||||
|
||||
it("finds a chunk's frames through the output offset", async () => {
|
||||
const onDisk = writeFrame(2);
|
||||
psnr.mockResolvedValue(50);
|
||||
await verifyDiskDrawElementSamples(
|
||||
sessionWith([12]),
|
||||
taskWith({ startFrame: 10, endFrame: 20, outputFrameOffset: 10 }),
|
||||
false,
|
||||
);
|
||||
expect(psnr.mock.calls).toEqual([[onDisk, truthFor(12)]]);
|
||||
});
|
||||
|
||||
it("reads png frames from a png capture", async () => {
|
||||
const onDisk = writeFrame(2, "png");
|
||||
psnr.mockResolvedValue(50);
|
||||
await verifyDiskDrawElementSamples(sessionWith([2], { format: "png" }), taskWith(), false);
|
||||
expect(psnr.mock.calls).toEqual([[onDisk, truthFor(2)]]);
|
||||
});
|
||||
|
||||
it("checks only the samples this worker captured", async () => {
|
||||
const mine = writeFrame(4);
|
||||
psnr.mockResolvedValue(50);
|
||||
await verifyDiskDrawElementSamples(
|
||||
sessionWith([1, 3, 4, 12]),
|
||||
taskWith({ startFrame: 0, endFrame: 10, frameStride: 2 }),
|
||||
false,
|
||||
);
|
||||
expect(psnr.mock.calls).toEqual([[mine, truthFor(4)]]);
|
||||
});
|
||||
|
||||
it.each([
|
||||
["a streaming capture, whose drain already checks", true, "drawelement"],
|
||||
["a session that fell back to screenshots", false, "screenshot"],
|
||||
] as const)("does not compare on %s", async (_name, streaming, captureMode) => {
|
||||
writeFrame(2);
|
||||
await verifyDiskDrawElementSamples(sessionWith([2], { captureMode }), taskWith(), streaming);
|
||||
expect(psnr).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it("skips a sample whose file cannot be read, and still checks the next", async () => {
|
||||
writeFrame(4);
|
||||
psnr.mockResolvedValue(FLOOR_DB - 1);
|
||||
const error = await verifyDiskDrawElementSamples(sessionWith([2, 4]), taskWith(), false).catch(
|
||||
(e: unknown) => e,
|
||||
);
|
||||
expect(error).toMatchObject({ frameIndex: 4 });
|
||||
});
|
||||
|
||||
it("skips a sample the compare could not score, and still checks the next", async () => {
|
||||
writeFrame(2);
|
||||
writeFrame(4);
|
||||
psnr.mockRejectedValueOnce(new Error("could not parse psnr")).mockResolvedValue(FLOOR_DB - 1);
|
||||
const error = await verifyDiskDrawElementSamples(sessionWith([2, 4]), taskWith(), false).catch(
|
||||
(e: unknown) => e,
|
||||
);
|
||||
expect(error).toMatchObject({ frameIndex: 4 });
|
||||
});
|
||||
|
||||
it.each([
|
||||
["a missing filter", new Error("No such filter: 'psnr'")],
|
||||
["a missing binary", Object.assign(new Error("spawn ffmpeg ENOENT"), { code: "ENOENT" })],
|
||||
])("aborts rather than ship unverified frames on %s", async (_name, failure) => {
|
||||
writeFrame(2);
|
||||
psnr.mockRejectedValue(failure);
|
||||
const error = await verifyDiskDrawElementSamples(sessionWith([2]), taskWith(), false).catch(
|
||||
(e: unknown) => e,
|
||||
);
|
||||
expect(isDrawElementVerificationError(error)).toBe(false);
|
||||
expect((error as Error).message).toContain("disk self-verify aborted");
|
||||
});
|
||||
});
|
||||
Reference in New Issue
Block a user