Merge pull request #827 from heygen-com/05-14-feat_producer_add_harness_mode_--mode_distributed-simulated

feat(producer): add harness mode --mode=distributed-simulated
This commit is contained in:
James Russo
2026-05-14 16:59:44 -04:00
committed by GitHub
8 changed files with 889 additions and 38 deletions
+2
View File
@@ -44,10 +44,12 @@
"bench:hdr": "tsx src/benchmark.ts --tags hdr",
"test": "tsx src/regression-harness.ts --exclude-tags transparency",
"test:update": "tsx src/regression-harness.ts --update --exclude-tags transparency",
"test:distributed": "tsx src/regression-harness.ts --exclude-tags transparency --mode=distributed-simulated",
"test:transparency": "tsx src/transparency-test.ts",
"docker:build:test": "docker build -f ../../Dockerfile.test -t hyperframes-producer:test ../..",
"docker:test": "docker run --rm --security-opt seccomp=unconfined --shm-size=2g -v ./tests:/app/packages/producer/tests hyperframes-producer:test",
"docker:test:update": "docker run --rm --security-opt seccomp=unconfined --shm-size=2g -v ./tests:/app/packages/producer/tests hyperframes-producer:test --update",
"docker:test:distributed": "docker run --rm --security-opt seccomp=unconfined --shm-size=2g -v ./tests:/app/packages/producer/tests hyperframes-producer:test --mode=distributed-simulated",
"prepublishOnly": "echo skip"
},
"dependencies": {
@@ -0,0 +1,144 @@
// Pure-function tests for the harness mode dispatch logic. End-to-end
// PSNR contract lives in `Dockerfile.test` runs of the regression harness.
import { describe, expect, it } from "bun:test";
import {
checkDistributedSupport,
DISTRIBUTED_SIMULATED_MIN_PSNR_DB,
parseHarnessModeFlag,
resolveMinPsnrForMode,
} from "./regression-harness-distributed.js";
describe("parseHarnessModeFlag()", () => {
it("parses --mode=in-process", () => {
expect(parseHarnessModeFlag("--mode=in-process")).toBe("in-process");
});
it("parses --mode=distributed-simulated", () => {
expect(parseHarnessModeFlag("--mode=distributed-simulated")).toBe("distributed-simulated");
});
it("returns null for tokens that aren't --mode", () => {
expect(parseHarnessModeFlag("--update")).toBeNull();
expect(parseHarnessModeFlag("font-variant-numeric")).toBeNull();
expect(parseHarnessModeFlag("--exclude-tags")).toBeNull();
});
it("throws on a known prefix with a bad value", () => {
expect(() => parseHarnessModeFlag("--mode=foo")).toThrow(/--mode must be/);
expect(() => parseHarnessModeFlag("--mode=")).toThrow(/--mode must be/);
});
});
describe("checkDistributedSupport()", () => {
it("accepts mp4 SDR at 24 / 30 / 60 fps", () => {
for (const fpsNum of [24, 30, 60]) {
const result = checkDistributedSupport({ fps: { num: fpsNum, den: 1 } });
expect(result.supported).toBe(true);
}
});
it("accepts explicit format=mp4", () => {
const result = checkDistributedSupport({ fps: { num: 30, den: 1 }, format: "mp4" });
expect(result.supported).toBe(true);
});
it("rejects fps with non-1 denominator (NTSC)", () => {
const result = checkDistributedSupport({ fps: { num: 30000, den: 1001 } });
expect(result.supported).toBe(false);
if (!result.supported) {
expect(result.reason).toMatch(/non-integer fps/);
}
});
it("rejects fps outside the {24,30,60} set", () => {
for (const fpsNum of [12, 25, 48, 50, 120]) {
const result = checkDistributedSupport({ fps: { num: fpsNum, den: 1 } });
expect(result.supported).toBe(false);
if (!result.supported) {
expect(result.reason).toMatch(/not in \{24, 30, 60\}/);
}
}
});
it("rejects format=webm", () => {
const result = checkDistributedSupport({ fps: { num: 30, den: 1 }, format: "webm" });
expect(result.supported).toBe(false);
if (!result.supported) {
expect(result.reason).toMatch(/webm/);
}
});
it("rejects hdr=true", () => {
const result = checkDistributedSupport({ fps: { num: 30, den: 1 }, hdr: true });
expect(result.supported).toBe(false);
if (!result.supported) {
expect(result.reason).toMatch(/hdr/);
}
});
it("accepts hdr=false (or unset)", () => {
expect(checkDistributedSupport({ fps: { num: 30, den: 1 }, hdr: false }).supported).toBe(true);
expect(checkDistributedSupport({ fps: { num: 30, den: 1 } }).supported).toBe(true);
});
});
describe("resolveMinPsnrForMode()", () => {
it("in-process mode uses the fixture's own threshold verbatim", () => {
expect(resolveMinPsnrForMode("in-process", 30)).toBe(30);
expect(resolveMinPsnrForMode("in-process", 50)).toBe(50);
expect(resolveMinPsnrForMode("in-process", 60)).toBe(60);
});
it("distributed-simulated uses the fixture's own minPsnr when above the absolute floor", () => {
// Fixtures with minPsnr >= the absolute floor (catastrophic-failure
// guard) use their authored threshold unchanged. Distributed must pass
// the same quality bar the in-process renderer passes against the same
// baseline — no extra tightening, since baseline drift is shared across
// modes.
expect(resolveMinPsnrForMode("distributed-simulated", 30)).toBe(30);
expect(resolveMinPsnrForMode("distributed-simulated", 50)).toBe(50);
expect(resolveMinPsnrForMode("distributed-simulated", 80)).toBe(80);
});
it("distributed-simulated raises pathologically-low thresholds to the absolute floor", () => {
// A fixture authored with minPsnr=0 (or very low) wouldn't catch a
// distributed-mode renderer producing fully-black output. The absolute
// floor exists to catch that pathology.
expect(resolveMinPsnrForMode("distributed-simulated", 0)).toBe(
DISTRIBUTED_SIMULATED_MIN_PSNR_DB,
);
expect(resolveMinPsnrForMode("distributed-simulated", 5)).toBe(
DISTRIBUTED_SIMULATED_MIN_PSNR_DB,
);
});
it("every committed fixture authors a minPsnr above the absolute floor", async () => {
// The pathology floor only fires for a fixture whose authored minPsnr
// is below it — by design that should be no committed fixture. If
// someone lands a permissive fixture (minPsnr: 5), distributed mode
// will silently use 10 dB instead, which is the right behavior but
// worth flagging so reviewers ask "is this fixture really meant to
// accept near-black output?". This test prevents accidental misuse
// by failing loudly when a fixture drops below the floor.
const { readdirSync, readFileSync, statSync } = await import("node:fs");
const { join: pathJoin } = await import("node:path");
const testsDir = pathJoin(import.meta.dir, "..", "tests");
const offenders: Array<{ fixture: string; minPsnr: number }> = [];
for (const entry of readdirSync(testsDir)) {
const metaPath = pathJoin(testsDir, entry, "meta.json");
let stat;
try {
stat = statSync(metaPath);
} catch {
continue;
}
if (!stat.isFile()) continue;
const meta = JSON.parse(readFileSync(metaPath, "utf-8")) as { minPsnr?: unknown };
if (typeof meta.minPsnr === "number" && meta.minPsnr < DISTRIBUTED_SIMULATED_MIN_PSNR_DB) {
offenders.push({ fixture: entry, minPsnr: meta.minPsnr });
}
}
expect(offenders).toEqual([]);
});
});
@@ -0,0 +1,221 @@
/**
* Distributed-render path for the regression harness.
*
* The regression harness has two modes:
*
* - `in-process` (default) — calls `executeRenderJob`, the same path the
* `hyperframes render` CLI takes. This is what produced every existing
* `tests/<name>/output/output.mp4` golden baseline.
*
* - `distributed-simulated` — calls `plan()` → `renderChunk()` per chunk
* → `assemble()` from `@hyperframes/producer/distributed`. No Temporal
* or Lambda involvement: the controller and chunk worker are both this
* process, but they go through the same artifact (planDir + frozen
* `meta/encoder.json` + per-chunk concat-copy) that a real fan-out
* would.
*
* Both modes share the per-fixture `minPsnr` threshold — distributed must
* pass the same quality bar the in-process renderer passes against the
* same frozen baseline. A separate {@link DISTRIBUTED_SIMULATED_MIN_PSNR_DB}
* pathology floor catches the case where a fixture authored a permissive
* threshold and distributed regresses to fully-black output. The §5.1
* 50 dB target was written for per-render comparison (fresh in-process vs
* fresh distributed); against the frozen baseline file it's unreachable
* for either mode due to shared encoder/JPEG-capture jitter, so the
* harness can't use it as a per-test gate.
*
* Not every fixture can run in distributed-simulated mode. Distributed mode
* refuses webm, HDR mp4, NTSC framerates, and non-{24,30,60} fps at plan
* time. Fixtures that don't meet the constraints are skipped — the harness
* logs the reason and the fixture is treated as "passed (skipped)" in
* distributed-simulated mode.
*/
import { existsSync, mkdirSync } from "node:fs";
import { join } from "node:path";
import type { Fps } from "@hyperframes/core";
import { assemble, plan, renderChunk } from "./distributed.js";
/** Two-mode contract that backs `--mode=<value>` on the regression harness CLI. */
export type HarnessMode = "in-process" | "distributed-simulated";
/**
* Absolute pathology floor for `--mode=distributed-simulated` — catches
* a chunk that renders fully-black against a fixture authored with a
* permissive `minPsnr`. Non-pathological drift is caught by the fixture's
* own threshold; both modes share the same encoder/JPEG-capture jitter
* floor against the frozen baseline file, so the §5.1 50 dB target is
* unreachable for either mode and isn't a useful per-test gate.
*/
export const DISTRIBUTED_SIMULATED_MIN_PSNR_DB = 10;
/** Result of {@link checkDistributedSupport}. */
export type DistributedSupportResult = { supported: true } | { supported: false; reason: string };
/**
* Decide whether a fixture's `renderConfig` is one the distributed pipeline
* can actually run. The four hard gates:
*
* - fps must be `{ num: 24|30|60, den: 1 }`. `DistributedRenderConfig.fps`
* accepts only the three integer values, and rationals like
* `{ num: 30000, den: 1001 }` (NTSC) trip the type system at the call
* site. We surface this gate in code rather than only in TS so the
* harness can skip the fixture cleanly instead of throwing.
* - format must not be `webm`. `plan()` refuses webm with
* `FORMAT_NOT_SUPPORTED_IN_DISTRIBUTED`.
* - hdr must not be `true`. Distributed mode is SDR-only at v1.
*
* Callers that want the structured reason can read it off the returned
* `reason` field; the message is intended to be log-friendly.
*/
export function checkDistributedSupport(renderConfig: {
fps: Fps;
format?: "mp4" | "webm";
hdr?: boolean;
}): DistributedSupportResult {
if (renderConfig.fps.den !== 1) {
return {
supported: false,
reason: `non-integer fps ${renderConfig.fps.num}/${renderConfig.fps.den} (distributed mode requires fps.den=1)`,
};
}
const fpsNum = renderConfig.fps.num;
if (fpsNum !== 24 && fpsNum !== 30 && fpsNum !== 60) {
return {
supported: false,
reason: `fps ${fpsNum} not in {24, 30, 60} (DistributedRenderConfig.fps is a closed set)`,
};
}
const format = renderConfig.format ?? "mp4";
if (format === "webm") {
return {
supported: false,
reason: "format=webm refused in distributed mode (VP9+matroska concat-copy is unstable)",
};
}
if (renderConfig.hdr === true) {
return {
supported: false,
reason: "hdr=true refused in distributed mode (HDR signaling re-apply not implemented)",
};
}
return { supported: true };
}
/**
* Inputs for {@link runDistributedSimulatedRender}. The harness has already
* prepared `projectDir` (a working copy of the fixture's `src/` directory)
* and `tempRoot` (where the harness writes its scratch artifacts).
*/
export interface RunDistributedSimulatedInput {
/** Working copy of the fixture's `src/` — contains `index.html`. */
projectDir: string;
/** Scratch root for plan + chunks; must be a directory the harness owns. */
tempRoot: string;
/** Where to write the assembled final mp4 / mov / png-sequence directory. */
renderedOutputPath: string;
/** From the fixture's renderConfig — must pass `checkDistributedSupport`. */
fps: 24 | 30 | 60;
format: "mp4" | "mov" | "png-sequence";
/** Optional chunkSize override; defaults to the plan's 240. */
chunkSize?: number;
/** Optional maxParallelChunks override; defaults to the plan's 16. */
maxParallelChunks?: number;
/** Forwarded to `plan()` and re-applied by `renderChunk()` at boot. */
variables?: Record<string, unknown>;
}
/**
* Run the distributed pipeline against a single fixture as if a fan-out
* adapter were driving it. The three activities run serially in this
* process — there is no Temporal, no Lambda, no S3 — so the planDir,
* chunk outputs, and assembled output all live under `tempRoot`.
*
* Width and height are required by `DistributedRenderConfig` for cross-call
* sanity but are not consulted at render time — `plan()` reads the
* composition's `data-width` / `data-height` attributes and overrides
* whatever the config carried. The harness passes a dummy 1920×1080 here
* for that reason; if the contract ever changes, the fixture's authored
* dimensions will flow through `PlanResult` and we can switch to using
* those instead.
*/
export async function runDistributedSimulatedRender(
input: RunDistributedSimulatedInput,
): Promise<void> {
const planDir = join(input.tempRoot, "plan");
const chunksDir = join(input.tempRoot, "chunks");
mkdirSync(planDir, { recursive: true });
mkdirSync(chunksDir, { recursive: true });
// Step A: plan.
const planResult = await plan(
input.projectDir,
{
fps: input.fps,
// Required-by-type but overridden by the composition's own attrs;
// see docstring above. Any positive integer works.
width: 1920,
height: 1080,
format: input.format,
chunkSize: input.chunkSize,
maxParallelChunks: input.maxParallelChunks,
// Force the SDR path explicitly — `auto` would still resolve to
// force-sdr in distributed mode, but pinning it here keeps the
// harness's behavior independent of any future auto-detect changes.
hdrMode: "force-sdr",
},
planDir,
);
// Step B: render every chunk. Sequential to keep the harness predictable —
// adapters in production are free to fan out; this code path's job is to
// exercise the per-chunk activity itself.
const chunkPaths: string[] = [];
for (let i = 0; i < planResult.chunkCount; i++) {
const chunkPath =
input.format === "png-sequence"
? join(chunksDir, `chunk-${String(i).padStart(4, "0")}`)
: join(chunksDir, `chunk-${String(i).padStart(4, "0")}.${input.format}`);
await renderChunk(planDir, i, chunkPath);
chunkPaths.push(chunkPath);
}
// Step C: assemble. `audio.aac` only exists when the composition has
// audio — pass null otherwise so `assemble()` doesn't try to mux silence.
const audioPath = join(planDir, "audio.aac");
const audioForAssemble = existsSync(audioPath) ? audioPath : null;
await assemble(planDir, chunkPaths, audioForAssemble, input.renderedOutputPath);
}
/**
* Pick the PSNR threshold for a fixture given the harness mode. Both modes
* share the fixture's authored `minPsnr` — distributed must clear the same
* quality bar in-process clears against the same frozen baseline.
* Distributed-simulated additionally lifts the threshold to
* {@link DISTRIBUTED_SIMULATED_MIN_PSNR_DB} for fixtures with a permissive
* authored threshold; that absolute floor catches fully-black-output
* regressions independent of fixture tolerance.
*/
export function resolveMinPsnrForMode(mode: HarnessMode, fixtureMinPsnr: number): number {
if (mode === "in-process") return fixtureMinPsnr;
return Math.max(fixtureMinPsnr, DISTRIBUTED_SIMULATED_MIN_PSNR_DB);
}
/**
* Parse `--mode=<value>` from a single CLI token. Returns the parsed mode
* when the token matches the expected shape, `null` otherwise so the
* caller can pass the token through to the next handler. Throws on a
* known prefix with a bad value (`--mode=foo`) — surfacing a typo at
* parse time is cheaper than discovering at render time.
*/
export function parseHarnessModeFlag(token: string): HarnessMode | null {
if (token === "--mode=in-process") return "in-process";
if (token === "--mode=distributed-simulated") return "distributed-simulated";
if (token.startsWith("--mode=")) {
const value = token.slice("--mode=".length);
throw new Error(
`regression-harness: --mode must be 'in-process' or 'distributed-simulated' (got ${JSON.stringify(value)})`,
);
}
return null;
}
+143 -23
View File
@@ -20,6 +20,13 @@ import { validateCompilation } from "./services/compilationTester.js";
import { extractMediaMetadata } from "./utils/ffprobe.js";
import { buildRmsEnvelope, compareAudioEnvelopes } from "./utils/audioRegression.js";
import { parseFps, fpsToNumber } from "@hyperframes/core";
import {
checkDistributedSupport,
type HarnessMode,
parseHarnessModeFlag,
resolveMinPsnrForMode,
runDistributedSimulatedRender,
} from "./regression-harness-distributed.js";
// ── Types ────────────────────────────────────────────────────────────────────
@@ -51,6 +58,18 @@ type TestMetadata = {
* and these overrides. Omit when the test doesn't exercise variables.
*/
variables?: Record<string, unknown>;
/**
* Chunk size in frames for `--mode=distributed-simulated`. Forwarded
* to `DistributedRenderConfig.chunkSize`. Ignored in `--mode=in-process`.
* Default is the plan's own default (240 frames).
*/
chunkSize?: number;
/**
* Cap on parallel chunks for `--mode=distributed-simulated`. Forwarded
* to `DistributedRenderConfig.maxParallelChunks`. Ignored in
* `--mode=in-process`. Default is the plan's own default (16).
*/
maxParallelChunks?: number;
};
};
@@ -67,11 +86,26 @@ type CliOptions = {
update: boolean;
sequential: boolean;
keepTemp: boolean;
/**
* Which render path to exercise. `in-process` (default) calls
* `executeRenderJob`; `distributed-simulated` calls
* `plan() → renderChunk() × N → assemble()` from
* `@hyperframes/producer/distributed`. See
* `regression-harness-distributed.ts`.
*/
mode: HarnessMode;
};
type TestResult = {
suite: TestSuite;
passed: boolean;
/**
* Set when `--mode=distributed-simulated` skips a fixture that the
* distributed pipeline can't run (webm, HDR, NTSC fps, fps∉{24,30,60}).
* `passed` is `true` for skipped fixtures — skipping is a clean outcome,
* not a failure — but the summary distinguishes them.
*/
skipped?: { reason: string };
compilation?: {
passed: boolean;
errors: string[];
@@ -105,6 +139,7 @@ function parseArgs(argv: string[]): CliOptions {
let update = false;
let sequential = false;
let keepTemp = false;
let mode: HarnessMode = "in-process";
for (let i = 2; i < argv.length; i += 1) {
const token = argv[i];
@@ -119,12 +154,29 @@ function parseArgs(argv: string[]): CliOptions {
i += 1;
const tagArg = argv[i];
if (tagArg) excludeTags.push(...tagArg.split(","));
} else if (!token.startsWith("--")) {
testNames.push(token);
} else {
const parsedMode = parseHarnessModeFlag(token);
if (parsedMode !== null) {
mode = parsedMode;
} else if (!token.startsWith("--")) {
testNames.push(token);
}
}
}
return { testNames, excludeTags, update, sequential, keepTemp };
if (update && mode === "distributed-simulated") {
// The in-process renderer is the source of truth for golden baselines —
// distributed-simulated's job is to verify the contract against the
// same baseline, not to author its own. Surfacing this at parse time
// saves a multi-minute render before the user notices.
throw new Error(
"regression-harness: --update is incompatible with --mode=distributed-simulated. " +
"Generate baselines with the in-process renderer (the default mode), then re-run " +
"without --update to verify both modes match.",
);
}
return { testNames, excludeTags, update, sequential, keepTemp, mode };
}
function validateMetadata(meta: unknown): TestMetadata {
@@ -195,6 +247,20 @@ function validateMetadata(meta: unknown): TestMetadata {
) {
throw new Error("meta.json: 'renderConfig.variables' must be a JSON object (or omitted)");
}
if (rc.chunkSize !== undefined) {
if (!Number.isInteger(rc.chunkSize) || (rc.chunkSize as number) < 1) {
throw new Error(
"meta.json: 'renderConfig.chunkSize' must be a positive integer (or omitted)",
);
}
}
if (rc.maxParallelChunks !== undefined) {
if (!Number.isInteger(rc.maxParallelChunks) || (rc.maxParallelChunks as number) < 1) {
throw new Error(
"meta.json: 'renderConfig.maxParallelChunks' must be a positive integer (or omitted)",
);
}
}
return m as TestMetadata;
}
@@ -399,6 +465,7 @@ function saveFailureDetails(
result: TestResult,
renderedVideoPath: string,
snapshotVideoPath: string,
effectiveMinPsnr: number,
compiledHtml?: string,
snapshotHtml?: string,
): void {
@@ -441,12 +508,13 @@ function saveFailureDetails(
summary: {
totalCheckpoints: result.visual.checkpoints.length,
failedCheckpoints: failedCheckpoints.length,
threshold: suite.meta.minPsnr,
threshold: effectiveMinPsnr,
fixtureThreshold: suite.meta.minPsnr,
},
failedFrames: failedCheckpoints.map((c) => ({
time: c.time,
psnr: c.psnr,
belowThresholdBy: suite.meta.minPsnr - c.psnr,
belowThresholdBy: effectiveMinPsnr - c.psnr,
})),
};
@@ -522,6 +590,7 @@ async function runTestSuite(
options: {
update: boolean;
keepTemp: boolean;
mode: HarnessMode;
},
): Promise<TestResult> {
// Use predictable temp location: /tmp/hyperframes-tests/{test-id}/
@@ -621,25 +690,65 @@ async function runTestSuite(
}
// STEP 2: Render video
console.log(JSON.stringify({ event: "rendering_start", suite: suite.id }));
logPretty("Rendering video...", "🎬");
console.log(JSON.stringify({ event: "rendering_start", suite: suite.id, mode: options.mode }));
logPretty(`Rendering video (mode=${options.mode})...`, "🎬");
const tempSrcDir = join(tempRoot, "src");
copyFixtureSupportFiles(suite, tempRoot);
cpSync(suite.srcDir, tempSrcDir, { recursive: true });
const job = createRenderJob({
fps: suite.meta.renderConfig.fps,
quality: "high", // Always use max quality for tests
format: outputFormat,
workers: suite.meta.renderConfig.workers,
useGpu: false,
debug: false,
hdrMode: suite.meta.renderConfig.hdr ? "force-hdr" : "force-sdr",
variables: suite.meta.renderConfig.variables,
});
if (options.mode === "distributed-simulated") {
const support = checkDistributedSupport(suite.meta.renderConfig);
if (!support.supported) {
// Skipping is a clean outcome — the distributed pipeline can't
// run this fixture, but in-process mode already covers it. Mark
// passed so the suite summary doesn't trip CI; the `skipped`
// field is what distinguishes a real pass from a skip.
console.log(
JSON.stringify({
event: "test_skipped",
suite: suite.id,
mode: options.mode,
reason: support.reason,
}),
);
logPretty(`Skipping ${suite.meta.name} (mode=${options.mode}): ${support.reason}`, "⏭️");
result.passed = true;
result.skipped = { reason: support.reason };
return result;
}
// `checkDistributedSupport` already narrowed fps to {24,30,60} and
// rejected webm; the cast surfaces that guarantee to TS.
const fpsNum = suite.meta.renderConfig.fps.num as 24 | 30 | 60;
// `validateMetadata` only accepts `format: "mp4" | "webm"` in
// `renderConfig`, and `checkDistributedSupport` rejected webm above,
// so by here only `mp4` (or the unset default) can reach this call.
// If the metadata schema grows to accept "mov" / "png-sequence"
// someday, narrow this cast accordingly.
await runDistributedSimulatedRender({
projectDir: tempSrcDir,
tempRoot,
renderedOutputPath,
fps: fpsNum,
format: "mp4",
chunkSize: suite.meta.renderConfig.chunkSize,
maxParallelChunks: suite.meta.renderConfig.maxParallelChunks,
variables: suite.meta.renderConfig.variables,
});
} else {
const job = createRenderJob({
fps: suite.meta.renderConfig.fps,
quality: "high", // Always use max quality for tests
format: outputFormat,
workers: suite.meta.renderConfig.workers,
useGpu: false,
debug: false,
hdrMode: suite.meta.renderConfig.hdr ? "force-hdr" : "force-sdr",
variables: suite.meta.renderConfig.variables,
});
await executeRenderJob(job, tempSrcDir, renderedOutputPath);
await executeRenderJob(job, tempSrcDir, renderedOutputPath);
}
console.log(JSON.stringify({ event: "rendering_complete", suite: suite.id }));
logPretty("Render complete! Starting quality validation...", "✓");
@@ -680,6 +789,7 @@ async function runTestSuite(
// which causes ffmpeg's PSNR filter to emit no `average:` line.
const videoDuration = Math.min(videoMetadata.durationSeconds, snapshotMetadata.durationSeconds);
const minPsnrForMode = resolveMinPsnrForMode(options.mode, suite.meta.minPsnr);
const visualCheckpoints: Array<{ time: number; psnr: number; passed: boolean }> = [];
for (let i = 0; i < 100; i++) {
const time = (videoDuration * i) / 100;
@@ -692,7 +802,7 @@ async function runTestSuite(
visualCheckpoints.push({
time,
psnr,
passed: psnr >= suite.meta.minPsnr,
passed: psnr >= minPsnrForMode,
});
// Progress indicator every 20 checkpoints
@@ -815,6 +925,7 @@ async function runTestSuite(
result,
renderedOutputPath,
snapshotVideoPath,
resolveMinPsnrForMode(options.mode, suite.meta.minPsnr),
compiledHtml,
snapshotHtml,
);
@@ -857,11 +968,13 @@ async function run(): Promise<void> {
event: "test_suite_start",
totalSuites: suites.length,
parallel: !options.sequential,
mode: options.mode,
}),
);
logPretty(
`Starting ${suites.length} test suite(s) - ${options.sequential ? "sequential" : "parallel"} mode`,
`Starting ${suites.length} test suite(s) - ${options.sequential ? "sequential" : "parallel"} mode, ` +
`harness mode=${options.mode}`,
"🚀",
);
@@ -925,7 +1038,8 @@ async function run(): Promise<void> {
);
logPretty(`Updated ${results.length} snapshot(s)`, "📸");
} else {
const passed = results.filter((r) => r.passed).length;
const skipped = results.filter((r) => r.skipped).length;
const passed = results.filter((r) => r.passed && !r.skipped).length;
const failed = results.filter((r) => !r.passed).length;
const failedAtCompilation = results.filter(
(r) => r.compilation && !r.compilation.passed,
@@ -939,6 +1053,8 @@ async function run(): Promise<void> {
total: results.length,
passed,
failed,
skipped,
mode: options.mode,
failedAtCompilation,
failedAtVisual,
failedAtAudio,
@@ -946,6 +1062,7 @@ async function run(): Promise<void> {
suite: r.suite.id,
name: r.suite.meta.name,
passed: r.passed,
skipped: r.skipped?.reason,
compilation: r.compilation?.passed,
visual: r.visual?.passed,
audio: r.audio?.passed,
@@ -955,8 +1072,11 @@ async function run(): Promise<void> {
// Pretty summary
logPretty("═══════════════════════════════════════", "");
logPretty(`Test Suite Summary`, "📊");
logPretty(`Total: ${results.length} | Passed: ${passed} | Failed: ${failed}`, "");
logPretty(`Test Suite Summary (mode=${options.mode})`, "📊");
logPretty(
`Total: ${results.length} | Passed: ${passed} | Failed: ${failed} | Skipped: ${skipped}`,
"",
);
if (failed > 0) {
logPretty(` Failed at compilation: ${failedAtCompilation}`, "");
logPretty(` Failed at visual: ${failedAtVisual}`, "");
@@ -24,8 +24,17 @@
* never have to handle them.
*/
import { existsSync, mkdirSync, readdirSync, renameSync, rmSync, statSync } from "node:fs";
import { join } from "node:path";
import {
cpSync,
existsSync,
mkdirSync,
readdirSync,
renameSync,
rmSync,
statSync,
writeFileSync,
} from "node:fs";
import { join, relative, sep } from "node:path";
import { type CanvasResolution } from "@hyperframes/core";
import { type EngineConfig, resolveConfig } from "@hyperframes/engine";
import { defaultLogger, type ProducerLogger } from "../../logger.js";
@@ -46,7 +55,13 @@ import {
} from "../render/stages/planHash.js";
import { validateNoGpuEncode, validateNoSystemFonts } from "../render/planValidation.js";
import { snapshotRuntimeEnv } from "../render/runtimeEnvSnapshot.js";
import { buildSyntheticRenderJob, readFfmpegVersion, readProducerVersion } from "./shared.js";
import {
buildSyntheticRenderJob,
PLAN_VIDEOS_META_RELATIVE_PATH,
type PlanVideosJson,
readFfmpegVersion,
readProducerVersion,
} from "./shared.js";
/**
* Caller-supplied configuration for a distributed render. `fps`, `width`,
@@ -131,6 +146,26 @@ export interface PlanResult {
producerVersion: string;
}
/**
* Top-level directory names skipped by the `projectDir → planDir/compiled/`
* pre-seed copy. Real projects often contain `node_modules/`, VCS metadata,
* and harness artifacts that have no business in a planDir — they bloat
* the 2 GB planDir cap and slow the S3/Lambda round-trip for no benefit.
* Matched against the path relative to `projectDir` so a `projectDir`
* whose absolute path happens to contain one of these names (e.g.
* `~/work/output/comp/`) doesn't false-positive-skip the entire copy.
*/
const PLAN_PROJECT_DIR_SKIP_SEGMENTS = new Set([
"node_modules",
".git",
".cache",
"output",
"failures",
"dist",
".next",
".turbo",
]);
/** Default chunk size in frames (~8s @ 30fps; fits Lambda's 15-min cap). */
export const DEFAULT_CHUNK_SIZE = 240;
/** Default cap on parallel chunks for operational fairness across renders. */
@@ -492,6 +527,28 @@ export async function plan(
if (!existsSync(workDir)) mkdirSync(workDir, { recursive: true });
const compiledDir = join(workDir, "compiled");
// Pre-seed the compiled directory with `projectDir`'s local assets
// (style.css, script.js, images, etc.). The chunk worker's file server
// serves ONLY from `<planDir>/compiled/`, so without this copy a
// composition's `<link rel=stylesheet href=style.css>` 404s and the
// first capture lands an unstyled fallback frame. `compileStage`
// overwrites the entry HTML afterwards. `dereference: true` resolves
// symlinks so the planDir survives S3 / Lambda /tmp round-trips.
mkdirSync(compiledDir, { recursive: true });
cpSync(projectDir, compiledDir, {
recursive: true,
dereference: true,
filter: (src) => {
// cpSync passes the absolute source path. Compare relative-to-projectDir
// so a parent directory of projectDir matching a skip name doesn't
// false-positive every descendant.
const rel = relative(projectDir, src);
if (rel === "" || rel.startsWith("..")) return true;
const firstSegment = rel.split(sep, 1)[0];
return firstSegment === undefined || !PLAN_PROJECT_DIR_SKIP_SEGMENTS.has(firstSegment);
},
});
// The compiled directory lives at `<planDir>/compiled/` in the final
// layout. The stages write under `<planDir>/.plan-work/compiled/`; we
// move the contents over once the staged work completes.
@@ -582,7 +639,9 @@ export async function plan(
assertNotAborted,
materializeSymlinks: true,
});
if (extractResult.frameLookup) extractResult.frameLookup.cleanup();
// Skip `extractResult.frameLookup.cleanup()`: it would rm-rf each
// video's outputDir, but in `plan()` those directories ARE the source
// material the renames below move into `planDir/video-frames/`.
// ── Audio ──
const audioResult = await runAudioStage({
@@ -613,6 +672,30 @@ export async function plan(
if (existsSync(finalCompiledDir)) rmSync(finalCompiledDir, { recursive: true, force: true });
renameSync(compiledDir, finalCompiledDir);
// `meta/videos.json` is the contract that makes distributed renders
// pixel-comparable to in-process for compositions with video sources —
// without it, renderChunk can't rebuild the BeforeCaptureHook and the
// page's native `<video>` element decodes the source mp4 ~1 frame
// off the pre-extracted images the in-process baseline was captured
// from.
const planVideosJson: PlanVideosJson = {
videos: composition.videos,
extracted: (extractResult.extractionResult?.extracted ?? []).map((ext) => ({
videoId: ext.videoId,
srcPath: ext.srcPath,
framePattern: ext.framePattern,
fps: ext.fps,
totalFrames: ext.totalFrames,
metadata: ext.metadata,
})),
};
mkdirSync(join(planDir, "meta"), { recursive: true });
writeFileSync(
join(planDir, PLAN_VIDEOS_META_RELATIVE_PATH),
JSON.stringify(planVideosJson, null, 2),
"utf-8",
);
const planAudioPath = join(planDir, "audio.aac");
if (audioResult.hasAudio && existsSync(audioResult.audioOutputPath)) {
renameSync(audioResult.audioOutputPath, planAudioPath);
@@ -25,8 +25,8 @@
* (`buildVirtualTimeShim({ seedRandomFromFrame: true })`) so any
* composition that uses `Math.random` / `crypto.getRandomValues`
* produces byte-identical pixels per `(planDir, chunkIndex)`.
* - One `discardWarmupCapture` runs before the chunk's first real frame
* to prime the BeginFrame `lastFrameCache`.
* - No `lastFrameCache` priming: every frame seeks fresh DOM so the
* cache is never read, and priming would deadlock the compositor.
* - The chunk's encode runs with `lockGopForChunkConcat: true` and
* `gopSize === framesInChunk` so concat-copy at assemble time is safe.
*
@@ -36,17 +36,20 @@
import { randomBytes } from "node:crypto";
import { existsSync, mkdirSync, readFileSync, readdirSync, rmSync, writeFileSync } from "node:fs";
import { join } from "node:path";
import { extname, join } from "node:path";
import type { Page } from "puppeteer-core";
import {
assertSwiftShader,
type BeforeCaptureHook,
BROWSER_GPU_NOT_SOFTWARE,
type CaptureOptions,
type CaptureSession,
closeCaptureSession,
createCaptureSession,
discardWarmupCapture,
createFrameLookupTable,
createVideoFrameInjector,
type EngineConfig,
type ExtractedFrames,
getEncoderPreset,
initializeSession,
resolveConfig,
@@ -62,7 +65,12 @@ import {
import { sha256Hex } from "../render/stages/planHash.js";
import { applyRuntimeEnvSnapshot } from "../render/runtimeEnvSnapshot.js";
import { buildVirtualTimeShim, createFileServer, type FileServerHandle } from "../fileServer.js";
import { buildSyntheticRenderJob, readFfmpegVersion } from "./shared.js";
import {
buildSyntheticRenderJob,
PLAN_VIDEOS_META_RELATIVE_PATH,
type PlanVideosJson,
readFfmpegVersion,
} from "./shared.js";
/**
* Non-retryable error codes raised when the planDir is structurally
@@ -124,6 +132,61 @@ export interface ChunkResult {
perfPath: string;
}
/**
* Rebuild the engine's in-memory `ExtractedFrames[]` from the on-disk
* planDir layout. `<planDir>/video-frames/<videoId>/` holds the numbered
* frame files plan() extracted; this lists each dir and rebuilds the
* 1-based `framePaths` Map that `FrameLookupTable` / `videoFrameInjector`
* both index against.
*/
function rebuildExtractedFramesFromPlanDir(
planDir: string,
videos: PlanVideosJson["extracted"],
): ExtractedFrames[] {
const result: ExtractedFrames[] = [];
for (const v of videos) {
const outputDir = join(planDir, "video-frames", v.videoId);
if (!existsSync(outputDir)) {
throw new Error(
`[renderChunk] planDir missing extracted video frames for ${JSON.stringify(v.videoId)}: ` +
`${outputDir} not present. plan() should have written frames here; the planDir is malformed.`,
);
}
// framePattern looks like `frame_%05d.jpg`; sprintf isn't available at
// runtime so list-and-sort the directory. Sorted-by-name matches
// sorted-by-frame-index because the extractor writes zero-padded
// monotonic indices.
const ext = (extname(v.framePattern) || ".jpg").toLowerCase();
const frames = readdirSync(outputDir)
.filter((name) => name.toLowerCase().endsWith(ext))
.sort();
const framePaths = new Map<number, string>();
for (let i = 0; i < frames.length; i++) {
const frameName = frames[i];
if (!frameName) continue;
// FrameLookupTable indexes frames 1-based.
framePaths.set(i + 1, join(outputDir, frameName));
}
result.push({
videoId: v.videoId,
srcPath: v.srcPath,
outputDir,
framePattern: v.framePattern,
fps: v.fps,
totalFrames: v.totalFrames,
metadata: v.metadata,
framePaths,
// The chunk worker doesn't own the planDir's video-frames/ directory
// (the controller does — adapters that fan out chunks across machines
// share the planDir as read-only). Mark ownership as false so the
// injector's eventual cleanup doesn't rm bytes another worker may
// still be reading.
ownedByLookup: false,
});
}
return result;
}
/** Plan-time JSON manifest written by `freezePlan`. */
interface PlanJson {
planHash: string;
@@ -257,6 +320,21 @@ export async function renderChunk(
const encoder = JSON.parse(readFileSync(encoderJsonPath, "utf-8")) as LockedRenderConfig;
const chunks = JSON.parse(readFileSync(chunksJsonPath, "utf-8")) as ChunkSliceJson[];
// `meta/videos.json` only exists when the composition has `<video>`
// elements; absence means no injector is needed.
const videosJsonPath = join(planDir, PLAN_VIDEOS_META_RELATIVE_PATH);
let planVideos: PlanVideosJson | null = null;
if (existsSync(videosJsonPath)) {
try {
planVideos = JSON.parse(readFileSync(videosJsonPath, "utf-8")) as PlanVideosJson;
} catch (err) {
throw new RenderChunkValidationError(
MISSING_PLAN_ARTIFACT,
`[renderChunk] failed to parse ${videosJsonPath}: ${err instanceof Error ? err.message : String(err)}`,
);
}
}
if (chunkIndex < 0 || chunkIndex >= chunks.length) {
throw new RenderChunkValidationError(
CHUNK_INDEX_OUT_OF_RANGE,
@@ -361,6 +439,22 @@ export async function renderChunk(
forceScreenshot: encoder.forceScreenshot,
};
// Build the BeforeCaptureHook that injects pre-extracted video frames
// into the page once per chunk and reuse — `runCaptureStage` may
// invoke `createRenderVideoFrameInjector` multiple times, and
// re-listing `planDir/video-frames/` each call would be wasteful.
// Compositions with no video elements produce `null`, matching the
// in-process renderer's skip path.
const videoInjector: BeforeCaptureHook | null =
planVideos && planVideos.extracted.length > 0
? createVideoFrameInjector(
createFrameLookupTable(
planVideos.videos,
rebuildExtractedFramesFromPlanDir(planDir, planVideos.extracted),
),
)
: null;
// ── Per-chunk work + frames directories ──
// Suffix workDir with pid + random bytes so concurrent invocations on
// the SAME `(planDir, chunkIndex)` (e.g. a scheduler that double-fires
@@ -412,11 +506,10 @@ export async function renderChunk(
await assertSwiftShader(session.page, readWebGlVendorInfoFromCanvas);
await initializeSession(session);
// Prime BeginFrame's `lastFrameCache` so the chunk's first real capture
// reports `hasDamage` the same as an in-process render at the same
// absolute frame would. Time is the chunk's first-frame absolute time.
const startTime = (slice.startFrame * plan.dimensions.fpsDen) / plan.dimensions.fpsNum;
await discardWarmupCapture(session, slice.startFrame, startTime);
// `discardWarmupCapture` is intentionally NOT called: every frame
// seeks fresh DOM, so `lastFrameCache` is never read; priming it
// would deadlock Chrome's compositor by issuing a second beginFrame
// at a `frameTimeTicks` it had just advanced to.
// ── Capture the chunk's range via runCaptureStage ──
await runCaptureStage({
@@ -437,7 +530,7 @@ export async function renderChunk(
needsAlpha: plan.dimensions.format !== "mp4",
captureAttempts: [],
buildCaptureOptions: () => captureOptions,
createRenderVideoFrameInjector: () => null,
createRenderVideoFrameInjector: () => videoInjector,
abortSignal: undefined,
assertNotAborted: () => {},
frameRange: { startFrame: slice.startFrame, endFrame: slice.endFrame },
@@ -10,9 +10,41 @@ import { existsSync, readFileSync } from "node:fs";
import { fileURLToPath } from "node:url";
import { promisify } from "node:util";
import { type Fps } from "@hyperframes/core";
import { type VideoElement, type VideoMetadata } from "@hyperframes/engine";
import { type RenderConfig, type RenderJob, createRenderJob } from "../renderOrchestrator.js";
import { defaultLogger, type ProducerLogger } from "../../logger.js";
/**
* Filename of the per-video extraction manifest written by `plan()` into
* `<planDir>/meta/` and consumed by `renderChunk()` to rebuild the
* BeforeCaptureHook that injects pre-extracted frames into the page.
* Absence is fine compositions with no `<video>` elements never
* produce the file.
*/
export const PLAN_VIDEOS_META_RELATIVE_PATH = "meta/videos.json";
/**
* On-disk shape of `<planDir>/meta/videos.json`. The engine's
* `ExtractedFrames` shape carries an absolute `outputDir`, a `framePaths`
* Map, and potentially an open file descriptor none of those survive
* a serialize re-deserialize round trip across processes. The
* serialized form keeps only what plan-time produced; `renderChunk` re-
* derives `outputDir` (always `<planDir>/video-frames/<videoId>`) and
* `framePaths` (re-listed from that directory) when reconstructing the
* `FrameLookupTable`.
*/
export interface PlanVideosJson {
videos: VideoElement[];
extracted: Array<{
videoId: string;
srcPath: string;
framePattern: string;
fps: number;
totalFrames: number;
metadata: VideoMetadata;
}>;
}
const execFile = promisify(execFileCallback);
/**
+156
View File
@@ -0,0 +1,156 @@
# Producer regression test fixtures
Each subdirectory under this folder is a **regression fixture** for the
HTML-to-video pipeline. The harness at
`packages/producer/src/regression-harness.ts` walks every subdirectory,
runs the composition, and PSNR-compares the rendered output against a
checked-in golden baseline.
## Fixture layout
```
<fixture-name>/
├── meta.json # name, tags, PSNR threshold, renderConfig
├── src/
│ ├── index.html # composition entry point
│ └── assets/... # any locally-referenced media
└── output/
├── compiled.html # golden compiled HTML (validated as a snapshot)
└── output.mp4 # golden rendered video
```
`meta.json` is validated by `validateMetadata` in
`src/regression-harness.ts`. The required fields are:
- `name` (string), `description` (string), `tags` (string[])
- `minPsnr` (number, dB)
- `maxFrameFailures` (integer)
- `minAudioCorrelation` (0..1), `maxAudioLagWindows` (integer ≥1)
- `renderConfig.fps` (integer like `30` or a rational string like `"30000/1001"`)
Optional `renderConfig` fields:
- `format``"mp4"` (default) or `"webm"`
- `workers` — integer ≥ 1
- `hdr` — boolean (default `false`)
- `variables` — JSON object of render-time variable overrides
- `chunkSize` — integer ≥ 1 (used by `--mode=distributed-simulated`)
- `maxParallelChunks` — integer ≥ 1 (used by `--mode=distributed-simulated`)
## Generating / updating a baseline
**Always inside Docker.** Host Chrome / FFmpeg versions drift across
distros, so a baseline captured on the host won't match the bytes CI
renders.
```bash
# From the repo root.
docker build -t hyperframes-producer:test -f Dockerfile.test .
# Generate a baseline (single fixture):
bun run --cwd packages/producer docker:test:update <fixture-name>
# Generate all baselines (rarely needed):
bun run --cwd packages/producer docker:test:update
```
The `--update` flag writes `output/compiled.html` and `output/output.mp4`
from the current render. Without `--update`, the harness compares against
those baselines.
## Running the harness locally
```bash
# Run every fixture (parallel, in-process mode — the default).
bun run --cwd packages/producer docker:test
# Run a single fixture:
bun run --cwd packages/producer docker:test font-variant-numeric
# Run sequentially (lower memory):
bun run --cwd packages/producer docker:test -- --sequential
```
## Harness modes
`--mode=<value>` chooses which render path the harness exercises:
| Mode | What it calls | Use for |
|---|---|---|
| `in-process` (default) | `executeRenderJob` | Day-to-day baselines. This is the same path the `hyperframes render` CLI takes, and it is what produced every existing `output/output.mp4`. |
| `distributed-simulated` | `plan()``renderChunk()` × N → `assemble()` from `@hyperframes/producer/distributed` | Validates the distributed pipeline against the in-process baseline. No Temporal or Lambda involvement — the controller and chunk worker are both this process. |
### `--mode=distributed-simulated`
```bash
bun run --cwd packages/producer docker:test -- --mode=distributed-simulated
bun run --cwd packages/producer docker:test font-variant-numeric -- --mode=distributed-simulated
```
The distributed pipeline cannot run every fixture. Fixtures that fail any
of these gates are **skipped** with a clear log line (and counted as
passing in the summary):
- `fps.den !== 1` — distributed mode is integer-fps only (no NTSC).
- `fps.num ∉ {24, 30, 60}` — closed set per `DistributedRenderConfig`.
- `format === "webm"``plan()` refuses webm.
- `hdr === true` — distributed mode is SDR-only at v1.
Both modes use the fixture's authored `minPsnr` as the per-test
threshold — distributed must clear the same quality bar in-process
clears against the same frozen baseline. (`DISTRIBUTED-RENDERING-PLAN.md`
§5.1's 50 dB target is a per-render distributed-vs-in-process contract;
against the frozen baseline file, neither mode reaches it consistently
due to shared encoder/JPEG-capture jitter.) An absolute 10 dB pathology
floor catches fully-black-output regressions when a fixture authors a
permissive threshold. A distributed failure at the fixture's own
threshold means the distributed pipeline has drifted — file an issue
rather than relaxing the fixture.
`--update` is incompatible with `--mode=distributed-simulated`: the
in-process renderer is the source of truth for baselines, and the
distributed mode's job is to verify the contract against the same
baseline.
### Validating PR 4.1 (the harness mode itself)
The smallest fixtures (`font-variant-numeric`, `many-cuts`) are sufficient
to verify the mode plumbing end to end:
```bash
docker build -t hyperframes-producer:test -f Dockerfile.test .
# In-process: existing behavior, unchanged.
bun run --cwd packages/producer docker:test font-variant-numeric
bun run --cwd packages/producer docker:test many-cuts
# Distributed-simulated: same baselines, distributed pipeline.
bun run --cwd packages/producer docker:test font-variant-numeric -- --mode=distributed-simulated
bun run --cwd packages/producer docker:test many-cuts -- --mode=distributed-simulated
```
Both modes must pass at each fixture's authored `minPsnr` against the
existing baseline. If `--mode=distributed-simulated` fails where
`--mode=in-process` passes, the distributed primitive has a regression —
file an issue rather than relaxing the fixture's threshold.
## Distributed-only fixtures
Fixtures under `tests/distributed/<name>/` are authored specifically for
the distributed pipeline. They follow the same `meta.json` schema as the
top-level fixtures, but they always set `chunkSize` / `maxParallelChunks`
so a `plan()` over the fixture produces N>1 chunks. Each fixture
exercises one of:
- per-format chunk-boundary correctness (mp4 H.264, mp4 H.265, ProRes, png-sequence)
- per-adapter chunk-seam state preservation (GSAP, Anime.js, Three.js, Lottie, CSS, WAAPI)
See `DISTRIBUTED-RENDERING-PLAN.md` §10.2 for the equivalence axes each
distributed fixture covers.
## Tags
Common `tags` values control which fixtures the default `bun test`
invocation runs. `--exclude-tags transparency` (the default for
`bun test`) skips webm/png-sequence alpha fixtures that need a working
chrome-headless-shell alpha pipeline.