mirror of
https://github.com/heygen-com/hyperframes.git
synced 2026-09-01 19:42:03 +00:00
Merge pull request #827 from heygen-com/05-14-feat_producer_add_harness_mode_--mode_distributed-simulated
feat(producer): add harness mode --mode=distributed-simulated
This commit is contained in:
@@ -44,10 +44,12 @@
|
||||
"bench:hdr": "tsx src/benchmark.ts --tags hdr",
|
||||
"test": "tsx src/regression-harness.ts --exclude-tags transparency",
|
||||
"test:update": "tsx src/regression-harness.ts --update --exclude-tags transparency",
|
||||
"test:distributed": "tsx src/regression-harness.ts --exclude-tags transparency --mode=distributed-simulated",
|
||||
"test:transparency": "tsx src/transparency-test.ts",
|
||||
"docker:build:test": "docker build -f ../../Dockerfile.test -t hyperframes-producer:test ../..",
|
||||
"docker:test": "docker run --rm --security-opt seccomp=unconfined --shm-size=2g -v ./tests:/app/packages/producer/tests hyperframes-producer:test",
|
||||
"docker:test:update": "docker run --rm --security-opt seccomp=unconfined --shm-size=2g -v ./tests:/app/packages/producer/tests hyperframes-producer:test --update",
|
||||
"docker:test:distributed": "docker run --rm --security-opt seccomp=unconfined --shm-size=2g -v ./tests:/app/packages/producer/tests hyperframes-producer:test --mode=distributed-simulated",
|
||||
"prepublishOnly": "echo skip"
|
||||
},
|
||||
"dependencies": {
|
||||
|
||||
@@ -0,0 +1,144 @@
|
||||
// Pure-function tests for the harness mode dispatch logic. End-to-end
|
||||
// PSNR contract lives in `Dockerfile.test` runs of the regression harness.
|
||||
|
||||
import { describe, expect, it } from "bun:test";
|
||||
import {
|
||||
checkDistributedSupport,
|
||||
DISTRIBUTED_SIMULATED_MIN_PSNR_DB,
|
||||
parseHarnessModeFlag,
|
||||
resolveMinPsnrForMode,
|
||||
} from "./regression-harness-distributed.js";
|
||||
|
||||
describe("parseHarnessModeFlag()", () => {
|
||||
it("parses --mode=in-process", () => {
|
||||
expect(parseHarnessModeFlag("--mode=in-process")).toBe("in-process");
|
||||
});
|
||||
|
||||
it("parses --mode=distributed-simulated", () => {
|
||||
expect(parseHarnessModeFlag("--mode=distributed-simulated")).toBe("distributed-simulated");
|
||||
});
|
||||
|
||||
it("returns null for tokens that aren't --mode", () => {
|
||||
expect(parseHarnessModeFlag("--update")).toBeNull();
|
||||
expect(parseHarnessModeFlag("font-variant-numeric")).toBeNull();
|
||||
expect(parseHarnessModeFlag("--exclude-tags")).toBeNull();
|
||||
});
|
||||
|
||||
it("throws on a known prefix with a bad value", () => {
|
||||
expect(() => parseHarnessModeFlag("--mode=foo")).toThrow(/--mode must be/);
|
||||
expect(() => parseHarnessModeFlag("--mode=")).toThrow(/--mode must be/);
|
||||
});
|
||||
});
|
||||
|
||||
describe("checkDistributedSupport()", () => {
|
||||
it("accepts mp4 SDR at 24 / 30 / 60 fps", () => {
|
||||
for (const fpsNum of [24, 30, 60]) {
|
||||
const result = checkDistributedSupport({ fps: { num: fpsNum, den: 1 } });
|
||||
expect(result.supported).toBe(true);
|
||||
}
|
||||
});
|
||||
|
||||
it("accepts explicit format=mp4", () => {
|
||||
const result = checkDistributedSupport({ fps: { num: 30, den: 1 }, format: "mp4" });
|
||||
expect(result.supported).toBe(true);
|
||||
});
|
||||
|
||||
it("rejects fps with non-1 denominator (NTSC)", () => {
|
||||
const result = checkDistributedSupport({ fps: { num: 30000, den: 1001 } });
|
||||
expect(result.supported).toBe(false);
|
||||
if (!result.supported) {
|
||||
expect(result.reason).toMatch(/non-integer fps/);
|
||||
}
|
||||
});
|
||||
|
||||
it("rejects fps outside the {24,30,60} set", () => {
|
||||
for (const fpsNum of [12, 25, 48, 50, 120]) {
|
||||
const result = checkDistributedSupport({ fps: { num: fpsNum, den: 1 } });
|
||||
expect(result.supported).toBe(false);
|
||||
if (!result.supported) {
|
||||
expect(result.reason).toMatch(/not in \{24, 30, 60\}/);
|
||||
}
|
||||
}
|
||||
});
|
||||
|
||||
it("rejects format=webm", () => {
|
||||
const result = checkDistributedSupport({ fps: { num: 30, den: 1 }, format: "webm" });
|
||||
expect(result.supported).toBe(false);
|
||||
if (!result.supported) {
|
||||
expect(result.reason).toMatch(/webm/);
|
||||
}
|
||||
});
|
||||
|
||||
it("rejects hdr=true", () => {
|
||||
const result = checkDistributedSupport({ fps: { num: 30, den: 1 }, hdr: true });
|
||||
expect(result.supported).toBe(false);
|
||||
if (!result.supported) {
|
||||
expect(result.reason).toMatch(/hdr/);
|
||||
}
|
||||
});
|
||||
|
||||
it("accepts hdr=false (or unset)", () => {
|
||||
expect(checkDistributedSupport({ fps: { num: 30, den: 1 }, hdr: false }).supported).toBe(true);
|
||||
expect(checkDistributedSupport({ fps: { num: 30, den: 1 } }).supported).toBe(true);
|
||||
});
|
||||
});
|
||||
|
||||
describe("resolveMinPsnrForMode()", () => {
|
||||
it("in-process mode uses the fixture's own threshold verbatim", () => {
|
||||
expect(resolveMinPsnrForMode("in-process", 30)).toBe(30);
|
||||
expect(resolveMinPsnrForMode("in-process", 50)).toBe(50);
|
||||
expect(resolveMinPsnrForMode("in-process", 60)).toBe(60);
|
||||
});
|
||||
|
||||
it("distributed-simulated uses the fixture's own minPsnr when above the absolute floor", () => {
|
||||
// Fixtures with minPsnr >= the absolute floor (catastrophic-failure
|
||||
// guard) use their authored threshold unchanged. Distributed must pass
|
||||
// the same quality bar the in-process renderer passes against the same
|
||||
// baseline — no extra tightening, since baseline drift is shared across
|
||||
// modes.
|
||||
expect(resolveMinPsnrForMode("distributed-simulated", 30)).toBe(30);
|
||||
expect(resolveMinPsnrForMode("distributed-simulated", 50)).toBe(50);
|
||||
expect(resolveMinPsnrForMode("distributed-simulated", 80)).toBe(80);
|
||||
});
|
||||
|
||||
it("distributed-simulated raises pathologically-low thresholds to the absolute floor", () => {
|
||||
// A fixture authored with minPsnr=0 (or very low) wouldn't catch a
|
||||
// distributed-mode renderer producing fully-black output. The absolute
|
||||
// floor exists to catch that pathology.
|
||||
expect(resolveMinPsnrForMode("distributed-simulated", 0)).toBe(
|
||||
DISTRIBUTED_SIMULATED_MIN_PSNR_DB,
|
||||
);
|
||||
expect(resolveMinPsnrForMode("distributed-simulated", 5)).toBe(
|
||||
DISTRIBUTED_SIMULATED_MIN_PSNR_DB,
|
||||
);
|
||||
});
|
||||
|
||||
it("every committed fixture authors a minPsnr above the absolute floor", async () => {
|
||||
// The pathology floor only fires for a fixture whose authored minPsnr
|
||||
// is below it — by design that should be no committed fixture. If
|
||||
// someone lands a permissive fixture (minPsnr: 5), distributed mode
|
||||
// will silently use 10 dB instead, which is the right behavior but
|
||||
// worth flagging so reviewers ask "is this fixture really meant to
|
||||
// accept near-black output?". This test prevents accidental misuse
|
||||
// by failing loudly when a fixture drops below the floor.
|
||||
const { readdirSync, readFileSync, statSync } = await import("node:fs");
|
||||
const { join: pathJoin } = await import("node:path");
|
||||
const testsDir = pathJoin(import.meta.dir, "..", "tests");
|
||||
const offenders: Array<{ fixture: string; minPsnr: number }> = [];
|
||||
for (const entry of readdirSync(testsDir)) {
|
||||
const metaPath = pathJoin(testsDir, entry, "meta.json");
|
||||
let stat;
|
||||
try {
|
||||
stat = statSync(metaPath);
|
||||
} catch {
|
||||
continue;
|
||||
}
|
||||
if (!stat.isFile()) continue;
|
||||
const meta = JSON.parse(readFileSync(metaPath, "utf-8")) as { minPsnr?: unknown };
|
||||
if (typeof meta.minPsnr === "number" && meta.minPsnr < DISTRIBUTED_SIMULATED_MIN_PSNR_DB) {
|
||||
offenders.push({ fixture: entry, minPsnr: meta.minPsnr });
|
||||
}
|
||||
}
|
||||
expect(offenders).toEqual([]);
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,221 @@
|
||||
/**
|
||||
* Distributed-render path for the regression harness.
|
||||
*
|
||||
* The regression harness has two modes:
|
||||
*
|
||||
* - `in-process` (default) — calls `executeRenderJob`, the same path the
|
||||
* `hyperframes render` CLI takes. This is what produced every existing
|
||||
* `tests/<name>/output/output.mp4` golden baseline.
|
||||
*
|
||||
* - `distributed-simulated` — calls `plan()` → `renderChunk()` per chunk
|
||||
* → `assemble()` from `@hyperframes/producer/distributed`. No Temporal
|
||||
* or Lambda involvement: the controller and chunk worker are both this
|
||||
* process, but they go through the same artifact (planDir + frozen
|
||||
* `meta/encoder.json` + per-chunk concat-copy) that a real fan-out
|
||||
* would.
|
||||
*
|
||||
* Both modes share the per-fixture `minPsnr` threshold — distributed must
|
||||
* pass the same quality bar the in-process renderer passes against the
|
||||
* same frozen baseline. A separate {@link DISTRIBUTED_SIMULATED_MIN_PSNR_DB}
|
||||
* pathology floor catches the case where a fixture authored a permissive
|
||||
* threshold and distributed regresses to fully-black output. The §5.1
|
||||
* 50 dB target was written for per-render comparison (fresh in-process vs
|
||||
* fresh distributed); against the frozen baseline file it's unreachable
|
||||
* for either mode due to shared encoder/JPEG-capture jitter, so the
|
||||
* harness can't use it as a per-test gate.
|
||||
*
|
||||
* Not every fixture can run in distributed-simulated mode. Distributed mode
|
||||
* refuses webm, HDR mp4, NTSC framerates, and non-{24,30,60} fps at plan
|
||||
* time. Fixtures that don't meet the constraints are skipped — the harness
|
||||
* logs the reason and the fixture is treated as "passed (skipped)" in
|
||||
* distributed-simulated mode.
|
||||
*/
|
||||
|
||||
import { existsSync, mkdirSync } from "node:fs";
|
||||
import { join } from "node:path";
|
||||
import type { Fps } from "@hyperframes/core";
|
||||
import { assemble, plan, renderChunk } from "./distributed.js";
|
||||
|
||||
/** Two-mode contract that backs `--mode=<value>` on the regression harness CLI. */
|
||||
export type HarnessMode = "in-process" | "distributed-simulated";
|
||||
|
||||
/**
|
||||
* Absolute pathology floor for `--mode=distributed-simulated` — catches
|
||||
* a chunk that renders fully-black against a fixture authored with a
|
||||
* permissive `minPsnr`. Non-pathological drift is caught by the fixture's
|
||||
* own threshold; both modes share the same encoder/JPEG-capture jitter
|
||||
* floor against the frozen baseline file, so the §5.1 50 dB target is
|
||||
* unreachable for either mode and isn't a useful per-test gate.
|
||||
*/
|
||||
export const DISTRIBUTED_SIMULATED_MIN_PSNR_DB = 10;
|
||||
|
||||
/** Result of {@link checkDistributedSupport}. */
|
||||
export type DistributedSupportResult = { supported: true } | { supported: false; reason: string };
|
||||
|
||||
/**
|
||||
* Decide whether a fixture's `renderConfig` is one the distributed pipeline
|
||||
* can actually run. The four hard gates:
|
||||
*
|
||||
* - fps must be `{ num: 24|30|60, den: 1 }`. `DistributedRenderConfig.fps`
|
||||
* accepts only the three integer values, and rationals like
|
||||
* `{ num: 30000, den: 1001 }` (NTSC) trip the type system at the call
|
||||
* site. We surface this gate in code rather than only in TS so the
|
||||
* harness can skip the fixture cleanly instead of throwing.
|
||||
* - format must not be `webm`. `plan()` refuses webm with
|
||||
* `FORMAT_NOT_SUPPORTED_IN_DISTRIBUTED`.
|
||||
* - hdr must not be `true`. Distributed mode is SDR-only at v1.
|
||||
*
|
||||
* Callers that want the structured reason can read it off the returned
|
||||
* `reason` field; the message is intended to be log-friendly.
|
||||
*/
|
||||
export function checkDistributedSupport(renderConfig: {
|
||||
fps: Fps;
|
||||
format?: "mp4" | "webm";
|
||||
hdr?: boolean;
|
||||
}): DistributedSupportResult {
|
||||
if (renderConfig.fps.den !== 1) {
|
||||
return {
|
||||
supported: false,
|
||||
reason: `non-integer fps ${renderConfig.fps.num}/${renderConfig.fps.den} (distributed mode requires fps.den=1)`,
|
||||
};
|
||||
}
|
||||
const fpsNum = renderConfig.fps.num;
|
||||
if (fpsNum !== 24 && fpsNum !== 30 && fpsNum !== 60) {
|
||||
return {
|
||||
supported: false,
|
||||
reason: `fps ${fpsNum} not in {24, 30, 60} (DistributedRenderConfig.fps is a closed set)`,
|
||||
};
|
||||
}
|
||||
const format = renderConfig.format ?? "mp4";
|
||||
if (format === "webm") {
|
||||
return {
|
||||
supported: false,
|
||||
reason: "format=webm refused in distributed mode (VP9+matroska concat-copy is unstable)",
|
||||
};
|
||||
}
|
||||
if (renderConfig.hdr === true) {
|
||||
return {
|
||||
supported: false,
|
||||
reason: "hdr=true refused in distributed mode (HDR signaling re-apply not implemented)",
|
||||
};
|
||||
}
|
||||
return { supported: true };
|
||||
}
|
||||
|
||||
/**
|
||||
* Inputs for {@link runDistributedSimulatedRender}. The harness has already
|
||||
* prepared `projectDir` (a working copy of the fixture's `src/` directory)
|
||||
* and `tempRoot` (where the harness writes its scratch artifacts).
|
||||
*/
|
||||
export interface RunDistributedSimulatedInput {
|
||||
/** Working copy of the fixture's `src/` — contains `index.html`. */
|
||||
projectDir: string;
|
||||
/** Scratch root for plan + chunks; must be a directory the harness owns. */
|
||||
tempRoot: string;
|
||||
/** Where to write the assembled final mp4 / mov / png-sequence directory. */
|
||||
renderedOutputPath: string;
|
||||
/** From the fixture's renderConfig — must pass `checkDistributedSupport`. */
|
||||
fps: 24 | 30 | 60;
|
||||
format: "mp4" | "mov" | "png-sequence";
|
||||
/** Optional chunkSize override; defaults to the plan's 240. */
|
||||
chunkSize?: number;
|
||||
/** Optional maxParallelChunks override; defaults to the plan's 16. */
|
||||
maxParallelChunks?: number;
|
||||
/** Forwarded to `plan()` and re-applied by `renderChunk()` at boot. */
|
||||
variables?: Record<string, unknown>;
|
||||
}
|
||||
|
||||
/**
|
||||
* Run the distributed pipeline against a single fixture as if a fan-out
|
||||
* adapter were driving it. The three activities run serially in this
|
||||
* process — there is no Temporal, no Lambda, no S3 — so the planDir,
|
||||
* chunk outputs, and assembled output all live under `tempRoot`.
|
||||
*
|
||||
* Width and height are required by `DistributedRenderConfig` for cross-call
|
||||
* sanity but are not consulted at render time — `plan()` reads the
|
||||
* composition's `data-width` / `data-height` attributes and overrides
|
||||
* whatever the config carried. The harness passes a dummy 1920×1080 here
|
||||
* for that reason; if the contract ever changes, the fixture's authored
|
||||
* dimensions will flow through `PlanResult` and we can switch to using
|
||||
* those instead.
|
||||
*/
|
||||
export async function runDistributedSimulatedRender(
|
||||
input: RunDistributedSimulatedInput,
|
||||
): Promise<void> {
|
||||
const planDir = join(input.tempRoot, "plan");
|
||||
const chunksDir = join(input.tempRoot, "chunks");
|
||||
mkdirSync(planDir, { recursive: true });
|
||||
mkdirSync(chunksDir, { recursive: true });
|
||||
|
||||
// Step A: plan.
|
||||
const planResult = await plan(
|
||||
input.projectDir,
|
||||
{
|
||||
fps: input.fps,
|
||||
// Required-by-type but overridden by the composition's own attrs;
|
||||
// see docstring above. Any positive integer works.
|
||||
width: 1920,
|
||||
height: 1080,
|
||||
format: input.format,
|
||||
chunkSize: input.chunkSize,
|
||||
maxParallelChunks: input.maxParallelChunks,
|
||||
// Force the SDR path explicitly — `auto` would still resolve to
|
||||
// force-sdr in distributed mode, but pinning it here keeps the
|
||||
// harness's behavior independent of any future auto-detect changes.
|
||||
hdrMode: "force-sdr",
|
||||
},
|
||||
planDir,
|
||||
);
|
||||
|
||||
// Step B: render every chunk. Sequential to keep the harness predictable —
|
||||
// adapters in production are free to fan out; this code path's job is to
|
||||
// exercise the per-chunk activity itself.
|
||||
const chunkPaths: string[] = [];
|
||||
for (let i = 0; i < planResult.chunkCount; i++) {
|
||||
const chunkPath =
|
||||
input.format === "png-sequence"
|
||||
? join(chunksDir, `chunk-${String(i).padStart(4, "0")}`)
|
||||
: join(chunksDir, `chunk-${String(i).padStart(4, "0")}.${input.format}`);
|
||||
await renderChunk(planDir, i, chunkPath);
|
||||
chunkPaths.push(chunkPath);
|
||||
}
|
||||
|
||||
// Step C: assemble. `audio.aac` only exists when the composition has
|
||||
// audio — pass null otherwise so `assemble()` doesn't try to mux silence.
|
||||
const audioPath = join(planDir, "audio.aac");
|
||||
const audioForAssemble = existsSync(audioPath) ? audioPath : null;
|
||||
await assemble(planDir, chunkPaths, audioForAssemble, input.renderedOutputPath);
|
||||
}
|
||||
|
||||
/**
|
||||
* Pick the PSNR threshold for a fixture given the harness mode. Both modes
|
||||
* share the fixture's authored `minPsnr` — distributed must clear the same
|
||||
* quality bar in-process clears against the same frozen baseline.
|
||||
* Distributed-simulated additionally lifts the threshold to
|
||||
* {@link DISTRIBUTED_SIMULATED_MIN_PSNR_DB} for fixtures with a permissive
|
||||
* authored threshold; that absolute floor catches fully-black-output
|
||||
* regressions independent of fixture tolerance.
|
||||
*/
|
||||
export function resolveMinPsnrForMode(mode: HarnessMode, fixtureMinPsnr: number): number {
|
||||
if (mode === "in-process") return fixtureMinPsnr;
|
||||
return Math.max(fixtureMinPsnr, DISTRIBUTED_SIMULATED_MIN_PSNR_DB);
|
||||
}
|
||||
|
||||
/**
|
||||
* Parse `--mode=<value>` from a single CLI token. Returns the parsed mode
|
||||
* when the token matches the expected shape, `null` otherwise so the
|
||||
* caller can pass the token through to the next handler. Throws on a
|
||||
* known prefix with a bad value (`--mode=foo`) — surfacing a typo at
|
||||
* parse time is cheaper than discovering at render time.
|
||||
*/
|
||||
export function parseHarnessModeFlag(token: string): HarnessMode | null {
|
||||
if (token === "--mode=in-process") return "in-process";
|
||||
if (token === "--mode=distributed-simulated") return "distributed-simulated";
|
||||
if (token.startsWith("--mode=")) {
|
||||
const value = token.slice("--mode=".length);
|
||||
throw new Error(
|
||||
`regression-harness: --mode must be 'in-process' or 'distributed-simulated' (got ${JSON.stringify(value)})`,
|
||||
);
|
||||
}
|
||||
return null;
|
||||
}
|
||||
@@ -20,6 +20,13 @@ import { validateCompilation } from "./services/compilationTester.js";
|
||||
import { extractMediaMetadata } from "./utils/ffprobe.js";
|
||||
import { buildRmsEnvelope, compareAudioEnvelopes } from "./utils/audioRegression.js";
|
||||
import { parseFps, fpsToNumber } from "@hyperframes/core";
|
||||
import {
|
||||
checkDistributedSupport,
|
||||
type HarnessMode,
|
||||
parseHarnessModeFlag,
|
||||
resolveMinPsnrForMode,
|
||||
runDistributedSimulatedRender,
|
||||
} from "./regression-harness-distributed.js";
|
||||
|
||||
// ── Types ────────────────────────────────────────────────────────────────────
|
||||
|
||||
@@ -51,6 +58,18 @@ type TestMetadata = {
|
||||
* and these overrides. Omit when the test doesn't exercise variables.
|
||||
*/
|
||||
variables?: Record<string, unknown>;
|
||||
/**
|
||||
* Chunk size in frames for `--mode=distributed-simulated`. Forwarded
|
||||
* to `DistributedRenderConfig.chunkSize`. Ignored in `--mode=in-process`.
|
||||
* Default is the plan's own default (240 frames).
|
||||
*/
|
||||
chunkSize?: number;
|
||||
/**
|
||||
* Cap on parallel chunks for `--mode=distributed-simulated`. Forwarded
|
||||
* to `DistributedRenderConfig.maxParallelChunks`. Ignored in
|
||||
* `--mode=in-process`. Default is the plan's own default (16).
|
||||
*/
|
||||
maxParallelChunks?: number;
|
||||
};
|
||||
};
|
||||
|
||||
@@ -67,11 +86,26 @@ type CliOptions = {
|
||||
update: boolean;
|
||||
sequential: boolean;
|
||||
keepTemp: boolean;
|
||||
/**
|
||||
* Which render path to exercise. `in-process` (default) calls
|
||||
* `executeRenderJob`; `distributed-simulated` calls
|
||||
* `plan() → renderChunk() × N → assemble()` from
|
||||
* `@hyperframes/producer/distributed`. See
|
||||
* `regression-harness-distributed.ts`.
|
||||
*/
|
||||
mode: HarnessMode;
|
||||
};
|
||||
|
||||
type TestResult = {
|
||||
suite: TestSuite;
|
||||
passed: boolean;
|
||||
/**
|
||||
* Set when `--mode=distributed-simulated` skips a fixture that the
|
||||
* distributed pipeline can't run (webm, HDR, NTSC fps, fps∉{24,30,60}).
|
||||
* `passed` is `true` for skipped fixtures — skipping is a clean outcome,
|
||||
* not a failure — but the summary distinguishes them.
|
||||
*/
|
||||
skipped?: { reason: string };
|
||||
compilation?: {
|
||||
passed: boolean;
|
||||
errors: string[];
|
||||
@@ -105,6 +139,7 @@ function parseArgs(argv: string[]): CliOptions {
|
||||
let update = false;
|
||||
let sequential = false;
|
||||
let keepTemp = false;
|
||||
let mode: HarnessMode = "in-process";
|
||||
|
||||
for (let i = 2; i < argv.length; i += 1) {
|
||||
const token = argv[i];
|
||||
@@ -119,12 +154,29 @@ function parseArgs(argv: string[]): CliOptions {
|
||||
i += 1;
|
||||
const tagArg = argv[i];
|
||||
if (tagArg) excludeTags.push(...tagArg.split(","));
|
||||
} else if (!token.startsWith("--")) {
|
||||
testNames.push(token);
|
||||
} else {
|
||||
const parsedMode = parseHarnessModeFlag(token);
|
||||
if (parsedMode !== null) {
|
||||
mode = parsedMode;
|
||||
} else if (!token.startsWith("--")) {
|
||||
testNames.push(token);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return { testNames, excludeTags, update, sequential, keepTemp };
|
||||
if (update && mode === "distributed-simulated") {
|
||||
// The in-process renderer is the source of truth for golden baselines —
|
||||
// distributed-simulated's job is to verify the contract against the
|
||||
// same baseline, not to author its own. Surfacing this at parse time
|
||||
// saves a multi-minute render before the user notices.
|
||||
throw new Error(
|
||||
"regression-harness: --update is incompatible with --mode=distributed-simulated. " +
|
||||
"Generate baselines with the in-process renderer (the default mode), then re-run " +
|
||||
"without --update to verify both modes match.",
|
||||
);
|
||||
}
|
||||
|
||||
return { testNames, excludeTags, update, sequential, keepTemp, mode };
|
||||
}
|
||||
|
||||
function validateMetadata(meta: unknown): TestMetadata {
|
||||
@@ -195,6 +247,20 @@ function validateMetadata(meta: unknown): TestMetadata {
|
||||
) {
|
||||
throw new Error("meta.json: 'renderConfig.variables' must be a JSON object (or omitted)");
|
||||
}
|
||||
if (rc.chunkSize !== undefined) {
|
||||
if (!Number.isInteger(rc.chunkSize) || (rc.chunkSize as number) < 1) {
|
||||
throw new Error(
|
||||
"meta.json: 'renderConfig.chunkSize' must be a positive integer (or omitted)",
|
||||
);
|
||||
}
|
||||
}
|
||||
if (rc.maxParallelChunks !== undefined) {
|
||||
if (!Number.isInteger(rc.maxParallelChunks) || (rc.maxParallelChunks as number) < 1) {
|
||||
throw new Error(
|
||||
"meta.json: 'renderConfig.maxParallelChunks' must be a positive integer (or omitted)",
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
return m as TestMetadata;
|
||||
}
|
||||
@@ -399,6 +465,7 @@ function saveFailureDetails(
|
||||
result: TestResult,
|
||||
renderedVideoPath: string,
|
||||
snapshotVideoPath: string,
|
||||
effectiveMinPsnr: number,
|
||||
compiledHtml?: string,
|
||||
snapshotHtml?: string,
|
||||
): void {
|
||||
@@ -441,12 +508,13 @@ function saveFailureDetails(
|
||||
summary: {
|
||||
totalCheckpoints: result.visual.checkpoints.length,
|
||||
failedCheckpoints: failedCheckpoints.length,
|
||||
threshold: suite.meta.minPsnr,
|
||||
threshold: effectiveMinPsnr,
|
||||
fixtureThreshold: suite.meta.minPsnr,
|
||||
},
|
||||
failedFrames: failedCheckpoints.map((c) => ({
|
||||
time: c.time,
|
||||
psnr: c.psnr,
|
||||
belowThresholdBy: suite.meta.minPsnr - c.psnr,
|
||||
belowThresholdBy: effectiveMinPsnr - c.psnr,
|
||||
})),
|
||||
};
|
||||
|
||||
@@ -522,6 +590,7 @@ async function runTestSuite(
|
||||
options: {
|
||||
update: boolean;
|
||||
keepTemp: boolean;
|
||||
mode: HarnessMode;
|
||||
},
|
||||
): Promise<TestResult> {
|
||||
// Use predictable temp location: /tmp/hyperframes-tests/{test-id}/
|
||||
@@ -621,25 +690,65 @@ async function runTestSuite(
|
||||
}
|
||||
|
||||
// STEP 2: Render video
|
||||
console.log(JSON.stringify({ event: "rendering_start", suite: suite.id }));
|
||||
logPretty("Rendering video...", "🎬");
|
||||
console.log(JSON.stringify({ event: "rendering_start", suite: suite.id, mode: options.mode }));
|
||||
logPretty(`Rendering video (mode=${options.mode})...`, "🎬");
|
||||
|
||||
const tempSrcDir = join(tempRoot, "src");
|
||||
copyFixtureSupportFiles(suite, tempRoot);
|
||||
cpSync(suite.srcDir, tempSrcDir, { recursive: true });
|
||||
|
||||
const job = createRenderJob({
|
||||
fps: suite.meta.renderConfig.fps,
|
||||
quality: "high", // Always use max quality for tests
|
||||
format: outputFormat,
|
||||
workers: suite.meta.renderConfig.workers,
|
||||
useGpu: false,
|
||||
debug: false,
|
||||
hdrMode: suite.meta.renderConfig.hdr ? "force-hdr" : "force-sdr",
|
||||
variables: suite.meta.renderConfig.variables,
|
||||
});
|
||||
if (options.mode === "distributed-simulated") {
|
||||
const support = checkDistributedSupport(suite.meta.renderConfig);
|
||||
if (!support.supported) {
|
||||
// Skipping is a clean outcome — the distributed pipeline can't
|
||||
// run this fixture, but in-process mode already covers it. Mark
|
||||
// passed so the suite summary doesn't trip CI; the `skipped`
|
||||
// field is what distinguishes a real pass from a skip.
|
||||
console.log(
|
||||
JSON.stringify({
|
||||
event: "test_skipped",
|
||||
suite: suite.id,
|
||||
mode: options.mode,
|
||||
reason: support.reason,
|
||||
}),
|
||||
);
|
||||
logPretty(`Skipping ${suite.meta.name} (mode=${options.mode}): ${support.reason}`, "⏭️");
|
||||
result.passed = true;
|
||||
result.skipped = { reason: support.reason };
|
||||
return result;
|
||||
}
|
||||
// `checkDistributedSupport` already narrowed fps to {24,30,60} and
|
||||
// rejected webm; the cast surfaces that guarantee to TS.
|
||||
const fpsNum = suite.meta.renderConfig.fps.num as 24 | 30 | 60;
|
||||
// `validateMetadata` only accepts `format: "mp4" | "webm"` in
|
||||
// `renderConfig`, and `checkDistributedSupport` rejected webm above,
|
||||
// so by here only `mp4` (or the unset default) can reach this call.
|
||||
// If the metadata schema grows to accept "mov" / "png-sequence"
|
||||
// someday, narrow this cast accordingly.
|
||||
await runDistributedSimulatedRender({
|
||||
projectDir: tempSrcDir,
|
||||
tempRoot,
|
||||
renderedOutputPath,
|
||||
fps: fpsNum,
|
||||
format: "mp4",
|
||||
chunkSize: suite.meta.renderConfig.chunkSize,
|
||||
maxParallelChunks: suite.meta.renderConfig.maxParallelChunks,
|
||||
variables: suite.meta.renderConfig.variables,
|
||||
});
|
||||
} else {
|
||||
const job = createRenderJob({
|
||||
fps: suite.meta.renderConfig.fps,
|
||||
quality: "high", // Always use max quality for tests
|
||||
format: outputFormat,
|
||||
workers: suite.meta.renderConfig.workers,
|
||||
useGpu: false,
|
||||
debug: false,
|
||||
hdrMode: suite.meta.renderConfig.hdr ? "force-hdr" : "force-sdr",
|
||||
variables: suite.meta.renderConfig.variables,
|
||||
});
|
||||
|
||||
await executeRenderJob(job, tempSrcDir, renderedOutputPath);
|
||||
await executeRenderJob(job, tempSrcDir, renderedOutputPath);
|
||||
}
|
||||
|
||||
console.log(JSON.stringify({ event: "rendering_complete", suite: suite.id }));
|
||||
logPretty("Render complete! Starting quality validation...", "✓");
|
||||
@@ -680,6 +789,7 @@ async function runTestSuite(
|
||||
// which causes ffmpeg's PSNR filter to emit no `average:` line.
|
||||
const videoDuration = Math.min(videoMetadata.durationSeconds, snapshotMetadata.durationSeconds);
|
||||
|
||||
const minPsnrForMode = resolveMinPsnrForMode(options.mode, suite.meta.minPsnr);
|
||||
const visualCheckpoints: Array<{ time: number; psnr: number; passed: boolean }> = [];
|
||||
for (let i = 0; i < 100; i++) {
|
||||
const time = (videoDuration * i) / 100;
|
||||
@@ -692,7 +802,7 @@ async function runTestSuite(
|
||||
visualCheckpoints.push({
|
||||
time,
|
||||
psnr,
|
||||
passed: psnr >= suite.meta.minPsnr,
|
||||
passed: psnr >= minPsnrForMode,
|
||||
});
|
||||
|
||||
// Progress indicator every 20 checkpoints
|
||||
@@ -815,6 +925,7 @@ async function runTestSuite(
|
||||
result,
|
||||
renderedOutputPath,
|
||||
snapshotVideoPath,
|
||||
resolveMinPsnrForMode(options.mode, suite.meta.minPsnr),
|
||||
compiledHtml,
|
||||
snapshotHtml,
|
||||
);
|
||||
@@ -857,11 +968,13 @@ async function run(): Promise<void> {
|
||||
event: "test_suite_start",
|
||||
totalSuites: suites.length,
|
||||
parallel: !options.sequential,
|
||||
mode: options.mode,
|
||||
}),
|
||||
);
|
||||
|
||||
logPretty(
|
||||
`Starting ${suites.length} test suite(s) - ${options.sequential ? "sequential" : "parallel"} mode`,
|
||||
`Starting ${suites.length} test suite(s) - ${options.sequential ? "sequential" : "parallel"} mode, ` +
|
||||
`harness mode=${options.mode}`,
|
||||
"🚀",
|
||||
);
|
||||
|
||||
@@ -925,7 +1038,8 @@ async function run(): Promise<void> {
|
||||
);
|
||||
logPretty(`Updated ${results.length} snapshot(s)`, "📸");
|
||||
} else {
|
||||
const passed = results.filter((r) => r.passed).length;
|
||||
const skipped = results.filter((r) => r.skipped).length;
|
||||
const passed = results.filter((r) => r.passed && !r.skipped).length;
|
||||
const failed = results.filter((r) => !r.passed).length;
|
||||
const failedAtCompilation = results.filter(
|
||||
(r) => r.compilation && !r.compilation.passed,
|
||||
@@ -939,6 +1053,8 @@ async function run(): Promise<void> {
|
||||
total: results.length,
|
||||
passed,
|
||||
failed,
|
||||
skipped,
|
||||
mode: options.mode,
|
||||
failedAtCompilation,
|
||||
failedAtVisual,
|
||||
failedAtAudio,
|
||||
@@ -946,6 +1062,7 @@ async function run(): Promise<void> {
|
||||
suite: r.suite.id,
|
||||
name: r.suite.meta.name,
|
||||
passed: r.passed,
|
||||
skipped: r.skipped?.reason,
|
||||
compilation: r.compilation?.passed,
|
||||
visual: r.visual?.passed,
|
||||
audio: r.audio?.passed,
|
||||
@@ -955,8 +1072,11 @@ async function run(): Promise<void> {
|
||||
|
||||
// Pretty summary
|
||||
logPretty("═══════════════════════════════════════", "");
|
||||
logPretty(`Test Suite Summary`, "📊");
|
||||
logPretty(`Total: ${results.length} | Passed: ${passed} | Failed: ${failed}`, "");
|
||||
logPretty(`Test Suite Summary (mode=${options.mode})`, "📊");
|
||||
logPretty(
|
||||
`Total: ${results.length} | Passed: ${passed} | Failed: ${failed} | Skipped: ${skipped}`,
|
||||
"",
|
||||
);
|
||||
if (failed > 0) {
|
||||
logPretty(` Failed at compilation: ${failedAtCompilation}`, "");
|
||||
logPretty(` Failed at visual: ${failedAtVisual}`, "");
|
||||
|
||||
@@ -24,8 +24,17 @@
|
||||
* never have to handle them.
|
||||
*/
|
||||
|
||||
import { existsSync, mkdirSync, readdirSync, renameSync, rmSync, statSync } from "node:fs";
|
||||
import { join } from "node:path";
|
||||
import {
|
||||
cpSync,
|
||||
existsSync,
|
||||
mkdirSync,
|
||||
readdirSync,
|
||||
renameSync,
|
||||
rmSync,
|
||||
statSync,
|
||||
writeFileSync,
|
||||
} from "node:fs";
|
||||
import { join, relative, sep } from "node:path";
|
||||
import { type CanvasResolution } from "@hyperframes/core";
|
||||
import { type EngineConfig, resolveConfig } from "@hyperframes/engine";
|
||||
import { defaultLogger, type ProducerLogger } from "../../logger.js";
|
||||
@@ -46,7 +55,13 @@ import {
|
||||
} from "../render/stages/planHash.js";
|
||||
import { validateNoGpuEncode, validateNoSystemFonts } from "../render/planValidation.js";
|
||||
import { snapshotRuntimeEnv } from "../render/runtimeEnvSnapshot.js";
|
||||
import { buildSyntheticRenderJob, readFfmpegVersion, readProducerVersion } from "./shared.js";
|
||||
import {
|
||||
buildSyntheticRenderJob,
|
||||
PLAN_VIDEOS_META_RELATIVE_PATH,
|
||||
type PlanVideosJson,
|
||||
readFfmpegVersion,
|
||||
readProducerVersion,
|
||||
} from "./shared.js";
|
||||
|
||||
/**
|
||||
* Caller-supplied configuration for a distributed render. `fps`, `width`,
|
||||
@@ -131,6 +146,26 @@ export interface PlanResult {
|
||||
producerVersion: string;
|
||||
}
|
||||
|
||||
/**
|
||||
* Top-level directory names skipped by the `projectDir → planDir/compiled/`
|
||||
* pre-seed copy. Real projects often contain `node_modules/`, VCS metadata,
|
||||
* and harness artifacts that have no business in a planDir — they bloat
|
||||
* the 2 GB planDir cap and slow the S3/Lambda round-trip for no benefit.
|
||||
* Matched against the path relative to `projectDir` so a `projectDir`
|
||||
* whose absolute path happens to contain one of these names (e.g.
|
||||
* `~/work/output/comp/`) doesn't false-positive-skip the entire copy.
|
||||
*/
|
||||
const PLAN_PROJECT_DIR_SKIP_SEGMENTS = new Set([
|
||||
"node_modules",
|
||||
".git",
|
||||
".cache",
|
||||
"output",
|
||||
"failures",
|
||||
"dist",
|
||||
".next",
|
||||
".turbo",
|
||||
]);
|
||||
|
||||
/** Default chunk size in frames (~8s @ 30fps; fits Lambda's 15-min cap). */
|
||||
export const DEFAULT_CHUNK_SIZE = 240;
|
||||
/** Default cap on parallel chunks for operational fairness across renders. */
|
||||
@@ -492,6 +527,28 @@ export async function plan(
|
||||
if (!existsSync(workDir)) mkdirSync(workDir, { recursive: true });
|
||||
const compiledDir = join(workDir, "compiled");
|
||||
|
||||
// Pre-seed the compiled directory with `projectDir`'s local assets
|
||||
// (style.css, script.js, images, etc.). The chunk worker's file server
|
||||
// serves ONLY from `<planDir>/compiled/`, so without this copy a
|
||||
// composition's `<link rel=stylesheet href=style.css>` 404s and the
|
||||
// first capture lands an unstyled fallback frame. `compileStage`
|
||||
// overwrites the entry HTML afterwards. `dereference: true` resolves
|
||||
// symlinks so the planDir survives S3 / Lambda /tmp round-trips.
|
||||
mkdirSync(compiledDir, { recursive: true });
|
||||
cpSync(projectDir, compiledDir, {
|
||||
recursive: true,
|
||||
dereference: true,
|
||||
filter: (src) => {
|
||||
// cpSync passes the absolute source path. Compare relative-to-projectDir
|
||||
// so a parent directory of projectDir matching a skip name doesn't
|
||||
// false-positive every descendant.
|
||||
const rel = relative(projectDir, src);
|
||||
if (rel === "" || rel.startsWith("..")) return true;
|
||||
const firstSegment = rel.split(sep, 1)[0];
|
||||
return firstSegment === undefined || !PLAN_PROJECT_DIR_SKIP_SEGMENTS.has(firstSegment);
|
||||
},
|
||||
});
|
||||
|
||||
// The compiled directory lives at `<planDir>/compiled/` in the final
|
||||
// layout. The stages write under `<planDir>/.plan-work/compiled/`; we
|
||||
// move the contents over once the staged work completes.
|
||||
@@ -582,7 +639,9 @@ export async function plan(
|
||||
assertNotAborted,
|
||||
materializeSymlinks: true,
|
||||
});
|
||||
if (extractResult.frameLookup) extractResult.frameLookup.cleanup();
|
||||
// Skip `extractResult.frameLookup.cleanup()`: it would rm-rf each
|
||||
// video's outputDir, but in `plan()` those directories ARE the source
|
||||
// material the renames below move into `planDir/video-frames/`.
|
||||
|
||||
// ── Audio ──
|
||||
const audioResult = await runAudioStage({
|
||||
@@ -613,6 +672,30 @@ export async function plan(
|
||||
if (existsSync(finalCompiledDir)) rmSync(finalCompiledDir, { recursive: true, force: true });
|
||||
renameSync(compiledDir, finalCompiledDir);
|
||||
|
||||
// `meta/videos.json` is the contract that makes distributed renders
|
||||
// pixel-comparable to in-process for compositions with video sources —
|
||||
// without it, renderChunk can't rebuild the BeforeCaptureHook and the
|
||||
// page's native `<video>` element decodes the source mp4 ~1 frame
|
||||
// off the pre-extracted images the in-process baseline was captured
|
||||
// from.
|
||||
const planVideosJson: PlanVideosJson = {
|
||||
videos: composition.videos,
|
||||
extracted: (extractResult.extractionResult?.extracted ?? []).map((ext) => ({
|
||||
videoId: ext.videoId,
|
||||
srcPath: ext.srcPath,
|
||||
framePattern: ext.framePattern,
|
||||
fps: ext.fps,
|
||||
totalFrames: ext.totalFrames,
|
||||
metadata: ext.metadata,
|
||||
})),
|
||||
};
|
||||
mkdirSync(join(planDir, "meta"), { recursive: true });
|
||||
writeFileSync(
|
||||
join(planDir, PLAN_VIDEOS_META_RELATIVE_PATH),
|
||||
JSON.stringify(planVideosJson, null, 2),
|
||||
"utf-8",
|
||||
);
|
||||
|
||||
const planAudioPath = join(planDir, "audio.aac");
|
||||
if (audioResult.hasAudio && existsSync(audioResult.audioOutputPath)) {
|
||||
renameSync(audioResult.audioOutputPath, planAudioPath);
|
||||
|
||||
@@ -25,8 +25,8 @@
|
||||
* (`buildVirtualTimeShim({ seedRandomFromFrame: true })`) so any
|
||||
* composition that uses `Math.random` / `crypto.getRandomValues`
|
||||
* produces byte-identical pixels per `(planDir, chunkIndex)`.
|
||||
* - One `discardWarmupCapture` runs before the chunk's first real frame
|
||||
* to prime the BeginFrame `lastFrameCache`.
|
||||
* - No `lastFrameCache` priming: every frame seeks fresh DOM so the
|
||||
* cache is never read, and priming would deadlock the compositor.
|
||||
* - The chunk's encode runs with `lockGopForChunkConcat: true` and
|
||||
* `gopSize === framesInChunk` so concat-copy at assemble time is safe.
|
||||
*
|
||||
@@ -36,17 +36,20 @@
|
||||
|
||||
import { randomBytes } from "node:crypto";
|
||||
import { existsSync, mkdirSync, readFileSync, readdirSync, rmSync, writeFileSync } from "node:fs";
|
||||
import { join } from "node:path";
|
||||
import { extname, join } from "node:path";
|
||||
import type { Page } from "puppeteer-core";
|
||||
import {
|
||||
assertSwiftShader,
|
||||
type BeforeCaptureHook,
|
||||
BROWSER_GPU_NOT_SOFTWARE,
|
||||
type CaptureOptions,
|
||||
type CaptureSession,
|
||||
closeCaptureSession,
|
||||
createCaptureSession,
|
||||
discardWarmupCapture,
|
||||
createFrameLookupTable,
|
||||
createVideoFrameInjector,
|
||||
type EngineConfig,
|
||||
type ExtractedFrames,
|
||||
getEncoderPreset,
|
||||
initializeSession,
|
||||
resolveConfig,
|
||||
@@ -62,7 +65,12 @@ import {
|
||||
import { sha256Hex } from "../render/stages/planHash.js";
|
||||
import { applyRuntimeEnvSnapshot } from "../render/runtimeEnvSnapshot.js";
|
||||
import { buildVirtualTimeShim, createFileServer, type FileServerHandle } from "../fileServer.js";
|
||||
import { buildSyntheticRenderJob, readFfmpegVersion } from "./shared.js";
|
||||
import {
|
||||
buildSyntheticRenderJob,
|
||||
PLAN_VIDEOS_META_RELATIVE_PATH,
|
||||
type PlanVideosJson,
|
||||
readFfmpegVersion,
|
||||
} from "./shared.js";
|
||||
|
||||
/**
|
||||
* Non-retryable error codes raised when the planDir is structurally
|
||||
@@ -124,6 +132,61 @@ export interface ChunkResult {
|
||||
perfPath: string;
|
||||
}
|
||||
|
||||
/**
|
||||
* Rebuild the engine's in-memory `ExtractedFrames[]` from the on-disk
|
||||
* planDir layout. `<planDir>/video-frames/<videoId>/` holds the numbered
|
||||
* frame files plan() extracted; this lists each dir and rebuilds the
|
||||
* 1-based `framePaths` Map that `FrameLookupTable` / `videoFrameInjector`
|
||||
* both index against.
|
||||
*/
|
||||
function rebuildExtractedFramesFromPlanDir(
|
||||
planDir: string,
|
||||
videos: PlanVideosJson["extracted"],
|
||||
): ExtractedFrames[] {
|
||||
const result: ExtractedFrames[] = [];
|
||||
for (const v of videos) {
|
||||
const outputDir = join(planDir, "video-frames", v.videoId);
|
||||
if (!existsSync(outputDir)) {
|
||||
throw new Error(
|
||||
`[renderChunk] planDir missing extracted video frames for ${JSON.stringify(v.videoId)}: ` +
|
||||
`${outputDir} not present. plan() should have written frames here; the planDir is malformed.`,
|
||||
);
|
||||
}
|
||||
// framePattern looks like `frame_%05d.jpg`; sprintf isn't available at
|
||||
// runtime so list-and-sort the directory. Sorted-by-name matches
|
||||
// sorted-by-frame-index because the extractor writes zero-padded
|
||||
// monotonic indices.
|
||||
const ext = (extname(v.framePattern) || ".jpg").toLowerCase();
|
||||
const frames = readdirSync(outputDir)
|
||||
.filter((name) => name.toLowerCase().endsWith(ext))
|
||||
.sort();
|
||||
const framePaths = new Map<number, string>();
|
||||
for (let i = 0; i < frames.length; i++) {
|
||||
const frameName = frames[i];
|
||||
if (!frameName) continue;
|
||||
// FrameLookupTable indexes frames 1-based.
|
||||
framePaths.set(i + 1, join(outputDir, frameName));
|
||||
}
|
||||
result.push({
|
||||
videoId: v.videoId,
|
||||
srcPath: v.srcPath,
|
||||
outputDir,
|
||||
framePattern: v.framePattern,
|
||||
fps: v.fps,
|
||||
totalFrames: v.totalFrames,
|
||||
metadata: v.metadata,
|
||||
framePaths,
|
||||
// The chunk worker doesn't own the planDir's video-frames/ directory
|
||||
// (the controller does — adapters that fan out chunks across machines
|
||||
// share the planDir as read-only). Mark ownership as false so the
|
||||
// injector's eventual cleanup doesn't rm bytes another worker may
|
||||
// still be reading.
|
||||
ownedByLookup: false,
|
||||
});
|
||||
}
|
||||
return result;
|
||||
}
|
||||
|
||||
/** Plan-time JSON manifest written by `freezePlan`. */
|
||||
interface PlanJson {
|
||||
planHash: string;
|
||||
@@ -257,6 +320,21 @@ export async function renderChunk(
|
||||
const encoder = JSON.parse(readFileSync(encoderJsonPath, "utf-8")) as LockedRenderConfig;
|
||||
const chunks = JSON.parse(readFileSync(chunksJsonPath, "utf-8")) as ChunkSliceJson[];
|
||||
|
||||
// `meta/videos.json` only exists when the composition has `<video>`
|
||||
// elements; absence means no injector is needed.
|
||||
const videosJsonPath = join(planDir, PLAN_VIDEOS_META_RELATIVE_PATH);
|
||||
let planVideos: PlanVideosJson | null = null;
|
||||
if (existsSync(videosJsonPath)) {
|
||||
try {
|
||||
planVideos = JSON.parse(readFileSync(videosJsonPath, "utf-8")) as PlanVideosJson;
|
||||
} catch (err) {
|
||||
throw new RenderChunkValidationError(
|
||||
MISSING_PLAN_ARTIFACT,
|
||||
`[renderChunk] failed to parse ${videosJsonPath}: ${err instanceof Error ? err.message : String(err)}`,
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
if (chunkIndex < 0 || chunkIndex >= chunks.length) {
|
||||
throw new RenderChunkValidationError(
|
||||
CHUNK_INDEX_OUT_OF_RANGE,
|
||||
@@ -361,6 +439,22 @@ export async function renderChunk(
|
||||
forceScreenshot: encoder.forceScreenshot,
|
||||
};
|
||||
|
||||
// Build the BeforeCaptureHook that injects pre-extracted video frames
|
||||
// into the page once per chunk and reuse — `runCaptureStage` may
|
||||
// invoke `createRenderVideoFrameInjector` multiple times, and
|
||||
// re-listing `planDir/video-frames/` each call would be wasteful.
|
||||
// Compositions with no video elements produce `null`, matching the
|
||||
// in-process renderer's skip path.
|
||||
const videoInjector: BeforeCaptureHook | null =
|
||||
planVideos && planVideos.extracted.length > 0
|
||||
? createVideoFrameInjector(
|
||||
createFrameLookupTable(
|
||||
planVideos.videos,
|
||||
rebuildExtractedFramesFromPlanDir(planDir, planVideos.extracted),
|
||||
),
|
||||
)
|
||||
: null;
|
||||
|
||||
// ── Per-chunk work + frames directories ──
|
||||
// Suffix workDir with pid + random bytes so concurrent invocations on
|
||||
// the SAME `(planDir, chunkIndex)` (e.g. a scheduler that double-fires
|
||||
@@ -412,11 +506,10 @@ export async function renderChunk(
|
||||
await assertSwiftShader(session.page, readWebGlVendorInfoFromCanvas);
|
||||
await initializeSession(session);
|
||||
|
||||
// Prime BeginFrame's `lastFrameCache` so the chunk's first real capture
|
||||
// reports `hasDamage` the same as an in-process render at the same
|
||||
// absolute frame would. Time is the chunk's first-frame absolute time.
|
||||
const startTime = (slice.startFrame * plan.dimensions.fpsDen) / plan.dimensions.fpsNum;
|
||||
await discardWarmupCapture(session, slice.startFrame, startTime);
|
||||
// `discardWarmupCapture` is intentionally NOT called: every frame
|
||||
// seeks fresh DOM, so `lastFrameCache` is never read; priming it
|
||||
// would deadlock Chrome's compositor by issuing a second beginFrame
|
||||
// at a `frameTimeTicks` it had just advanced to.
|
||||
|
||||
// ── Capture the chunk's range via runCaptureStage ──
|
||||
await runCaptureStage({
|
||||
@@ -437,7 +530,7 @@ export async function renderChunk(
|
||||
needsAlpha: plan.dimensions.format !== "mp4",
|
||||
captureAttempts: [],
|
||||
buildCaptureOptions: () => captureOptions,
|
||||
createRenderVideoFrameInjector: () => null,
|
||||
createRenderVideoFrameInjector: () => videoInjector,
|
||||
abortSignal: undefined,
|
||||
assertNotAborted: () => {},
|
||||
frameRange: { startFrame: slice.startFrame, endFrame: slice.endFrame },
|
||||
|
||||
@@ -10,9 +10,41 @@ import { existsSync, readFileSync } from "node:fs";
|
||||
import { fileURLToPath } from "node:url";
|
||||
import { promisify } from "node:util";
|
||||
import { type Fps } from "@hyperframes/core";
|
||||
import { type VideoElement, type VideoMetadata } from "@hyperframes/engine";
|
||||
import { type RenderConfig, type RenderJob, createRenderJob } from "../renderOrchestrator.js";
|
||||
import { defaultLogger, type ProducerLogger } from "../../logger.js";
|
||||
|
||||
/**
|
||||
* Filename of the per-video extraction manifest written by `plan()` into
|
||||
* `<planDir>/meta/` and consumed by `renderChunk()` to rebuild the
|
||||
* BeforeCaptureHook that injects pre-extracted frames into the page.
|
||||
* Absence is fine — compositions with no `<video>` elements never
|
||||
* produce the file.
|
||||
*/
|
||||
export const PLAN_VIDEOS_META_RELATIVE_PATH = "meta/videos.json";
|
||||
|
||||
/**
|
||||
* On-disk shape of `<planDir>/meta/videos.json`. The engine's
|
||||
* `ExtractedFrames` shape carries an absolute `outputDir`, a `framePaths`
|
||||
* Map, and potentially an open file descriptor — none of those survive
|
||||
* a serialize → re-deserialize round trip across processes. The
|
||||
* serialized form keeps only what plan-time produced; `renderChunk` re-
|
||||
* derives `outputDir` (always `<planDir>/video-frames/<videoId>`) and
|
||||
* `framePaths` (re-listed from that directory) when reconstructing the
|
||||
* `FrameLookupTable`.
|
||||
*/
|
||||
export interface PlanVideosJson {
|
||||
videos: VideoElement[];
|
||||
extracted: Array<{
|
||||
videoId: string;
|
||||
srcPath: string;
|
||||
framePattern: string;
|
||||
fps: number;
|
||||
totalFrames: number;
|
||||
metadata: VideoMetadata;
|
||||
}>;
|
||||
}
|
||||
|
||||
const execFile = promisify(execFileCallback);
|
||||
|
||||
/**
|
||||
|
||||
@@ -0,0 +1,156 @@
|
||||
# Producer regression test fixtures
|
||||
|
||||
Each subdirectory under this folder is a **regression fixture** for the
|
||||
HTML-to-video pipeline. The harness at
|
||||
`packages/producer/src/regression-harness.ts` walks every subdirectory,
|
||||
runs the composition, and PSNR-compares the rendered output against a
|
||||
checked-in golden baseline.
|
||||
|
||||
## Fixture layout
|
||||
|
||||
```
|
||||
<fixture-name>/
|
||||
├── meta.json # name, tags, PSNR threshold, renderConfig
|
||||
├── src/
|
||||
│ ├── index.html # composition entry point
|
||||
│ └── assets/... # any locally-referenced media
|
||||
└── output/
|
||||
├── compiled.html # golden compiled HTML (validated as a snapshot)
|
||||
└── output.mp4 # golden rendered video
|
||||
```
|
||||
|
||||
`meta.json` is validated by `validateMetadata` in
|
||||
`src/regression-harness.ts`. The required fields are:
|
||||
|
||||
- `name` (string), `description` (string), `tags` (string[])
|
||||
- `minPsnr` (number, dB)
|
||||
- `maxFrameFailures` (integer)
|
||||
- `minAudioCorrelation` (0..1), `maxAudioLagWindows` (integer ≥1)
|
||||
- `renderConfig.fps` (integer like `30` or a rational string like `"30000/1001"`)
|
||||
|
||||
Optional `renderConfig` fields:
|
||||
|
||||
- `format` — `"mp4"` (default) or `"webm"`
|
||||
- `workers` — integer ≥ 1
|
||||
- `hdr` — boolean (default `false`)
|
||||
- `variables` — JSON object of render-time variable overrides
|
||||
- `chunkSize` — integer ≥ 1 (used by `--mode=distributed-simulated`)
|
||||
- `maxParallelChunks` — integer ≥ 1 (used by `--mode=distributed-simulated`)
|
||||
|
||||
## Generating / updating a baseline
|
||||
|
||||
**Always inside Docker.** Host Chrome / FFmpeg versions drift across
|
||||
distros, so a baseline captured on the host won't match the bytes CI
|
||||
renders.
|
||||
|
||||
```bash
|
||||
# From the repo root.
|
||||
docker build -t hyperframes-producer:test -f Dockerfile.test .
|
||||
|
||||
# Generate a baseline (single fixture):
|
||||
bun run --cwd packages/producer docker:test:update <fixture-name>
|
||||
|
||||
# Generate all baselines (rarely needed):
|
||||
bun run --cwd packages/producer docker:test:update
|
||||
```
|
||||
|
||||
The `--update` flag writes `output/compiled.html` and `output/output.mp4`
|
||||
from the current render. Without `--update`, the harness compares against
|
||||
those baselines.
|
||||
|
||||
## Running the harness locally
|
||||
|
||||
```bash
|
||||
# Run every fixture (parallel, in-process mode — the default).
|
||||
bun run --cwd packages/producer docker:test
|
||||
|
||||
# Run a single fixture:
|
||||
bun run --cwd packages/producer docker:test font-variant-numeric
|
||||
|
||||
# Run sequentially (lower memory):
|
||||
bun run --cwd packages/producer docker:test -- --sequential
|
||||
```
|
||||
|
||||
## Harness modes
|
||||
|
||||
`--mode=<value>` chooses which render path the harness exercises:
|
||||
|
||||
| Mode | What it calls | Use for |
|
||||
|---|---|---|
|
||||
| `in-process` (default) | `executeRenderJob` | Day-to-day baselines. This is the same path the `hyperframes render` CLI takes, and it is what produced every existing `output/output.mp4`. |
|
||||
| `distributed-simulated` | `plan()` → `renderChunk()` × N → `assemble()` from `@hyperframes/producer/distributed` | Validates the distributed pipeline against the in-process baseline. No Temporal or Lambda involvement — the controller and chunk worker are both this process. |
|
||||
|
||||
### `--mode=distributed-simulated`
|
||||
|
||||
```bash
|
||||
bun run --cwd packages/producer docker:test -- --mode=distributed-simulated
|
||||
bun run --cwd packages/producer docker:test font-variant-numeric -- --mode=distributed-simulated
|
||||
```
|
||||
|
||||
The distributed pipeline cannot run every fixture. Fixtures that fail any
|
||||
of these gates are **skipped** with a clear log line (and counted as
|
||||
passing in the summary):
|
||||
|
||||
- `fps.den !== 1` — distributed mode is integer-fps only (no NTSC).
|
||||
- `fps.num ∉ {24, 30, 60}` — closed set per `DistributedRenderConfig`.
|
||||
- `format === "webm"` — `plan()` refuses webm.
|
||||
- `hdr === true` — distributed mode is SDR-only at v1.
|
||||
|
||||
Both modes use the fixture's authored `minPsnr` as the per-test
|
||||
threshold — distributed must clear the same quality bar in-process
|
||||
clears against the same frozen baseline. (`DISTRIBUTED-RENDERING-PLAN.md`
|
||||
§5.1's 50 dB target is a per-render distributed-vs-in-process contract;
|
||||
against the frozen baseline file, neither mode reaches it consistently
|
||||
due to shared encoder/JPEG-capture jitter.) An absolute 10 dB pathology
|
||||
floor catches fully-black-output regressions when a fixture authors a
|
||||
permissive threshold. A distributed failure at the fixture's own
|
||||
threshold means the distributed pipeline has drifted — file an issue
|
||||
rather than relaxing the fixture.
|
||||
|
||||
`--update` is incompatible with `--mode=distributed-simulated`: the
|
||||
in-process renderer is the source of truth for baselines, and the
|
||||
distributed mode's job is to verify the contract against the same
|
||||
baseline.
|
||||
|
||||
### Validating PR 4.1 (the harness mode itself)
|
||||
|
||||
The smallest fixtures (`font-variant-numeric`, `many-cuts`) are sufficient
|
||||
to verify the mode plumbing end to end:
|
||||
|
||||
```bash
|
||||
docker build -t hyperframes-producer:test -f Dockerfile.test .
|
||||
|
||||
# In-process: existing behavior, unchanged.
|
||||
bun run --cwd packages/producer docker:test font-variant-numeric
|
||||
bun run --cwd packages/producer docker:test many-cuts
|
||||
|
||||
# Distributed-simulated: same baselines, distributed pipeline.
|
||||
bun run --cwd packages/producer docker:test font-variant-numeric -- --mode=distributed-simulated
|
||||
bun run --cwd packages/producer docker:test many-cuts -- --mode=distributed-simulated
|
||||
```
|
||||
|
||||
Both modes must pass at each fixture's authored `minPsnr` against the
|
||||
existing baseline. If `--mode=distributed-simulated` fails where
|
||||
`--mode=in-process` passes, the distributed primitive has a regression —
|
||||
file an issue rather than relaxing the fixture's threshold.
|
||||
|
||||
## Distributed-only fixtures
|
||||
|
||||
Fixtures under `tests/distributed/<name>/` are authored specifically for
|
||||
the distributed pipeline. They follow the same `meta.json` schema as the
|
||||
top-level fixtures, but they always set `chunkSize` / `maxParallelChunks`
|
||||
so a `plan()` over the fixture produces N>1 chunks. Each fixture
|
||||
exercises one of:
|
||||
|
||||
- per-format chunk-boundary correctness (mp4 H.264, mp4 H.265, ProRes, png-sequence)
|
||||
- per-adapter chunk-seam state preservation (GSAP, Anime.js, Three.js, Lottie, CSS, WAAPI)
|
||||
|
||||
See `DISTRIBUTED-RENDERING-PLAN.md` §10.2 for the equivalence axes each
|
||||
distributed fixture covers.
|
||||
|
||||
## Tags
|
||||
|
||||
Common `tags` values control which fixtures the default `bun test`
|
||||
invocation runs. `--exclude-tags transparency` (the default for
|
||||
`bun test`) skips webm/png-sequence alpha fixtures that need a working
|
||||
chrome-headless-shell alpha pipeline.
|
||||
Reference in New Issue
Block a user