From 2974bcb1a9149de8c74eb07bf7ec6115fa8aec37 Mon Sep 17 00:00:00 2001 From: James Date: Thu, 14 May 2026 21:37:15 +0000 Subject: [PATCH 1/3] test(producer): add mp4 H.264 SDR distributed fixture --- packages/producer/src/regression-harness.ts | 67 ++++++---- packages/producer/tests/README.md | 24 ++++ .../tests/distributed/mp4-h264-sdr/meta.json | 16 +++ .../distributed/mp4-h264-sdr/src/index.html | 121 ++++++++++++++++++ 4 files changed, 200 insertions(+), 28 deletions(-) create mode 100644 packages/producer/tests/distributed/mp4-h264-sdr/meta.json create mode 100644 packages/producer/tests/distributed/mp4-h264-sdr/src/index.html diff --git a/packages/producer/src/regression-harness.ts b/packages/producer/src/regression-harness.ts index 1cacd5c50..da5f4f402 100644 --- a/packages/producer/src/regression-harness.ts +++ b/packages/producer/src/regression-harness.ts @@ -274,63 +274,74 @@ function discoverTestSuites( throw new Error(`Tests directory not found: ${testsDir}`); } - const entries = readdirSync(testsDir); const suites: TestSuite[] = []; - for (const entry of entries) { - const dir = join(testsDir, entry); - if (!statSync(dir).isDirectory()) continue; - if (entry === "node_modules" || entry.startsWith(".")) continue; - - // If filter is specified, skip non-matching tests - if (filterNames.length > 0 && !filterNames.includes(entry)) { - continue; - } + // Validate + push a single candidate fixture directory. Logs the reason + // and returns silently if the directory doesn't look like a fixture, so + // callers can blindly hand over every candidate. + const tryAddSuite = (id: string, dir: string): void => { + if (filterNames.length > 0 && !filterNames.includes(id)) return; const srcDir = join(dir, "src"); const metaPath = join(dir, "meta.json"); - // Validate structure if (!existsSync(srcDir) || !statSync(srcDir).isDirectory()) { - console.warn(`⚠️ Skipping ${entry}: missing src/ directory`); - continue; + console.warn(`⚠️ Skipping ${id}: missing src/ directory`); + return; } if (!existsSync(join(srcDir, "index.html"))) { - console.warn(`⚠️ Skipping ${entry}: missing src/index.html`); - continue; + console.warn(`⚠️ Skipping ${id}: missing src/index.html`); + return; } if (!existsSync(metaPath)) { - console.warn(`⚠️ Skipping ${entry}: missing meta.json`); - continue; + console.warn(`⚠️ Skipping ${id}: missing meta.json`); + return; } - // Parse and validate meta.json let meta: TestMetadata; try { const metaRaw = JSON.parse(readFileSync(metaPath, "utf-8")); meta = validateMetadata(metaRaw); } catch (error) { console.warn( - `⚠️ Skipping ${entry}: invalid meta.json - ${error instanceof Error ? error.message : String(error)}`, + `⚠️ Skipping ${id}: invalid meta.json - ${error instanceof Error ? error.message : String(error)}`, ); - continue; + return; } - // Skip tests with excluded tags if (excludeTags.length > 0 && meta.tags.some((t) => excludeTags.includes(t))) { logPretty( - `Skipping ${entry}: excluded by tags [${meta.tags.filter((t) => excludeTags.includes(t)).join(", ")}]`, + `Skipping ${id}: excluded by tags [${meta.tags.filter((t) => excludeTags.includes(t)).join(", ")}]`, "⏭️", ); + return; + } + + suites.push({ id, dir, srcDir, meta }); + }; + + for (const entry of readdirSync(testsDir)) { + const dir = join(testsDir, entry); + if (!statSync(dir).isDirectory()) continue; + if (entry === "node_modules" || entry.startsWith(".")) continue; + + // `tests/distributed//` is the home for fixtures authored + // specifically for the distributed pipeline (see tests/README.md and + // DISTRIBUTED-RENDERING-PLAN.md §10.2). Recurse one level deeper so + // each `` becomes a first-class fixture ID (`mp4-h264-sdr`, + // `mov-prores`, …) the user can target on the CLI without their + // namespace prefix. + if (entry === "distributed") { + for (const sub of readdirSync(dir)) { + const subDir = join(dir, sub); + if (!statSync(subDir).isDirectory()) continue; + if (sub === "node_modules" || sub.startsWith(".")) continue; + tryAddSuite(sub, subDir); + } continue; } - suites.push({ - id: entry, - dir, - srcDir, - meta, - }); + tryAddSuite(entry, dir); } return suites; diff --git a/packages/producer/tests/README.md b/packages/producer/tests/README.md index c7347f9ae..906b7ac41 100644 --- a/packages/producer/tests/README.md +++ b/packages/producer/tests/README.md @@ -148,6 +148,30 @@ exercises one of: See `DISTRIBUTED-RENDERING-PLAN.md` §10.2 for the equivalence axes each distributed fixture covers. +### Fixture pattern (4.2 onward) + +Each `tests/distributed//` fixture has the same structure as a +top-level fixture (`meta.json` + `src/index.html` + `output/output.mp4`). +Differences worth knowing: + +- `renderConfig.chunkSize` is **required** — pick a value that yields + N≥2 chunks for your fixture's frame count (e.g. 60 frames at + `chunkSize: 15` produces N=4). Without this the fixture renders in a + single chunk and never exercises the seam. +- The fixture's ID on the CLI is just `` (no `distributed/` + prefix). `bun run --cwd packages/producer docker:test mp4-h264-sdr` + works the same as for a top-level fixture. +- The `distributed` tag is informational — it doesn't gate any tag-based + filter today. Add it so the fixture is easy to find by tag. +- The composition should stress *state continuity* across the chunk + seams: an animation crossing a seam, a counter, a rotation. A + fully-static composition would pass even if chunk-boundary state was + broken. +- Baselines must be generated inside Docker — see the section above. + The baseline is rendered by the in-process renderer (the source of + truth for golden output); `--mode=distributed-simulated` is validated + against the same baseline. + ## Tags Common `tags` values control which fixtures the default `bun test` diff --git a/packages/producer/tests/distributed/mp4-h264-sdr/meta.json b/packages/producer/tests/distributed/mp4-h264-sdr/meta.json new file mode 100644 index 000000000..17c99a6b1 --- /dev/null +++ b/packages/producer/tests/distributed/mp4-h264-sdr/meta.json @@ -0,0 +1,16 @@ +{ + "name": "Distributed: mp4 H.264 SDR", + "description": "60-frame composition (2s @ 30fps) with text, a crossfade transition, and a small inline-SVG image. renderConfig.chunkSize=15 produces exactly N=4 chunks, exercising libx264's closed-GOP + concat-copy contract end-to-end.", + "tags": ["distributed", "mp4", "h264", "sdr"], + + "minPsnr": 30, + "maxFrameFailures": 0, + + "minAudioCorrelation": 0.9, + "maxAudioLagWindows": 120, + + "renderConfig": { + "fps": 30, + "chunkSize": 15 + } +} diff --git a/packages/producer/tests/distributed/mp4-h264-sdr/src/index.html b/packages/producer/tests/distributed/mp4-h264-sdr/src/index.html new file mode 100644 index 000000000..b0b6ecb64 --- /dev/null +++ b/packages/producer/tests/distributed/mp4-h264-sdr/src/index.html @@ -0,0 +1,121 @@ + + + + + + mp4 H.264 SDR distributed fixture + + + + +
+
+
CHUNK
+
PHASE ONE
+
+
+
CHUNK
+
PHASE TWO
+
+ + + + + + +
+ + + + From 447d42845242aa6719be00585f1bf0ba446c27c2 Mon Sep 17 00:00:00 2001 From: James Date: Thu, 14 May 2026 21:38:44 +0000 Subject: [PATCH 2/3] test(producer): track distributed fixture baselines via LFS --- .gitattributes | 3 + .gitignore | 2 + .../mp4-h264-sdr/output/compiled.html | 139 ++++++++++++++++++ .../mp4-h264-sdr/output/output.mp4 | 3 + 4 files changed, 147 insertions(+) create mode 100644 packages/producer/tests/distributed/mp4-h264-sdr/output/compiled.html create mode 100644 packages/producer/tests/distributed/mp4-h264-sdr/output/output.mp4 diff --git a/.gitattributes b/.gitattributes index ec68cb840..cc0c61197 100644 --- a/.gitattributes +++ b/.gitattributes @@ -6,12 +6,15 @@ # Golden baseline videos for regression tests packages/producer/tests/*/output/output.mp4 filter=lfs diff=lfs merge=lfs -text +packages/producer/tests/distributed/*/output/output.mp4 filter=lfs diff=lfs merge=lfs -text # Source video clips for regression test fixtures (HDR samples, etc.) packages/producer/tests/*/src/*.mp4 filter=lfs diff=lfs merge=lfs -text +packages/producer/tests/distributed/*/src/*.mp4 filter=lfs diff=lfs merge=lfs -text # Source image assets for regression test fixtures (HDR PNGs, screenshot fixtures, etc.) packages/producer/tests/*/src/*.png filter=lfs diff=lfs merge=lfs -text +packages/producer/tests/distributed/*/src/*.png filter=lfs diff=lfs merge=lfs -text # GitHub Linguist overrides — HTML files are compositions (user content / templates), # not the framework source. Hide them from the repo language stats so TypeScript, diff --git a/.gitignore b/.gitignore index eaf23805f..76b9858be 100644 --- a/.gitignore +++ b/.gitignore @@ -32,6 +32,7 @@ coverage/ # Producer regression test failures (generated debugging artifacts) packages/producer/tests/*/failures/ +packages/producer/tests/distributed/*/failures/ packages/producer/tests/parity/fixtures/hyperframe.runtime.iife.js # Player perf test results (generated each run, attached as CI artifact) @@ -41,6 +42,7 @@ packages/player/tests/perf/results/ output/ renders/ !packages/producer/tests/*/output/ +!packages/producer/tests/distributed/*/output/ # Composition source media (large binaries) compositions/**/*.mp4 diff --git a/packages/producer/tests/distributed/mp4-h264-sdr/output/compiled.html b/packages/producer/tests/distributed/mp4-h264-sdr/output/compiled.html new file mode 100644 index 000000000..514f4f83c --- /dev/null +++ b/packages/producer/tests/distributed/mp4-h264-sdr/output/compiled.html @@ -0,0 +1,139 @@ + + + + + + mp4 H.264 SDR distributed fixture + + + + +
+
+
CHUNK
+
PHASE ONE
+
+
+
CHUNK
+
PHASE TWO
+
+ + + + + + +
+ + + + diff --git a/packages/producer/tests/distributed/mp4-h264-sdr/output/output.mp4 b/packages/producer/tests/distributed/mp4-h264-sdr/output/output.mp4 new file mode 100644 index 000000000..f348d4717 --- /dev/null +++ b/packages/producer/tests/distributed/mp4-h264-sdr/output/output.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6c6137ec9cb1117e09d8da79b0dc18d160bdc750d5311e3752c353046ccac7f2 +size 62079 From 6c98393ec91dd9f53390f265ffce3cc1b705e670 Mon Sep 17 00:00:00 2001 From: James Date: Thu, 14 May 2026 23:41:51 +0000 Subject: [PATCH 3/3] fix(producer): detect duplicate fixture IDs across tests// and tests/distributed// MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Address @vanceingalls review on #845: the new discoverTestSuites dispatch was silently allowing a future tests/distributed// fixture to collide with an existing tests// fixture of the same name. Both would push under the same suite.id and stomp each other's failures/ output, baseline lookup, and CLI --filter match. Detect the collision at discovery time and throw with both source dirs named, so the conflict is fixable at author time. Easier to enforce now (one fixture in the new namespace) than after the rest of the Phase 4 fixtures land. 🤖 Generated with [Claude Code](https://claude.com/claude-code) --- packages/producer/src/regression-harness.ts | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/packages/producer/src/regression-harness.ts b/packages/producer/src/regression-harness.ts index da5f4f402..a6a1b4e23 100644 --- a/packages/producer/src/regression-harness.ts +++ b/packages/producer/src/regression-harness.ts @@ -344,6 +344,25 @@ function discoverTestSuites( tryAddSuite(entry, dir); } + // CLI filter, failures/ output, baselines, and the suite summary all key + // off `suite.id`. If a future fixture lands at `tests/distributed//` + // while a top-level `tests//` already exists they would silently + // collide: both pushed with the same `id`, both running under one name, + // and the second to write `failures/` overwrites the first. Fail fast + // here naming both source dirs so the conflict is fixable at author time. + const seen = new Map(); + for (const suite of suites) { + const prior = seen.get(suite.id); + if (prior !== undefined) { + throw new Error( + `[regression-harness] duplicate fixture id ${JSON.stringify(suite.id)}: ` + + `${prior} and ${suite.dir}. Rename one of the directories so the CLI ` + + `--filter, failures/ output, and summary key onto a single suite.`, + ); + } + seen.set(suite.id, suite.dir); + } + return suites; }