diff --git a/packages/producer/src/regression-harness.ts b/packages/producer/src/regression-harness.ts index 1cacd5c50..da5f4f402 100644 --- a/packages/producer/src/regression-harness.ts +++ b/packages/producer/src/regression-harness.ts @@ -274,63 +274,74 @@ function discoverTestSuites( throw new Error(`Tests directory not found: ${testsDir}`); } - const entries = readdirSync(testsDir); const suites: TestSuite[] = []; - for (const entry of entries) { - const dir = join(testsDir, entry); - if (!statSync(dir).isDirectory()) continue; - if (entry === "node_modules" || entry.startsWith(".")) continue; - - // If filter is specified, skip non-matching tests - if (filterNames.length > 0 && !filterNames.includes(entry)) { - continue; - } + // Validate + push a single candidate fixture directory. Logs the reason + // and returns silently if the directory doesn't look like a fixture, so + // callers can blindly hand over every candidate. + const tryAddSuite = (id: string, dir: string): void => { + if (filterNames.length > 0 && !filterNames.includes(id)) return; const srcDir = join(dir, "src"); const metaPath = join(dir, "meta.json"); - // Validate structure if (!existsSync(srcDir) || !statSync(srcDir).isDirectory()) { - console.warn(`⚠️ Skipping ${entry}: missing src/ directory`); - continue; + console.warn(`⚠️ Skipping ${id}: missing src/ directory`); + return; } if (!existsSync(join(srcDir, "index.html"))) { - console.warn(`⚠️ Skipping ${entry}: missing src/index.html`); - continue; + console.warn(`⚠️ Skipping ${id}: missing src/index.html`); + return; } if (!existsSync(metaPath)) { - console.warn(`⚠️ Skipping ${entry}: missing meta.json`); - continue; + console.warn(`⚠️ Skipping ${id}: missing meta.json`); + return; } - // Parse and validate meta.json let meta: TestMetadata; try { const metaRaw = JSON.parse(readFileSync(metaPath, "utf-8")); meta = validateMetadata(metaRaw); } catch (error) { console.warn( - `⚠️ Skipping ${entry}: invalid meta.json - ${error instanceof Error ? error.message : String(error)}`, + `⚠️ Skipping ${id}: invalid meta.json - ${error instanceof Error ? error.message : String(error)}`, ); - continue; + return; } - // Skip tests with excluded tags if (excludeTags.length > 0 && meta.tags.some((t) => excludeTags.includes(t))) { logPretty( - `Skipping ${entry}: excluded by tags [${meta.tags.filter((t) => excludeTags.includes(t)).join(", ")}]`, + `Skipping ${id}: excluded by tags [${meta.tags.filter((t) => excludeTags.includes(t)).join(", ")}]`, "⏭️", ); + return; + } + + suites.push({ id, dir, srcDir, meta }); + }; + + for (const entry of readdirSync(testsDir)) { + const dir = join(testsDir, entry); + if (!statSync(dir).isDirectory()) continue; + if (entry === "node_modules" || entry.startsWith(".")) continue; + + // `tests/distributed//` is the home for fixtures authored + // specifically for the distributed pipeline (see tests/README.md and + // DISTRIBUTED-RENDERING-PLAN.md §10.2). Recurse one level deeper so + // each `` becomes a first-class fixture ID (`mp4-h264-sdr`, + // `mov-prores`, …) the user can target on the CLI without their + // namespace prefix. + if (entry === "distributed") { + for (const sub of readdirSync(dir)) { + const subDir = join(dir, sub); + if (!statSync(subDir).isDirectory()) continue; + if (sub === "node_modules" || sub.startsWith(".")) continue; + tryAddSuite(sub, subDir); + } continue; } - suites.push({ - id: entry, - dir, - srcDir, - meta, - }); + tryAddSuite(entry, dir); } return suites; diff --git a/packages/producer/tests/README.md b/packages/producer/tests/README.md index c7347f9ae..906b7ac41 100644 --- a/packages/producer/tests/README.md +++ b/packages/producer/tests/README.md @@ -148,6 +148,30 @@ exercises one of: See `DISTRIBUTED-RENDERING-PLAN.md` §10.2 for the equivalence axes each distributed fixture covers. +### Fixture pattern (4.2 onward) + +Each `tests/distributed//` fixture has the same structure as a +top-level fixture (`meta.json` + `src/index.html` + `output/output.mp4`). +Differences worth knowing: + +- `renderConfig.chunkSize` is **required** — pick a value that yields + N≥2 chunks for your fixture's frame count (e.g. 60 frames at + `chunkSize: 15` produces N=4). Without this the fixture renders in a + single chunk and never exercises the seam. +- The fixture's ID on the CLI is just `` (no `distributed/` + prefix). `bun run --cwd packages/producer docker:test mp4-h264-sdr` + works the same as for a top-level fixture. +- The `distributed` tag is informational — it doesn't gate any tag-based + filter today. Add it so the fixture is easy to find by tag. +- The composition should stress *state continuity* across the chunk + seams: an animation crossing a seam, a counter, a rotation. A + fully-static composition would pass even if chunk-boundary state was + broken. +- Baselines must be generated inside Docker — see the section above. + The baseline is rendered by the in-process renderer (the source of + truth for golden output); `--mode=distributed-simulated` is validated + against the same baseline. + ## Tags Common `tags` values control which fixtures the default `bun test` diff --git a/packages/producer/tests/distributed/mp4-h264-sdr/meta.json b/packages/producer/tests/distributed/mp4-h264-sdr/meta.json new file mode 100644 index 000000000..17c99a6b1 --- /dev/null +++ b/packages/producer/tests/distributed/mp4-h264-sdr/meta.json @@ -0,0 +1,16 @@ +{ + "name": "Distributed: mp4 H.264 SDR", + "description": "60-frame composition (2s @ 30fps) with text, a crossfade transition, and a small inline-SVG image. renderConfig.chunkSize=15 produces exactly N=4 chunks, exercising libx264's closed-GOP + concat-copy contract end-to-end.", + "tags": ["distributed", "mp4", "h264", "sdr"], + + "minPsnr": 30, + "maxFrameFailures": 0, + + "minAudioCorrelation": 0.9, + "maxAudioLagWindows": 120, + + "renderConfig": { + "fps": 30, + "chunkSize": 15 + } +} diff --git a/packages/producer/tests/distributed/mp4-h264-sdr/src/index.html b/packages/producer/tests/distributed/mp4-h264-sdr/src/index.html new file mode 100644 index 000000000..b0b6ecb64 --- /dev/null +++ b/packages/producer/tests/distributed/mp4-h264-sdr/src/index.html @@ -0,0 +1,121 @@ + + + + + + mp4 H.264 SDR distributed fixture + + + + +
+
+
CHUNK
+
PHASE ONE
+
+
+
CHUNK
+
PHASE TWO
+
+ + + + + + +
+ + + +