From b8e8617f80a167f1f009cabdd0b6fc7bb50975e3 Mon Sep 17 00:00:00 2001 From: James Date: Thu, 14 May 2026 19:51:18 +0000 Subject: [PATCH] refactor(producer): apply /simplify cleanups across distributed harness stack MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Code reuse: - Move `PlanVideosJson` interface + `meta/videos.json` path constant into `services/distributed/shared.ts`; plan.ts and renderChunk.ts import from there instead of redeclaring the same shape with the "duplicated here so renderChunk doesn't import from plan.ts" comment. - Replace hand-rolled `framePattern.slice(lastIndexOf("."))` with `extname()` from `node:path` in `rebuildExtractedFramesFromPlanDir`. Efficiency: - Hoist `rebuildExtractedFramesFromPlanDir` + `createFrameLookupTable` + `createVideoFrameInjector` out of the per-chunk closure in renderChunk. Computed once per chunk now, not once per `createRenderVideoFrameInjector` callsite (which `runCaptureStage` may invoke multiple times). - Add a regex filter to `cpSync(projectDir → planDir/compiled/)` so `node_modules`, `.git`, `output/`, `failures/`, `dist/`, etc. are not copied. Real projects can have hundreds of MB in those directories; shipping them to S3/Lambda /tmp on every render bloats cost and time. - Drop redundant `if (!existsSync(metaDir)) mkdirSync(metaDir, {recursive:true})` guards; `mkdirSync({recursive:true})` is already idempotent. Quality: - Strip narrative comments that told the story of debugging: - renderChunk's 30-line "Two failure modes made the call actively harmful" block → 4-line invariant on why `discardWarmupCapture` is omitted. - plan.ts's "DO NOT call cleanup()" block → 3 lines naming the invariant. - plan.ts's pre-seed-projectDir block → 7 lines on the file-server invariant. - renderChunk.ts top docstring's discardWarmupCapture paragraph. - regression-harness-distributed.ts's PSNR-drift table (belongs in DISTRIBUTED-RENDERING-PLAN.md, not the source). - test file's docstring about which tests live where. - Drop the unreachable IIFE-throw on `format === "webm"` in the harness (the support check above rejected webm); replace with a plain `as` cast. - Replace dynamic `await import("node:fs")` with a top-level import in `regression-harness-distributed.ts`. All 54 distributed unit tests still pass in Docker. Full fixture sweep in `docker:test --mode=distributed-simulated` (font-variant-numeric, many-cuts, gsap-letters-render-compat, style-1-prod, sub-composition-video) all PASSED. Co-Authored-By: Claude Opus 4.7 (1M context) --- .../regression-harness-distributed.test.ts | 13 +- .../src/regression-harness-distributed.ts | 32 +--- packages/producer/src/regression-harness.ts | 16 +- .../producer/src/services/distributed/plan.ts | 127 ++++++---------- .../src/services/distributed/renderChunk.ts | 141 ++++++------------ .../src/services/distributed/shared.ts | 32 ++++ 6 files changed, 135 insertions(+), 226 deletions(-) diff --git a/packages/producer/src/regression-harness-distributed.test.ts b/packages/producer/src/regression-harness-distributed.test.ts index a3c5600465..10621a84d0 100644 --- a/packages/producer/src/regression-harness-distributed.test.ts +++ b/packages/producer/src/regression-harness-distributed.test.ts @@ -1,14 +1,5 @@ -/** - * Unit tests for the harness mode plumbing — `parseHarnessModeFlag`, - * `checkDistributedSupport`, and `resolveMinPsnrForMode`. - * - * These do NOT exercise the render pipeline itself. The byte-identical-retry - * contract is covered by `services/distributed/renderChunk.test.ts` and the - * PSNR contract is covered by `Dockerfile.test` running the harness with - * `--mode=distributed-simulated` against the smallest existing fixture. - * What lives here is the dispatch logic that decides which mode runs and - * which fixtures skip. - */ +// Pure-function tests for the harness mode dispatch logic. End-to-end +// PSNR contract lives in `Dockerfile.test` runs of the regression harness. import { describe, expect, it } from "bun:test"; import { diff --git a/packages/producer/src/regression-harness-distributed.ts b/packages/producer/src/regression-harness-distributed.ts index 871581f0b1..a421bfd5d1 100644 --- a/packages/producer/src/regression-harness-distributed.ts +++ b/packages/producer/src/regression-harness-distributed.ts @@ -26,7 +26,7 @@ * format and adapter. */ -import { existsSync } from "node:fs"; +import { existsSync, mkdirSync } from "node:fs"; import { join } from "node:path"; import type { Fps } from "@hyperframes/core"; import { assemble, plan, renderChunk } from "./distributed.js"; @@ -35,29 +35,12 @@ import { assemble, plan, renderChunk } from "./distributed.js"; export type HarnessMode = "in-process" | "distributed-simulated"; /** - * Minimum PSNR (in dB) at which a distributed-simulated render is considered - * equivalent to its in-process baseline. - * - * `DISTRIBUTED-RENDERING-PLAN.md` §5.1 names ≥50 dB as the - * distributed-vs-in-process equivalence floor, but that target was written - * against per-render comparisons (one fresh in-process render vs one fresh - * distributed render). The regression harness compares against a frozen - * baseline file, and the in-process renderer itself drifts against that - * baseline by varying amounts depending on the composition's dynamics: - * - Static compositions (font-variant-numeric): ~48 dB drift - * - Rapid transitions (many-cuts): 34-44 dB drift - * Both modes share the same encoder/JPEG-capture jitter floor, so this is - * a property of the fixture's content, not of either renderer. - * - * The harness therefore uses the fixture's own `minPsnr` for both modes: - * distributed must pass the same quality threshold the in-process renderer - * passes against the same baseline. A distributed regression that pushes - * PSNR below the fixture's tolerance is caught the same way an in-process - * regression would be. - * - * This constant is retained as a non-zero floor for absolute pathology - * (e.g. a chunk renders as fully black), independent of how lenient the - * fixture's authored threshold is. + * Absolute pathology floor for `--mode=distributed-simulated` — catches + * a chunk that renders fully-black against a fixture authored with a + * permissive `minPsnr`. Non-pathological drift is caught by the fixture's + * own threshold; both modes share the same encoder/JPEG-capture jitter + * floor against the frozen baseline file, so the §5.1 50 dB target is + * unreachable for either mode and isn't a useful per-test gate. */ export const DISTRIBUTED_SIMULATED_MIN_PSNR_DB = 10; @@ -156,7 +139,6 @@ export async function runDistributedSimulatedRender( ): Promise { const planDir = join(input.tempRoot, "plan"); const chunksDir = join(input.tempRoot, "chunks"); - const { mkdirSync } = await import("node:fs"); mkdirSync(planDir, { recursive: true }); mkdirSync(chunksDir, { recursive: true }); diff --git a/packages/producer/src/regression-harness.ts b/packages/producer/src/regression-harness.ts index d7e0a4698b..fe7ac53811 100644 --- a/packages/producer/src/regression-harness.ts +++ b/packages/producer/src/regression-harness.ts @@ -717,17 +717,13 @@ async function runTestSuite( result.skipped = { reason: support.reason }; return result; } - // The support check narrows `fps.num` to 24|30|60; assert here so - // TS sees the literal type that `DistributedRenderConfig.fps` - // requires. The check itself rejected every other value above. + // `checkDistributedSupport` already narrowed fps to {24,30,60} and + // rejected webm; the `as` casts surface those guarantees to TS. const fpsNum = suite.meta.renderConfig.fps.num as 24 | 30 | 60; - const distributedFormat: "mp4" | "mov" | "png-sequence" = - suite.meta.renderConfig.format === "webm" - ? // Unreachable — checkDistributedSupport rejected webm above. - (() => { - throw new Error("webm reached distributed render path"); - })() - : (suite.meta.renderConfig.format ?? "mp4"); + const distributedFormat = (suite.meta.renderConfig.format ?? "mp4") as + | "mp4" + | "mov" + | "png-sequence"; await runDistributedSimulatedRender({ projectDir: tempSrcDir, tempRoot, diff --git a/packages/producer/src/services/distributed/plan.ts b/packages/producer/src/services/distributed/plan.ts index 249bac1437..206b8d1461 100644 --- a/packages/producer/src/services/distributed/plan.ts +++ b/packages/producer/src/services/distributed/plan.ts @@ -36,12 +36,7 @@ import { } from "node:fs"; import { join } from "node:path"; import { type CanvasResolution } from "@hyperframes/core"; -import { - type EngineConfig, - type ExtractedFrames, - resolveConfig, - type VideoElement, -} from "@hyperframes/engine"; +import { type EngineConfig, resolveConfig } from "@hyperframes/engine"; import { defaultLogger, type ProducerLogger } from "../../logger.js"; import { runAudioStage } from "../render/stages/audioStage.js"; import { runCompileStage } from "../render/stages/compileStage.js"; @@ -60,7 +55,13 @@ import { } from "../render/stages/planHash.js"; import { validateNoGpuEncode, validateNoSystemFonts } from "../render/planValidation.js"; import { snapshotRuntimeEnv } from "../render/runtimeEnvSnapshot.js"; -import { buildSyntheticRenderJob, readFfmpegVersion, readProducerVersion } from "./shared.js"; +import { + buildSyntheticRenderJob, + PLAN_VIDEOS_META_RELATIVE_PATH, + type PlanVideosJson, + readFfmpegVersion, + readProducerVersion, +} from "./shared.js"; /** * Caller-supplied configuration for a distributed render. `fps`, `width`, @@ -145,6 +146,15 @@ export interface PlanResult { producerVersion: string; } +/** + * Skip patterns for the `projectDir → planDir/compiled/` pre-seed copy. + * Real projects often contain `node_modules/`, VCS metadata, and harness + * artifacts that have no business in a planDir — they bloat the 2 GB + * planDir cap and slow the S3/Lambda round-trip for no benefit. + */ +const PLAN_PROJECT_DIR_COPY_SKIP = + /(^|\/)(node_modules|\.git|\.cache|output|failures|dist|\.next|\.turbo)(\/|$)/; + /** Default chunk size in frames (~8s @ 30fps; fits Lambda's 15-min cap). */ export const DEFAULT_CHUNK_SIZE = 240; /** Default cap on parallel chunks for operational fairness across renders. */ @@ -433,42 +443,6 @@ function buildLockedRenderConfig(input: { }; } -/** - * Persisted shape of the video-extraction outputs from `runExtractVideosStage`, - * written to `/meta/videos.json` for `renderChunk` to reconstruct - * a `FrameLookupTable` from. The in-process renderer keeps these structures - * in memory; the distributed pipeline writes them out so a separate chunk - * worker process can rebuild the video-frame injector without re-running - * the extract stage. - * - * `ExtractedFrames` from the engine carries an absolute `outputDir`, - * an open file descriptor in some paths, and a `framePaths` map — none of - * those survive a serialize → re-deserialize round trip across processes. - * The serialized form keeps only what plan-time produced and lets - * `renderChunk` re-derive `outputDir` (always `/video-frames/`) - * and `framePaths` (re-listed from that directory). - */ -interface PlanVideosJson { - /** - * Composition's `