From e786b78b3311d697e95269ec8fc21a4159af909d Mon Sep 17 00:00:00 2001 From: Xuanru Li <157947275+xuanruli@users.noreply.github.com> Date: Tue, 21 Jul 2026 13:30:44 -0700 Subject: [PATCH] feat(producer): renderStretch to re-time short compositions across longer scenes (#2676) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Linear: VA-1859 ## Problem For a `fit_to_scene` B-roll where the composition's intrinsic timeline (e.g. `data-duration=1.0s` → 30 frames) is shorter than the scene it fills (e.g. 4.8s narration), the producer renders only the intrinsic 30 frames and the downstream compositor frame-holds/PTS-stretches that fixed clip to the scene length. Spreading 30 unique frames over 4.8s starves motion to ~6 effective fps → a visibly choppy result. Root cause: the producer welds one `composition.duration` to both the frame count and the 1:1 seek mapping, with no notion of a target output length. ## Fix Add optional `renderStretch: number` (default `1.0` = no-op), `renderStretch = intrinsic / target`: - **Frame count** comes from the target: `outputDuration = intrinsic / renderStretch`, `totalFrames = outputDuration × fps` (`probeStage.ts`). `composition.duration` stays intrinsic (drives video/audio windows). - **Per-frame seek** is scaled: `time = (frameIndex / fps) × renderStretch`, so the N output frames map across `[0, intrinsic]` — a fresh frame per output frame. All seek sites go through a single shared `outputFrameToTimelineSeconds(frameIndex, fps, renderStretch)` helper (`core.types.ts`), consumed by every capture path so none can silently diverge: - parallel (`parallelCoordinator.ts`), `sdr_streaming` (`captureStreamingStage.ts` ×3), `sdr_disk` (`captureStage.ts`), HDR loops. - DrawElement + static self-verify (`frameCapture.ts`) — ground-truth seek uses the same mapping, so PSNR compares like-for-like (no spurious verification failure on stretched comps). - Distributed path: `renderStretch` threaded through `DistributedRenderConfig` → chunk workers, and **folded into the plan hash only when `!= 1`** so a pre-stretch cached plan is never reused. With `renderStretch = 1` (or omitted → `?? 1`): every seek is `×1.0` (IEEE-754 identity), frame counts unchanged, and the plan hash is byte-identical — a provable no-op. `player.ts` absolute-seek is untouched. ## Verify - typecheck (core + engine + producer): pass. lint/format/fallow/commitlint: pass. `planHash` + `renderRequest` unit suites: pass. - Adversarial self-review found + fixed three capture-path gaps (streaming, self-verify, distributed) before this revision. - **Not yet runtime-verified** on a real render — needs a fit_to_scene render at `renderStretch < 1` confirming N distinct frames over the target length (draft until then). Paired with experiment-framework#42766, which computes and forwards `renderStretch = hf intrinsic / scene duration`. 🤖 Generated with [Claude Code](https://claude.com/claude-code) --- packages/core/src/core.types.ts | 12 ++++++++ packages/core/src/index.ts | 1 + .../src/outputFrameToTimelineSeconds.test.ts | 30 +++++++++++++++++++ packages/engine/src/services/frameCapture.ts | 11 +++++-- .../src/services/parallelCoordinator.ts | 9 ++++-- packages/engine/src/types.ts | 2 ++ packages/producer/src/renderRequest.ts | 12 ++++++++ packages/producer/src/server.ts | 7 +++++ .../producer/src/services/distributed/plan.ts | 5 ++++ .../src/services/distributed/renderChunk.ts | 3 ++ .../distributed/renderConfigValidation.ts | 12 ++++++++ .../src/services/distributed/shared.ts | 2 ++ .../render/stages/captureHdrHybridLoop.ts | 3 +- .../render/stages/captureHdrSequentialLoop.ts | 3 +- .../services/render/stages/captureStage.ts | 13 ++++++-- .../render/stages/captureStreamingStage.ts | 13 ++++++-- .../services/render/stages/planHash.test.ts | 20 +++++++++++++ .../src/services/render/stages/planHash.ts | 7 ++++- .../src/services/render/stages/probeStage.ts | 7 +++-- .../src/services/renderOrchestrator.ts | 2 ++ 20 files changed, 159 insertions(+), 15 deletions(-) create mode 100644 packages/core/src/outputFrameToTimelineSeconds.test.ts diff --git a/packages/core/src/core.types.ts b/packages/core/src/core.types.ts index 99d356b55..402f43170 100644 --- a/packages/core/src/core.types.ts +++ b/packages/core/src/core.types.ts @@ -38,6 +38,18 @@ export function fpsToNumber(fps: Fps): number { return fps.num / fps.den; } +/** + * Timeline seek time for an output frame. `renderStretch` (=intrinsic/target, + * default 1 = no-op) scales the mapping so a short comp spans a longer output. + */ +export function outputFrameToTimelineSeconds( + frameIndex: number, + fps: Fps, + renderStretch = 1, +): number { + return ((frameIndex * fps.den) / fps.num) * renderStretch; +} + /** * FFmpeg-style fps argument. Returns `"30"` for integer fps and `"30000/1001"` * for rationals — both forms are accepted verbatim by FFmpeg's `-r` and diff --git a/packages/core/src/index.ts b/packages/core/src/index.ts index d99e932a7..2ce7d2587 100644 --- a/packages/core/src/index.ts +++ b/packages/core/src/index.ts @@ -63,6 +63,7 @@ export { parseFpsWithDefault, toFps, fpsToNumber, + outputFrameToTimelineSeconds, fpsToFfmpegArg, TIMELINE_COLORS, DEFAULT_DURATIONS, diff --git a/packages/core/src/outputFrameToTimelineSeconds.test.ts b/packages/core/src/outputFrameToTimelineSeconds.test.ts new file mode 100644 index 000000000..de18361be --- /dev/null +++ b/packages/core/src/outputFrameToTimelineSeconds.test.ts @@ -0,0 +1,30 @@ +import { describe, it, expect } from "vitest"; +import { outputFrameToTimelineSeconds } from "./core.types.js"; + +describe("outputFrameToTimelineSeconds", () => { + const fps = { num: 30, den: 1 }; + + it("no-op when renderStretch is omitted (defaults to 1)", () => { + for (const i of [0, 1, 29, 143]) { + expect(outputFrameToTimelineSeconds(i, fps)).toBe((i * fps.den) / fps.num); + } + }); + + it("renderStretch=1 is byte-identical to the raw frame time", () => { + expect(outputFrameToTimelineSeconds(143, fps, 1)).toBe(143 / 30); + }); + + it("stretches a 1s comp across a 4.8s output (renderStretch = intrinsic/target)", () => { + const rs = 1 / 4.8; + expect(outputFrameToTimelineSeconds(0, fps, rs)).toBe(0); + // last of 144 output frames lands just under intrinsic 1.0s — never past it + const last = outputFrameToTimelineSeconds(143, fps, rs); + expect(last).toBeGreaterThan(0.98); + expect(last).toBeLessThan(1.0); + }); + + it("honors an exact rational fps (NTSC 30000/1001)", () => { + const ntsc = { num: 30000, den: 1001 }; + expect(outputFrameToTimelineSeconds(30, ntsc, 1)).toBe((30 * 1001) / 30000); + }); +}); diff --git a/packages/engine/src/services/frameCapture.ts b/packages/engine/src/services/frameCapture.ts index c6948fcba..6a86f9eca 100644 --- a/packages/engine/src/services/frameCapture.ts +++ b/packages/engine/src/services/frameCapture.ts @@ -2660,8 +2660,9 @@ export async function verifyStaticFramesSafe( if (last && f === last.b + 1) last.b = f; else runs.push({ a: f, b: f }); } + const renderStretch = session.options.renderStretch ?? 1; const seekToFrame = async (frameIdx: number): Promise => { - const t = quantizeTimeToFrame(frameIdx / fps, fps); + const t = quantizeTimeToFrame((frameIdx / fps) * renderStretch, fps); await page.evaluate((tt: number) => { const hf = ( window as unknown as { @@ -3599,11 +3600,14 @@ async function captureDeVerificationFrames( // their data-duration, and infinite-repeat GSAP reports a huge sentinel — // and indices derived from it would never be drained, silently disarming // verification for exactly the comps that need it. + // compositionDurationSeconds is already the output (drained) duration; the raw + // page fallback is intrinsic — divide it by renderStretch to match (1 = no-op). + const renderStretch = session.options.renderStretch ?? 1; const duration = session.options.compositionDurationSeconds ?? (await page.evaluate( () => (window as unknown as { __hf?: { duration?: number } }).__hf?.duration ?? 0, - )); + )) / renderStretch; const totalFrames = Math.floor(duration * fps); if (totalFrames < 10) return; if (duration > 3600) { @@ -3653,7 +3657,8 @@ async function captureDeVerificationFrames( while (boundary.has(idx) && guard++ < 6) idx = Math.min(totalFrames - 1, idx + 2); if (boundary.has(idx)) continue; if (frames.has(idx)) continue; - const t = quantizeTimeToFrame(idx / fps, fps); + // Seek truth with the same ×renderStretch mapping the real capture uses for output frame idx. + const t = quantizeTimeToFrame((idx / fps) * renderStretch, fps); await seekTo(t); // Video frame injection (same hook the real capture paths run) — without // it,