mirror of
https://github.com/heygen-com/hyperframes.git
synced 2026-09-04 16:42:27 +00:00
feat(producer): renderStretch to re-time short compositions across longer scenes (#2676)
Linear: VA-1859 ## Problem For a `fit_to_scene` B-roll where the composition's intrinsic timeline (e.g. `data-duration=1.0s` → 30 frames) is shorter than the scene it fills (e.g. 4.8s narration), the producer renders only the intrinsic 30 frames and the downstream compositor frame-holds/PTS-stretches that fixed clip to the scene length. Spreading 30 unique frames over 4.8s starves motion to ~6 effective fps → a visibly choppy result. Root cause: the producer welds one `composition.duration` to both the frame count and the 1:1 seek mapping, with no notion of a target output length. ## Fix Add optional `renderStretch: number` (default `1.0` = no-op), `renderStretch = intrinsic / target`: - **Frame count** comes from the target: `outputDuration = intrinsic / renderStretch`, `totalFrames = outputDuration × fps` (`probeStage.ts`). `composition.duration` stays intrinsic (drives video/audio windows). - **Per-frame seek** is scaled: `time = (frameIndex / fps) × renderStretch`, so the N output frames map across `[0, intrinsic]` — a fresh frame per output frame. All seek sites go through a single shared `outputFrameToTimelineSeconds(frameIndex, fps, renderStretch)` helper (`core.types.ts`), consumed by every capture path so none can silently diverge: - parallel (`parallelCoordinator.ts`), `sdr_streaming` (`captureStreamingStage.ts` ×3), `sdr_disk` (`captureStage.ts`), HDR loops. - DrawElement + static self-verify (`frameCapture.ts`) — ground-truth seek uses the same mapping, so PSNR compares like-for-like (no spurious verification failure on stretched comps). - Distributed path: `renderStretch` threaded through `DistributedRenderConfig` → chunk workers, and **folded into the plan hash only when `!= 1`** so a pre-stretch cached plan is never reused. With `renderStretch = 1` (or omitted → `?? 1`): every seek is `×1.0` (IEEE-754 identity), frame counts unchanged, and the plan hash is byte-identical — a provable no-op. `player.ts` absolute-seek is untouched. ## Verify - typecheck (core + engine + producer): pass. lint/format/fallow/commitlint: pass. `planHash` + `renderRequest` unit suites: pass. - Adversarial self-review found + fixed three capture-path gaps (streaming, self-verify, distributed) before this revision. - **Not yet runtime-verified** on a real render — needs a fit_to_scene render at `renderStretch < 1` confirming N distinct frames over the target length (draft until then). Paired with experiment-framework#42766, which computes and forwards `renderStretch = hf intrinsic / scene duration`. 🤖 Generated with [Claude Code](https://claude.com/claude-code)
This commit is contained in:
@@ -2660,8 +2660,9 @@ export async function verifyStaticFramesSafe(
|
||||
if (last && f === last.b + 1) last.b = f;
|
||||
else runs.push({ a: f, b: f });
|
||||
}
|
||||
const renderStretch = session.options.renderStretch ?? 1;
|
||||
const seekToFrame = async (frameIdx: number): Promise<void> => {
|
||||
const t = quantizeTimeToFrame(frameIdx / fps, fps);
|
||||
const t = quantizeTimeToFrame((frameIdx / fps) * renderStretch, fps);
|
||||
await page.evaluate((tt: number) => {
|
||||
const hf = (
|
||||
window as unknown as {
|
||||
@@ -3599,11 +3600,14 @@ async function captureDeVerificationFrames(
|
||||
// their data-duration, and infinite-repeat GSAP reports a huge sentinel —
|
||||
// and indices derived from it would never be drained, silently disarming
|
||||
// verification for exactly the comps that need it.
|
||||
// compositionDurationSeconds is already the output (drained) duration; the raw
|
||||
// page fallback is intrinsic — divide it by renderStretch to match (1 = no-op).
|
||||
const renderStretch = session.options.renderStretch ?? 1;
|
||||
const duration =
|
||||
session.options.compositionDurationSeconds ??
|
||||
(await page.evaluate(
|
||||
() => (window as unknown as { __hf?: { duration?: number } }).__hf?.duration ?? 0,
|
||||
));
|
||||
)) / renderStretch;
|
||||
const totalFrames = Math.floor(duration * fps);
|
||||
if (totalFrames < 10) return;
|
||||
if (duration > 3600) {
|
||||
@@ -3653,7 +3657,8 @@ async function captureDeVerificationFrames(
|
||||
while (boundary.has(idx) && guard++ < 6) idx = Math.min(totalFrames - 1, idx + 2);
|
||||
if (boundary.has(idx)) continue;
|
||||
if (frames.has(idx)) continue;
|
||||
const t = quantizeTimeToFrame(idx / fps, fps);
|
||||
// Seek truth with the same ×renderStretch mapping the real capture uses for output frame idx.
|
||||
const t = quantizeTimeToFrame((idx / fps) * renderStretch, fps);
|
||||
await seekTo(t);
|
||||
// Video frame injection (same hook the real capture paths run) — without
|
||||
// it, <video> elements screenshot black and every video comp would
|
||||
|
||||
@@ -23,6 +23,7 @@ import {
|
||||
type CapturePerfSummary,
|
||||
type BeforeCaptureHook,
|
||||
} from "./frameCapture.js";
|
||||
import { outputFrameToTimelineSeconds } from "@hyperframes/core";
|
||||
import { DEFAULT_CONFIG, type EngineConfig } from "../config.js";
|
||||
import { assertSwiftShader } from "../utils/assertSwiftShader.js";
|
||||
import { readWebGlVendorInfoFromCanvas } from "../utils/readWebGlVendorInfoFromCanvas.js";
|
||||
@@ -368,6 +369,10 @@ async function captureFrameRange(
|
||||
let framesCaptured = 0;
|
||||
const outputOffset = task.outputFrameOffset ?? 0;
|
||||
const stride = task.frameStride ?? 1;
|
||||
// Per-frame seek ×renderStretch maps frames across [0, intrinsic] (1 = no-op).
|
||||
const renderStretch = captureOptions.renderStretch ?? 1;
|
||||
const seekTime = (i: number): number =>
|
||||
outputFrameToTimelineSeconds(i, captureOptions.fps, renderStretch);
|
||||
// Depth-2 pipelined drawElement produce (HF_DE_PARALLEL_STREAM spike): frame
|
||||
// k's in-page worker encode overlaps frame k+stride's produce phase — the
|
||||
// same shape as the sequential worker-encode loop. Only engaged when the
|
||||
@@ -387,7 +392,7 @@ async function captureFrameRange(
|
||||
let prev: { idx: number; encodeResult: Promise<Buffer> } | null = null;
|
||||
for (let i = task.startFrame; i < task.endFrame; i += stride) {
|
||||
if (signal?.aborted) throw new Error("Parallel worker cancelled");
|
||||
const time = (i * captureOptions.fps.den) / captureOptions.fps.num;
|
||||
const time = seekTime(i);
|
||||
if (dbg && i < task.startFrame + dbgWin) {
|
||||
console.log(`[par:w${task.workerId}] +${Date.now() - dbgT0}ms produce ${i} start`);
|
||||
}
|
||||
@@ -431,7 +436,7 @@ async function captureFrameRange(
|
||||
}
|
||||
for (let i = task.startFrame; i < task.endFrame; i += stride) {
|
||||
if (signal?.aborted) throw new Error("Parallel worker cancelled");
|
||||
const time = (i * captureOptions.fps.den) / captureOptions.fps.num;
|
||||
const time = seekTime(i);
|
||||
const fileFrameIdx = i - outputOffset;
|
||||
|
||||
if (onFrameBuffer) {
|
||||
|
||||
Reference in New Issue
Block a user