From 35cbca99274907c914e4e083f30684060e3b4c58 Mon Sep 17 00:00:00 2001 From: Vance Ingalls Date: Thu, 5 Nov 2026 09:45:12 -0800 Subject: [PATCH] refactor(producer): deferred frame extraction moves out of the distributed code (#7240) * refactor(producer): deferred frame extraction moves out of the distributed code Co-Authored-By: Claude Opus 5.5 * test(producer): the real-ffmpeg deferred extraction test moves to the integration lane Co-Authored-By: Claude Opus 5.5 --------- Co-authored-by: Claude Opus 5.5 --- .../producer/scripts/integration-tests.json | 3 +- .../src/services/distributed/chunkFrames.ts | 106 ++++++------------ .../src/services/distributed/renderChunk.ts | 4 +- .../deferredVideoFrames.integration.test.ts | 98 ++++++++++++++++ .../render/deferredVideoFrames.test.ts | 59 ++++++++++ .../services/render/deferredVideoFrames.ts | 98 ++++++++++++++++ 6 files changed, 293 insertions(+), 75 deletions(-) create mode 100644 packages/producer/src/services/render/deferredVideoFrames.integration.test.ts create mode 100644 packages/producer/src/services/render/deferredVideoFrames.test.ts create mode 100644 packages/producer/src/services/render/deferredVideoFrames.ts diff --git a/packages/producer/scripts/integration-tests.json b/packages/producer/scripts/integration-tests.json index c07020bb7a..08c5c33794 100644 --- a/packages/producer/scripts/integration-tests.json +++ b/packages/producer/scripts/integration-tests.json @@ -48,5 +48,6 @@ "src/utils/audioRegression.test.ts", "src/utils/streamDurationParity.test.ts", "src/services/distributed/referenceBackground.integration.test.ts", - "src/services/render/planReferenceBackground.polling.test.ts" + "src/services/render/planReferenceBackground.polling.test.ts", + "src/services/render/deferredVideoFrames.integration.test.ts" ] diff --git a/packages/producer/src/services/distributed/chunkFrames.ts b/packages/producer/src/services/distributed/chunkFrames.ts index 7480e42872..d5af14615f 100644 --- a/packages/producer/src/services/distributed/chunkFrames.ts +++ b/packages/producer/src/services/distributed/chunkFrames.ts @@ -6,19 +6,16 @@ import { BROWSER_GPU_NOT_SOFTWARE, compositionRequiresWebGpu, type CaptureOptions, - createFrameLookupTable, createVideoFrameInjector, type EngineConfig, type ExtractedFrames, - extractVideoFramesRange, - type FrameRange, type FrameLookupTable, type ShownBox, } from "@hyperframes/engine"; import { resolveVideoCaptureBeyondViewport } from "../render/captureBeyondViewport.js"; import { resolveCaptureJpegQuality } from "../render/captureImageFormat.js"; import type { LockedRenderConfig } from "../render/stages/freezePlan.js"; -import { resolveRenderFpsConfig } from "../fileServer.js"; +import { extractDeferredFrames, shownDeferredFrameIndexes } from "../render/deferredVideoFrames.js"; import { defaultLogger } from "../../logger.js"; import type { PlanV2RemoteSources } from "./planV2.js"; import { type DistributedFormat, INVALID_VIDEO_METADATA, type PlanVideosJson } from "./shared.js"; @@ -128,16 +125,6 @@ export function rebuildExtractedFramesFromPlanDir( return result; } -function contiguousRuns(sortedIndexes: readonly number[]): FrameRange[] { - const runs: FrameRange[] = []; - for (const index of sortedIndexes) { - const last = runs[runs.length - 1]; - if (last && last.firstFrame + last.frames === index) last.frames++; - else runs.push({ firstFrame: index, frames: 1 }); - } - return runs; -} - /** Capture options a chunk session uses; the planner measures frame cost with the same ones. */ export function buildChunkCaptureOptions(input: { dimensions: { @@ -180,33 +167,24 @@ export function buildChunkCaptureOptions(input: { }; } -/** Extracts the given frames of deferred videos under the names a full extraction gives them. */ -export function shownDeferredFrameIndexes( +function planExtractedFrames(planVideos: PlanVideosJson): ExtractedFrames[] { + return planVideos.extracted.map((video) => ({ + ...video, + outputDir: "", + framePaths: new Map(), + })); +} + +export function shownPlanFrameIndexes( planVideos: PlanVideosJson, frames: Iterable, fps: { num: number; den: number }, ): Map> { - const shown = new Map>(); - const deferred = planVideos.extracted.filter((video) => video.deferredRange); - if (deferred.length === 0) return shown; - const table = createFrameLookupTable( - planVideos.videos, - deferred.map((video) => ({ - ...video, - outputDir: "", - framePaths: new Map(Array.from({ length: video.totalFrames }, (_, i) => [i, String(i)])), - })), - resolveRenderFpsConfig(fps).value, + return shownDeferredFrameIndexes( + { videos: planVideos.videos, extracted: planExtractedFrames(planVideos) }, + frames, + fps, ); - for (const frame of frames) { - const globalTime = (frame * fps.den) / fps.num; - for (const [videoId, payload] of table.getActiveFramePayloads(globalTime)) { - const indexes = shown.get(videoId) ?? new Set(); - indexes.add(payload.frameIndex); - shown.set(videoId, indexes); - } - } - return shown; } export async function extractDeferredVideoFramesForChunk(input: { @@ -234,16 +212,17 @@ export async function extractDeferredVideoFramesForChunk(input: { url: remoteSources[path]!.split("?")[0], }); } - if (deferred.length === 0) return []; - const shown = shownDeferredFrameIndexes(input.planVideos, input.frames, input.fps); - return Promise.all( - deferred.map(async (video) => { - const outputDir = join(input.outputRoot, video.videoId); - const indexes = [...(shown.get(video.videoId) ?? [])].sort((a, b) => a - b); - if (indexes.length === 0) { - return { ...video, outputDir, framePaths: new Map(), ownedByLookup: false }; - } - const range = video.deferredRange!; + const planned = new Map(input.planVideos.extracted.map((video) => [video.videoId, video])); + return extractDeferredFrames({ + videos: input.planVideos.videos, + extracted: planExtractedFrames(input.planVideos), + frames: input.frames, + fps: input.fps, + outputRoot: input.outputRoot, + cfg: input.cfg, + shownBoxes: input.shownBoxes, + sourceOf: (video) => { + const range = planned.get(video.videoId)!.deferredRange!; const remoteUrl = remoteSources[range.sourcePath]; const sourcePath = remoteUrl ?? join(input.planDir, ...range.sourcePath.split("/")); if (remoteUrl === undefined && !existsSync(sourcePath)) { @@ -252,30 +231,13 @@ export async function extractDeferredVideoFramesForChunk(input: { `[renderChunk] planDir is missing the source for video ${JSON.stringify(video.videoId)}: ${range.sourcePath}`, ); } - const extracted = await extractVideoFramesRange( - sourcePath, - video.videoId, - range.startTime, - range.durationSeconds, - { - fps: input.fps, - outputDir: input.outputRoot, - format: range.format, - frameRanges: contiguousRuns(indexes), - shownBox: input.shownBoxes?.get(video.videoId), - }, - undefined, - input.cfg, - ); - const missing = indexes.find((index) => !extracted.framePaths.has(index)); - if (missing !== undefined) { - throw new RenderChunkValidationError( - INVALID_VIDEO_METADATA, - `[renderChunk] video ${JSON.stringify(video.videoId)} has no frame ${missing}; ` + - `the plan expected ${video.totalFrames} frames from ${range.sourcePath}.`, - ); - } - return { ...video, outputDir, framePaths: extracted.framePaths, ownedByLookup: false }; - }), - ); + return sourcePath; + }, + onMissingFrame: (video, index) => + new RenderChunkValidationError( + INVALID_VIDEO_METADATA, + `[renderChunk] video ${JSON.stringify(video.videoId)} has no frame ${index}; ` + + `the plan expected ${video.totalFrames} frames from ${planned.get(video.videoId)!.deferredRange!.sourcePath}.`, + ), + }); } diff --git a/packages/producer/src/services/distributed/renderChunk.ts b/packages/producer/src/services/distributed/renderChunk.ts index 9b44b1e936..83c17fb451 100644 --- a/packages/producer/src/services/distributed/renderChunk.ts +++ b/packages/producer/src/services/distributed/renderChunk.ts @@ -117,7 +117,7 @@ import { MISSING_RUNTIME_ENV_SNAPSHOT, rebuildExtractedFramesFromPlanDir, RenderChunkValidationError, - shownDeferredFrameIndexes, + shownPlanFrameIndexes, } from "./chunkFrames.js"; export { @@ -697,7 +697,7 @@ export async function renderChunk( const chunkFps = { num: plan.dimensions.fpsNum, den: plan.dimensions.fpsDen }; const deferredVideos = planVideos?.extracted.filter((video) => video.deferredRange) ?? []; const shownFrames = planVideos - ? shownDeferredFrameIndexes(planVideos, chunkFrameNumbers, chunkFps) + ? shownPlanFrameIndexes(planVideos, chunkFrameNumbers, chunkFps) : new Map>(); // The browser cannot load a source read by URL; size its