refactor(producer): deferred frame extraction moves out of the distributed code (#7240)

* refactor(producer): deferred frame extraction moves out of the distributed code

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

* test(producer): the real-ffmpeg deferred extraction test moves to the integration lane

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

---------

Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
Vance Ingalls
2026-11-05 09:45:12 -08:00
committed by GitHub
co-authored by Claude Opus 5.5
parent b108f85120
commit 35cbca9927
6 changed files with 293 additions and 75 deletions
@@ -48,5 +48,6 @@
"src/utils/audioRegression.test.ts",
"src/utils/streamDurationParity.test.ts",
"src/services/distributed/referenceBackground.integration.test.ts",
"src/services/render/planReferenceBackground.polling.test.ts"
"src/services/render/planReferenceBackground.polling.test.ts",
"src/services/render/deferredVideoFrames.integration.test.ts"
]
@@ -6,19 +6,16 @@ import {
BROWSER_GPU_NOT_SOFTWARE,
compositionRequiresWebGpu,
type CaptureOptions,
createFrameLookupTable,
createVideoFrameInjector,
type EngineConfig,
type ExtractedFrames,
extractVideoFramesRange,
type FrameRange,
type FrameLookupTable,
type ShownBox,
} from "@hyperframes/engine";
import { resolveVideoCaptureBeyondViewport } from "../render/captureBeyondViewport.js";
import { resolveCaptureJpegQuality } from "../render/captureImageFormat.js";
import type { LockedRenderConfig } from "../render/stages/freezePlan.js";
import { resolveRenderFpsConfig } from "../fileServer.js";
import { extractDeferredFrames, shownDeferredFrameIndexes } from "../render/deferredVideoFrames.js";
import { defaultLogger } from "../../logger.js";
import type { PlanV2RemoteSources } from "./planV2.js";
import { type DistributedFormat, INVALID_VIDEO_METADATA, type PlanVideosJson } from "./shared.js";
@@ -128,16 +125,6 @@ export function rebuildExtractedFramesFromPlanDir(
return result;
}
function contiguousRuns(sortedIndexes: readonly number[]): FrameRange[] {
const runs: FrameRange[] = [];
for (const index of sortedIndexes) {
const last = runs[runs.length - 1];
if (last && last.firstFrame + last.frames === index) last.frames++;
else runs.push({ firstFrame: index, frames: 1 });
}
return runs;
}
/** Capture options a chunk session uses; the planner measures frame cost with the same ones. */
export function buildChunkCaptureOptions(input: {
dimensions: {
@@ -180,33 +167,24 @@ export function buildChunkCaptureOptions(input: {
};
}
/** Extracts the given frames of deferred videos under the names a full extraction gives them. */
export function shownDeferredFrameIndexes(
function planExtractedFrames(planVideos: PlanVideosJson): ExtractedFrames[] {
return planVideos.extracted.map((video) => ({
...video,
outputDir: "",
framePaths: new Map<number, string>(),
}));
}
export function shownPlanFrameIndexes(
planVideos: PlanVideosJson,
frames: Iterable<number>,
fps: { num: number; den: number },
): Map<string, Set<number>> {
const shown = new Map<string, Set<number>>();
const deferred = planVideos.extracted.filter((video) => video.deferredRange);
if (deferred.length === 0) return shown;
const table = createFrameLookupTable(
planVideos.videos,
deferred.map((video) => ({
...video,
outputDir: "",
framePaths: new Map(Array.from({ length: video.totalFrames }, (_, i) => [i, String(i)])),
})),
resolveRenderFpsConfig(fps).value,
return shownDeferredFrameIndexes(
{ videos: planVideos.videos, extracted: planExtractedFrames(planVideos) },
frames,
fps,
);
for (const frame of frames) {
const globalTime = (frame * fps.den) / fps.num;
for (const [videoId, payload] of table.getActiveFramePayloads(globalTime)) {
const indexes = shown.get(videoId) ?? new Set<number>();
indexes.add(payload.frameIndex);
shown.set(videoId, indexes);
}
}
return shown;
}
export async function extractDeferredVideoFramesForChunk(input: {
@@ -234,16 +212,17 @@ export async function extractDeferredVideoFramesForChunk(input: {
url: remoteSources[path]!.split("?")[0],
});
}
if (deferred.length === 0) return [];
const shown = shownDeferredFrameIndexes(input.planVideos, input.frames, input.fps);
return Promise.all(
deferred.map(async (video) => {
const outputDir = join(input.outputRoot, video.videoId);
const indexes = [...(shown.get(video.videoId) ?? [])].sort((a, b) => a - b);
if (indexes.length === 0) {
return { ...video, outputDir, framePaths: new Map<number, string>(), ownedByLookup: false };
}
const range = video.deferredRange!;
const planned = new Map(input.planVideos.extracted.map((video) => [video.videoId, video]));
return extractDeferredFrames({
videos: input.planVideos.videos,
extracted: planExtractedFrames(input.planVideos),
frames: input.frames,
fps: input.fps,
outputRoot: input.outputRoot,
cfg: input.cfg,
shownBoxes: input.shownBoxes,
sourceOf: (video) => {
const range = planned.get(video.videoId)!.deferredRange!;
const remoteUrl = remoteSources[range.sourcePath];
const sourcePath = remoteUrl ?? join(input.planDir, ...range.sourcePath.split("/"));
if (remoteUrl === undefined && !existsSync(sourcePath)) {
@@ -252,30 +231,13 @@ export async function extractDeferredVideoFramesForChunk(input: {
`[renderChunk] planDir is missing the source for video ${JSON.stringify(video.videoId)}: ${range.sourcePath}`,
);
}
const extracted = await extractVideoFramesRange(
sourcePath,
video.videoId,
range.startTime,
range.durationSeconds,
{
fps: input.fps,
outputDir: input.outputRoot,
format: range.format,
frameRanges: contiguousRuns(indexes),
shownBox: input.shownBoxes?.get(video.videoId),
},
undefined,
input.cfg,
);
const missing = indexes.find((index) => !extracted.framePaths.has(index));
if (missing !== undefined) {
throw new RenderChunkValidationError(
INVALID_VIDEO_METADATA,
`[renderChunk] video ${JSON.stringify(video.videoId)} has no frame ${missing}; ` +
`the plan expected ${video.totalFrames} frames from ${range.sourcePath}.`,
);
}
return { ...video, outputDir, framePaths: extracted.framePaths, ownedByLookup: false };
}),
);
return sourcePath;
},
onMissingFrame: (video, index) =>
new RenderChunkValidationError(
INVALID_VIDEO_METADATA,
`[renderChunk] video ${JSON.stringify(video.videoId)} has no frame ${index}; ` +
`the plan expected ${video.totalFrames} frames from ${planned.get(video.videoId)!.deferredRange!.sourcePath}.`,
),
});
}
@@ -117,7 +117,7 @@ import {
MISSING_RUNTIME_ENV_SNAPSHOT,
rebuildExtractedFramesFromPlanDir,
RenderChunkValidationError,
shownDeferredFrameIndexes,
shownPlanFrameIndexes,
} from "./chunkFrames.js";
export {
@@ -697,7 +697,7 @@ export async function renderChunk(
const chunkFps = { num: plan.dimensions.fpsNum, den: plan.dimensions.fpsDen };
const deferredVideos = planVideos?.extracted.filter((video) => video.deferredRange) ?? [];
const shownFrames = planVideos
? shownDeferredFrameIndexes(planVideos, chunkFrameNumbers, chunkFps)
? shownPlanFrameIndexes(planVideos, chunkFrameNumbers, chunkFps)
: new Map<string, Set<number>>();
// The browser cannot load a source read by URL; size its <video> from the plan instead.
const remoteVideoIds = new Set(
@@ -0,0 +1,98 @@
import { spawnSync } from "node:child_process";
import { mkdirSync, mkdtempSync, rmSync } from "node:fs";
import { tmpdir } from "node:os";
import { join } from "node:path";
import { afterAll, beforeAll, describe, expect, it } from "bun:test";
import type { ExtractedFrames, VideoElement } from "@hyperframes/engine";
import { extractDeferredFrames } from "./deferredVideoFrames.js";
const metadata = {
durationSeconds: 4,
videoStreamDurationSeconds: 4,
videoStreamStartSeconds: 0,
width: 64,
height: 36,
fps: 30,
videoCodec: "h264",
hasAudio: false,
isVFR: false,
hasAlpha: false,
colorSpace: null,
};
const deferredVideo = (videoId: string, totalFrames: number): ExtractedFrames => ({
videoId,
srcPath: `/planner/${videoId}.mp4`,
outputDir: "",
framePattern: "frame_%05d.jpg",
fps: 30,
totalFrames,
metadata,
framePaths: new Map(),
deferredRange: { startTime: 1.5, durationSeconds: totalFrames / 30, format: "jpg" },
});
const element = (id: string, start: number, end: number): VideoElement => ({
id,
src: `${id}.mp4`,
start,
end,
mediaStart: 1.5,
loop: false,
hasAudio: false,
});
const fps = { num: 30, den: 1 };
const span = (start: number, end: number) =>
Array.from({ length: end - start }, (_, i) => start + i);
describe("extractDeferredFrames", () => {
const root = mkdtempSync(join(tmpdir(), "hf-deferred-frames-"));
const sourcePath = join(root, "short.mp4");
beforeAll(() => {
mkdirSync(root, { recursive: true });
const synth = spawnSync("ffmpeg", [
"-v",
"error",
"-y",
"-f",
"lavfi",
"-i",
"testsrc2=s=64x36:r=30:d=3",
"-c:v",
"libx264",
"-pix_fmt",
"yuv420p",
sourcePath,
]);
if (synth.status !== 0) throw new Error(synth.stderr.toString());
});
afterAll(() => {
rmSync(root, { recursive: true, force: true });
});
it("calls onMissingFrame when the extractor returns fewer frames than the plan expected", async () => {
const planned = deferredVideo("clip", 75);
const missingError = new Error("plan expected more frames");
const reported: Array<{ videoId: string; index: number }> = [];
const extraction = extractDeferredFrames({
videos: [element("clip", 0, 4)],
extracted: [planned],
frames: span(40, 62),
fps,
outputRoot: join(root, "out"),
cfg: {},
sourceOf: () => sourcePath,
onMissingFrame: (video, index) => {
reported.push({ videoId: video.videoId, index });
return missingError;
},
});
await expect(extraction).rejects.toBe(missingError);
expect(reported).toEqual([{ videoId: "clip", index: 45 }]);
});
});
@@ -0,0 +1,59 @@
import { describe, expect, it } from "bun:test";
import type { ExtractedFrames, VideoElement } from "@hyperframes/engine";
import { shownDeferredFrameIndexes } from "./deferredVideoFrames.js";
const metadata = {
durationSeconds: 4,
videoStreamDurationSeconds: 4,
videoStreamStartSeconds: 0,
width: 64,
height: 36,
fps: 30,
videoCodec: "h264",
hasAudio: false,
isVFR: false,
hasAlpha: false,
colorSpace: null,
};
const deferredVideo = (videoId: string, totalFrames: number): ExtractedFrames => ({
videoId,
srcPath: `/planner/${videoId}.mp4`,
outputDir: "",
framePattern: "frame_%05d.jpg",
fps: 30,
totalFrames,
metadata,
framePaths: new Map(),
deferredRange: { startTime: 1.5, durationSeconds: totalFrames / 30, format: "jpg" },
});
const element = (id: string, start: number, end: number): VideoElement => ({
id,
src: `${id}.mp4`,
start,
end,
mediaStart: 1.5,
loop: false,
hasAudio: false,
});
const fps = { num: 30, den: 1 };
const span = (start: number, end: number) =>
Array.from({ length: end - start }, (_, i) => start + i);
describe("shownDeferredFrameIndexes", () => {
it("names the frames each deferred video shows across a frame range, and none for a video off screen", () => {
const shown = shownDeferredFrameIndexes(
{
videos: [element("clip", 0, 4), element("later", 10, 12)],
extracted: [deferredVideo("clip", 75), deferredVideo("later", 75)],
},
span(30, 45),
fps,
);
expect([...shown.get("clip")!].sort((a, b) => a - b)).toEqual(span(30, 45));
expect(shown.has("later")).toBe(false);
});
});
@@ -0,0 +1,98 @@
import { join } from "node:path";
import {
createFrameLookupTable,
type EngineConfig,
type ExtractedFrames,
extractVideoFramesRange,
type FrameRange,
type ShownBox,
type VideoElement,
} from "@hyperframes/engine";
import { resolveRenderFpsConfig } from "../fileServer.js";
function contiguousRuns(sortedIndexes: readonly number[]): FrameRange[] {
const runs: FrameRange[] = [];
for (const index of sortedIndexes) {
const last = runs[runs.length - 1];
if (last && last.firstFrame + last.frames === index) last.frames++;
else runs.push({ firstFrame: index, frames: 1 });
}
return runs;
}
/** The frame indexes each deferred video shows across `frames`, by video id. */
export function shownDeferredFrameIndexes(
source: { videos: VideoElement[]; extracted: ExtractedFrames[] },
frames: Iterable<number>,
fps: { num: number; den: number },
): Map<string, Set<number>> {
const shown = new Map<string, Set<number>>();
const deferred = source.extracted.filter((video) => video.deferredRange);
if (deferred.length === 0) return shown;
const table = createFrameLookupTable(
source.videos,
deferred.map((video) => ({
...video,
outputDir: "",
framePaths: new Map(Array.from({ length: video.totalFrames }, (_, i) => [i, String(i)])),
})),
resolveRenderFpsConfig(fps).value,
);
for (const frame of frames) {
const globalTime = (frame * fps.den) / fps.num;
for (const [videoId, payload] of table.getActiveFramePayloads(globalTime)) {
const indexes = shown.get(videoId) ?? new Set<number>();
indexes.add(payload.frameIndex);
shown.set(videoId, indexes);
}
}
return shown;
}
/** Extracts the given frames of deferred videos under the names a full extraction gives them. */
export async function extractDeferredFrames(input: {
videos: VideoElement[];
extracted: ExtractedFrames[];
frames: Iterable<number>;
fps: { num: number; den: number };
outputRoot: string;
cfg: Pick<EngineConfig, "ffmpegProcessTimeout">;
signal?: AbortSignal;
/** Where the video's source is read from: a local path or a URL. May throw. */
sourceOf: (video: ExtractedFrames) => string;
shownBoxes?: ReadonlyMap<string, ShownBox>;
/** The error to throw when the extractor returns no frame for an index the video shows. */
onMissingFrame: (video: ExtractedFrames, index: number) => Error;
}): Promise<ExtractedFrames[]> {
const deferred = input.extracted.filter((video) => video.deferredRange);
if (deferred.length === 0) return [];
const shown = shownDeferredFrameIndexes(input, input.frames, input.fps);
return Promise.all(
deferred.map(async (video) => {
const outputDir = join(input.outputRoot, video.videoId);
const indexes = [...(shown.get(video.videoId) ?? [])].sort((a, b) => a - b);
if (indexes.length === 0) {
return { ...video, outputDir, framePaths: new Map<number, string>(), ownedByLookup: false };
}
const range = video.deferredRange!;
const extracted = await extractVideoFramesRange(
input.sourceOf(video),
video.videoId,
range.startTime,
range.durationSeconds,
{
fps: input.fps,
outputDir: input.outputRoot,
format: range.format,
frameRanges: contiguousRuns(indexes),
shownBox: input.shownBoxes?.get(video.videoId),
},
input.signal,
input.cfg,
);
const missing = indexes.find((index) => !extracted.framePaths.has(index));
if (missing !== undefined) throw input.onMissingFrame(video, missing);
return { ...video, outputDir, framePaths: extracted.framePaths, ownedByLookup: false };
}),
);
}