mirror of
https://github.com/heygen-com/hyperframes.git
synced 2026-11-18 16:08:56 +00:00
refactor(producer): deferred frame extraction moves out of the distributed code (#7240)
* refactor(producer): deferred frame extraction moves out of the distributed code Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> * test(producer): the real-ffmpeg deferred extraction test moves to the integration lane Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> --------- Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
co-authored by
Claude Opus 5.5
parent
b108f85120
commit
35cbca9927
@@ -48,5 +48,6 @@
|
||||
"src/utils/audioRegression.test.ts",
|
||||
"src/utils/streamDurationParity.test.ts",
|
||||
"src/services/distributed/referenceBackground.integration.test.ts",
|
||||
"src/services/render/planReferenceBackground.polling.test.ts"
|
||||
"src/services/render/planReferenceBackground.polling.test.ts",
|
||||
"src/services/render/deferredVideoFrames.integration.test.ts"
|
||||
]
|
||||
|
||||
@@ -6,19 +6,16 @@ import {
|
||||
BROWSER_GPU_NOT_SOFTWARE,
|
||||
compositionRequiresWebGpu,
|
||||
type CaptureOptions,
|
||||
createFrameLookupTable,
|
||||
createVideoFrameInjector,
|
||||
type EngineConfig,
|
||||
type ExtractedFrames,
|
||||
extractVideoFramesRange,
|
||||
type FrameRange,
|
||||
type FrameLookupTable,
|
||||
type ShownBox,
|
||||
} from "@hyperframes/engine";
|
||||
import { resolveVideoCaptureBeyondViewport } from "../render/captureBeyondViewport.js";
|
||||
import { resolveCaptureJpegQuality } from "../render/captureImageFormat.js";
|
||||
import type { LockedRenderConfig } from "../render/stages/freezePlan.js";
|
||||
import { resolveRenderFpsConfig } from "../fileServer.js";
|
||||
import { extractDeferredFrames, shownDeferredFrameIndexes } from "../render/deferredVideoFrames.js";
|
||||
import { defaultLogger } from "../../logger.js";
|
||||
import type { PlanV2RemoteSources } from "./planV2.js";
|
||||
import { type DistributedFormat, INVALID_VIDEO_METADATA, type PlanVideosJson } from "./shared.js";
|
||||
@@ -128,16 +125,6 @@ export function rebuildExtractedFramesFromPlanDir(
|
||||
return result;
|
||||
}
|
||||
|
||||
function contiguousRuns(sortedIndexes: readonly number[]): FrameRange[] {
|
||||
const runs: FrameRange[] = [];
|
||||
for (const index of sortedIndexes) {
|
||||
const last = runs[runs.length - 1];
|
||||
if (last && last.firstFrame + last.frames === index) last.frames++;
|
||||
else runs.push({ firstFrame: index, frames: 1 });
|
||||
}
|
||||
return runs;
|
||||
}
|
||||
|
||||
/** Capture options a chunk session uses; the planner measures frame cost with the same ones. */
|
||||
export function buildChunkCaptureOptions(input: {
|
||||
dimensions: {
|
||||
@@ -180,33 +167,24 @@ export function buildChunkCaptureOptions(input: {
|
||||
};
|
||||
}
|
||||
|
||||
/** Extracts the given frames of deferred videos under the names a full extraction gives them. */
|
||||
export function shownDeferredFrameIndexes(
|
||||
function planExtractedFrames(planVideos: PlanVideosJson): ExtractedFrames[] {
|
||||
return planVideos.extracted.map((video) => ({
|
||||
...video,
|
||||
outputDir: "",
|
||||
framePaths: new Map<number, string>(),
|
||||
}));
|
||||
}
|
||||
|
||||
export function shownPlanFrameIndexes(
|
||||
planVideos: PlanVideosJson,
|
||||
frames: Iterable<number>,
|
||||
fps: { num: number; den: number },
|
||||
): Map<string, Set<number>> {
|
||||
const shown = new Map<string, Set<number>>();
|
||||
const deferred = planVideos.extracted.filter((video) => video.deferredRange);
|
||||
if (deferred.length === 0) return shown;
|
||||
const table = createFrameLookupTable(
|
||||
planVideos.videos,
|
||||
deferred.map((video) => ({
|
||||
...video,
|
||||
outputDir: "",
|
||||
framePaths: new Map(Array.from({ length: video.totalFrames }, (_, i) => [i, String(i)])),
|
||||
})),
|
||||
resolveRenderFpsConfig(fps).value,
|
||||
return shownDeferredFrameIndexes(
|
||||
{ videos: planVideos.videos, extracted: planExtractedFrames(planVideos) },
|
||||
frames,
|
||||
fps,
|
||||
);
|
||||
for (const frame of frames) {
|
||||
const globalTime = (frame * fps.den) / fps.num;
|
||||
for (const [videoId, payload] of table.getActiveFramePayloads(globalTime)) {
|
||||
const indexes = shown.get(videoId) ?? new Set<number>();
|
||||
indexes.add(payload.frameIndex);
|
||||
shown.set(videoId, indexes);
|
||||
}
|
||||
}
|
||||
return shown;
|
||||
}
|
||||
|
||||
export async function extractDeferredVideoFramesForChunk(input: {
|
||||
@@ -234,16 +212,17 @@ export async function extractDeferredVideoFramesForChunk(input: {
|
||||
url: remoteSources[path]!.split("?")[0],
|
||||
});
|
||||
}
|
||||
if (deferred.length === 0) return [];
|
||||
const shown = shownDeferredFrameIndexes(input.planVideos, input.frames, input.fps);
|
||||
return Promise.all(
|
||||
deferred.map(async (video) => {
|
||||
const outputDir = join(input.outputRoot, video.videoId);
|
||||
const indexes = [...(shown.get(video.videoId) ?? [])].sort((a, b) => a - b);
|
||||
if (indexes.length === 0) {
|
||||
return { ...video, outputDir, framePaths: new Map<number, string>(), ownedByLookup: false };
|
||||
}
|
||||
const range = video.deferredRange!;
|
||||
const planned = new Map(input.planVideos.extracted.map((video) => [video.videoId, video]));
|
||||
return extractDeferredFrames({
|
||||
videos: input.planVideos.videos,
|
||||
extracted: planExtractedFrames(input.planVideos),
|
||||
frames: input.frames,
|
||||
fps: input.fps,
|
||||
outputRoot: input.outputRoot,
|
||||
cfg: input.cfg,
|
||||
shownBoxes: input.shownBoxes,
|
||||
sourceOf: (video) => {
|
||||
const range = planned.get(video.videoId)!.deferredRange!;
|
||||
const remoteUrl = remoteSources[range.sourcePath];
|
||||
const sourcePath = remoteUrl ?? join(input.planDir, ...range.sourcePath.split("/"));
|
||||
if (remoteUrl === undefined && !existsSync(sourcePath)) {
|
||||
@@ -252,30 +231,13 @@ export async function extractDeferredVideoFramesForChunk(input: {
|
||||
`[renderChunk] planDir is missing the source for video ${JSON.stringify(video.videoId)}: ${range.sourcePath}`,
|
||||
);
|
||||
}
|
||||
const extracted = await extractVideoFramesRange(
|
||||
sourcePath,
|
||||
video.videoId,
|
||||
range.startTime,
|
||||
range.durationSeconds,
|
||||
{
|
||||
fps: input.fps,
|
||||
outputDir: input.outputRoot,
|
||||
format: range.format,
|
||||
frameRanges: contiguousRuns(indexes),
|
||||
shownBox: input.shownBoxes?.get(video.videoId),
|
||||
},
|
||||
undefined,
|
||||
input.cfg,
|
||||
);
|
||||
const missing = indexes.find((index) => !extracted.framePaths.has(index));
|
||||
if (missing !== undefined) {
|
||||
throw new RenderChunkValidationError(
|
||||
INVALID_VIDEO_METADATA,
|
||||
`[renderChunk] video ${JSON.stringify(video.videoId)} has no frame ${missing}; ` +
|
||||
`the plan expected ${video.totalFrames} frames from ${range.sourcePath}.`,
|
||||
);
|
||||
}
|
||||
return { ...video, outputDir, framePaths: extracted.framePaths, ownedByLookup: false };
|
||||
}),
|
||||
);
|
||||
return sourcePath;
|
||||
},
|
||||
onMissingFrame: (video, index) =>
|
||||
new RenderChunkValidationError(
|
||||
INVALID_VIDEO_METADATA,
|
||||
`[renderChunk] video ${JSON.stringify(video.videoId)} has no frame ${index}; ` +
|
||||
`the plan expected ${video.totalFrames} frames from ${planned.get(video.videoId)!.deferredRange!.sourcePath}.`,
|
||||
),
|
||||
});
|
||||
}
|
||||
|
||||
@@ -117,7 +117,7 @@ import {
|
||||
MISSING_RUNTIME_ENV_SNAPSHOT,
|
||||
rebuildExtractedFramesFromPlanDir,
|
||||
RenderChunkValidationError,
|
||||
shownDeferredFrameIndexes,
|
||||
shownPlanFrameIndexes,
|
||||
} from "./chunkFrames.js";
|
||||
|
||||
export {
|
||||
@@ -697,7 +697,7 @@ export async function renderChunk(
|
||||
const chunkFps = { num: plan.dimensions.fpsNum, den: plan.dimensions.fpsDen };
|
||||
const deferredVideos = planVideos?.extracted.filter((video) => video.deferredRange) ?? [];
|
||||
const shownFrames = planVideos
|
||||
? shownDeferredFrameIndexes(planVideos, chunkFrameNumbers, chunkFps)
|
||||
? shownPlanFrameIndexes(planVideos, chunkFrameNumbers, chunkFps)
|
||||
: new Map<string, Set<number>>();
|
||||
// The browser cannot load a source read by URL; size its <video> from the plan instead.
|
||||
const remoteVideoIds = new Set(
|
||||
|
||||
@@ -0,0 +1,98 @@
|
||||
import { spawnSync } from "node:child_process";
|
||||
import { mkdirSync, mkdtempSync, rmSync } from "node:fs";
|
||||
import { tmpdir } from "node:os";
|
||||
import { join } from "node:path";
|
||||
import { afterAll, beforeAll, describe, expect, it } from "bun:test";
|
||||
import type { ExtractedFrames, VideoElement } from "@hyperframes/engine";
|
||||
import { extractDeferredFrames } from "./deferredVideoFrames.js";
|
||||
|
||||
const metadata = {
|
||||
durationSeconds: 4,
|
||||
videoStreamDurationSeconds: 4,
|
||||
videoStreamStartSeconds: 0,
|
||||
width: 64,
|
||||
height: 36,
|
||||
fps: 30,
|
||||
videoCodec: "h264",
|
||||
hasAudio: false,
|
||||
isVFR: false,
|
||||
hasAlpha: false,
|
||||
colorSpace: null,
|
||||
};
|
||||
|
||||
const deferredVideo = (videoId: string, totalFrames: number): ExtractedFrames => ({
|
||||
videoId,
|
||||
srcPath: `/planner/${videoId}.mp4`,
|
||||
outputDir: "",
|
||||
framePattern: "frame_%05d.jpg",
|
||||
fps: 30,
|
||||
totalFrames,
|
||||
metadata,
|
||||
framePaths: new Map(),
|
||||
deferredRange: { startTime: 1.5, durationSeconds: totalFrames / 30, format: "jpg" },
|
||||
});
|
||||
|
||||
const element = (id: string, start: number, end: number): VideoElement => ({
|
||||
id,
|
||||
src: `${id}.mp4`,
|
||||
start,
|
||||
end,
|
||||
mediaStart: 1.5,
|
||||
loop: false,
|
||||
hasAudio: false,
|
||||
});
|
||||
|
||||
const fps = { num: 30, den: 1 };
|
||||
const span = (start: number, end: number) =>
|
||||
Array.from({ length: end - start }, (_, i) => start + i);
|
||||
|
||||
describe("extractDeferredFrames", () => {
|
||||
const root = mkdtempSync(join(tmpdir(), "hf-deferred-frames-"));
|
||||
const sourcePath = join(root, "short.mp4");
|
||||
|
||||
beforeAll(() => {
|
||||
mkdirSync(root, { recursive: true });
|
||||
const synth = spawnSync("ffmpeg", [
|
||||
"-v",
|
||||
"error",
|
||||
"-y",
|
||||
"-f",
|
||||
"lavfi",
|
||||
"-i",
|
||||
"testsrc2=s=64x36:r=30:d=3",
|
||||
"-c:v",
|
||||
"libx264",
|
||||
"-pix_fmt",
|
||||
"yuv420p",
|
||||
sourcePath,
|
||||
]);
|
||||
if (synth.status !== 0) throw new Error(synth.stderr.toString());
|
||||
});
|
||||
|
||||
afterAll(() => {
|
||||
rmSync(root, { recursive: true, force: true });
|
||||
});
|
||||
|
||||
it("calls onMissingFrame when the extractor returns fewer frames than the plan expected", async () => {
|
||||
const planned = deferredVideo("clip", 75);
|
||||
const missingError = new Error("plan expected more frames");
|
||||
const reported: Array<{ videoId: string; index: number }> = [];
|
||||
|
||||
const extraction = extractDeferredFrames({
|
||||
videos: [element("clip", 0, 4)],
|
||||
extracted: [planned],
|
||||
frames: span(40, 62),
|
||||
fps,
|
||||
outputRoot: join(root, "out"),
|
||||
cfg: {},
|
||||
sourceOf: () => sourcePath,
|
||||
onMissingFrame: (video, index) => {
|
||||
reported.push({ videoId: video.videoId, index });
|
||||
return missingError;
|
||||
},
|
||||
});
|
||||
|
||||
await expect(extraction).rejects.toBe(missingError);
|
||||
expect(reported).toEqual([{ videoId: "clip", index: 45 }]);
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,59 @@
|
||||
import { describe, expect, it } from "bun:test";
|
||||
import type { ExtractedFrames, VideoElement } from "@hyperframes/engine";
|
||||
import { shownDeferredFrameIndexes } from "./deferredVideoFrames.js";
|
||||
|
||||
const metadata = {
|
||||
durationSeconds: 4,
|
||||
videoStreamDurationSeconds: 4,
|
||||
videoStreamStartSeconds: 0,
|
||||
width: 64,
|
||||
height: 36,
|
||||
fps: 30,
|
||||
videoCodec: "h264",
|
||||
hasAudio: false,
|
||||
isVFR: false,
|
||||
hasAlpha: false,
|
||||
colorSpace: null,
|
||||
};
|
||||
|
||||
const deferredVideo = (videoId: string, totalFrames: number): ExtractedFrames => ({
|
||||
videoId,
|
||||
srcPath: `/planner/${videoId}.mp4`,
|
||||
outputDir: "",
|
||||
framePattern: "frame_%05d.jpg",
|
||||
fps: 30,
|
||||
totalFrames,
|
||||
metadata,
|
||||
framePaths: new Map(),
|
||||
deferredRange: { startTime: 1.5, durationSeconds: totalFrames / 30, format: "jpg" },
|
||||
});
|
||||
|
||||
const element = (id: string, start: number, end: number): VideoElement => ({
|
||||
id,
|
||||
src: `${id}.mp4`,
|
||||
start,
|
||||
end,
|
||||
mediaStart: 1.5,
|
||||
loop: false,
|
||||
hasAudio: false,
|
||||
});
|
||||
|
||||
const fps = { num: 30, den: 1 };
|
||||
const span = (start: number, end: number) =>
|
||||
Array.from({ length: end - start }, (_, i) => start + i);
|
||||
|
||||
describe("shownDeferredFrameIndexes", () => {
|
||||
it("names the frames each deferred video shows across a frame range, and none for a video off screen", () => {
|
||||
const shown = shownDeferredFrameIndexes(
|
||||
{
|
||||
videos: [element("clip", 0, 4), element("later", 10, 12)],
|
||||
extracted: [deferredVideo("clip", 75), deferredVideo("later", 75)],
|
||||
},
|
||||
span(30, 45),
|
||||
fps,
|
||||
);
|
||||
|
||||
expect([...shown.get("clip")!].sort((a, b) => a - b)).toEqual(span(30, 45));
|
||||
expect(shown.has("later")).toBe(false);
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,98 @@
|
||||
import { join } from "node:path";
|
||||
import {
|
||||
createFrameLookupTable,
|
||||
type EngineConfig,
|
||||
type ExtractedFrames,
|
||||
extractVideoFramesRange,
|
||||
type FrameRange,
|
||||
type ShownBox,
|
||||
type VideoElement,
|
||||
} from "@hyperframes/engine";
|
||||
import { resolveRenderFpsConfig } from "../fileServer.js";
|
||||
|
||||
function contiguousRuns(sortedIndexes: readonly number[]): FrameRange[] {
|
||||
const runs: FrameRange[] = [];
|
||||
for (const index of sortedIndexes) {
|
||||
const last = runs[runs.length - 1];
|
||||
if (last && last.firstFrame + last.frames === index) last.frames++;
|
||||
else runs.push({ firstFrame: index, frames: 1 });
|
||||
}
|
||||
return runs;
|
||||
}
|
||||
|
||||
/** The frame indexes each deferred video shows across `frames`, by video id. */
|
||||
export function shownDeferredFrameIndexes(
|
||||
source: { videos: VideoElement[]; extracted: ExtractedFrames[] },
|
||||
frames: Iterable<number>,
|
||||
fps: { num: number; den: number },
|
||||
): Map<string, Set<number>> {
|
||||
const shown = new Map<string, Set<number>>();
|
||||
const deferred = source.extracted.filter((video) => video.deferredRange);
|
||||
if (deferred.length === 0) return shown;
|
||||
const table = createFrameLookupTable(
|
||||
source.videos,
|
||||
deferred.map((video) => ({
|
||||
...video,
|
||||
outputDir: "",
|
||||
framePaths: new Map(Array.from({ length: video.totalFrames }, (_, i) => [i, String(i)])),
|
||||
})),
|
||||
resolveRenderFpsConfig(fps).value,
|
||||
);
|
||||
for (const frame of frames) {
|
||||
const globalTime = (frame * fps.den) / fps.num;
|
||||
for (const [videoId, payload] of table.getActiveFramePayloads(globalTime)) {
|
||||
const indexes = shown.get(videoId) ?? new Set<number>();
|
||||
indexes.add(payload.frameIndex);
|
||||
shown.set(videoId, indexes);
|
||||
}
|
||||
}
|
||||
return shown;
|
||||
}
|
||||
|
||||
/** Extracts the given frames of deferred videos under the names a full extraction gives them. */
|
||||
export async function extractDeferredFrames(input: {
|
||||
videos: VideoElement[];
|
||||
extracted: ExtractedFrames[];
|
||||
frames: Iterable<number>;
|
||||
fps: { num: number; den: number };
|
||||
outputRoot: string;
|
||||
cfg: Pick<EngineConfig, "ffmpegProcessTimeout">;
|
||||
signal?: AbortSignal;
|
||||
/** Where the video's source is read from: a local path or a URL. May throw. */
|
||||
sourceOf: (video: ExtractedFrames) => string;
|
||||
shownBoxes?: ReadonlyMap<string, ShownBox>;
|
||||
/** The error to throw when the extractor returns no frame for an index the video shows. */
|
||||
onMissingFrame: (video: ExtractedFrames, index: number) => Error;
|
||||
}): Promise<ExtractedFrames[]> {
|
||||
const deferred = input.extracted.filter((video) => video.deferredRange);
|
||||
if (deferred.length === 0) return [];
|
||||
const shown = shownDeferredFrameIndexes(input, input.frames, input.fps);
|
||||
return Promise.all(
|
||||
deferred.map(async (video) => {
|
||||
const outputDir = join(input.outputRoot, video.videoId);
|
||||
const indexes = [...(shown.get(video.videoId) ?? [])].sort((a, b) => a - b);
|
||||
if (indexes.length === 0) {
|
||||
return { ...video, outputDir, framePaths: new Map<number, string>(), ownedByLookup: false };
|
||||
}
|
||||
const range = video.deferredRange!;
|
||||
const extracted = await extractVideoFramesRange(
|
||||
input.sourceOf(video),
|
||||
video.videoId,
|
||||
range.startTime,
|
||||
range.durationSeconds,
|
||||
{
|
||||
fps: input.fps,
|
||||
outputDir: input.outputRoot,
|
||||
format: range.format,
|
||||
frameRanges: contiguousRuns(indexes),
|
||||
shownBox: input.shownBoxes?.get(video.videoId),
|
||||
},
|
||||
input.signal,
|
||||
input.cfg,
|
||||
);
|
||||
const missing = indexes.find((index) => !extracted.framePaths.has(index));
|
||||
if (missing !== undefined) throw input.onMissingFrame(video, missing);
|
||||
return { ...video, outputDir, framePaths: extracted.framePaths, ownedByLookup: false };
|
||||
}),
|
||||
);
|
||||
}
|
||||
Reference in New Issue
Block a user