mirror of
https://github.com/heygen-com/hyperframes.git
synced 2026-10-30 09:16:55 +00:00
feat(producer): render a distributed chunk as frame sub-ranges (#5916)
* feat(producer): render a distributed chunk as frame sub-ranges renderChunk and renderChunkV2 take an optional frameRange inside the chunk's slice, so a slow chunk can be retried as smaller parts rendered in parallel. A part renders as a chunk of its own: capture, deferred video frames, GOP size, frame names and the audio packets it carries all follow the range. Omitting it renders the whole slice as before. assemble and assembleV2 accept the ordered parts of a chunk in place of its single output. DISTRIBUTED_RENDER_CAPABILITIES.chunkFrameRange advertises both. Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com> * fix(producer): keep the chunk sub-range comments within the comment checks Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com> --------- Co-authored-by: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
This commit is contained in:
James Russo
co-authored by
Claude Opus 5.5
Claude Opus 5.5
parent
44ac4c4bbb
commit
fcd93ce184
@@ -98,10 +98,13 @@ export {
|
||||
readWebGlVendorInfoFromCanvas,
|
||||
renderChunk,
|
||||
// Types
|
||||
type ChunkFrameRange,
|
||||
type ChunkRenderer,
|
||||
type ChunkResult,
|
||||
type EffectiveChunkResult,
|
||||
type RenderChunkOptions,
|
||||
// Error codes + classes
|
||||
CHUNK_FRAME_RANGE_INVALID,
|
||||
FFMPEG_VERSION_MISMATCH,
|
||||
INVALID_VIDEO_METADATA,
|
||||
PLAN_HASH_MISMATCH,
|
||||
@@ -112,6 +115,7 @@ export {
|
||||
export {
|
||||
assemble,
|
||||
chunksAreStitchedMp4,
|
||||
type AssembleChunkPaths,
|
||||
type AssembleResult,
|
||||
} from "./services/distributed/assemble.js";
|
||||
export { muxChunkWithAudio } from "./services/distributed/chunkAudio.js";
|
||||
|
||||
@@ -83,6 +83,8 @@ interface PlanJsonForAssemble {
|
||||
};
|
||||
}
|
||||
|
||||
export type AssembleChunkPaths = readonly (string | readonly string[])[];
|
||||
|
||||
type CfrRateControlSource = Partial<Pick<LockedRenderConfig, "quality" | "crf" | "bitrate">>;
|
||||
|
||||
export function cfrRateControlArgs(locked: CfrRateControlSource): string[] {
|
||||
@@ -99,7 +101,8 @@ export function cfrRateControlArgs(locked: CfrRateControlSource): string[] {
|
||||
* @param planDir — absolute path to the planDir produced by `plan()`.
|
||||
* @param chunkPaths — ordered chunk outputs, length === `chunks.json` length.
|
||||
* For mp4/mov each entry is a path to an encoded chunk file; for
|
||||
* png-sequence each entry is a path to a directory of frames.
|
||||
* png-sequence each entry is a path to a directory of frames. An entry
|
||||
* may instead be the ordered parts of that chunk (see {@link AssembleChunkPaths}).
|
||||
* @param audioPath — `<planDir>/<MIXED_AUDIO_FILENAME>` for mux'd formats. Pass `null`
|
||||
* when the composition has no audio (or `assemble` is being called for a
|
||||
* format whose audio is muxed elsewhere). `assemble` always normalizes
|
||||
@@ -110,7 +113,7 @@ export function cfrRateControlArgs(locked: CfrRateControlSource): string[] {
|
||||
*/
|
||||
export async function assemble(
|
||||
planDir: string,
|
||||
chunkPaths: readonly string[],
|
||||
chunkOutputs: AssembleChunkPaths,
|
||||
audioPath: string | null,
|
||||
outputPath: string,
|
||||
options?: {
|
||||
@@ -136,7 +139,8 @@ export async function assemble(
|
||||
const log = options?.logger ?? defaultLogger;
|
||||
const cfr = options?.cfr === true;
|
||||
|
||||
const plan = readAssemblePlan(planDir, chunkPaths);
|
||||
const plan = readAssemblePlan(planDir, chunkOutputs);
|
||||
const chunkPaths = chunkOutputs.flat();
|
||||
if (plan.dimensions.format === "png-sequence") {
|
||||
// ── 2a. png-sequence: merge frame directories with global re-numbering
|
||||
return mergePngFrameDirs(chunkPaths, outputPath, plan.totalFrames, audioPath, start);
|
||||
@@ -208,7 +212,7 @@ export async function assemble(
|
||||
};
|
||||
}
|
||||
|
||||
function readAssemblePlan(planDir: string, chunkPaths: readonly string[]): PlanJsonForAssemble {
|
||||
function readAssemblePlan(planDir: string, chunkPaths: AssembleChunkPaths): PlanJsonForAssemble {
|
||||
// ── 1. Validate planDir manifest matches chunkPaths shape ──────────────
|
||||
const planJsonPath = join(planDir, "plan.json");
|
||||
const chunksJsonPath = join(planDir, "meta", "chunks.json");
|
||||
@@ -229,7 +233,12 @@ function readAssemblePlan(planDir: string, chunkPaths: readonly string[]): PlanJ
|
||||
`per chunk, ordered by index.`,
|
||||
);
|
||||
}
|
||||
for (const path of chunkPaths) {
|
||||
for (const [index, entry] of chunkPaths.entries()) {
|
||||
if (typeof entry !== "string" && entry.length === 0) {
|
||||
throw new Error(`[assemble] chunk ${index} has an empty list of parts`);
|
||||
}
|
||||
}
|
||||
for (const path of chunkPaths.flat()) {
|
||||
if (!existsSync(path)) {
|
||||
throw new Error(`[assemble] chunk path does not exist: ${path}`);
|
||||
}
|
||||
|
||||
@@ -7,6 +7,7 @@ import { assemble } from "./assemble.js";
|
||||
import {
|
||||
audioCutForChunk,
|
||||
chunkAudioSource,
|
||||
chunksWithSubRange,
|
||||
muxChunkWithAudio,
|
||||
writeChunkAudioSlices,
|
||||
} from "./chunkAudio.js";
|
||||
@@ -40,6 +41,49 @@ describe("audioCutForChunk", () => {
|
||||
});
|
||||
});
|
||||
|
||||
describe("chunksWithSubRange", () => {
|
||||
const grid = { sampleRate: 48000, firstStart: -1024, packetSamples: 1024, packetCount: 564 };
|
||||
const chunks = [0, 120, 240].map((startFrame, index) => ({
|
||||
index,
|
||||
startFrame,
|
||||
endFrame: startFrame + 120,
|
||||
}));
|
||||
const packetsOf = (list: typeof chunks, index: number) => {
|
||||
const cut = audioCutForChunk(grid, list, index, FPS);
|
||||
return cut ? [cut.firstPacket, cut.firstPacket + cut.packetCount] : null;
|
||||
};
|
||||
|
||||
it("splits only the chunk, and the range is a chunk of its own", () => {
|
||||
expect(chunksWithSubRange(chunks, 1, { startFrame: 150, endFrame: 200 })).toEqual({
|
||||
chunks: [
|
||||
{ index: 0, startFrame: 0, endFrame: 120 },
|
||||
{ index: 1, startFrame: 120, endFrame: 150 },
|
||||
{ index: 2, startFrame: 150, endFrame: 200 },
|
||||
{ index: 3, startFrame: 200, endFrame: 240 },
|
||||
{ index: 4, startFrame: 240, endFrame: 360 },
|
||||
],
|
||||
index: 2,
|
||||
});
|
||||
expect(chunksWithSubRange(chunks, 1, { startFrame: 120, endFrame: 240 })).toEqual({
|
||||
chunks,
|
||||
index: 1,
|
||||
});
|
||||
});
|
||||
|
||||
it.each([0, 1, 2])("gives chunk %i's parts its packets, each once and in order", (index) => {
|
||||
const slice = chunks[index]!;
|
||||
const edges = [slice.startFrame, slice.startFrame + 7, slice.startFrame + 61, slice.endFrame];
|
||||
const parts = edges.slice(1).map((endFrame, i) => {
|
||||
const split = chunksWithSubRange(chunks, index, { startFrame: edges[i]!, endFrame });
|
||||
return packetsOf(split.chunks, split.index);
|
||||
});
|
||||
const owned = parts.filter((part) => part !== null);
|
||||
expect(owned[0]![0]).toBe(packetsOf(chunks, index)![0]);
|
||||
expect(owned.at(-1)![1]).toBe(packetsOf(chunks, index)![1]);
|
||||
for (let i = 1; i < owned.length; i++) expect(owned[i]![0]).toBe(owned[i - 1]![1]);
|
||||
});
|
||||
});
|
||||
|
||||
const hasFfmpeg = spawnSync("ffmpeg", ["-version"]).status === 0;
|
||||
|
||||
describe.skipIf(!hasFfmpeg)("chunks that carry their audio", () => {
|
||||
@@ -195,4 +239,110 @@ describe.skipIf(!hasFfmpeg)("chunks that carry their audio", () => {
|
||||
},
|
||||
60_000,
|
||||
);
|
||||
|
||||
// Chunks 0 and 1 rendered as parts (chunk 0's first part keeps the priming), chunk 2 whole.
|
||||
it("chunks rendered as frame sub-ranges assemble into the same audio and joined frames", async () => {
|
||||
const totalFrames = 91;
|
||||
const CHUNKS = [0, 47, 90].map((startFrame, index) => ({
|
||||
index,
|
||||
startFrame,
|
||||
endFrame: [47, 90, 91][index]!,
|
||||
}));
|
||||
const PARTS = [
|
||||
[0, 20, 47],
|
||||
[47, 60, 90],
|
||||
[90, 91],
|
||||
];
|
||||
const dir = join(root, "parts");
|
||||
mkdirSync(dir, { recursive: true });
|
||||
const audioPath = join(dir, "audio.m4a");
|
||||
ffmpeg(
|
||||
`-f lavfi -i aevalsrc=0.4*sin(2*PI*(220+40*t)*t):s=48000:d=${totalFrames / 30} -c:a aac -b:a 192k -y`
|
||||
.split(" ")
|
||||
.concat(audioPath),
|
||||
);
|
||||
const planDir = join(dir, "plan");
|
||||
mkdirSync(join(planDir, "meta"), { recursive: true });
|
||||
writeFileSync(
|
||||
join(planDir, "plan.json"),
|
||||
JSON.stringify({
|
||||
planHash: "fake",
|
||||
totalFrames,
|
||||
hasAudio: true,
|
||||
dimensions: { fpsNum: 30, fpsDen: 1, width: 160, height: 120, format: "mp4" },
|
||||
}),
|
||||
);
|
||||
writeFileSync(join(planDir, "meta", "chunks.json"), JSON.stringify(CHUNKS));
|
||||
const slicedPlanDir = join(dir, "sliced-plan");
|
||||
await writeChunkAudioSlices({ audioPath, planDir: slicedPlanDir, chunks: CHUNKS, fps: FPS });
|
||||
|
||||
// Encode and mux one frame range of chunk `index`, as renderChunk does with `frameRange`.
|
||||
const renderRange = async (index: number, startFrame: number, endFrame: number) => {
|
||||
const name = `${index}-${startFrame}-${endFrame}`;
|
||||
const videoPath = join(dir, `video-${name}.mp4`);
|
||||
const n = String(endFrame - startFrame);
|
||||
ffmpeg(
|
||||
`-f lavfi -i testsrc2=size=160x120:rate=30,trim=start_frame=${startFrame}:end_frame=${endFrame} -c:v libx264 -preset veryfast -g ${n} -keyint_min ${n} -sc_threshold 0 -bf 0 -x264-params scenecut=0:open-gop=0:repeat-headers=1 -pix_fmt yuv420p -avoid_negative_ts make_zero -y`
|
||||
.split(" ")
|
||||
.concat(videoPath),
|
||||
);
|
||||
const chunkPath = join(dir, `chunk-${name}.mp4`);
|
||||
const split = chunksWithSubRange(CHUNKS, index, { startFrame, endFrame });
|
||||
await muxChunkWithAudio({
|
||||
videoPath,
|
||||
audioPath: chunkAudioSource(slicedPlanDir, index, join(dir, "rebuilt.m4a")),
|
||||
outputPath: chunkPath,
|
||||
chunks: split.chunks,
|
||||
chunkIndex: split.index,
|
||||
fps: FPS,
|
||||
timeoutMs: 60_000,
|
||||
});
|
||||
return { videoPath, chunkPath };
|
||||
};
|
||||
const wholes = await Promise.all(
|
||||
CHUNKS.map((slice) => renderRange(slice.index, slice.startFrame, slice.endFrame)),
|
||||
);
|
||||
const parts = await Promise.all(
|
||||
PARTS.map((edges, index) =>
|
||||
Promise.all(edges.slice(1).map((end, i) => renderRange(index, edges[i]!, end))),
|
||||
),
|
||||
);
|
||||
|
||||
writeFileSync(
|
||||
join(planDir, "meta", "encoder.json"),
|
||||
JSON.stringify({ chunksCarryAudio: true }),
|
||||
);
|
||||
const fromWholes = join(dir, "from-wholes.mp4");
|
||||
const fromParts = join(dir, "from-parts.mp4");
|
||||
await assemble(
|
||||
planDir,
|
||||
wholes.map((whole) => whole.chunkPath),
|
||||
audioPath,
|
||||
fromWholes,
|
||||
);
|
||||
await assemble(
|
||||
planDir,
|
||||
parts.map((chunk) => chunk.map((part) => part.chunkPath)),
|
||||
audioPath,
|
||||
fromParts,
|
||||
);
|
||||
// The parts carry the same audio packets the whole chunks do, so the audio is unchanged.
|
||||
expect(pcm(fromParts) === pcm(fromWholes)).toBe(true);
|
||||
const audioPacketsOf = (path: string) =>
|
||||
streams(path).find((stream) => stream.codec_type === "audio")!.nb_read_packets;
|
||||
expect(audioPacketsOf(fromParts)).toBe(audioPacketsOf(fromWholes));
|
||||
expect(audioPacketsOf(fromParts)).toBe(audioPacketsOf(audioPath));
|
||||
|
||||
// Frames are the parts' own frames in order: today's video-only join of the same encodes.
|
||||
const videoOnly = join(dir, "parts-video-only.mp4");
|
||||
writeFileSync(join(planDir, "meta", "encoder.json"), "{}");
|
||||
await assemble(
|
||||
planDir,
|
||||
parts.map((chunk) => chunk.map((part) => part.videoPath)),
|
||||
audioPath,
|
||||
videoOnly,
|
||||
);
|
||||
expect(frames(fromParts)).toEqual(frames(videoOnly));
|
||||
expect(frames(fromParts)).toHaveLength(totalFrames);
|
||||
}, 60_000);
|
||||
});
|
||||
|
||||
@@ -57,6 +57,24 @@ export function audioCutForChunk(
|
||||
return { seekSeconds, durationSeconds, firstPacket: first, packetCount: end - first };
|
||||
}
|
||||
|
||||
/** Chunk `index` split around `range`, so `audioCutForChunk` treats `range` as a chunk. */
|
||||
export function chunksWithSubRange(
|
||||
chunks: readonly ChunkSliceJson[],
|
||||
index: number,
|
||||
range: { startFrame: number; endFrame: number },
|
||||
): { chunks: ChunkSliceJson[]; index: number } {
|
||||
const slice = chunks[index]!;
|
||||
const parts = [
|
||||
{ startFrame: slice.startFrame, endFrame: range.startFrame },
|
||||
range,
|
||||
{ startFrame: range.endFrame, endFrame: slice.endFrame },
|
||||
].filter((part) => part.endFrame > part.startFrame);
|
||||
const split = [...chunks.slice(0, index), ...parts, ...chunks.slice(index + 1)].map(
|
||||
(chunk, i) => ({ index: i, startFrame: chunk.startFrame, endFrame: chunk.endFrame }),
|
||||
);
|
||||
return { chunks: split, index: range.startFrame > slice.startFrame ? index + 1 : index };
|
||||
}
|
||||
|
||||
interface FfprobeAudioGrid {
|
||||
streams?: { sample_rate?: string; time_base?: string; nb_frames?: string }[];
|
||||
packets?: { pts?: number; duration?: number }[];
|
||||
|
||||
@@ -27,6 +27,7 @@ export const FFMPEG_VERSION_MISMATCH = "FFMPEG_VERSION_MISMATCH";
|
||||
export const PLAN_HASH_MISMATCH = "PLAN_HASH_MISMATCH";
|
||||
export const MISSING_PLAN_ARTIFACT = "MISSING_PLAN_ARTIFACT";
|
||||
export const CHUNK_INDEX_OUT_OF_RANGE = "CHUNK_INDEX_OUT_OF_RANGE";
|
||||
export const CHUNK_FRAME_RANGE_INVALID = "CHUNK_FRAME_RANGE_INVALID";
|
||||
export const MISSING_RUNTIME_ENV_SNAPSHOT = "MISSING_RUNTIME_ENV_SNAPSHOT";
|
||||
export { INVALID_VIDEO_METADATA };
|
||||
|
||||
@@ -35,6 +36,7 @@ export type RenderChunkValidationCode =
|
||||
| typeof PLAN_HASH_MISMATCH
|
||||
| typeof MISSING_PLAN_ARTIFACT
|
||||
| typeof CHUNK_INDEX_OUT_OF_RANGE
|
||||
| typeof CHUNK_FRAME_RANGE_INVALID
|
||||
| typeof MISSING_RUNTIME_ENV_SNAPSHOT
|
||||
| typeof INVALID_VIDEO_METADATA
|
||||
| typeof BROWSER_GPU_NOT_SOFTWARE;
|
||||
|
||||
@@ -199,6 +199,7 @@ describe("getDistributedRenderCapabilities()", () => {
|
||||
acceptsLegacyV1WithoutDescriptor: true,
|
||||
},
|
||||
},
|
||||
chunkFrameRange: true,
|
||||
});
|
||||
});
|
||||
|
||||
|
||||
@@ -71,6 +71,7 @@ export interface DistributedRenderCapabilities {
|
||||
chunk: Readonly<PlanProtocolConsumerCapabilities>;
|
||||
assembler: Readonly<PlanProtocolConsumerCapabilities>;
|
||||
}>;
|
||||
readonly chunkFrameRange?: boolean;
|
||||
}
|
||||
|
||||
/** Serializable capability payload for fleet rollout and worker handshakes. */
|
||||
@@ -89,6 +90,7 @@ export const DISTRIBUTED_RENDER_CAPABILITIES: Readonly<DistributedRenderCapabili
|
||||
acceptsLegacyV1WithoutDescriptor: true,
|
||||
}),
|
||||
}),
|
||||
chunkFrameRange: true,
|
||||
});
|
||||
|
||||
export function getDistributedRenderCapabilities(): Readonly<DistributedRenderCapabilities> {
|
||||
|
||||
@@ -1,9 +1,9 @@
|
||||
import { mkdtempSync, rmSync } from "node:fs";
|
||||
import { tmpdir } from "node:os";
|
||||
import { join } from "node:path";
|
||||
import { assemble, type AssembleResult } from "./assemble.js";
|
||||
import { assemble, type AssembleChunkPaths, type AssembleResult } from "./assemble.js";
|
||||
import { materializePlanV2Target } from "./planV2.js";
|
||||
import { renderChunk, type EffectiveChunkResult } from "./renderChunk.js";
|
||||
import { renderChunk, type EffectiveChunkResult, type RenderChunkOptions } from "./renderChunk.js";
|
||||
|
||||
/**
|
||||
* Direct v2 chunk-role entry point. Storage adapters may instead download the
|
||||
@@ -13,12 +13,13 @@ export async function renderChunkV2(
|
||||
planV2Dir: string,
|
||||
chunkIndex: number,
|
||||
outputChunkPath: string,
|
||||
options?: RenderChunkOptions,
|
||||
): Promise<EffectiveChunkResult> {
|
||||
const workRoot = mkdtempSync(join(tmpdir(), "hf-plan-v2-chunk-"));
|
||||
const materializedPlanDir = join(workRoot, "plan");
|
||||
try {
|
||||
materializePlanV2Target(planV2Dir, { role: "chunk", chunkIndex }, materializedPlanDir);
|
||||
return await renderChunk(materializedPlanDir, chunkIndex, outputChunkPath);
|
||||
return await renderChunk(materializedPlanDir, chunkIndex, outputChunkPath, options);
|
||||
} finally {
|
||||
rmSync(workRoot, { recursive: true, force: true });
|
||||
}
|
||||
@@ -30,7 +31,7 @@ export async function renderChunkV2(
|
||||
*/
|
||||
export async function assembleV2(
|
||||
planV2Dir: string,
|
||||
chunkPaths: readonly string[],
|
||||
chunkPaths: AssembleChunkPaths,
|
||||
outputPath: string,
|
||||
options?: Parameters<typeof assemble>[4],
|
||||
): Promise<AssembleResult> {
|
||||
|
||||
@@ -20,6 +20,7 @@
|
||||
|
||||
import { afterAll, beforeAll, describe, expect, it } from "bun:test";
|
||||
import { spawnSync } from "node:child_process";
|
||||
import { createHash } from "node:crypto";
|
||||
import {
|
||||
copyFileSync,
|
||||
cpSync,
|
||||
@@ -41,8 +42,10 @@ import { type CaptureOptions, extractVideoFramesRange } from "@hyperframes/engin
|
||||
import { HOST_CHROME_FAILURE_PATTERNS } from "./__test_utils__/hostChromeFailures.js";
|
||||
import { defaultLogger } from "../../logger.js";
|
||||
import { RenderQualityError } from "../renderOrchestrator.js";
|
||||
import { assemble } from "./assemble.js";
|
||||
import { plan } from "./plan.js";
|
||||
import { materializePlanV2Target, planV2, readPlanV2Manifest } from "./planV2.js";
|
||||
import { resolvePlanAudioPath } from "./shared.js";
|
||||
import {
|
||||
CHUNK_INDEX_OUT_OF_RANGE,
|
||||
distributedCaptureSessionDependencies,
|
||||
@@ -677,6 +680,215 @@ describe("renderChunk() — a video the chunk does not show", () => {
|
||||
}, 120_000);
|
||||
});
|
||||
|
||||
describe("renderChunk() — frameRange", () => {
|
||||
// A box sliding across the frame, so every frame differs and a misplaced,
|
||||
// dropped or repeated frame changes the assembled output.
|
||||
const ANIMATED_HTML = (duration: number, audio: boolean) => `<!doctype html>
|
||||
<html>
|
||||
<head><meta charset="utf-8"><style>
|
||||
@keyframes slide { from { transform: translateX(0px); } to { transform: translateX(120px); } }
|
||||
#box { position:absolute; top:40px; width:40px; height:40px; background:#f97316;
|
||||
animation: slide ${duration}s linear forwards; }
|
||||
</style></head>
|
||||
<body style="margin:0;background:#0f172a">
|
||||
<div data-composition-id="root" data-no-timeline data-width="160" data-height="120" data-duration="${duration}">
|
||||
<div id="box"></div>
|
||||
${audio ? '<audio data-composition-audio src="tone.wav"></audio>' : ""}
|
||||
</div>
|
||||
</body>
|
||||
</html>`;
|
||||
|
||||
function run(cmd: string, args: string[]): string {
|
||||
const result = spawnSync(cmd, args, { maxBuffer: 1 << 28, encoding: "latin1" });
|
||||
if (result.status !== 0) throw new Error(`${cmd} ${args.join(" ")}: ${result.stderr}`);
|
||||
return result.stdout;
|
||||
}
|
||||
// Decoded video frame hashes in presentation order.
|
||||
const frameHashes = (path: string) =>
|
||||
run("ffmpeg", ["-nostdin", "-v", "error", "-i", path, "-map", "0:v", "-f", "framemd5", "-"])
|
||||
.split("\n")
|
||||
.filter((line) => line && !line.startsWith("#"))
|
||||
.map((line) => line.split(",").at(-1)!.trim());
|
||||
const pcm = (path: string) =>
|
||||
run("ffmpeg", ["-nostdin", "-v", "error", "-i", path, "-map", "0:a", "-f", "s16le", "-"]);
|
||||
const videoPts = (path: string) =>
|
||||
run("ffprobe", [
|
||||
...["-v", "error", "-select_streams", "v:0", "-show_entries", "packet=pts_time"],
|
||||
...["-of", "csv=p=0", "--", path],
|
||||
])
|
||||
.split("\n")
|
||||
.filter(Boolean)
|
||||
.map((line) => Number.parseFloat(line))
|
||||
.sort((x, y) => x - y);
|
||||
const sha = (path: string) => createHash("sha256").update(readFileSync(path)).digest("hex");
|
||||
const minPsnr = (a: string, b: string) => {
|
||||
const log = spawnSync(
|
||||
"ffmpeg",
|
||||
[
|
||||
...["-nostdin", "-i", a, "-i", b, "-lavfi"],
|
||||
// Pair frames by index, not timestamp: webm's millisecond timestamps round differently.
|
||||
"[0:v]setpts=N/(30*TB)[a];[1:v]setpts=N/(30*TB)[b];[a][b]psnr=stats_file=-",
|
||||
...["-f", "null", "-"],
|
||||
],
|
||||
{ maxBuffer: 1 << 28, encoding: "latin1" },
|
||||
).stdout;
|
||||
const values = [...log.matchAll(/psnr_avg:(\S+)/g)].map((m) => Number(m[1]));
|
||||
return Math.min(...values.map((v) => (Number.isFinite(v) ? v : 100)));
|
||||
};
|
||||
|
||||
async function renderOrSkip<T>(label: string, render: () => Promise<T>): Promise<T | null> {
|
||||
try {
|
||||
return await render();
|
||||
} catch (err) {
|
||||
const message = err instanceof Error ? err.message : String(err);
|
||||
if (!HOST_CHROME_FAILURE_PATTERNS.test(message)) throw err;
|
||||
console.warn(
|
||||
`[renderChunk.test] skipping ${label} — host Chrome stack can't render. Diagnostic:`,
|
||||
message.slice(0, 240),
|
||||
);
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
it("renders png-sequence halves that assemble into the whole chunk's frames", async () => {
|
||||
if (!hasChrome) return;
|
||||
const dir = join(runRoot, "frame-range-png");
|
||||
const project = join(dir, "project");
|
||||
mkdirSync(project, { recursive: true });
|
||||
writeFileSync(join(project, "index.html"), ANIMATED_HTML(0.3, false), "utf-8");
|
||||
const planPng = join(dir, "plan");
|
||||
await plan(project, { fps: 30, width: 160, height: 120, format: "png-sequence" }, planPng);
|
||||
const [slice] = JSON.parse(readFileSync(join(planPng, "meta", "chunks.json"), "utf-8"));
|
||||
expect(slice).toMatchObject({ startFrame: 0, endFrame: 9 });
|
||||
|
||||
const whole = await renderOrSkip("png frameRange test", () =>
|
||||
renderChunk(planPng, 0, join(dir, "whole")),
|
||||
);
|
||||
if (whole === null) return;
|
||||
const full = await renderChunk(planPng, 0, join(dir, "full"), {
|
||||
frameRange: { startFrame: 0, endFrame: 9 },
|
||||
});
|
||||
const first = await renderChunk(planPng, 0, join(dir, "part-a"), {
|
||||
frameRange: { startFrame: 0, endFrame: 4 },
|
||||
});
|
||||
const second = await renderChunk(planPng, 0, join(dir, "part-b"), {
|
||||
frameRange: { startFrame: 4, endFrame: 9 },
|
||||
});
|
||||
|
||||
// The whole slice as an explicit range is today's output, byte for byte.
|
||||
expect(whole.frameRange).toEqual({ startFrame: 0, endFrame: 9 });
|
||||
expect(full.sha256).toBe(whole.sha256);
|
||||
expect([first.framesEncoded, second.framesEncoded]).toEqual([4, 5]);
|
||||
expect(second.frameRange).toEqual({ startFrame: 4, endFrame: 9 });
|
||||
expect(JSON.parse(readFileSync(second.perfPath, "utf-8"))).toMatchObject({
|
||||
startFrame: 4,
|
||||
endFrame: 9,
|
||||
framesEncoded: 5,
|
||||
});
|
||||
|
||||
const fromWhole = join(dir, "out-whole");
|
||||
const fromParts = join(dir, "out-parts");
|
||||
await assemble(planPng, [whole.outputPath], null, fromWhole);
|
||||
const result = await assemble(
|
||||
planPng,
|
||||
[[first.outputPath, second.outputPath]],
|
||||
null,
|
||||
fromParts,
|
||||
);
|
||||
expect(result.framesEncoded).toBe(9);
|
||||
const names = readdirSync(fromWhole).sort();
|
||||
expect(readdirSync(fromParts).sort()).toEqual(names);
|
||||
for (const name of names) {
|
||||
expect(readFileSync(join(fromParts, name)).equals(readFileSync(join(fromWhole, name)))).toBe(
|
||||
true,
|
||||
);
|
||||
}
|
||||
// The animation moves, so equal frames above are not equal by accident.
|
||||
expect(new Set(names.map((name) => sha(join(fromWhole, name)))).size).toBe(names.length);
|
||||
}, 120_000);
|
||||
|
||||
// Two chunks of 9 frames; chunk 0 also renders as [0, 4) + [4, 9).
|
||||
it.each([
|
||||
["mp4, chunks carrying audio (stitched)", "mp4", true],
|
||||
["mp4, audio muxed at assemble (joined)", "mp4", false],
|
||||
["webm (joined)", "webm", false],
|
||||
] as const)(
|
||||
"%s: halves assemble seamlessly into the whole chunk's frames and audio",
|
||||
async (label, format, chunksCarryAudio) => {
|
||||
if (!hasChrome) return;
|
||||
const dir = join(
|
||||
runRoot,
|
||||
`frame-range-${format}-${chunksCarryAudio ? "stitched" : "joined"}`,
|
||||
);
|
||||
const project = join(dir, "project");
|
||||
mkdirSync(project, { recursive: true });
|
||||
run("ffmpeg", [
|
||||
...["-nostdin", "-v", "error", "-y", "-f", "lavfi"],
|
||||
...["-i", "aevalsrc=0.3*sin(2*PI*440*t):s=48000:d=0.6", "-c:a", "pcm_s16le"],
|
||||
join(project, "tone.wav"),
|
||||
]);
|
||||
writeFileSync(join(project, "index.html"), ANIMATED_HTML(0.6, true), "utf-8");
|
||||
const planEnc = join(dir, "plan");
|
||||
await plan(
|
||||
project,
|
||||
{ fps: 30, width: 160, height: 120, format, chunkSize: 9 },
|
||||
planEnc,
|
||||
chunksCarryAudio ? { chunksCarryAudio: true } : {},
|
||||
);
|
||||
expect(JSON.parse(readFileSync(join(planEnc, "meta", "chunks.json"), "utf-8"))).toHaveLength(
|
||||
2,
|
||||
);
|
||||
|
||||
const out = (name: string) => join(dir, `${name}.${format}`);
|
||||
const whole0 = await renderOrSkip(label, () => renderChunk(planEnc, 0, out("whole-0")));
|
||||
if (whole0 === null) return;
|
||||
const whole1 = await renderChunk(planEnc, 1, out("whole-1"));
|
||||
const first = await renderChunk(planEnc, 0, out("part-0a"), {
|
||||
frameRange: { startFrame: 0, endFrame: 4 },
|
||||
});
|
||||
const second = await renderChunk(planEnc, 0, out("part-0b"), {
|
||||
frameRange: { startFrame: 4, endFrame: 9 },
|
||||
});
|
||||
expect([first.framesEncoded, second.framesEncoded]).toEqual([4, 5]);
|
||||
// Each part starts on its own keyframe, as a chunk does.
|
||||
for (const part of [first, second]) {
|
||||
const keyframes = run("ffprobe", [
|
||||
...["-v", "error", "-select_streams", "v:0", "-show_entries", "packet=flags"],
|
||||
...["-of", "csv=p=0", "--", part.outputPath],
|
||||
]);
|
||||
expect(keyframes.trim().split("\n")[0]).toStartWith("K");
|
||||
}
|
||||
|
||||
const audio = chunksCarryAudio ? null : resolvePlanAudioPath(planEnc);
|
||||
expect(chunksCarryAudio || audio !== null).toBe(true);
|
||||
const fromWhole = out("from-whole");
|
||||
const fromParts = out("from-parts");
|
||||
await assemble(planEnc, [whole0.outputPath, whole1.outputPath], audio, fromWhole);
|
||||
await assemble(
|
||||
planEnc,
|
||||
[[first.outputPath, second.outputPath], whole1.outputPath],
|
||||
audio,
|
||||
fromParts,
|
||||
);
|
||||
|
||||
// Same frame count, and exactly the parts' own frames in order: nothing
|
||||
// dropped, repeated or reordered at the seams.
|
||||
const partFrames = [first, second, whole1].flatMap((r) => frameHashes(r.outputPath));
|
||||
expect(frameHashes(fromParts)).toEqual(partFrames);
|
||||
expect(frameHashes(fromParts)).toHaveLength(18);
|
||||
expect(frameHashes(fromWhole)).toHaveLength(18);
|
||||
// Frames keep their slot on the timeline: mp4 timestamps are exact; webm's
|
||||
// millisecond timebase truncates at every join, part seams as chunk seams.
|
||||
if (format === "mp4") expect(videoPts(fromParts)).toEqual(videoPts(fromWhole));
|
||||
videoPts(fromParts).forEach((pts, i) => expect(Math.abs(pts - i / 30)).toBeLessThan(0.002));
|
||||
// The parts are separate encodes, so compare their pixels to the whole's.
|
||||
expect(minPsnr(fromParts, fromWhole)).toBeGreaterThan(35);
|
||||
expect(pcm(fromParts) === pcm(fromWhole)).toBe(true);
|
||||
},
|
||||
180_000,
|
||||
);
|
||||
});
|
||||
|
||||
describe("resolvePresetForLockedEncoder", () => {
|
||||
// Tiny fast tests for the codec-override helper. No Chrome, no ffmpeg —
|
||||
// exists so a refactor that moves the override (e.g. into
|
||||
|
||||
@@ -85,7 +85,7 @@ import {
|
||||
type PlanVideosJson,
|
||||
readFfmpegVersion,
|
||||
} from "./shared.js";
|
||||
import { chunkAudioSource, muxChunkWithAudio } from "./chunkAudio.js";
|
||||
import { chunkAudioSource, chunksWithSubRange, muxChunkWithAudio } from "./chunkAudio.js";
|
||||
import { DISTRIBUTED_RENDER_CAPABILITIES, readPlanProtocolV1 } from "./planProtocol.js";
|
||||
import { readValidatedPlanV2Materialization } from "./planV2.js";
|
||||
import {
|
||||
@@ -95,6 +95,7 @@ import {
|
||||
MISSING_PLAN_ARTIFACT,
|
||||
PLAN_HASH_MISMATCH,
|
||||
FFMPEG_VERSION_MISMATCH,
|
||||
CHUNK_FRAME_RANGE_INVALID,
|
||||
CHUNK_INDEX_OUT_OF_RANGE,
|
||||
MISSING_RUNTIME_ENV_SNAPSHOT,
|
||||
rebuildExtractedFramesFromPlanDir,
|
||||
@@ -102,6 +103,7 @@ import {
|
||||
} from "./chunkFrames.js";
|
||||
|
||||
export {
|
||||
CHUNK_FRAME_RANGE_INVALID,
|
||||
CHUNK_INDEX_OUT_OF_RANGE,
|
||||
createChunkVideoFrameInjectorFactory,
|
||||
extractDeferredVideoFramesForChunk,
|
||||
@@ -140,6 +142,7 @@ export interface ChunkResult {
|
||||
/** `"file"` for mp4/mov; `"frame-dir"` for png-sequence. */
|
||||
outputKind: "file" | "frame-dir";
|
||||
framesEncoded: number;
|
||||
frameRange?: ChunkFrameRange;
|
||||
sha256: string;
|
||||
durationMs: number;
|
||||
/**
|
||||
@@ -182,6 +185,49 @@ export interface ChunkResult {
|
||||
/** Result returned by the built-in renderer, which always observes its mode. */
|
||||
export interface EffectiveChunkResult extends ChunkResult {
|
||||
captureMode: CaptureMode;
|
||||
frameRange: ChunkFrameRange;
|
||||
}
|
||||
|
||||
/** Absolute composition frame indices, end exclusive, like a chunk's slice. */
|
||||
export interface ChunkFrameRange {
|
||||
startFrame: number;
|
||||
endFrame: number;
|
||||
}
|
||||
|
||||
export interface RenderChunkOptions {
|
||||
/**
|
||||
* Render only these frames of the chunk, e.g. to retry a slow chunk as
|
||||
* parts rendered in parallel. Must be a non-empty range inside the chunk's
|
||||
* slice. Each part is a self-contained output (its own IDR keyframe, frame
|
||||
* names from zero, the audio packets it owns); pass the parts of a chunk,
|
||||
* in order, as one `chunkPaths` entry to `assemble`. Omitted renders the
|
||||
* whole slice.
|
||||
*/
|
||||
frameRange?: ChunkFrameRange;
|
||||
}
|
||||
|
||||
/** The frames a chunk renders: its slice, or a validated sub-range of it. */
|
||||
export function resolveChunkFrameRange(
|
||||
slice: ChunkSliceJson,
|
||||
chunkIndex: number,
|
||||
frameRange: ChunkFrameRange | undefined,
|
||||
): ChunkFrameRange {
|
||||
if (frameRange === undefined) return { startFrame: slice.startFrame, endFrame: slice.endFrame };
|
||||
const { startFrame, endFrame } = frameRange;
|
||||
if (
|
||||
!Number.isSafeInteger(startFrame) ||
|
||||
!Number.isSafeInteger(endFrame) ||
|
||||
startFrame < slice.startFrame ||
|
||||
endFrame > slice.endFrame ||
|
||||
endFrame <= startFrame
|
||||
) {
|
||||
throw new RenderChunkValidationError(
|
||||
CHUNK_FRAME_RANGE_INVALID,
|
||||
`[renderChunk] frameRange ${JSON.stringify(frameRange)} must be a non-empty integer range ` +
|
||||
`inside chunk ${chunkIndex}'s slice [${slice.startFrame}, ${slice.endFrame})`,
|
||||
);
|
||||
}
|
||||
return { startFrame, endFrame };
|
||||
}
|
||||
|
||||
/** Compatibility-safe adapter seam for built-in or injected chunk renderers. */
|
||||
@@ -398,6 +444,7 @@ export async function renderChunk(
|
||||
planDir: string,
|
||||
chunkIndex: number,
|
||||
outputChunkPath: string,
|
||||
options?: RenderChunkOptions,
|
||||
): Promise<EffectiveChunkResult> {
|
||||
const start = Date.now();
|
||||
const log = defaultLogger;
|
||||
@@ -460,13 +507,17 @@ export async function renderChunk(
|
||||
if (slice === undefined) {
|
||||
throw new RenderChunkValidationError(CHUNK_INDEX_OUT_OF_RANGE, "[renderChunk] missing slice");
|
||||
}
|
||||
const framesInChunk = slice.endFrame - slice.startFrame;
|
||||
if (framesInChunk <= 0) {
|
||||
const sliceFrames = slice.endFrame - slice.startFrame;
|
||||
if (sliceFrames <= 0) {
|
||||
throw new RenderChunkValidationError(
|
||||
CHUNK_INDEX_OUT_OF_RANGE,
|
||||
`[renderChunk] chunk ${chunkIndex} has non-positive frame count: ${framesInChunk}`,
|
||||
`[renderChunk] chunk ${chunkIndex} has non-positive frame count: ${sliceFrames}`,
|
||||
);
|
||||
}
|
||||
// Every frame-bound below (capture, video frames, GOP, audio packets, perf)
|
||||
// reads `range`, so a sub-range renders exactly as a chunk of its own would.
|
||||
const range = resolveChunkFrameRange(slice, chunkIndex, options?.frameRange);
|
||||
const framesInChunk = range.endFrame - range.startFrame;
|
||||
|
||||
const compiledDir = join(planDir, "compiled");
|
||||
if (!existsSync(compiledDir)) {
|
||||
@@ -580,10 +631,7 @@ export async function renderChunk(
|
||||
? await extractDeferredVideoFramesForChunk({
|
||||
planDir,
|
||||
planVideos,
|
||||
frames: Array.from(
|
||||
{ length: slice.endFrame - slice.startFrame },
|
||||
(_, i) => slice.startFrame + i,
|
||||
),
|
||||
frames: Array.from({ length: framesInChunk }, (_, i) => range.startFrame + i),
|
||||
fps: { num: plan.dimensions.fpsNum, den: plan.dimensions.fpsDen },
|
||||
outputRoot: videoFramesDir,
|
||||
cfg,
|
||||
@@ -792,7 +840,7 @@ export async function renderChunk(
|
||||
createRenderVideoFrameInjector: createChunkVideoFrameInjector,
|
||||
abortSignal: undefined,
|
||||
assertNotAborted: () => {},
|
||||
frameRange: { startFrame: slice.startFrame, endFrame: slice.endFrame },
|
||||
frameRange: range,
|
||||
});
|
||||
|
||||
const observedModes = new Set(capturePerfs.map((perf) => perf.captureMode));
|
||||
@@ -907,12 +955,16 @@ export async function renderChunk(
|
||||
});
|
||||
if (encoder.chunksCarryAudio) {
|
||||
const audioScratchPath = `${outputChunkPath}.audio.m4a`;
|
||||
// A sub-range muxes the packets it owns as if it were a chunk of its own.
|
||||
const audioChunks = options?.frameRange
|
||||
? chunksWithSubRange(chunks, chunkIndex, range)
|
||||
: { chunks, index: chunkIndex };
|
||||
await muxChunkWithAudio({
|
||||
videoPath: videoOnlyPath,
|
||||
audioPath: chunkAudioSource(planDir, chunkIndex, audioScratchPath),
|
||||
outputPath: outputChunkPath,
|
||||
chunks,
|
||||
chunkIndex,
|
||||
chunks: audioChunks.chunks,
|
||||
chunkIndex: audioChunks.index,
|
||||
fps: { num: plan.dimensions.fpsNum, den: plan.dimensions.fpsDen },
|
||||
timeoutMs: cfg.ffmpegProcessTimeout,
|
||||
});
|
||||
@@ -947,8 +999,8 @@ export async function renderChunk(
|
||||
const perfPayload = {
|
||||
planHash: plan.planHash,
|
||||
chunkIndex,
|
||||
startFrame: slice.startFrame,
|
||||
endFrame: slice.endFrame,
|
||||
startFrame: range.startFrame,
|
||||
endFrame: range.endFrame,
|
||||
framesEncoded,
|
||||
durationMs,
|
||||
planHashMs,
|
||||
@@ -980,6 +1032,7 @@ export async function renderChunk(
|
||||
outputPath: outputChunkPath,
|
||||
outputKind,
|
||||
framesEncoded,
|
||||
frameRange: range,
|
||||
sha256,
|
||||
durationMs,
|
||||
planHashMs,
|
||||
|
||||
@@ -0,0 +1,80 @@
|
||||
import { afterAll, beforeAll, describe, expect, it } from "bun:test";
|
||||
import { mkdirSync, mkdtempSync, rmSync, writeFileSync } from "node:fs";
|
||||
import { tmpdir } from "node:os";
|
||||
import { join } from "node:path";
|
||||
import {
|
||||
CHUNK_FRAME_RANGE_INVALID,
|
||||
renderChunk,
|
||||
RenderChunkValidationError,
|
||||
resolveChunkFrameRange,
|
||||
} from "./renderChunk.js";
|
||||
|
||||
const SLICE = { index: 1, startFrame: 10, endFrame: 20 };
|
||||
|
||||
describe("resolveChunkFrameRange", () => {
|
||||
it("is the whole slice when omitted", () => {
|
||||
expect(resolveChunkFrameRange(SLICE, 1, undefined)).toEqual({ startFrame: 10, endFrame: 20 });
|
||||
});
|
||||
|
||||
it.each([
|
||||
[10, 20],
|
||||
[10, 11],
|
||||
[13, 17],
|
||||
[19, 20],
|
||||
])("accepts [%i, %i) inside the slice", (startFrame, endFrame) => {
|
||||
expect(resolveChunkFrameRange(SLICE, 1, { startFrame, endFrame })).toEqual({
|
||||
startFrame,
|
||||
endFrame,
|
||||
});
|
||||
});
|
||||
|
||||
it.each([
|
||||
["starts before the slice", 9, 15],
|
||||
["ends past the slice", 15, 21],
|
||||
["is empty", 15, 15],
|
||||
["is reversed", 16, 15],
|
||||
["is not whole frames", 12.5, 15],
|
||||
["is not a number", Number.NaN, 15],
|
||||
])("rejects a range that %s", (_, startFrame, endFrame) => {
|
||||
let caught: unknown;
|
||||
try {
|
||||
resolveChunkFrameRange(SLICE, 1, { startFrame, endFrame });
|
||||
} catch (err) {
|
||||
caught = err;
|
||||
}
|
||||
expect(caught).toBeInstanceOf(RenderChunkValidationError);
|
||||
expect((caught as RenderChunkValidationError).code).toBe(CHUNK_FRAME_RANGE_INVALID);
|
||||
expect((caught as Error).message).toContain("chunk 1's slice [10, 20)");
|
||||
});
|
||||
});
|
||||
|
||||
describe("renderChunk() frameRange validation", () => {
|
||||
let root: string;
|
||||
let planDir: string;
|
||||
|
||||
beforeAll(() => {
|
||||
root = mkdtempSync(join(tmpdir(), "hf-chunk-frame-range-"));
|
||||
planDir = join(root, "plan");
|
||||
mkdirSync(join(planDir, "meta"), { recursive: true });
|
||||
writeFileSync(join(planDir, "plan.json"), JSON.stringify({ planHash: "fake" }));
|
||||
writeFileSync(join(planDir, "meta", "encoder.json"), "{}");
|
||||
writeFileSync(
|
||||
join(planDir, "meta", "chunks.json"),
|
||||
JSON.stringify([{ index: 0, startFrame: 0, endFrame: 10 }, SLICE]),
|
||||
);
|
||||
});
|
||||
afterAll(() => {
|
||||
rmSync(root, { recursive: true, force: true });
|
||||
});
|
||||
|
||||
// The check runs before ffmpeg, the plan hash or a browser, so a bad range fails fast.
|
||||
it("rejects a range outside the chunk's slice before rendering", async () => {
|
||||
const out = join(root, "out.mp4");
|
||||
await expect(
|
||||
renderChunk(planDir, 1, out, { frameRange: { startFrame: 5, endFrame: 15 } }),
|
||||
).rejects.toMatchObject({ code: CHUNK_FRAME_RANGE_INVALID });
|
||||
await expect(
|
||||
renderChunk(planDir, 1, out, { frameRange: { startFrame: 12, endFrame: 12 } }),
|
||||
).rejects.toMatchObject({ code: CHUNK_FRAME_RANGE_INVALID });
|
||||
});
|
||||
});
|
||||
Reference in New Issue
Block a user