fix: bound HDR and video extraction resources (#2955)

* fix: bound HDR and video extraction resources

* fix: trim negative video extraction preroll

* fix: skip invisible video extraction windows

* fix: preserve negative-start loop and held tails

* fix: cap finite video slots to source duration

* fix: bound held-tail frame extraction

* fix: plan from playable video duration

* fix: preserve open-ended held video tails

* fix: resolve held tails from decoded frames

* fix: normalize final-frame probe timestamps

* fix: handle unseekable final-frame sources

* fix: dedupe final-frame probes per render

* refactor: clarify output dynamic range contract
This commit is contained in:
James Russo
2026-08-02 20:21:54 -07:00
committed by GitHub
parent 67ffafb11c
commit 2339757377
22 changed files with 2541 additions and 156 deletions
+8
View File
@@ -66,6 +66,7 @@ export {
normalizeVp9CpuUsed, normalizeVp9CpuUsed,
} from "./services/vp9Options.js"; } from "./services/vp9Options.js";
export { export {
getCgroupMemoryLimitMb,
getSystemTotalMb, getSystemTotalMb,
isLowMemorySystem, isLowMemorySystem,
LOW_MEMORY_TOTAL_MB_THRESHOLD, LOW_MEMORY_TOTAL_MB_THRESHOLD,
@@ -179,6 +180,11 @@ export {
parseImageElements, parseImageElements,
extractVideoFramesRange, extractVideoFramesRange,
extractAllVideoFrames, extractAllVideoFrames,
resolveTimelineExtractionWindow,
resolveVideoExtractionWindow,
resolveFinalFrameExtractionWindow,
resolveVideoExtractionDuration,
resolvePlayableVideoDuration,
resolveProjectRelativeSrc, resolveProjectRelativeSrc,
getFrameAtTime, getFrameAtTime,
createFrameLookupTable, createFrameLookupTable,
@@ -194,6 +200,7 @@ export {
type ExtractionOptions, type ExtractionOptions,
type ExtractionResult, type ExtractionResult,
type ExtractionPhaseBreakdown, type ExtractionPhaseBreakdown,
type TimelineExtractionWindow,
type VideoExtractionFailure, type VideoExtractionFailure,
type VideoExtractionFailureKind, type VideoExtractionFailureKind,
type VideoFrameFormat, type VideoFrameFormat,
@@ -255,6 +262,7 @@ export { readWebGlVendorInfoFromCanvas } from "./utils/readWebGlVendorInfoFromCa
export { export {
extractMediaMetadata, extractMediaMetadata,
extractVideoMetadata, extractVideoMetadata,
extractFinalVideoFrameTimestamp,
extractAudioMetadata, extractAudioMetadata,
analyzeKeyframeIntervals, analyzeKeyframeIntervals,
type VideoMetadata, type VideoMetadata,
@@ -1,3 +1,4 @@
// fallow-ignore-file code-duplication
import { afterEach, beforeEach, describe, it, expect, vi } from "vitest"; import { afterEach, beforeEach, describe, it, expect, vi } from "vitest";
import { import {
_resetCgroupLimitCacheForTests, _resetCgroupLimitCacheForTests,
@@ -154,6 +155,30 @@ describe("parseCgroupLimitMb", () => {
}); });
}); });
describe("getCgroupMemoryLimitMb", () => {
it("returns only an actual cgroup limit and never host RAM", async () => {
await withSystemMemoryMocks(
{
files: { [CGROUP_V2_MEMORY_MAX_PATH]: `${24576 * BYTES_PER_MIB}` },
hostTotalMb: 65536,
},
({ getCgroupMemoryLimitMb }) => {
expect(getCgroupMemoryLimitMb()).toBe(24576);
},
);
await withSystemMemoryMocks(
{
files: { [CGROUP_V2_MEMORY_MAX_PATH]: "max" },
hostTotalMb: 65536,
},
({ getCgroupMemoryLimitMb }) => {
expect(getCgroupMemoryLimitMb()).toBeNull();
},
);
});
});
describe("getSystemTotalMb", () => { describe("getSystemTotalMb", () => {
it("caches cgroup probes until the test reset hook clears the cache", async () => { it("caches cgroup probes until the test reset hook clears the cache", async () => {
const readCalls: string[] = []; const readCalls: string[] = [];
+6 -2
View File
@@ -80,7 +80,11 @@ export function _resetCgroupLimitCacheForTests(): void {
_warnedCgroupReadFailure = false; _warnedCgroupReadFailure = false;
} }
function getCgroupLimitMb(): number | null { /**
* Actual Linux cgroup memory ceiling in MiB, or null when the process is not
* cgroup-limited. Unlike getSystemTotalMb this never falls back to host RAM.
*/
export function getCgroupMemoryLimitMb(): number | null {
if (_cachedCgroupLimitMb !== undefined) return _cachedCgroupLimitMb; if (_cachedCgroupLimitMb !== undefined) return _cachedCgroupLimitMb;
if (process.platform !== "linux") { if (process.platform !== "linux") {
@@ -142,7 +146,7 @@ function warnCgroupReadFailure(path: string, error: unknown): void {
/** Total physical RAM in MiB. */ /** Total physical RAM in MiB. */
export function getSystemTotalMb(): number { export function getSystemTotalMb(): number {
const hostTotalMb = Math.floor(totalmem() / BYTES_PER_MIB); const hostTotalMb = Math.floor(totalmem() / BYTES_PER_MIB);
const cgroupLimitMb = getCgroupLimitMb(); const cgroupLimitMb = getCgroupMemoryLimitMb();
return cgroupLimitMb === null ? hostTotalMb : Math.min(hostTotalMb, cgroupLimitMb); return cgroupLimitMb === null ? hostTotalMb : Math.min(hostTotalMb, cgroupLimitMb);
} }
@@ -24,6 +24,8 @@ import {
resolveFrameFormat, resolveFrameFormat,
codecMayHaveAlpha, codecMayHaveAlpha,
decoderForCodec, decoderForCodec,
resolveVideoExtractionWindow,
resolveVideoExtractionDuration,
getFrameAtTime, getFrameAtTime,
analyzeClipMediaFit, analyzeClipMediaFit,
classifyVideoExtractionError, classifyVideoExtractionError,
@@ -33,9 +35,14 @@ import {
type ExtractedFrames, type ExtractedFrames,
type ExtractionResult, type ExtractionResult,
} from "./videoFrameExtractor.js"; } from "./videoFrameExtractor.js";
import { extractVideoMetadata, type VideoMetadata } from "../utils/ffprobe.js"; import {
extractFinalVideoFrameTimestamp,
extractVideoMetadata,
type VideoMetadata,
} from "../utils/ffprobe.js";
import { runFfmpeg } from "../utils/runFfmpeg.js"; import { runFfmpeg } from "../utils/runFfmpeg.js";
import { COMPLETE_SENTINEL, GC_MARKER, SCHEMA_PREFIX } from "./extractionCache.js"; import { COMPLETE_SENTINEL, GC_MARKER, SCHEMA_PREFIX } from "./extractionCache.js";
import { resolveRuntimeMediaClipDuration } from "../../../core/src/runtime/media.js";
// ffmpeg is not preinstalled on GitHub's ubuntu-24.04 runners. The producer // ffmpeg is not preinstalled on GitHub's ubuntu-24.04 runners. The producer
// regression test at packages/producer/tests/vfr-screen-recording/ runs inside // regression test at packages/producer/tests/vfr-screen-recording/ runs inside
@@ -45,6 +52,261 @@ import { COMPLETE_SENTINEL, GC_MARKER, SCHEMA_PREFIX } from "./extractionCache.j
// synthesized VFR fixture. // synthesized VFR fixture.
const HAS_FFMPEG = spawnSync("ffmpeg", ["-version"]).status === 0; const HAS_FFMPEG = spawnSync("ffmpeg", ["-version"]).status === 0;
describe("resolveVideoExtractionDuration", () => {
const metadata = (
durationSeconds: number,
videoStreamDurationSeconds = durationSeconds,
): VideoMetadata => ({
durationSeconds,
videoStreamDurationSeconds,
width: 1920,
height: 1080,
fps: 30,
videoCodec: "h264",
hasAudio: false,
isVFR: false,
hasAlpha: false,
colorSpace: null,
});
const video = (overrides: Partial<VideoElement> = {}): VideoElement => ({
id: "root-video",
src: "video.mp4",
start: 0,
end: Number.POSITIVE_INFINITY,
mediaStart: 0,
loop: false,
hasAudio: false,
...overrides,
});
it("caps an open 60-second root source to a two-second composition", () => {
expect(resolveVideoExtractionDuration(video(), metadata(60), 2)).toBe(2);
});
it("keeps a shorter natural source duration inside a longer composition", () => {
expect(resolveVideoExtractionDuration(video(), metadata(2), 10)).toBe(2);
});
it("falls back to container duration when stream duration is unavailable", () => {
expect(resolveVideoExtractionDuration(video(), metadata(2, 0), 10)).toBe(2);
});
it("preserves explicit bounds and loop flags while applying the timeline ceiling", () => {
const explicitLoop = video({ end: 8, loop: true });
expect(resolveVideoExtractionDuration(explicitLoop, metadata(60), 10)).toBe(8);
expect(explicitLoop.loop).toBe(true);
});
it("trims materially negative preroll and advances the source offset", () => {
const preroll = video({ start: -60, end: 120, mediaStart: 0 });
expect(resolveVideoExtractionWindow(preroll, metadata(120), 2)).toEqual({
compositionStart: 0,
mediaStart: 60,
durationSeconds: 2,
});
expect(resolveVideoExtractionDuration(preroll, metadata(120), 2)).toBe(2);
});
it("returns an empty window for a clip entirely before composition time zero", () => {
expect(resolveVideoExtractionWindow(video({ start: -60, end: -10 }), metadata(120), 2)).toEqual(
{ compositionStart: 0, mediaStart: 60, durationSeconds: 0 },
);
});
it("preserves a short source cycle when negative preroll crosses a loop boundary", () => {
expect(
resolveVideoExtractionWindow(
video({ start: -5, end: 10, mediaStart: 0, loop: true }),
metadata(3),
2,
),
).toEqual({
compositionStart: -5,
mediaStart: 0,
durationSeconds: 3,
preserveTimelinePhase: true,
});
});
it("marks an entirely held interval for exact final-frame resolution", () => {
expect(
resolveVideoExtractionWindow(
video({ start: -5, end: 10, mediaStart: 0, loop: false }),
metadata(3),
2,
),
).toEqual({
compositionStart: -2.000001,
mediaStart: 2.999999,
durationSeconds: 0.000001,
preserveTimelineEnd: true,
ensureFinalFrame: true,
});
});
it.each([
{ loop: true, preservation: { preserveTimelinePhase: true }, label: "loop" },
{
loop: false,
preservation: { preserveTimelineEnd: true, ensureFinalFrame: true },
label: "held tail",
},
])(
"caps a finite long slot to one short source range for $label playback",
({ loop, preservation }) => {
expect(resolveVideoExtractionWindow(video({ end: 60, loop }), metadata(3), 60)).toEqual({
compositionStart: 0,
mediaStart: 0,
durationSeconds: 3,
...preservation,
});
},
);
it.each([
{ loop: true, preservation: { preserveTimelinePhase: true }, label: "loop" },
{
loop: false,
preservation: { preserveTimelineEnd: true, ensureFinalFrame: true },
label: "held tail",
},
])(
"uses the playable video-stream duration for a long-audio mux in $label playback",
({ loop, preservation }) => {
expect(resolveVideoExtractionWindow(video({ end: 60, loop }), metadata(60, 3), 60)).toEqual({
compositionStart: 0,
mediaStart: 0,
durationSeconds: 3,
...preservation,
});
},
);
it("preserves authored timing when the visible interval partially crosses a held tail", () => {
expect(
resolveVideoExtractionWindow(
video({ start: -2, end: 10, mediaStart: 0, loop: false }),
metadata(3),
2,
),
).toEqual({
compositionStart: 0,
mediaStart: 2,
durationSeconds: 1,
preserveTimelineEnd: true,
ensureFinalFrame: true,
});
});
it.each([
{ start: 0, expected: { compositionStart: 0, mediaStart: 0, durationSeconds: 3 } },
{ start: -2, expected: { compositionStart: 0, mediaStart: 2, durationSeconds: 1 } },
{ start: -5, expected: { compositionStart: 0, mediaStart: 5, durationSeconds: 0 } },
])(
"keeps an omitted-duration clip source-bounded like runtime (start=$start)",
({ start, expected }) => {
for (const loop of [false, true]) {
const parsed = parseVideoElements(
`<video id="natural" src="video.mp4"${loop ? " loop" : ""}></video>`,
)[0]!;
const planned = { ...parsed, start };
const runtimeDuration = resolveRuntimeMediaClipDuration({
isVideo: true,
sourceDuration: 3,
hostRemaining: 15 - start,
explicitDuration: null,
});
expect(runtimeDuration).toBe(3);
expect(parsed.end).toBe(Number.POSITIVE_INFINITY);
expect(resolveVideoExtractionWindow(planned, metadata(3), 15)).toEqual(expected);
}
},
);
it("bounds an entirely held long source to its final-frame sample", () => {
expect(
resolveVideoExtractionWindow(
video({ start: -600, end: 10, mediaStart: 0, loop: false }),
metadata(120),
2,
),
).toEqual({
compositionStart: -480.000001,
mediaStart: 119.999999,
durationSeconds: 0.000001,
preserveTimelineEnd: true,
ensureFinalFrame: true,
});
});
it("preserves a complete loop cycle when visibility ends exactly on a wrap boundary", () => {
expect(
resolveVideoExtractionWindow(
video({ start: -1, end: 10, mediaStart: 0, loop: true }),
metadata(3),
2,
),
).toEqual({
compositionStart: -1,
mediaStart: 0,
durationSeconds: 3,
preserveTimelinePhase: true,
});
});
it("keeps an open-ended loop source-bounded instead of inventing a longer slot", () => {
expect(resolveVideoExtractionWindow(video({ loop: true }), metadata(3), 10)).toEqual({
compositionStart: 0,
mediaStart: 0,
durationSeconds: 3,
});
});
it("never plans more extraction than the playable source range", () => {
for (const loop of [false, true]) {
for (const sourceDuration of [0.5, 3, 120]) {
for (const mediaStart of [0, sourceDuration / 3]) {
for (const start of [-600, -5, -1, 0, 2]) {
const window = resolveVideoExtractionWindow(
video({ start, end: start + 60, mediaStart, loop }),
metadata(sourceDuration),
10,
);
expect(window.durationSeconds).toBeLessThanOrEqual(sourceDuration - mediaStart);
expect(window.durationSeconds).toBeGreaterThanOrEqual(0);
}
}
}
}
});
it("rejects a media start at source EOF before planning extraction", () => {
expect(() =>
resolveVideoExtractionWindow(video({ mediaStart: 3 }), metadata(3), 10),
).toThrowError(expect.objectContaining({ kind: "media_start_out_of_range", retryable: false }));
});
it("rejects a media start at video-stream EOF even when the container continues", () => {
expect(() =>
resolveVideoExtractionWindow(video({ mediaStart: 3 }), metadata(60, 3), 10),
).toThrowError(expect.objectContaining({ kind: "media_start_out_of_range", retryable: false }));
});
it("rebases a loop phase when the visible window stays within one cycle", () => {
expect(
resolveVideoExtractionWindow(
video({ start: -5, end: 10, mediaStart: 0, loop: true }),
metadata(3),
0.5,
),
).toEqual({ compositionStart: 0, mediaStart: 2, durationSeconds: 0.5 });
});
it("retains legacy behavior when no timeline end is supplied", () => {
expect(resolveVideoExtractionDuration(video(), metadata(60))).toBe(60);
});
});
describe("video extraction failure taxonomy and bounded retry", () => { describe("video extraction failure taxonomy and bounded retry", () => {
it("classifies missing and transient HTTP sources without exposing retry ambiguity", () => { it("classifies missing and transient HTTP sources without exposing retry ambiguity", () => {
expect(classifyVideoExtractionError(new Error("HTTP 404: Not Found"))).toMatchObject({ expect(classifyVideoExtractionError(new Error("HTTP 404: Not Found"))).toMatchObject({
@@ -507,6 +769,28 @@ describe("FrameLookupTable", () => {
expect(table.getActiveFramePayloads(4.5).get("hero")?.frameIndex).toBe(15); expect(table.getActiveFramePayloads(4.5).get("hero")?.frameIndex).toBe(15);
}); });
it("wraps at video-stream EOF when a mux container has longer audio", () => {
const extracted = fakeExtracted(6, 2);
extracted.metadata.durationSeconds = 60;
extracted.metadata.videoStreamDurationSeconds = 3;
const table = createFrameLookupTable(
[
{
id: "hero",
src: "clip.webm",
start: 0,
end: 60,
mediaStart: 0,
loop: true,
hasAudio: false,
},
],
[extracted],
);
expect(table.getActiveFramePayloads(4).get("hero")?.frameIndex).toBe(2);
});
it("holds the last frame for a non-looping clip until its authored slot ends", () => { it("holds the last frame for a non-looping clip until its authored slot ends", () => {
const table = createFrameLookupTable( const table = createFrameLookupTable(
[ [
@@ -958,6 +1242,137 @@ describe.skipIf(!HAS_FFMPEG)("video frame extraction format", () => {
}, 60_000); }, 60_000);
}); });
describe.skipIf(!HAS_FFMPEG)("held tails on sparse-timestamp sources", () => {
const fixtureDir = mkdtempSync(join(tmpdir(), "hf-sparse-held-tail-"));
const cfrFixture = join(fixtureDir, "sub-1fps-cfr.mp4");
const vfrFixture = join(fixtureDir, "sparse-vfr.mp4");
const nonZeroStartFixture = join(fixtureDir, "nonzero-start.mp4");
const negativeStartTransportFixture = join(fixtureDir, "negative-start.ts");
beforeAll(async () => {
const fixtures = [
{
path: cfrFixture,
input: "testsrc2=s=64x64:d=10:rate=1/5",
filters: [] as string[],
},
{
path: vfrFixture,
input: "testsrc2=s=64x64:d=10:rate=1/2",
filters: ["-vf", "select='eq(n,0)+eq(n,2)'", "-vsync", "vfr"],
},
{
path: nonZeroStartFixture,
input: "testsrc2=s=64x64:d=3:rate=1",
filters: ["-output_ts_offset", "5"],
},
{
path: negativeStartTransportFixture,
input: "testsrc2=s=64x64:d=3:rate=1",
filters: [
"-mpegts_copyts",
"1",
"-muxdelay",
"0",
"-avoid_negative_ts",
"disabled",
"-output_ts_offset",
"-2",
],
},
];
for (const fixture of fixtures) {
const result = await runFfmpeg([
"-hide_banner",
"-loglevel",
"error",
"-f",
"lavfi",
"-i",
fixture.input,
...fixture.filters,
"-c:v",
"libx264",
"-pix_fmt",
"yuv420p",
"-y",
fixture.path,
]);
if (!result.success) {
throw new Error(`sparse fixture synthesis failed: ${result.stderr.slice(-400)}`);
}
}
}, 30_000);
afterAll(() => {
rmSync(fixtureDir, { recursive: true, force: true });
});
it.each([
{
label: "sub-1fps CFR",
src: cfrFixture,
expectedVfr: false,
finalTimestamp: 5,
streamStart: 0,
},
{
label: "sparse VFR",
src: vfrFixture,
expectedVfr: true,
finalTimestamp: 4,
streamStart: 0,
},
{
label: "non-zero stream start",
src: nonZeroStartFixture,
expectedVfr: false,
finalTimestamp: 2,
streamStart: 5,
},
{
label: "unindexed negative-base MPEG-TS",
src: negativeStartTransportFixture,
expectedVfr: false,
finalTimestamp: 2,
streamStart: -2,
},
])(
"extracts one real final SDR frame for $label",
async ({ src, expectedVfr, finalTimestamp, streamStart }) => {
const metadata = await extractVideoMetadata(src);
expect(metadata.fps).toBeLessThanOrEqual(1);
expect(metadata.isVFR).toBe(expectedVfr);
expect(metadata.videoStreamStartSeconds).toBeCloseTo(streamStart, 6);
await expect(extractFinalVideoFrameTimestamp(src, metadata)).resolves.toBe(finalTimestamp);
const outputDir = mkdtempSync(join(fixtureDir, "out-"));
const video: VideoElement = {
id: `held-${String(expectedVfr)}`,
src,
start: -15,
end: 5,
mediaStart: 0,
loop: false,
hasAudio: false,
};
const result = await extractAllVideoFrames([video], fixtureDir, {
fps: 30,
format: "png",
outputDir,
timelineEnd: 2,
});
expect(result.errors).toEqual([]);
expect(result.extracted).toHaveLength(1);
expect(result.extracted[0]?.totalFrames).toBe(1);
expect(video).toMatchObject({ start: 0, end: 5, loop: false });
expect(video.mediaStart).toBeCloseTo(metadata.videoStreamDurationSeconds - 0.000001, 7);
},
30_000,
);
});
// Regression test for the VFR (variable frame rate) freeze bug. // Regression test for the VFR (variable frame rate) freeze bug.
// Screen recordings and phone videos often have irregular timestamps. // Screen recordings and phone videos often have irregular timestamps.
// When such inputs hit `extractVideoFramesRange`'s `-ss <start> -i ... -t <dur> // When such inputs hit `extractVideoFramesRange`'s `-ss <start> -i ... -t <dur>
@@ -1017,6 +1432,90 @@ describe.skipIf(!HAS_FFMPEG)("extractAllVideoFrames on a VFR source", () => {
if (existsSync(FIXTURE_DIR)) rmSync(FIXTURE_DIR, { recursive: true, force: true }); if (existsSync(FIXTURE_DIR)) rmSync(FIXTURE_DIR, { recursive: true, force: true });
}); });
it("skips a clip entirely before time zero without reporting an extraction error", async () => {
const outputDir = join(FIXTURE_DIR, "out-before-timeline");
mkdirSync(outputDir, { recursive: true });
const video: VideoElement = {
id: "before-timeline",
src: VFR_FIXTURE,
start: -2,
end: -1,
mediaStart: 0,
loop: false,
hasAudio: false,
};
const result = await extractAllVideoFrames([video], FIXTURE_DIR, {
fps: 1,
outputDir,
timelineEnd: 2,
});
expect(result).toMatchObject({ success: true, extracted: [], errors: [] });
});
it("preserves loop phase when negative preroll crosses the source boundary", async () => {
const outputDir = join(FIXTURE_DIR, "out-negative-loop");
mkdirSync(outputDir, { recursive: true });
const video: VideoElement = {
id: "negative-loop",
src: VFR_FIXTURE,
start: -19,
end: 5,
mediaStart: 0,
loop: true,
hasAudio: false,
};
const result = await extractAllVideoFrames([video], FIXTURE_DIR, {
fps: 1,
outputDir,
timelineEnd: 2,
});
expect(result.errors).toEqual([]);
const extracted = result.extracted[0];
if (!extracted) throw new Error("expected loop source frames");
const lookup = createFrameLookupTable([video], result.extracted);
expect(video).toMatchObject({ start: -19, mediaStart: 0, loop: true });
expect(lookup.getFrame("negative-loop", 0)).toBe(
extracted.framePaths.get(extracted.totalFrames - 1),
);
}, 30_000);
it("preserves the held final frame after negative preroll exhausts a source", async () => {
const outputDir = join(FIXTURE_DIR, "out-negative-held-tail");
mkdirSync(outputDir, { recursive: true });
const video: VideoElement = {
id: "negative-held-tail",
src: VFR_FIXTURE,
start: -15,
end: 5,
mediaStart: 0,
loop: false,
hasAudio: false,
};
const result = await extractAllVideoFrames([video], FIXTURE_DIR, {
fps: 1,
outputDir,
timelineEnd: 2,
});
expect(result.errors).toEqual([]);
const extracted = result.extracted[0];
if (!extracted) throw new Error("expected held-tail source frames");
const lookup = createFrameLookupTable([video], result.extracted);
// The authored slot remains active through end=5, but lookup is rebased to
// one exact final frame instead of assuming the last second contains a
// timestamp or materializing the full source.
expect(video).toMatchObject({ start: 0, end: 5, mediaStart: 9.999999, loop: false });
expect(extracted.totalFrames).toBe(1);
expect(lookup.getFrame("negative-held-tail", 0)).toBe(
extracted.framePaths.get(extracted.totalFrames - 1),
);
}, 30_000);
it("detects the synthesized fixture as VFR", async () => { it("detects the synthesized fixture as VFR", async () => {
const md = await extractVideoMetadata(VFR_FIXTURE); const md = await extractVideoMetadata(VFR_FIXTURE);
expect(md.isVFR).toBe(true); expect(md.isVFR).toBe(true);
@@ -1236,6 +1735,52 @@ describe.skipIf(!HAS_FFMPEG)("extractAllVideoFrames on a VFR source", () => {
rmSync(CACHE_DIR, { recursive: true, force: true }); rmSync(CACHE_DIR, { recursive: true, force: true });
}, 60_000); }, 60_000);
it("reuses one-cycle loop extraction across different authored starts", async () => {
const cacheDir = mkdtempSync(join(tmpdir(), "hf-extract-loop-phase-cache-test-"));
const src = await synthCfrClip("cache-loop-phase-src.mp4", 3);
try {
const firstOutputDir = join(FIXTURE_DIR, "out-cache-loop-phase-first");
const secondOutputDir = join(FIXTURE_DIR, "out-cache-loop-phase-second");
mkdirSync(firstOutputDir, { recursive: true });
mkdirSync(secondOutputDir, { recursive: true });
const first = await extractAllVideoFrames(
[
{
...cfrClipElement("loop-cache-first", src, 60),
loop: true,
},
],
FIXTURE_DIR,
{ fps: 30, outputDir: firstOutputDir, timelineEnd: 60 },
undefined,
{ extractCacheDir: cacheDir },
);
expect(first.errors).toEqual([]);
expect(first.phaseBreakdown.cacheMisses).toBe(1);
const second = await extractAllVideoFrames(
[
{
...cfrClipElement("loop-cache-second", src, 66),
start: -6,
loop: true,
},
],
FIXTURE_DIR,
{ fps: 30, outputDir: secondOutputDir, timelineEnd: 60 },
undefined,
{ extractCacheDir: cacheDir },
);
expect(second.errors).toEqual([]);
expect(second.phaseBreakdown.cacheHits).toBe(1);
expect(second.phaseBreakdown.cacheMisses).toBe(0);
expect(second.extracted[0]?.totalFrames).toBe(first.extracted[0]?.totalFrames);
} finally {
rmSync(cacheDir, { recursive: true, force: true });
}
}, 60_000);
it("updates the cache sentinel mtime on a hit", async () => { it("updates the cache sentinel mtime on a hit", async () => {
const CACHE_DIR = mkdtempSync(join(tmpdir(), "hf-extract-cache-touch-test-")); const CACHE_DIR = mkdtempSync(join(tmpdir(), "hf-extract-cache-touch-test-"));
const SRC = await synthCfrClip("cache-touch-src.mp4", 1); const SRC = await synthCfrClip("cache-touch-src.mp4", 1);
@@ -11,7 +11,11 @@ import { isAbsolute, join, posix, resolve, sep } from "path";
import { parseHTML } from "linkedom"; import { parseHTML } from "linkedom";
import { decodeUrlPathVariants, MEDIA_DURATION_CLAMP_EPSILON_SECONDS } from "@hyperframes/core"; import { decodeUrlPathVariants, MEDIA_DURATION_CLAMP_EPSILON_SECONDS } from "@hyperframes/core";
import { resolveReferencedStart, type RefResolverEl } from "./referenceResolver.js"; import { resolveReferencedStart, type RefResolverEl } from "./referenceResolver.js";
import { extractMediaMetadata, type VideoMetadata } from "../utils/ffprobe.js"; import {
extractFinalVideoFrameTimestamp,
extractMediaMetadata,
type VideoMetadata,
} from "../utils/ffprobe.js";
import { import {
analyzeCompositionHdr, analyzeCompositionHdr,
isHdrColorSpace as isHdrColorSpaceUtil, isHdrColorSpace as isHdrColorSpaceUtil,
@@ -83,6 +87,15 @@ export interface ExtractionOptions {
quality?: number; quality?: number;
format?: VideoFrameFormat; format?: VideoFrameFormat;
sdrToHdrTransfer?: HdrTransfer; sdrToHdrTransfer?: HdrTransfer;
/** Extract exactly one frame at `startTime`. Used only after ffprobe has
* resolved the actual final decoded-frame timestamp for a held tail. */
finalFrameOnly?: boolean;
/**
* Absolute composition/timeline end in seconds. Applied only after source
* metadata resolves open-ended/natural-duration media. Invisible negative
* preroll is trimmed while advancing mediaStart to preserve source alignment.
*/
timelineEnd?: number;
/** /**
* Bounded per-source FFmpeg retries. Default 0 preserves stable behavior; * Bounded per-source FFmpeg retries. Default 0 preserves stable behavior;
* the producer may canary at most one retry after observing typed failures. * the producer may canary at most one retry after observing typed failures.
@@ -444,7 +457,10 @@ export function parseVideoElements(html: string): VideoElement[] {
// reference; the resolver handles both. // reference; the resolver handles both.
const start = startAttr ? resolveReferencedStart(document, el, startCache, visiting) : 0; const start = startAttr ? resolveReferencedStart(document, el, startCache, visiting) : 0;
// Derive end from data-end → data-start+data-duration → Infinity (natural duration). // Derive end from data-end → data-start+data-duration → Infinity (natural duration).
// The caller (htmlCompiler) clamps Infinity to the composition's absoluteEnd. // Static compilation cannot always clamp root media because GSAP may supply
// the root duration at runtime. The producer passes the resolved timeline
// end into frame extraction, which caps the source duration only after the
// natural duration is known without rewriting authored timing metadata.
let end = 0; let end = 0;
if (endAttr) { if (endAttr) {
end = parseFloat(endAttr); end = parseFloat(endAttr);
@@ -542,20 +558,21 @@ export async function extractVideoFramesRange(
} catch (error) { } catch (error) {
throw classifyVideoExtractionError(error); throw classifyVideoExtractionError(error);
} }
if (!(metadata.durationSeconds > 0)) { const playableDuration = resolvePlayableVideoDuration(metadata);
if (!(playableDuration > 0)) {
throw new VideoSourceExtractionError( throw new VideoSourceExtractionError(
"invalid_media", "invalid_media",
false, false,
"Video source has no positive duration", "Video source has no positive duration",
`Video source duration is ${metadata.durationSeconds}s`, `Playable video stream duration is ${playableDuration}s`,
); );
} }
if (startTime >= metadata.durationSeconds) { if (startTime >= playableDuration) {
throw new VideoSourceExtractionError( throw new VideoSourceExtractionError(
"media_start_out_of_range", "media_start_out_of_range",
false, false,
"Video media start is outside the source duration", "Video media start is outside the source duration",
`Video media start ${startTime}s is outside source duration ${metadata.durationSeconds}s`, `Video media start ${startTime}s is outside playable video duration ${playableDuration}s`,
); );
} }
const format = resolveFrameFormat(metadata, options.format); const format = resolveFrameFormat(metadata, options.format);
@@ -585,14 +602,22 @@ export async function extractVideoFramesRange(
if (codecMayHaveAlpha(metadata.videoCodec)) { if (codecMayHaveAlpha(metadata.videoCodec)) {
args.push("-c:v", decoderForCodec(metadata.videoCodec)); args.push("-c:v", decoderForCodec(metadata.videoCodec));
} }
args.push("-ss", String(startTime), "-i", videoPath, "-t", String(duration)); if (options.finalFrameOnly) {
// Output-side seek decodes from the start before selecting the final
// sample. This is intentionally reserved for the one-frame path: input
// seeking is faster, but valid unindexed transports (notably MPEG-TS with
// a negative timestamp base) can seek to EOF and emit zero frames.
args.push("-i", videoPath, "-ss", String(startTime), "-frames:v", "1");
} else {
args.push("-ss", String(startTime), "-i", videoPath, "-t", String(duration));
}
const vfFilters: string[] = []; const vfFilters: string[] = [];
if (isHdr && isMacOS) { if (isHdr && isMacOS) {
// VideoToolbox tone-maps during decode; force output to bt709 SDR format // VideoToolbox tone-maps during decode; force output to bt709 SDR format
vfFilters.push("format=nv12"); vfFilters.push("format=nv12");
} }
if (!metadata.isVFR) { if (!options.finalFrameOnly && !metadata.isVFR) {
vfFilters.push(`fps=${fps}`); vfFilters.push(`fps=${fps}`);
} }
if (options.sdrToHdrTransfer) { if (options.sdrToHdrTransfer) {
@@ -606,7 +631,9 @@ export async function extractVideoFramesRange(
vfFilters.push(SDR_TO_HDR_COLORSPACE_FILTER); vfFilters.push(SDR_TO_HDR_COLORSPACE_FILTER);
} }
if (vfFilters.length > 0) args.push("-vf", vfFilters.join(",")); if (vfFilters.length > 0) args.push("-vf", vfFilters.join(","));
if (metadata.isVFR) args.push("-fps_mode", "cfr", "-r", String(fps)); if (!options.finalFrameOnly && metadata.isVFR) {
args.push("-fps_mode", "cfr", "-r", String(fps));
}
args.push("-q:v", format === "jpg" ? String(Math.ceil((100 - quality) / 3)) : "0"); args.push("-q:v", format === "jpg" ? String(Math.ceil((100 - quality) / 3)) : "0");
// Render-scoped temp frames are read once; level 1 measured 3-5x faster for ~14% larger files. // Render-scoped temp frames are read once; level 1 measured 3-5x faster for ~14% larger files.
@@ -718,11 +745,223 @@ export function classifyFfmpegSpawnError(error: unknown, stderr = ""): VideoSour
function resolveSegmentDuration( function resolveSegmentDuration(
requested: number, requested: number,
mediaStart: number, mediaStart: number,
metadata: VideoMetadata, sourceDuration: number,
): number { ): number {
if (Number.isFinite(requested) && requested > 0) return requested; if (Number.isFinite(requested) && requested > 0) return requested;
const sourceRemaining = metadata.durationSeconds - mediaStart; const sourceRemaining = sourceDuration - mediaStart;
return sourceRemaining > 0 ? sourceRemaining : metadata.durationSeconds; return sourceRemaining > 0 ? sourceRemaining : sourceDuration;
}
/**
* Return the range that can actually produce video frames.
*
* Container duration may include a longer audio stream or mux padding. Using
* it for video extraction planning can reserve raw-frame scratch for seconds
* where no video frames exist. `extractMediaMetadata` already falls back to
* the container duration when ffprobe omits the stream duration; keep the
* explicit fallback here for callers supplying older/manual metadata.
*/
export function resolvePlayableVideoDuration(metadata: VideoMetadata): number {
return Number.isFinite(metadata.videoStreamDurationSeconds) &&
metadata.videoStreamDurationSeconds > 0
? metadata.videoStreamDurationSeconds
: metadata.durationSeconds;
}
export interface TimelineExtractionWindow {
compositionStart: number;
mediaStart: number;
durationSeconds: number;
/**
* Preserve the authored timeline origin and mediaStart for lookup. This is
* required when a looped visible interval crosses a source boundary and
* still needs modulo phase against the complete extracted source cycle.
*/
preserveTimelinePhase?: boolean;
/**
* Keep the authored end while rebasing start/mediaStart to the extracted
* source suffix. Non-looping lookup then holds the suffix's final frame
* through the remainder of the authored slot.
*/
preserveTimelineEnd?: boolean;
/** This window reaches a held tail and must be checked against the actual
* final decoded-frame timestamp before extraction. */
ensureFinalFrame?: boolean;
/** Source timestamp used by FFmpeg when it differs from the logical lookup
* mediaStart (the one-frame held-tail representation). */
extractionMediaStart?: number;
/** FFmpeg emits one decoded frame; lookup then holds that frame. */
finalFrameOnly?: boolean;
}
type TimelineWindowVideo = Pick<VideoElement, "start" | "end" | "mediaStart"> &
Partial<Pick<VideoElement, "loop">>;
// Logical duration assigned to a one-frame held-tail representation. This is
// deliberately below any supported output frame interval: coverage expects
// one frame, while FFmpeg seeks to the separately probed real frame timestamp.
const FINAL_FRAME_LOGICAL_DURATION_SECONDS = 1e-6;
/**
* Intersect an authored slot with the render timeline, then select the
* smallest playable source range that preserves timeline lookup semantics.
*
* A finite authored slot can outlive the source. In that case FFmpeg should
* still extract at most one source range: lookup either wraps that range for
* loops or holds its final frame for non-looping video. Keeping the authored
* timeline origin separate from the extracted range is what makes both
* behaviours survive the source-duration cap.
*/
export function resolveTimelineExtractionWindow(
video: TimelineWindowVideo,
resolvedDuration: number,
timelineEnd?: number,
sourceDuration?: number,
): TimelineExtractionWindow {
if (timelineEnd === undefined) {
return {
compositionStart: video.start,
mediaStart: video.mediaStart,
durationSeconds: resolvedDuration,
};
}
if (!Number.isFinite(timelineEnd)) {
throw new Error(`Video extraction timelineEnd must be finite; got ${String(timelineEnd)}`);
}
const compositionStart = Math.max(0, video.start);
const trimmedPreroll = compositionStart - video.start;
const timelineDuration = Math.max(0, timelineEnd - compositionStart);
// Infinity means "natural source duration", not an authored infinite slot.
// Explicit finite slots may outlive the source (loop or held tail), while an
// omitted duration remains source-bounded exactly like the browser runtime.
const resolvedVisibleDuration = resolvedDuration - trimmedPreroll;
const visibleDuration = Math.max(0, Math.min(resolvedVisibleDuration, timelineDuration));
let mediaStart = video.mediaStart + trimmedPreroll;
if (visibleDuration > 0 && sourceDuration !== undefined) {
const sourceRemaining = Math.max(0, sourceDuration - video.mediaStart);
if (sourceRemaining > 0 && video.loop && Number.isFinite(video.end)) {
const phaseOffset = trimmedPreroll % sourceRemaining;
const phaseRemaining = sourceRemaining - phaseOffset;
// The element visibility contract includes its end boundary. Preserve a
// complete cycle on equality as well, otherwise a rebased suffix would
// wrap to its own first frame instead of the source cycle's first frame.
if (visibleDuration >= phaseRemaining) {
return {
compositionStart: video.start,
mediaStart: video.mediaStart,
durationSeconds: sourceRemaining,
preserveTimelinePhase: true,
};
}
mediaStart = video.mediaStart + phaseOffset;
} else if (sourceRemaining > 0) {
const sourceVisibleAfterPreroll = Math.max(0, sourceRemaining - trimmedPreroll);
if (visibleDuration <= sourceVisibleAfterPreroll) {
return {
compositionStart,
mediaStart,
durationSeconds: visibleDuration,
};
}
// The visible interval enters (or is entirely inside) the held tail.
// Extract the visible source suffix. If preroll is already at/past the
// final decoded timestamp, the async resolver below replaces this tiny
// provisional suffix with one exact final frame.
const extractionDuration = Math.min(
sourceRemaining,
Math.max(sourceVisibleAfterPreroll, FINAL_FRAME_LOGICAL_DURATION_SECONDS),
);
const extractionOffset = sourceRemaining - extractionDuration;
return {
compositionStart: video.start + extractionOffset,
mediaStart: video.mediaStart + extractionOffset,
durationSeconds: extractionDuration,
preserveTimelineEnd: true,
ensureFinalFrame: true,
};
}
}
return {
compositionStart,
mediaStart,
durationSeconds: visibleDuration,
};
}
/**
* Replace a held-tail suffix that starts at/after the final decoded timestamp
* with one exact frame. This keeps raw HDR scratch O(one frame) without
* assuming a one-second seek window contains a CFR/VFR timestamp.
*/
export async function resolveFinalFrameExtractionWindow(
videoPath: string,
video: TimelineWindowVideo,
metadata: VideoMetadata,
window: TimelineExtractionWindow,
signal?: AbortSignal,
): Promise<TimelineExtractionWindow> {
if (!window.ensureFinalFrame) return window;
const playableDuration = resolvePlayableVideoDuration(metadata);
const finalFrameTimestamp = await extractFinalVideoFrameTimestamp(
videoPath,
{
videoStreamDurationSeconds: playableDuration,
videoStreamStartSeconds: metadata.videoStreamStartSeconds,
},
signal,
);
if (window.mediaStart < finalFrameTimestamp - 1e-9) return window;
const sourceRemaining = playableDuration - video.mediaStart;
const logicalDuration = Math.min(sourceRemaining, FINAL_FRAME_LOGICAL_DURATION_SECONDS);
return {
compositionStart: Math.max(0, video.start),
mediaStart: playableDuration - logicalDuration,
extractionMediaStart: finalFrameTimestamp,
durationSeconds: logicalDuration,
preserveTimelineEnd: true,
finalFrameOnly: true,
};
}
/** Resolve source duration first, then intersect it with the render timeline. */
export function resolveVideoExtractionWindow(
video: TimelineWindowVideo,
metadata: VideoMetadata,
timelineEnd?: number,
): TimelineExtractionWindow {
const playableDuration = resolvePlayableVideoDuration(metadata);
if (!(playableDuration > 0)) {
throw new VideoSourceExtractionError(
"invalid_media",
false,
"Video source has no positive duration",
`Playable video stream duration is ${playableDuration}s`,
);
}
if (video.mediaStart >= playableDuration) {
throw new VideoSourceExtractionError(
"media_start_out_of_range",
false,
"Video media start is outside the source duration",
`Video media start ${video.mediaStart}s is outside playable video duration ${playableDuration}s`,
);
}
const resolvedDuration = resolveSegmentDuration(
video.end - video.start,
video.mediaStart,
playableDuration,
);
return resolveTimelineExtractionWindow(video, resolvedDuration, timelineEnd, playableDuration);
}
export function resolveVideoExtractionDuration(
video: TimelineWindowVideo,
metadata: VideoMetadata,
timelineEnd?: number,
): number {
return resolveVideoExtractionWindow(video, metadata, timelineEnd).durationSeconds;
} }
/** /**
@@ -761,6 +1000,8 @@ type PreparedExtraction = {
index: number; index: number;
metadata: VideoMetadata; metadata: VideoMetadata;
videoDuration: number; videoDuration: number;
extractionMediaStart: number;
finalFrameOnly: boolean;
format: CacheFrameFormat; format: CacheFrameFormat;
sdrToHdrTransfer?: HdrTransfer; sdrToHdrTransfer?: HdrTransfer;
dedupeKey: string; dedupeKey: string;
@@ -831,7 +1072,13 @@ function linkOrCopyFrame(src: string, dest: string): void {
} }
function supersetGroupingKey(work: PreparedExtraction, fps: number): string { function supersetGroupingKey(work: PreparedExtraction, fps: number): string {
return [work.videoPath, String(fps), work.format, work.sdrToHdrTransfer ?? ""].join("\0"); return [
work.videoPath,
String(fps),
work.format,
work.sdrToHdrTransfer ?? "",
work.finalFrameOnly ? "final" : "range",
].join("\0");
} }
function isIntegralFrameOffset(offsetSeconds: number, fps: number): boolean { function isIntegralFrameOffset(offsetSeconds: number, fps: number): boolean {
@@ -854,6 +1101,7 @@ function buildSupersetGroup(
fps: number, fps: number,
): SupersetGroupPlan | null { ): SupersetGroupPlan | null {
if (misses.length < 2) return null; if (misses.length < 2) return null;
if (misses.some(({ work }) => work.finalFrameOnly)) return null;
const baseStart = Math.min(...misses.map(({ work }) => work.video.mediaStart)); const baseStart = Math.min(...misses.map(({ work }) => work.video.mediaStart));
if (!misses.every(({ work }) => isIntegralFrameOffset(work.video.mediaStart - baseStart, fps))) { if (!misses.every(({ work }) => isIntegralFrameOffset(work.video.mediaStart - baseStart, fps))) {
return null; return null;
@@ -1029,6 +1277,11 @@ export async function extractAllVideoFrames(
>, >,
compiledDir?: string, compiledDir?: string,
): Promise<ExtractionResult> { ): Promise<ExtractionResult> {
if (options.timelineEnd !== undefined && !Number.isFinite(options.timelineEnd)) {
throw new Error(
`Video extraction timelineEnd must be finite; got ${String(options.timelineEnd)}`,
);
}
const startTime = Date.now(); const startTime = Date.now();
const extracted: ExtractedFrames[] = []; const extracted: ExtractedFrames[] = [];
const errors: VideoExtractionFailure[] = []; const errors: VideoExtractionFailure[] = [];
@@ -1062,6 +1315,7 @@ export async function extractAllVideoFrames(
const warnedSrcs = new Set<string>(); const warnedSrcs = new Set<string>();
for (const video of videos) { for (const video of videos) {
if (signal?.aborted) break; if (signal?.aborted) break;
if (options.timelineEnd !== undefined && video.start >= options.timelineEnd) continue;
try { try {
let videoPath = video.src; let videoPath = video.src;
if (!isHttpUrl(videoPath)) { if (!isHttpUrl(videoPath)) {
@@ -1113,10 +1367,11 @@ export async function extractAllVideoFrames(
breakdown.resolveMs = Date.now() - phase1Start; breakdown.resolveMs = Date.now() - phase1Start;
// Snapshot the pre-preflight key inputs so the extraction cache keys on the // Snapshot the pre-preflight key inputs so the extraction cache keys on the
// user-visible source (original path, original mediaStart, original segment // user-visible source path rather than the
// bounds) rather than the workDir-local normalized file produced by the // workDir-local normalized file produced by the
// HDR preflight. Without this, every render would write a new // HDR preflight. Without this, every render would write a new
// normalized file with a fresh mtime → fresh cache key → perpetual misses. // normalized file with a fresh mtime → fresh cache key → perpetual misses.
// Phase 3 updates mediaStart after trimming any invisible negative preroll.
const cacheKeyInputs = resolvedVideos.map(({ video, videoPath }) => { const cacheKeyInputs = resolvedVideos.map(({ video, videoPath }) => {
const stat = readKeyStat(videoPath); const stat = readKeyStat(videoPath);
// Missing files return null — skip the cache path for that entry. The // Missing files return null — skip the cache path for that entry. The
@@ -1129,8 +1384,6 @@ export async function extractAllVideoFrames(
mtimeMs: stat.mtimeMs, mtimeMs: stat.mtimeMs,
size: stat.size, size: stat.size,
mediaStart: video.mediaStart, mediaStart: video.mediaStart,
start: video.start,
end: video.end,
}; };
}); });
@@ -1217,12 +1470,13 @@ export async function extractAllVideoFrames(
// Guard against mediaStart past EOF — FFmpeg's `-ss` silently produces // Guard against mediaStart past EOF — FFmpeg's `-ss` silently produces
// a 0-byte file when seeking beyond the source duration, and the // a 0-byte file when seeking beyond the source duration, and the
// downstream extractor then points at a broken input. // downstream extractor then points at a broken input.
if (entry.video.mediaStart >= metadata.durationSeconds) { const playableDuration = resolvePlayableVideoDuration(metadata);
if (entry.video.mediaStart >= playableDuration) {
errors.push({ errors.push({
videoId: entry.video.id, videoId: entry.video.id,
kind: "media_start_out_of_range", kind: "media_start_out_of_range",
retryable: false, retryable: false,
error: `SDR→HDR conversion skipped: mediaStart (${entry.video.mediaStart}s) ≥ source duration (${metadata.durationSeconds}s)`, error: `SDR→HDR conversion skipped: mediaStart (${entry.video.mediaStart}s) ≥ playable video duration (${playableDuration}s)`,
}); });
hdrSkippedIndices.add(i); hdrSkippedIndices.add(i);
continue; continue;
@@ -1289,12 +1543,20 @@ export async function extractAllVideoFrames(
}; };
} }
type PreparedExtractionResult = { work: PreparedExtraction } | { error: VideoExtractionFailure }; type PreparedExtractionResult =
| { work: PreparedExtraction }
| { error: VideoExtractionFailure }
| { skipped: true };
type ExtractionOutcome = { result: ExtractedFrames } | { error: VideoExtractionFailure }; type ExtractionOutcome = { result: ExtractedFrames } | { error: VideoExtractionFailure };
function scopedExtractionOptions(work: PreparedExtraction): ExtractionOptions { function scopedExtractionOptions(work: PreparedExtraction): ExtractionOptions {
return { ...options, format: work.format, sdrToHdrTransfer: work.sdrToHdrTransfer }; return {
...options,
format: work.format,
sdrToHdrTransfer: work.sdrToHdrTransfer,
finalFrameOnly: work.finalFrameOnly,
};
} }
function rehydratePublishedCache(work: PreparedExtraction, target: CacheMissTarget) { function rehydratePublishedCache(work: PreparedExtraction, target: CacheMissTarget) {
@@ -1312,21 +1574,18 @@ export async function extractAllVideoFrames(
if (!cacheRootDir) return { work }; if (!cacheRootDir) return { work };
const keyInput = cacheKeyInputs[work.index]; const keyInput = cacheKeyInputs[work.index];
if (!keyInput) return { work }; if (!keyInput) return { work };
const transform = work.sdrToHdrTransfer const transformParts = [
? sdrToHdrTransformKey(work.sdrToHdrTransfer) work.sdrToHdrTransfer ? sdrToHdrTransformKey(work.sdrToHdrTransfer) : undefined,
: undefined; work.finalFrameOnly ? "final-frame" : undefined,
].filter((part): part is string => part !== undefined);
const transform = transformParts.length > 0 ? transformParts.join("+") : undefined;
const keyDuration = resolveSegmentDuration(
keyInput.end - keyInput.start,
keyInput.mediaStart,
work.metadata,
);
const lookup = lookupCacheEntry(cacheRootDir, { const lookup = lookupCacheEntry(cacheRootDir, {
videoPath: keyInput.videoPath, videoPath: keyInput.videoPath,
mtimeMs: keyInput.mtimeMs, mtimeMs: keyInput.mtimeMs,
size: keyInput.size, size: keyInput.size,
mediaStart: keyInput.mediaStart, mediaStart: keyInput.mediaStart,
duration: keyDuration, duration: work.videoDuration,
fps: options.fps, fps: options.fps,
format: work.format, format: work.format,
transform, transform,
@@ -1356,7 +1615,7 @@ export async function extractAllVideoFrames(
extractVideoFramesRange( extractVideoFramesRange(
work.videoPath, work.videoPath,
work.video.id, work.video.id,
work.video.mediaStart, work.extractionMediaStart,
work.videoDuration, work.videoDuration,
scopedExtractionOptions(work), scopedExtractionOptions(work),
signal, signal,
@@ -1382,7 +1641,7 @@ export async function extractAllVideoFrames(
extractVideoFramesRange( extractVideoFramesRange(
work.videoPath, work.videoPath,
work.video.id, work.video.id,
work.video.mediaStart, work.extractionMediaStart,
work.videoDuration, work.videoDuration,
scopedExtractionOptions(work), scopedExtractionOptions(work),
signal, signal,
@@ -1516,18 +1775,33 @@ export async function extractAllVideoFrames(
} }
try { try {
const metadata = videoMetadata[index] ?? (await extractMediaMetadata(videoPath)); const metadata = videoMetadata[index] ?? (await extractMediaMetadata(videoPath));
const videoDuration = resolveSegmentDuration( const initialWindow = resolveVideoExtractionWindow(video, metadata, options.timelineEnd);
video.end - video.start, const window = await resolveFinalFrameExtractionWindow(
video.mediaStart, videoPath,
video,
metadata, metadata,
initialWindow,
signal,
); );
if (video.end - video.start !== videoDuration) { const videoDuration = window.durationSeconds;
video.end = video.start + videoDuration; if (videoDuration <= 0) {
return { skipped: true };
} }
if (!window.preserveTimelinePhase) {
video.start = window.compositionStart;
if (!window.preserveTimelineEnd) {
video.end = window.compositionStart + videoDuration;
}
video.mediaStart = window.mediaStart;
}
const keyInput = cacheKeyInputs[index];
const extractionMediaStart = window.extractionMediaStart ?? window.mediaStart;
if (keyInput) keyInput.mediaStart = extractionMediaStart;
const format = resolveFrameFormat(metadata, options.format); const format = resolveFrameFormat(metadata, options.format);
const sdrToHdrTransfer = sdrToHdrTransfers[index]; const sdrToHdrTransfer = sdrToHdrTransfers[index];
const dedupeKey = `${videoPath}\0${video.mediaStart}\0${videoDuration}\0${options.fps}\0${format}\0${sdrToHdrTransfer ?? ""}`; const finalFrameOnly = window.finalFrameOnly === true;
const dedupeKey = `${videoPath}\0${extractionMediaStart}\0${videoDuration}\0${options.fps}\0${format}\0${sdrToHdrTransfer ?? ""}\0${finalFrameOnly ? "final" : "range"}`;
return { return {
work: { work: {
@@ -1536,6 +1810,8 @@ export async function extractAllVideoFrames(
index, index,
metadata, metadata,
videoDuration, videoDuration,
extractionMediaStart,
finalFrameOnly,
format, format,
sdrToHdrTransfer, sdrToHdrTransfer,
dedupeKey, dedupeKey,
@@ -1581,11 +1857,20 @@ export async function extractAllVideoFrames(
for (const [key, outcome] of groupOutcomes) uniqueOutcomes.set(key, outcome); for (const [key, outcome] of groupOutcomes) uniqueOutcomes.set(key, outcome);
} }
const results: ExtractionOutcome[] = preparedExtractions.map((prepared) => { const results: ExtractionOutcome[] = [];
if ("error" in prepared) return prepared; for (const prepared of preparedExtractions) {
if ("skipped" in prepared) continue;
if ("error" in prepared) {
results.push(prepared);
continue;
}
const outcome = uniqueOutcomes.get(prepared.work.dedupeKey); const outcome = uniqueOutcomes.get(prepared.work.dedupeKey);
if (!outcome) if (!outcome) {
return { error: extractionError(prepared.work.video.id, "missing extraction result") }; results.push({
error: extractionError(prepared.work.video.id, "missing extraction result"),
});
continue;
}
if ("error" in outcome) { if ("error" in outcome) {
// A shared (deduped/superset) failure fans out to every element with the // A shared (deduped/superset) failure fans out to every element with the
// same key; annotate followers with the leader's videoId so N copies of // same key; annotate followers with the leader's videoId so N copies of
@@ -1594,17 +1879,18 @@ export async function extractAllVideoFrames(
const message = isFollower const message = isFollower
? `[shared extraction, leader ${outcome.error.videoId}] ${outcome.error.error}` ? `[shared extraction, leader ${outcome.error.videoId}] ${outcome.error.error}`
: outcome.error.error; : outcome.error.error;
return { results.push({
error: { error: {
videoId: prepared.work.video.id, videoId: prepared.work.video.id,
kind: outcome.error.kind, kind: outcome.error.kind,
retryable: outcome.error.retryable, retryable: outcome.error.retryable,
error: message, error: message,
}, },
}; });
continue;
} }
return { result: { ...outcome.result, videoId: prepared.work.video.id } }; results.push({ result: { ...outcome.result, videoId: prepared.work.video.id } });
}); }
breakdown.extractMs = Date.now() - phase3Start; breakdown.extractMs = Date.now() - phase3Start;
@@ -1653,7 +1939,7 @@ function getFrameIndexAtTime(
): number | null { ): number | null {
let localTime = globalTime - videoStart; let localTime = globalTime - videoStart;
if (localTime < 0) return null; if (localTime < 0) return null;
const loopDuration = Math.max(0, extracted.metadata.durationSeconds - mediaStart); const loopDuration = Math.max(0, resolvePlayableVideoDuration(extracted.metadata) - mediaStart);
if (loop && loopDuration > 0 && localTime >= loopDuration) { if (loop && loopDuration > 0 && localTime >= loopDuration) {
localTime %= loopDuration; localTime %= loopDuration;
} }
+150
View File
@@ -972,6 +972,156 @@ describe("AAC duration refinement must never fail or distort the call", () => {
}); });
}); });
describe("final video frame timestamp probes", () => {
afterEach(() => {
vi.resetModules();
vi.doUnmock("child_process");
});
it("normalizes absolute frame PTS by the selected video stream start", async () => {
const { spawn, calls } = createSpawnSpy([
{
kind: "exit",
code: 0,
stdout: JSON.stringify({
streams: [
{
codec_type: "video",
codec_name: "h264",
width: 64,
height: 64,
duration: "3",
start_time: "5",
r_frame_rate: "1/1",
avg_frame_rate: "1/1",
},
],
format: { duration: "8" },
}),
},
{ kind: "exit", code: 0, stdout: "5.000000,\n6.000000\n7.000000\n" },
]);
vi.resetModules();
vi.doMock("child_process", () => ({ spawn }));
const { extractFinalVideoFrameTimestamp, extractMediaMetadata } = await import("./ffprobe.js");
const metadata = await extractMediaMetadata("/tmp/nonzero-start.mp4");
expect(metadata.videoStreamDurationSeconds).toBe(3);
expect(metadata.videoStreamStartSeconds).toBe(5);
await expect(extractFinalVideoFrameTimestamp("/tmp/nonzero-start.mp4", metadata)).resolves.toBe(
2,
);
const intervalIndex = calls[1]?.args.indexOf("-read_intervals") ?? -1;
expect(calls[1]?.args[intervalIndex + 1]).toBe("7%8");
});
it("falls back to a bounded-output full scan when a transport cannot interval-seek", async () => {
const { spawn, calls } = createSpawnSpy([
{ kind: "exit", code: 0, stdout: "" },
{ kind: "exit", code: 0, stdout: "-2.000000,\n-1.000000\n0.000000\n" },
]);
vi.resetModules();
vi.doMock("child_process", () => ({ spawn }));
const { extractFinalVideoFrameTimestamp } = await import("./ffprobe.js");
await expect(
extractFinalVideoFrameTimestamp("/tmp/unindexed-negative-base.ts", {
videoStreamDurationSeconds: 3,
videoStreamStartSeconds: -2,
}),
).resolves.toBe(2);
const intervalIndex = calls[0]?.args.indexOf("-read_intervals") ?? -1;
expect(calls[0]?.args[intervalIndex + 1]).toBe("0%1");
expect(calls[1]?.args).not.toContain("-read_intervals");
});
it("does not share a caller-cancellable probe across render consumers", async () => {
type KillableFakeProc = FakeProc & { kill: (signal?: NodeJS.Signals) => boolean };
const processes: KillableFakeProc[] = [];
const spawn = () => {
const proc = new EventEmitter() as KillableFakeProc;
proc.stdout = new EventEmitter();
proc.stderr = new EventEmitter();
proc.kill = vi.fn(() => {
process.nextTick(() => proc.emit("close", null, "SIGTERM"));
return true;
});
processes.push(proc);
process.nextTick(() => proc.emit("spawn"));
return proc;
};
vi.resetModules();
vi.doMock("child_process", () => ({ spawn }));
const { extractFinalVideoFrameTimestamp } = await import("./ffprobe.js");
const metadata = { videoStreamDurationSeconds: 3, videoStreamStartSeconds: 0 };
const firstController = new AbortController();
const secondController = new AbortController();
const first = extractFinalVideoFrameTimestamp(
"/tmp/shared-source.mp4",
metadata,
firstController.signal,
);
const second = extractFinalVideoFrameTimestamp(
"/tmp/shared-source.mp4",
metadata,
secondController.signal,
);
expect(processes).toHaveLength(2);
firstController.abort();
await expect(first).rejects.toThrow(/ffprobe abort/);
expect(processes[0]?.kill).toHaveBeenCalledWith("SIGTERM");
expect(processes[1]?.kill).not.toHaveBeenCalled();
processes[1]?.stdout.emit("data", Buffer.from("2.000000\n"));
processes[1]?.emit("close", 0, null);
await expect(second).resolves.toBe(2);
expect(secondController.signal.aborted).toBe(false);
});
it("deduplicates the interval and fallback chain within one cancellation scope", async () => {
const { spawn, calls } = createSpawnSpy([
{ kind: "exit", code: 0, stdout: "" },
{ kind: "exit", code: 0, stdout: "-2.000000\n-1.000000\n0.000000\n" },
]);
vi.resetModules();
vi.doMock("child_process", () => ({ spawn }));
const { extractFinalVideoFrameTimestamp } = await import("./ffprobe.js");
const metadata = { videoStreamDurationSeconds: 3, videoStreamStartSeconds: -2 };
const signal = new AbortController().signal;
await expect(
Promise.all([
extractFinalVideoFrameTimestamp("/tmp/repeated-held-tail.ts", metadata, signal),
extractFinalVideoFrameTimestamp("/tmp/repeated-held-tail.ts", metadata, signal),
]),
).resolves.toEqual([2, 2]);
expect(calls).toHaveLength(2);
expect(calls[0]?.args).toContain("-read_intervals");
expect(calls[1]?.args).not.toContain("-read_intervals");
});
it("still deduplicates cancellation-independent probes", async () => {
const { spawn, calls } = createSpawnSpy([{ kind: "exit", code: 0, stdout: "2.000000\n" }]);
vi.resetModules();
vi.doMock("child_process", () => ({ spawn }));
const { extractFinalVideoFrameTimestamp } = await import("./ffprobe.js");
const metadata = { videoStreamDurationSeconds: 3, videoStreamStartSeconds: 0 };
await Promise.all([
extractFinalVideoFrameTimestamp("/tmp/shared-source.mp4", metadata),
extractFinalVideoFrameTimestamp("/tmp/shared-source.mp4", metadata),
]);
expect(calls).toHaveLength(1);
});
});
describe("runFfprobe process and stream handling", () => { describe("runFfprobe process and stream handling", () => {
afterEach(() => { afterEach(() => {
vi.resetModules(); vi.resetModules();
+112 -4
View File
@@ -52,6 +52,7 @@ async function runFfprobe(
filePath: string, filePath: string,
argsWithoutInput: string[], argsWithoutInput: string[],
signal?: AbortSignal, signal?: AbortSignal,
stdoutOptions?: { retainTail?: boolean; maxChars?: number },
): Promise<string> { ): Promise<string> {
// `--` stops option parsing so a path like "-intro.mp4" is a filename, but // `--` stops option parsing so a path like "-intro.mp4" is a filename, but
// it does NOT cover a path of exactly "-": ffprobe rewrites that to `fd:` // it does NOT cover a path of exactly "-": ffprobe rewrites that to `fd:`
@@ -79,6 +80,7 @@ async function runFfprobe(
const decoder = new StringDecoder("utf8"); const decoder = new StringDecoder("utf8");
let stdout = ""; let stdout = "";
let stdoutTruncated = false; let stdoutTruncated = false;
const stdoutMaxChars = stdoutOptions?.maxChars ?? FFPROBE_STDOUT_MAX_CHARS;
proc.stdout.on("data", (data: Buffer) => { proc.stdout.on("data", (data: Buffer) => {
// stderr is capped by ManagedChildProcess; stdout had no bound at all, and // stderr is capped by ManagedChildProcess; stdout had no bound at all, and
// analyzeKeyframeIntervals emits one line per frame — an all-intra ProRes // analyzeKeyframeIntervals emits one line per frame — an all-intra ProRes
@@ -87,9 +89,13 @@ async function runFfprobe(
stdout += decoder.write(data); stdout += decoder.write(data);
// Checked AFTER appending: a single chunk can already exceed the bound, // Checked AFTER appending: a single chunk can already exceed the bound,
// so a pre-append check only ever stops the second one. // so a pre-append check only ever stops the second one.
if (stdout.length > FFPROBE_STDOUT_MAX_CHARS) { if (stdout.length > stdoutMaxChars) {
stdoutTruncated = true; if (stdoutOptions?.retainTail) {
stdout = ""; stdout = stdout.slice(-stdoutMaxChars);
} else {
stdoutTruncated = true;
stdout = "";
}
} }
}); });
const managed = new ManagedChildProcess(proc, { const managed = new ManagedChildProcess(proc, {
@@ -101,7 +107,7 @@ async function runFfprobe(
stdout += decoder.end(); stdout += decoder.end();
if (stdoutTruncated) { if (stdoutTruncated) {
throw new Error( throw new Error(
`[FFmpeg] ffprobe output exceeded ${FFPROBE_STDOUT_MAX_CHARS} characters; refusing to parse a truncated result.`, `[FFmpeg] ffprobe output exceeded ${stdoutMaxChars} characters; refusing to parse a truncated result.`,
); );
} }
if (outcome.reason === "spawn_error") { if (outcome.reason === "spawn_error") {
@@ -135,6 +141,11 @@ function parseProbeJson(stdout: string): FFProbeOutput {
} }
const videoMetadataCache = new Map<string, Promise<VideoMetadata>>(); const videoMetadataCache = new Map<string, Promise<VideoMetadata>>();
const finalVideoFrameTimestampCache = new Map<string, Promise<number>>();
const finalVideoFrameTimestampSignalCaches = new WeakMap<
AbortSignal,
Map<string, Promise<number>>
>();
const audioMetadataCache = new Map<string, Promise<AudioMetadata>>(); const audioMetadataCache = new Map<string, Promise<AudioMetadata>>();
// FFmpeg's built-in AAC encoder emits AAC-LC, which has 1024 samples per packet. // FFmpeg's built-in AAC encoder emits AAC-LC, which has 1024 samples per packet.
const AAC_LC_SAMPLES_PER_PACKET = 1024; const AAC_LC_SAMPLES_PER_PACKET = 1024;
@@ -151,6 +162,11 @@ export interface VideoColorSpace {
export interface VideoMetadata { export interface VideoMetadata {
durationSeconds: number; durationSeconds: number;
videoStreamDurationSeconds: number; videoStreamDurationSeconds: number;
/** Absolute presentation timestamp at which the selected video stream
* starts. FFmpeg input seeks are relative to this point, while ffprobe frame
* timestamps are absolute, so callers crossing those APIs must normalize by
* this value. Absent only in legacy/manually-constructed metadata. */
videoStreamStartSeconds?: number;
width: number; width: number;
height: number; height: number;
fps: number; fps: number;
@@ -185,6 +201,7 @@ interface FFProbeStream {
width?: number; width?: number;
height?: number; height?: number;
duration?: string; duration?: string;
start_time?: string;
nb_frames?: string; nb_frames?: string;
nb_read_packets?: string; nb_read_packets?: string;
pix_fmt?: string; pix_fmt?: string;
@@ -478,6 +495,7 @@ export async function extractMediaMetadata(filePath: string): Promise<VideoMetad
return { return {
durationSeconds: 0, durationSeconds: 0,
videoStreamDurationSeconds: 0, videoStreamDurationSeconds: 0,
videoStreamStartSeconds: 0,
width: stillImageMeta.width, width: stillImageMeta.width,
height: stillImageMeta.height, height: stillImageMeta.height,
fps: 0, fps: 0,
@@ -528,10 +546,13 @@ export async function extractMediaMetadata(filePath: string): Promise<VideoMetad
const containerDuration = output?.format.duration ? parseFloat(output.format.duration) : 0; const containerDuration = output?.format.duration ? parseFloat(output.format.duration) : 0;
const streamDuration = videoStream.duration ? parseFloat(videoStream.duration) : 0; const streamDuration = videoStream.duration ? parseFloat(videoStream.duration) : 0;
const parsedStreamStart = videoStream.start_time ? parseFloat(videoStream.start_time) : 0;
const streamStart = Number.isFinite(parsedStreamStart) ? parsedStreamStart : 0;
return { return {
durationSeconds: containerDuration, durationSeconds: containerDuration,
videoStreamDurationSeconds: streamDuration > 0 ? streamDuration : containerDuration, videoStreamDurationSeconds: streamDuration > 0 ? streamDuration : containerDuration,
videoStreamStartSeconds: streamStart,
width: videoStream.width || stillImage()?.width || 0, width: videoStream.width || stillImage()?.width || 0,
height: videoStream.height || stillImage()?.height || 0, height: videoStream.height || stillImage()?.height || 0,
fps, fps,
@@ -552,6 +573,93 @@ export async function extractMediaMetadata(filePath: string): Promise<VideoMetad
return probePromise; return probePromise;
} }
/**
* Return the FFmpeg input-seek position of the final decoded video frame.
*
* A fixed seek window near EOF is not sufficient: sub-1fps and sparse VFR
* sources can have no frame timestamp inside that window even though the last
* decoded frame remains displayed through the stream duration. ffprobe seeks
* to the preceding keyframe and walks forward; retaining only its stdout tail
* keeps memory bounded even for a pathological long GOP. ffprobe reports
* absolute presentation timestamps, but FFmpeg input `-ss` is relative to the
* stream start; the result is normalized into that relative seek domain. Some
* unindexed transports cannot decode after an interval seek, so an empty tail
* probe falls back to a bounded-output full scan rather than rejecting valid
* media. The scan may cost decode time, but retains only 64 KiB of timestamps.
*/
export async function extractFinalVideoFrameTimestamp(
filePath: string,
metadata: Pick<VideoMetadata, "videoStreamDurationSeconds" | "videoStreamStartSeconds">,
signal?: AbortSignal,
): Promise<number> {
const videoDurationSeconds = metadata.videoStreamDurationSeconds;
const candidateStreamStart = metadata.videoStreamStartSeconds ?? 0;
const videoStreamStartSeconds = Number.isFinite(candidateStreamStart) ? candidateStreamStart : 0;
const cacheKey = `${filePath}\0${String(videoStreamStartSeconds)}\0${String(videoDurationSeconds)}`;
// A caller-owned abort signal cannot safely own a globally shared process
// promise: aborting one render would fail unrelated consumers. Calls in the
// SAME cancellation scope should still share the expensive interval +
// fallback chain, though — duplicate held-tail elements in one render carry
// the same signal and otherwise fan out N full-file scans before extraction
// dedupe. Weakly key the cache by cancellation owner to preserve both
// aggregate work bounds and cross-render isolation.
let probeCache = finalVideoFrameTimestampCache;
if (signal) {
probeCache = finalVideoFrameTimestampSignalCaches.get(signal) ?? new Map();
finalVideoFrameTimestampSignalCaches.set(signal, probeCache);
}
const cached = probeCache.get(cacheKey);
if (cached) return cached;
const probePromise = (async () => {
if (!(videoDurationSeconds > 0) || !Number.isFinite(videoDurationSeconds)) {
throw new Error(
`[FFmpeg] Cannot locate final video frame for invalid duration ${String(videoDurationSeconds)}`,
);
}
const streamEnd = videoStreamStartSeconds + videoDurationSeconds;
const intervalStart = Math.max(videoStreamStartSeconds, streamEnd - 1);
const parseFinalTimestamp = (stdout: string): number | undefined =>
stdout
.split("\n")
.map((line) => line.trim().split(",")[0]?.trim() ?? "")
.filter((value) => value.length > 0)
.map((value) => Number(value))
.filter((timestamp) => Number.isFinite(timestamp))
.at(-1);
const probe = async (readInterval?: string): Promise<number | undefined> => {
const args = [
"-select_streams",
"v:0",
"-show_entries",
"frame=best_effort_timestamp_time",
"-of",
"csv=p=0",
];
if (readInterval) args.splice(2, 0, "-read_intervals", readInterval);
const stdout = await runFfprobe(filePath, args, signal, {
retainTail: true,
maxChars: 64 * 1024,
});
return parseFinalTimestamp(stdout);
};
const timestamp =
(await probe(`${intervalStart}%${streamEnd}`)) ?? (await probe(/* full scan */));
if (timestamp === undefined) {
throw new Error("[FFmpeg] ffprobe found no decodable final video frame");
}
return Math.min(Math.max(timestamp - videoStreamStartSeconds, 0), videoDurationSeconds);
})();
probeCache.set(cacheKey, probePromise);
probePromise.catch(() => {
if (probeCache.get(cacheKey) === probePromise) {
probeCache.delete(cacheKey);
}
});
return probePromise;
}
/** /**
* @deprecated Use `extractMediaMetadata` this name is kept for backward * @deprecated Use `extractMediaMetadata` this name is kept for backward
* compatibility with consumers that imported the original video-only name * compatibility with consumers that imported the original video-only name
@@ -0,0 +1,89 @@
import { Hono } from "hono";
import { beforeEach, describe, expect, it, vi } from "vitest";
const capturedRenderConfigs = vi.hoisted(() => new Array<Record<string, unknown>>());
vi.mock("./services/renderOrchestrator.js", () => {
class RenderCancelledError extends Error {}
return {
RenderCancelledError,
createRenderJob: (config: Record<string, unknown>) => {
capturedRenderConfigs.push(config);
return {
config,
progress: 0,
currentStage: "queued",
framesRendered: 0,
totalFrames: 0,
warnings: [],
};
},
executeRenderJob: async (job: Record<string, unknown>) => {
job.outcome = "completed";
job.currentStage = "complete";
},
};
});
import { createRenderHandlers } from "./server.js";
function createInternalStreamingApp(): Hono {
const app = new Hono();
const handlers = createRenderHandlers({
getRequestId: () => "hdr-mode-test",
maxConcurrentRenders: 1,
});
app.post("/v1/render-stream", handlers.renderStream);
return app;
}
function requestRender(overrides: Record<string, unknown>) {
return createInternalStreamingApp().request("/v1/render-stream", {
method: "POST",
headers: { "content-type": "application/json" },
body: JSON.stringify({ html: "<html><body></body></html>", ...overrides }),
});
}
describe("POST /v1/render-stream — outputDynamicRange", () => {
beforeEach(() => capturedRenderConfigs.splice(0));
it.each([
["auto", "auto"],
["hdr", "force-hdr"],
["sdr", "force-sdr"],
] as const)(
"maps %s through createRenderRequest to internal hdrMode %s",
async (outputDynamicRange, hdrMode) => {
const response = await requestRender({ outputDynamicRange });
expect(response.status).toBe(200);
expect(await response.text()).toContain('"type":"complete"');
expect(capturedRenderConfigs).toHaveLength(1);
expect(capturedRenderConfigs[0]?.hdrMode).toBe(hdrMode);
},
);
it("rejects an invalid mode before creating a render job", async () => {
const response = await requestRender({ outputDynamicRange: "force-sdr" });
expect(response.status).toBe(200);
expect(await response.text()).toContain(
'outputDynamicRange must be one of: \\"auto\\", \\"hdr\\", \\"sdr\\"',
);
expect(capturedRenderConfigs).toHaveLength(0);
});
it("accepts the matching legacy field during rolling deployment", async () => {
const response = await requestRender({
outputDynamicRange: "sdr",
hdrMode: "force-sdr",
});
expect(response.status).toBe(200);
expect(await response.text()).toContain('"type":"complete"');
expect(capturedRenderConfigs).toHaveLength(1);
expect(capturedRenderConfigs[0]?.hdrMode).toBe("force-sdr");
});
});
+50
View File
@@ -46,6 +46,33 @@ describe("parseRenderOptions — render strictness", () => {
}); });
}); });
describe("parseRenderOptions — outputDynamicRange", () => {
it.each(["auto", "hdr", "sdr"] as const)("forwards %s", (outputDynamicRange) => {
expect(parseRenderOptions({ outputDynamicRange }).outputDynamicRange).toBe(outputDynamicRange);
});
it("drops invalid values from the lenient parser", () => {
expect(
parseRenderOptions({ outputDynamicRange: "force-sdr" }).outputDynamicRange,
).toBeUndefined();
expect(parseRenderOptions({ outputDynamicRange: true }).outputDynamicRange).toBeUndefined();
});
it.each([
["auto", "auto"],
["force-hdr", "hdr"],
["force-sdr", "sdr"],
] as const)("maps legacy hdrMode %s to %s", (hdrMode, outputDynamicRange) => {
expect(parseRenderOptions({ hdrMode }).outputDynamicRange).toBe(outputDynamicRange);
});
it("prefers the canonical field when both equivalent fields are present", () => {
expect(
parseRenderOptions({ outputDynamicRange: "sdr", hdrMode: "force-sdr" }).outputDynamicRange,
).toBe("sdr");
});
});
describe("prepareRenderBody — validation", () => { describe("prepareRenderBody — validation", () => {
it.each(["", " "])( it.each(["", " "])(
"treats an empty projectDir as absent and uses inline HTML", "treats an empty projectDir as absent and uses inline HTML",
@@ -67,6 +94,29 @@ describe("prepareRenderBody — validation", () => {
expect((result as { error: string }).error).toContain("variables must be a JSON object"); expect((result as { error: string }).error).toContain("variables must be a JSON object");
}); });
it("rejects an explicitly-supplied invalid outputDynamicRange", async () => {
const result = await prepareRenderBody({
outputDynamicRange: "force-sdr",
html: "<html></html>",
});
expect(result).toHaveProperty("error");
expect((result as { error: string }).error).toContain(
'outputDynamicRange must be one of: "auto", "hdr", "sdr"',
);
});
it("rejects conflicting canonical and legacy policies", async () => {
const result = await prepareRenderBody({
outputDynamicRange: "sdr",
hdrMode: "force-hdr",
html: "<html></html>",
});
expect(result).toHaveProperty("error");
expect((result as { error: string }).error).toContain(
"outputDynamicRange and legacy hdrMode must describe the same output policy",
);
});
it("rejects an explicitly-supplied invalid outputResolution", async () => { it("rejects an explicitly-supplied invalid outputResolution", async () => {
const result = await prepareRenderBody({ outputResolution: "8k", html: "<html></html>" }); const result = await prepareRenderBody({ outputResolution: "8k", html: "<html></html>" });
expect(result).toHaveProperty("error"); expect(result).toHaveProperty("error");
+46
View File
@@ -84,6 +84,7 @@ interface RenderInput {
quality: "draft" | "standard" | "high"; quality: "draft" | "standard" | "high";
format?: "mp4" | "webm" | "mov"; format?: "mp4" | "webm" | "mov";
videoFrameFormat?: RenderConfig["videoFrameFormat"]; videoFrameFormat?: RenderConfig["videoFrameFormat"];
outputDynamicRange?: "auto" | "hdr" | "sdr";
workers?: number; workers?: number;
useGpu: boolean; useGpu: boolean;
debug: boolean; debug: boolean;
@@ -158,6 +159,28 @@ function parseServerFormat(value: unknown): RenderInput["format"] {
return value === "mp4" || value === "webm" || value === "mov" ? value : undefined; return value === "mp4" || value === "webm" || value === "mov" ? value : undefined;
} }
function parseServerOutputDynamicRange(value: unknown): RenderInput["outputDynamicRange"] {
return value === "auto" || value === "hdr" || value === "sdr" ? value : undefined;
}
function parseLegacyServerHdrMode(value: unknown): RenderConfig["hdrMode"] {
return value === "auto" || value === "force-hdr" || value === "force-sdr" ? value : undefined;
}
function fromRenderHdrMode(hdrMode: RenderConfig["hdrMode"]): RenderInput["outputDynamicRange"] {
if (hdrMode === "force-hdr") return "hdr";
if (hdrMode === "force-sdr") return "sdr";
return hdrMode;
}
function toRenderHdrMode(
outputDynamicRange: RenderInput["outputDynamicRange"],
): RenderConfig["hdrMode"] {
if (outputDynamicRange === "hdr") return "force-hdr";
if (outputDynamicRange === "sdr") return "force-sdr";
return outputDynamicRange;
}
export function parseRenderOptions(body: Record<string, unknown>): Omit<RenderInput, "projectDir"> { export function parseRenderOptions(body: Record<string, unknown>): Omit<RenderInput, "projectDir"> {
// Accept either a JSON `number` (integer fps) or a JSON `string` (rational // Accept either a JSON `number` (integer fps) or a JSON `string` (rational
// like "30000/1001"). Falls back to 30 fps on parse failure to preserve the // like "30000/1001"). Falls back to 30 fps on parse failure to preserve the
@@ -176,6 +199,9 @@ export function parseRenderOptions(body: Record<string, unknown>): Omit<RenderIn
const outputPath = parseOutputCandidate(body); const outputPath = parseOutputCandidate(body);
const entryFile = nonEmptyString(body.entryFile); const entryFile = nonEmptyString(body.entryFile);
const format = parseServerFormat(body.format); const format = parseServerFormat(body.format);
const outputDynamicRange =
parseServerOutputDynamicRange(body.outputDynamicRange) ??
fromRenderHdrMode(parseLegacyServerHdrMode(body.hdrMode));
const videoFrameFormat = isVideoFrameFormat(body.videoFrameFormat) const videoFrameFormat = isVideoFrameFormat(body.videoFrameFormat)
? body.videoFrameFormat ? body.videoFrameFormat
: undefined; : undefined;
@@ -193,6 +219,7 @@ export function parseRenderOptions(body: Record<string, unknown>): Omit<RenderIn
strictness, strictness,
entryFile, entryFile,
format, format,
outputDynamicRange,
variables, variables,
outputResolution, outputResolution,
outputResolutionAspectAgnostic, outputResolutionAspectAgnostic,
@@ -254,6 +281,7 @@ function buildRenderJobConfig(input: RenderInput, outputPath: string, log: Produ
outputResolution: input.outputResolution, outputResolution: input.outputResolution,
outputResolutionAspectAgnostic: input.outputResolutionAspectAgnostic, outputResolutionAspectAgnostic: input.outputResolutionAspectAgnostic,
videoFrameFormat: input.videoFrameFormat, videoFrameFormat: input.videoFrameFormat,
hdrMode: toRenderHdrMode(input.outputDynamicRange),
}, },
}); });
return renderConfigFromRequest(request, { logger: log }); return renderConfigFromRequest(request, { logger: log });
@@ -286,6 +314,24 @@ function validateRenderOverrides(body: Record<string, unknown>): string | undefi
if (body.variables !== undefined && !isPlainObject(body.variables)) { if (body.variables !== undefined && !isPlainObject(body.variables)) {
return 'variables must be a JSON object keyed by variable id (e.g. {"title":"Hello"})'; return 'variables must be a JSON object keyed by variable id (e.g. {"title":"Hello"})';
} }
if (
body.outputDynamicRange !== undefined &&
parseServerOutputDynamicRange(body.outputDynamicRange) === undefined
) {
return 'outputDynamicRange must be one of: "auto", "hdr", "sdr"';
}
const legacyHdrMode = parseLegacyServerHdrMode(body.hdrMode);
if (body.hdrMode !== undefined && legacyHdrMode === undefined) {
return 'legacy hdrMode must be one of: "auto", "force-hdr", "force-sdr"';
}
const outputDynamicRange = parseServerOutputDynamicRange(body.outputDynamicRange);
if (
outputDynamicRange !== undefined &&
legacyHdrMode !== undefined &&
outputDynamicRange !== fromRenderHdrMode(legacyHdrMode)
) {
return "outputDynamicRange and legacy hdrMode must describe the same output policy";
}
return validateOutputResolutionOverride(body); return validateOutputResolutionOverride(body);
} }
@@ -153,6 +153,13 @@ function readVideoMetadata(
record.videoStreamDurationSeconds, record.videoStreamDurationSeconds,
`${field}.videoStreamDurationSeconds`, `${field}.videoStreamDurationSeconds`,
), ),
// Plans written before stream-start metadata existed implicitly used the
// overwhelmingly common start-at-zero domain. Preserve that compatibility
// while carrying non-zero edit-list/transport timestamps in new plans.
videoStreamStartSeconds:
record.videoStreamStartSeconds === undefined
? 0
: readFiniteNumber(record.videoStreamStartSeconds, `${field}.videoStreamStartSeconds`),
width: readPositiveInteger(record.width, `${field}.width`), width: readPositiveInteger(record.width, `${field}.width`),
height: readPositiveInteger(record.height, `${field}.height`), height: readPositiveInteger(record.height, `${field}.height`),
fps: readFiniteNumber(record.fps, `${field}.fps`), fps: readFiniteNumber(record.fps, `${field}.fps`),
@@ -94,6 +94,26 @@ describe("distributed video metadata", () => {
expect(sourceDerived.videos[0]?.mediaStart).toBe(1); expect(sourceDerived.videos[0]?.mediaStart).toBe(1);
}); });
it("round-trips a non-zero video stream start and defaults legacy plans to zero", () => {
const withStart = buildPlanVideosJson({
videos: [video()],
extracted: [
extractedMetadata({
metadata: {
...extractedMetadata().metadata,
videoStreamStartSeconds: 5,
},
}),
],
compositionEnd: 8,
});
expect(parsePlanVideosJson(withStart).extracted[0]?.metadata.videoStreamStartSeconds).toBe(5);
const legacy = structuredClone(withStart);
delete legacy.extracted[0]?.metadata.videoStreamStartSeconds;
expect(parsePlanVideosJson(legacy).extracted[0]?.metadata.videoStreamStartSeconds).toBe(0);
});
it.each([Number.NaN, Number.POSITIVE_INFINITY, 0, 2])( it.each([Number.NaN, Number.POSITIVE_INFINITY, 0, 2])(
"fails closed when no safe composition boundary can be derived (%s)", "fails closed when no safe composition boundary can be derived (%s)",
(compositionEnd) => { (compositionEnd) => {
@@ -139,6 +139,8 @@ export interface HdrVideoFrameSource {
frameSize: number; frameSize: number;
frameCount: number; frameCount: number;
scratch: Buffer; scratch: Buffer;
/** The raw file contains one playable source cycle and must wrap at EOF. */
loop?: boolean;
} }
export function closeHdrVideoFrameSource(source: HdrVideoFrameSource, log?: ProducerLogger): void { export function closeHdrVideoFrameSource(source: HdrVideoFrameSource, log?: ProducerLogger): void {
@@ -152,6 +154,18 @@ export function closeHdrVideoFrameSource(source: HdrVideoFrameSource, log?: Prod
} }
} }
export function resolveHdrVideoFrameIndex(
time: number,
startTime: number,
fps: number,
frameCount: number,
loop = false,
): number | null {
const frameIndex = Math.round((time - startTime) * fps);
if (frameIndex < 0 || frameCount < 1) return null;
return loop ? frameIndex % frameCount : Math.min(frameIndex, frameCount - 1);
}
// fallow-ignore-next-line complexity // fallow-ignore-next-line complexity
export function blitHdrVideoLayer( export function blitHdrVideoLayer(
canvas: Buffer, canvas: Buffer,
@@ -173,14 +187,18 @@ export function blitHdrVideoLayer(
return; return;
} }
// Frame index within the video. Clamp to the extracted raw frame count so // Frame index within the extracted playable source range. Loops wrap one
// a composition that outlives the source clip freezes on the last frame, // extracted cycle; non-loops clamp to its final frame, matching Chrome's
// matching Chrome's <video> behavior. // held-tail behavior for authored slots that outlive the source.
const videoFrameIndex = Math.round((time - startTime) * fps) + 1; const effectiveIndex = resolveHdrVideoFrameIndex(
if (videoFrameIndex < 1) return; time,
const effectiveIndex = Math.min(videoFrameIndex, frameSource.frameCount); startTime,
if (effectiveIndex < 1) return; fps,
const frameOffset = (effectiveIndex - 1) * frameSource.frameSize; frameSource.frameCount,
frameSource.loop,
);
if (effectiveIndex === null) return;
const frameOffset = effectiveIndex * frameSource.frameSize;
try { try {
if (hdrPerf) hdrPerf.hdrVideoLayerBlits += 1; if (hdrPerf) hdrPerf.hdrVideoLayerBlits += 1;
@@ -9,7 +9,6 @@
* centralized here. * centralized here.
*/ */
import { rmSync } from "node:fs";
import { import {
type BeforeCaptureHook, type BeforeCaptureHook,
type CaptureSession, type CaptureSession,
@@ -28,9 +27,9 @@ import {
type TransitionRange, type TransitionRange,
blitHdrImageLayer, blitHdrImageLayer,
blitHdrVideoLayer, blitHdrVideoLayer,
closeHdrVideoFrameSource,
selectDomLayerShowIds, selectDomLayerShowIds,
} from "../../hdrCompositor.js"; } from "../../hdrCompositor.js";
import { cleanupHdrVideoFrameSource } from "./captureHdrResources.js";
import { import {
type HdrPerfCollector, type HdrPerfCollector,
type HdrPerfTimingKey, type HdrPerfTimingKey,
@@ -402,17 +401,7 @@ export function cleanupEndedHdrVideos(args: {
if (!stillNeeded) { if (!stillNeeded) {
const frameSource = hdrVideoFrameSources.get(videoId); const frameSource = hdrVideoFrameSources.get(videoId);
if (frameSource) { if (frameSource) {
closeHdrVideoFrameSource(frameSource, log); cleanupHdrVideoFrameSource(frameSource, log);
try {
rmSync(frameSource.dir, { recursive: true, force: true });
} catch (err) {
log.warn("Failed to clean up HDR raw frame directory", {
videoId,
frameDir: frameSource.dir,
rawPath: frameSource.rawPath,
error: err instanceof Error ? err.message : String(err),
});
}
hdrVideoFrameSources.delete(videoId); hdrVideoFrameSources.delete(videoId);
} }
cleanedUpVideos.add(videoId); cleanedUpVideos.add(videoId);
@@ -1,5 +1,119 @@
import { describe, expect, it } from "vitest"; import {
import { estimateHdrExtractionBytes } from "./captureHdrResources.js"; closeSync,
constants,
existsSync,
mkdtempSync,
openSync,
rmSync,
writeFileSync,
} from "node:fs";
import { tmpdir } from "node:os";
import { dirname, join } from "node:path";
import { spawnSync } from "node:child_process";
import {
extractMediaMetadata,
resolveFinalFrameExtractionWindow,
runFfmpeg,
type RunFfmpegResult,
type VideoElement,
type VideoMetadata,
} from "@hyperframes/engine";
import { afterAll, afterEach, beforeAll, describe, expect, it, vi } from "vitest";
import { createRenderJob } from "../../renderOrchestrator.js";
import { resolveHdrVideoFrameIndex } from "../../hdrCompositor.js";
import {
cleanupHdrVideoFrameSource,
estimateHdrExtractionBytes,
extractHdrVideoFrames,
getHdrExtractionReservedBytes,
reserveHdrExtractionBytes,
resolveHdrExtractionActiveBudgetBytes,
resolveHdrExtractionBudgetBytes,
resolveHdrExtractionWindow,
} from "./captureHdrResources.js";
afterEach(() => {
vi.unstubAllEnvs();
expect(getHdrExtractionReservedBytes()).toBe(0);
});
function ffmpegResult(success: boolean): RunFfmpegResult {
return {
success,
exitCode: success ? 0 : 1,
stderr: success ? "" : "mock extraction failure",
durationMs: 1,
terminationReason: "exit",
};
}
const HAS_FFMPEG = spawnSync("ffmpeg", ["-version"]).status === 0;
function hdrVideo(id: string, overrides: Partial<VideoElement> = {}): VideoElement {
return {
id,
src: `${id}.mov`,
start: 0,
end: Number.POSITIVE_INFINITY,
mediaStart: 0,
loop: false,
hasAudio: false,
...overrides,
};
}
function videoMetadata(
durationSeconds: number,
videoStreamDurationSeconds = durationSeconds,
): VideoMetadata {
return {
durationSeconds,
videoStreamDurationSeconds,
width: 1,
height: 1,
fps: 2,
videoCodec: "hevc",
hasAudio: false,
isVFR: false,
hasAlpha: false,
colorSpace: null,
};
}
function hdrExtractionFixture(videos: VideoElement[], framesDir: string) {
return {
job: createRenderJob({ fps: { num: 2, den: 1 }, quality: "standard" }),
log: {
error: vi.fn(),
warn: vi.fn(),
info: vi.fn(),
debug: vi.fn(),
},
framesDir,
composition: {
duration: 2,
videos,
audios: [],
images: [],
width: 1,
height: 1,
},
prep: {
hdrVideoIds: videos.map((video) => video.id),
hdrVideoSrcPaths: new Map(videos.map((video) => [video.id, video.src])),
hdrVideoStartTimes: new Map(videos.map((video) => [video.id, video.start])),
hdrImageStartTimes: new Map(),
hdrExtractionDims: new Map(videos.map((video) => [video.id, { width: 1, height: 1 }])),
hdrImageFitInfo: new Map(),
},
width: 1,
height: 1,
abortSignal: undefined,
hdrDiagnostics: { videoExtractionFailures: 0, imageDecodeFailures: 0 },
extractMediaMetadataImpl: async () => videoMetadata(120),
resolveFinalFrameExtractionWindowImpl: async (_srcPath, _video, _metadata, window) => window,
};
}
describe("estimateHdrExtractionBytes", () => { describe("estimateHdrExtractionBytes", () => {
it("sums 6 bytes per pixel per frame across videos", () => { it("sums 6 bytes per pixel per frame across videos", () => {
@@ -26,3 +140,551 @@ describe("estimateHdrExtractionBytes", () => {
); );
}); });
}); });
describe("resolveHdrExtractionWindow", () => {
it("bounds a two-second composition backed by an unbounded HDR source to 60 raw frames", () => {
const window = resolveHdrExtractionWindow(hdrVideo("long-hdr"), 2, videoMetadata(120));
if (!window) throw new Error("expected a visible HDR extraction window");
const { durationSeconds } = window;
expect(durationSeconds).toBe(2);
expect(estimateHdrExtractionBytes([{ durationSeconds, width: 3840, height: 2160 }], 30)).toBe(
60 * 3840 * 2160 * 6,
);
});
it("independently caps a stale finite media end at the composition duration", () => {
expect(resolveHdrExtractionWindow(hdrVideo("hdr", { end: 60 }), 2, videoMetadata(60))).toEqual({
compositionStart: 0,
mediaStart: 0,
durationSeconds: 2,
});
});
it("trims materially negative preroll and advances the source offset", () => {
expect(
resolveHdrExtractionWindow(hdrVideo("hdr", { start: -60, end: 120 }), 2, videoMetadata(120)),
).toEqual({ compositionStart: 0, mediaStart: 60, durationSeconds: 2 });
});
it("preserves a short loop cycle when negative preroll crosses EOF", () => {
expect(
resolveHdrExtractionWindow(
hdrVideo("loop", { start: -5, end: 10, loop: true }),
2,
videoMetadata(3),
),
).toEqual({
compositionStart: -5,
mediaStart: 0,
durationSeconds: 3,
preserveTimelinePhase: true,
});
});
it("marks an entirely held interval for exact final-frame resolution", () => {
expect(
resolveHdrExtractionWindow(hdrVideo("held", { start: -5, end: 10 }), 2, videoMetadata(3)),
).toEqual({
compositionStart: -2.000001,
mediaStart: 2.999999,
durationSeconds: 0.000001,
preserveTimelineEnd: true,
ensureFinalFrame: true,
});
});
it.each([
{ start: -5, compositionDuration: 2, expected: null, label: "already ended" },
{
start: -2,
compositionDuration: 15,
expected: { compositionStart: 0, mediaStart: 2, durationSeconds: 1 },
label: "partially visible",
},
])(
"keeps an open-ended HDR clip source-bounded when $label",
({ start, compositionDuration, expected }) => {
expect(
resolveHdrExtractionWindow(
hdrVideo("open-held", { start, end: Number.POSITIVE_INFINITY, loop: false }),
compositionDuration,
videoMetadata(3),
),
).toEqual(expected);
},
);
it.each([
{ loop: true, preservation: { preserveTimelinePhase: true }, label: "loop" },
{
loop: false,
preservation: { preserveTimelineEnd: true, ensureFinalFrame: true },
label: "held tail",
},
])(
"caps a finite 60-second $label slot to one 3-second source range",
({ loop, preservation }) => {
expect(
resolveHdrExtractionWindow(hdrVideo("short", { end: 60, loop }), 60, videoMetadata(3)),
).toEqual({
compositionStart: 0,
mediaStart: 0,
durationSeconds: 3,
...preservation,
});
},
);
it("bounds an entirely held 120-second source to an exact one-frame plan", () => {
expect(
resolveHdrExtractionWindow(
hdrVideo("long-held", { start: -600, end: 10 }),
2,
videoMetadata(120),
),
).toEqual({
compositionStart: -480.000001,
mediaStart: 119.999999,
durationSeconds: 0.000001,
preserveTimelineEnd: true,
ensureFinalFrame: true,
});
});
it("rejects mediaStart at source EOF before HDR budgeting", () => {
expect(() =>
resolveHdrExtractionWindow(hdrVideo("past-eof", { mediaStart: 3 }), 60, videoMetadata(3)),
).toThrowError(expect.objectContaining({ kind: "media_start_out_of_range", retryable: false }));
});
it("skips HDR media with no interval inside the composition", () => {
expect(
resolveHdrExtractionWindow(hdrVideo("hdr", { start: 3 }), 2, videoMetadata(60)),
).toBeNull();
});
});
describe.skipIf(!HAS_FFMPEG)("raw HDR held tails on sparse-timestamp sources", () => {
const fixtureDir = mkdtempSync(join(tmpdir(), "hf-hdr-sparse-held-tail-"));
const cfrFixture = join(fixtureDir, "sub-1fps-cfr.mp4");
const vfrFixture = join(fixtureDir, "sparse-vfr.mp4");
const nonZeroStartFixture = join(fixtureDir, "nonzero-start.mp4");
const negativeStartTransportFixture = join(fixtureDir, "negative-start.ts");
beforeAll(async () => {
const fixtures = [
{
path: cfrFixture,
input: "testsrc2=s=64x64:d=10:rate=1/5",
filters: [] as string[],
},
{
path: vfrFixture,
input: "testsrc2=s=64x64:d=10:rate=1/2",
filters: ["-vf", "select='eq(n,0)+eq(n,2)'", "-vsync", "vfr"],
},
{
path: nonZeroStartFixture,
input: "testsrc2=s=64x64:d=3:rate=1",
filters: ["-output_ts_offset", "5"],
},
{
path: negativeStartTransportFixture,
input: "testsrc2=s=64x64:d=3:rate=1",
filters: [
"-mpegts_copyts",
"1",
"-muxdelay",
"0",
"-avoid_negative_ts",
"disabled",
"-output_ts_offset",
"-2",
],
},
];
for (const fixture of fixtures) {
const result = await runFfmpeg([
"-hide_banner",
"-loglevel",
"error",
"-f",
"lavfi",
"-i",
fixture.input,
...fixture.filters,
"-c:v",
"libx264",
"-pix_fmt",
"yuv420p",
"-y",
fixture.path,
]);
if (!result.success) {
throw new Error(`sparse HDR fixture synthesis failed: ${result.stderr.slice(-400)}`);
}
}
}, 30_000);
afterAll(() => {
rmSync(fixtureDir, { recursive: true, force: true });
});
it.each([
{ label: "sub-1fps CFR", src: cfrFixture, expectedVfr: false, streamStart: 0 },
{ label: "sparse VFR", src: vfrFixture, expectedVfr: true, streamStart: 0 },
{
label: "non-zero stream start",
src: nonZeroStartFixture,
expectedVfr: false,
streamStart: 5,
},
{
label: "unindexed negative-base MPEG-TS",
src: negativeStartTransportFixture,
expectedVfr: false,
streamStart: -2,
},
])(
"writes exactly one raw HDR frame for $label",
async ({ src, expectedVfr, streamStart }) => {
const metadata = await extractMediaMetadata(src);
expect(metadata.fps).toBeLessThanOrEqual(1);
expect(metadata.isVFR).toBe(expectedVfr);
expect(metadata.videoStreamStartSeconds).toBeCloseTo(streamStart, 6);
const framesDir = mkdtempSync(join(fixtureDir, "raw-out-"));
const video = hdrVideo(`real-${String(expectedVfr)}`, {
src,
start: -15,
end: 5,
});
const fixture = hdrExtractionFixture([video], framesDir);
fixture.prep.hdrExtractionDims.set(video.id, { width: 64, height: 64 });
const extracted = await extractHdrVideoFrames({
...fixture,
width: 64,
height: 64,
runFfmpegImpl: runFfmpeg,
extractMediaMetadataImpl: extractMediaMetadata,
resolveFinalFrameExtractionWindowImpl: resolveFinalFrameExtractionWindow,
});
try {
expect(extracted.sources.get(video.id)?.frameCount).toBe(1);
expect(extracted.estimatedBytes).toBe(64 * 64 * 6);
expect(fixture.prep.hdrVideoStartTimes.get(video.id)).toBe(0);
} finally {
for (const source of extracted.sources.values()) cleanupHdrVideoFrameSource(source);
extracted.releaseReservation();
rmSync(framesDir, { recursive: true, force: true });
}
},
30_000,
);
});
describe("resolveHdrExtractionBudgetBytes", () => {
it("uses half an actual cgroup limit when no env budget is configured", () => {
expect(resolveHdrExtractionBudgetBytes(undefined, 24 * 1024)).toBe(12 * 1024 ** 3);
expect(resolveHdrExtractionBudgetBytes(undefined, null)).toBeUndefined();
});
it("uses the stricter of the environment and cgroup budgets", () => {
expect(resolveHdrExtractionBudgetBytes(String(8 * 1024 ** 3), 24 * 1024)).toBe(8 * 1024 ** 3);
expect(resolveHdrExtractionBudgetBytes(String(20 * 1024 ** 3), 24 * 1024)).toBe(12 * 1024 ** 3);
expect(resolveHdrExtractionBudgetBytes("1234.9", null)).toBe(1234);
});
it("rejects invalid budgets", () => {
expect(() => resolveHdrExtractionBudgetBytes("0", null)).toThrow("must be a positive finite");
expect(() => resolveHdrExtractionBudgetBytes("Infinity", null)).toThrow(
"must be a positive finite",
);
});
});
describe("reserveHdrExtractionBytes", () => {
it("prevents concurrent aggregate overcommit and releases idempotently", () => {
const releaseFirst = reserveHdrExtractionBytes(60, 100);
try {
expect(getHdrExtractionReservedBytes()).toBe(60);
expect(() => reserveHdrExtractionBytes(50, 100)).toThrow("Concurrent HDR pre-extractions");
} finally {
releaseFirst();
releaseFirst();
}
const releaseAfter = reserveHdrExtractionBytes(100, 100);
expect(getHdrExtractionReservedBytes()).toBe(100);
releaseAfter();
});
it("prevents two concurrent jobs from overcommitting a disk-only budget", () => {
const diskOnlyBudget = resolveHdrExtractionActiveBudgetBytes(undefined, 100);
expect(diskOnlyBudget).toBe(90);
const releaseFirstJob = reserveHdrExtractionBytes(60, diskOnlyBudget);
try {
expect(() => reserveHdrExtractionBytes(40, diskOnlyBudget)).toThrow(
"Concurrent HDR pre-extractions",
);
} finally {
releaseFirstJob();
}
});
});
describe("extractHdrVideoFrames", () => {
it("pins FFmpeg seek/duration, raw frame count, and reservation lifetime", async () => {
const framesDir = mkdtempSync(join(tmpdir(), "hf-hdr-extract-"));
const video = hdrVideo("preroll", { start: -60, end: 120, mediaStart: 0 });
const fixture = hdrExtractionFixture([video], framesDir);
const calls: string[][] = [];
try {
const extracted = await extractHdrVideoFrames({
...fixture,
runFfmpegImpl: async (args) => {
calls.push(args);
const rawPath = args.at(-1);
if (!rawPath) throw new Error("mock FFmpeg output path missing");
// The visible [0, 2] interval is two seconds at 2fps = 4 rgb48le 1x1 frames.
writeFileSync(rawPath, Buffer.alloc(4 * 6));
return ffmpegResult(true);
},
});
try {
expect(calls).toHaveLength(1);
const args = calls[0] ?? [];
expect(args.slice(args.indexOf("-ss"), args.indexOf("-ss") + 2)).toEqual(["-ss", "60"]);
expect(args.slice(args.indexOf("-t"), args.indexOf("-t") + 2)).toEqual(["-t", "2"]);
expect(fixture.prep.hdrVideoStartTimes.get("preroll")).toBe(0);
expect(extracted.sources.get("preroll")?.frameCount).toBe(4);
expect(extracted.estimatedBytes).toBe(24);
expect(getHdrExtractionReservedBytes()).toBe(24);
} finally {
for (const source of extracted.sources.values()) cleanupHdrVideoFrameSource(source);
extracted.releaseReservation();
}
} finally {
rmSync(framesDir, { recursive: true, force: true });
}
});
it.each([
{
loop: true,
expectedFrameIndex: 4,
expectedFrameCount: 6,
expectedStart: -5,
expectedSeek: "0",
expectedDuration: "3",
label: "loop phase",
},
{
loop: false,
expectedFrameIndex: 0,
expectedFrameCount: 1,
expectedStart: 0,
expectedSeek: "2",
expectedDuration: undefined,
label: "held final frame",
},
])("preserves $label after materially negative preroll", async (testCase) => {
const {
loop,
expectedFrameIndex,
expectedFrameCount,
expectedStart,
expectedSeek,
expectedDuration,
} = testCase;
const framesDir = mkdtempSync(join(tmpdir(), "hf-hdr-negative-source-"));
const video = hdrVideo(`negative-${String(loop)}`, { start: -5, end: 10, loop });
const fixture = hdrExtractionFixture([video], framesDir);
const calls: string[][] = [];
try {
const extracted = await extractHdrVideoFrames({
...fixture,
extractMediaMetadataImpl: async () => videoMetadata(3),
resolveFinalFrameExtractionWindowImpl: async (_srcPath, _video, _metadata, window) =>
window.ensureFinalFrame
? {
compositionStart: 0,
mediaStart: 2.999999,
extractionMediaStart: 2,
durationSeconds: 0.000001,
preserveTimelineEnd: true,
finalFrameOnly: true,
}
: window,
runFfmpegImpl: async (args) => {
calls.push(args);
const rawPath = args.at(-1);
if (!rawPath) throw new Error("mock FFmpeg output path missing");
writeFileSync(rawPath, Buffer.alloc(expectedFrameCount * 6));
return ffmpegResult(true);
},
});
try {
const args = calls[0] ?? [];
expect(args.slice(args.indexOf("-ss"), args.indexOf("-ss") + 2)).toEqual([
"-ss",
expectedSeek,
]);
if (expectedDuration === undefined) {
expect(args).not.toContain("-t");
expect(args.slice(args.indexOf("-frames:v"), args.indexOf("-frames:v") + 2)).toEqual([
"-frames:v",
"1",
]);
} else {
expect(args.slice(args.indexOf("-t"), args.indexOf("-t") + 2)).toEqual([
"-t",
expectedDuration,
]);
}
expect(fixture.prep.hdrVideoStartTimes.get(video.id)).toBe(expectedStart);
const source = extracted.sources.get(video.id);
expect(source?.loop).toBe(loop);
expect(
resolveHdrVideoFrameIndex(0, expectedStart, 2, source?.frameCount ?? 0, source?.loop),
).toBe(expectedFrameIndex);
} finally {
for (const source of extracted.sources.values()) cleanupHdrVideoFrameSource(source);
extracted.releaseReservation();
}
} finally {
rmSync(framesDir, { recursive: true, force: true });
}
});
it.each([
{ loop: true, expectedFrameIndex: 4, label: "loop" },
{ loop: false, expectedFrameIndex: 5, label: "held tail" },
])(
"reserves one playable video-stream range for a finite 60-second $label slot with longer audio",
async ({ loop, expectedFrameIndex }) => {
const framesDir = mkdtempSync(join(tmpdir(), "hf-hdr-finite-short-source-"));
const video = hdrVideo(`finite-${String(loop)}`, { end: 60, loop });
const fixture = hdrExtractionFixture([video], framesDir);
fixture.composition.duration = 60;
const calls: string[][] = [];
try {
const extracted = await extractHdrVideoFrames({
...fixture,
// A valid mux can have a short video stream and a much longer audio
// stream/container. Scratch planning must follow video EOF.
extractMediaMetadataImpl: async () => videoMetadata(60, 3),
runFfmpegImpl: async (args) => {
calls.push(args);
const rawPath = args.at(-1);
if (!rawPath) throw new Error("mock FFmpeg output path missing");
writeFileSync(rawPath, Buffer.alloc(6 * 6));
return ffmpegResult(true);
},
});
try {
const args = calls[0] ?? [];
expect(args.slice(args.indexOf("-ss"), args.indexOf("-ss") + 2)).toEqual(["-ss", "0"]);
expect(args.slice(args.indexOf("-t"), args.indexOf("-t") + 2)).toEqual(["-t", "3"]);
expect(extracted.estimatedBytes).toBe(36);
expect(getHdrExtractionReservedBytes()).toBe(36);
expect(fixture.prep.hdrVideoStartTimes.get(video.id)).toBe(0);
const source = extracted.sources.get(video.id);
expect(source?.loop).toBe(loop);
expect(resolveHdrVideoFrameIndex(59, 0, 2, source?.frameCount ?? 0, source?.loop)).toBe(
expectedFrameIndex,
);
} finally {
for (const source of extracted.sources.values()) cleanupHdrVideoFrameSource(source);
extracted.releaseReservation();
}
} finally {
rmSync(framesDir, { recursive: true, force: true });
}
},
);
it("closes/removes completed and partial sources and releases reservation on failure", async () => {
const framesDir = mkdtempSync(join(tmpdir(), "hf-hdr-partial-"));
const fixture = hdrExtractionFixture([hdrVideo("first"), hdrVideo("second")], framesDir);
const createdRawPaths: string[] = [];
let call = 0;
try {
await expect(
extractHdrVideoFrames({
...fixture,
runFfmpegImpl: async (args) => {
call += 1;
const rawPath = args.at(-1);
if (!rawPath) throw new Error("mock FFmpeg output path missing");
createdRawPaths.push(rawPath);
if (call === 2) return ffmpegResult(false);
writeFileSync(rawPath, Buffer.alloc(4 * 6));
return ffmpegResult(true);
},
}),
).rejects.toThrow('HDR frame extraction failed for video "second"');
expect(fixture.hdrDiagnostics.videoExtractionFailures).toBe(1);
expect(createdRawPaths).toHaveLength(2);
for (const rawPath of createdRawPaths) expect(existsSync(dirname(rawPath))).toBe(false);
expect(getHdrExtractionReservedBytes()).toBe(0);
} finally {
rmSync(framesDir, { recursive: true, force: true });
}
});
});
describe("cleanupHdrVideoFrameSource", () => {
it("closes the raw descriptor and immediately removes its directory", () => {
const dir = mkdtempSync(join(tmpdir(), "hf-hdr-cleanup-"));
const rawPath = join(dir, "frames.rgb48le");
writeFileSync(rawPath, Buffer.alloc(12));
const fd = openSync(rawPath, constants.O_RDONLY);
cleanupHdrVideoFrameSource({
dir,
rawPath,
fd,
width: 1,
height: 2,
frameSize: 12,
frameCount: 1,
scratch: Buffer.alloc(12),
});
expect(existsSync(dir)).toBe(false);
expect(() => closeSync(fd)).toThrow();
});
it("closes the descriptor but retains raw files with KEEP_TEMP=1", () => {
vi.stubEnv("KEEP_TEMP", "1");
const dir = mkdtempSync(join(tmpdir(), "hf-hdr-keep-temp-"));
const rawPath = join(dir, "frames.rgb48le");
writeFileSync(rawPath, Buffer.alloc(12));
const fd = openSync(rawPath, constants.O_RDONLY);
try {
cleanupHdrVideoFrameSource({
dir,
rawPath,
fd,
width: 1,
height: 2,
frameSize: 12,
frameCount: 1,
scratch: Buffer.alloc(12),
});
expect(existsSync(rawPath)).toBe(true);
expect(() => closeSync(fd)).toThrow();
} finally {
rmSync(dir, { recursive: true, force: true });
}
});
});
@@ -24,20 +24,31 @@ import {
mkdtempSync, mkdtempSync,
openSync, openSync,
readFileSync, readFileSync,
rmSync,
statfsSync, statfsSync,
} from "node:fs"; } from "node:fs";
import { join } from "node:path"; import { join } from "node:path";
import { import {
type CaptureSession, type CaptureSession,
decodePngToRgb48le, decodePngToRgb48le,
extractMediaMetadata,
getCgroupMemoryLimitMb,
normalizeObjectFit, normalizeObjectFit,
queryElementStacking, queryElementStacking,
resampleRgb48leObjectFit, resampleRgb48leObjectFit,
resolveFinalFrameExtractionWindow,
resolveVideoExtractionWindow,
runFfmpeg, runFfmpeg,
type TimelineExtractionWindow,
type VideoMetadata,
} from "@hyperframes/engine"; } from "@hyperframes/engine";
import { fpsToFfmpegArg, fpsToNumber } from "@hyperframes/core"; import { fpsToFfmpegArg, fpsToNumber } from "@hyperframes/core";
import type { ProducerLogger } from "../../../logger.js"; import type { ProducerLogger } from "../../../logger.js";
import type { HdrImageBuffer, HdrVideoFrameSource } from "../../hdrCompositor.js"; import {
closeHdrVideoFrameSource,
type HdrImageBuffer,
type HdrVideoFrameSource,
} from "../../hdrCompositor.js";
import type { HdrDiagnostics, RenderJob } from "../../renderOrchestrator.js"; import type { HdrDiagnostics, RenderJob } from "../../renderOrchestrator.js";
import type { CompositionMetadata } from "../shared.js"; import type { CompositionMetadata } from "../shared.js";
@@ -107,6 +118,7 @@ export function planHdrResources(args: {
* probe for HDR images whose `data-start` instant reports zero dims (GSAP * probe for HDR images whose `data-start` instant reports zero dims (GSAP
* `from` tweens animate the element in slightly later). * `from` tweens animate the element in slightly later).
*/ */
// fallow-ignore-next-line complexity code-duplication
export async function probeHdrExtractionDims(args: { export async function probeHdrExtractionDims(args: {
domSession: CaptureSession; domSession: CaptureSession;
nativeHdrIds: Set<string>; nativeHdrIds: Set<string>;
@@ -186,7 +198,129 @@ export function estimateHdrExtractionBytes(
} }
const HDR_EXTRACTION_HEADROOM_FRACTION = 0.9; const HDR_EXTRACTION_HEADROOM_FRACTION = 0.9;
const HDR_EXTRACTION_CGROUP_BUDGET_FRACTION = 0.5;
const HDR_EXTRACTION_WARN_BYTES = 10e9; const HDR_EXTRACTION_WARN_BYTES = 10e9;
const HDR_EXTRACTION_MAX_BYTES_ENV = "HDR_EXTRACTION_MAX_BYTES";
const BYTES_PER_MIB = 1024 * 1024;
let aggregateHdrExtractionReservedBytes = 0;
export function resolveHdrExtractionBudgetBytes(
raw: string | undefined,
cgroupLimitMb: number | null = getCgroupMemoryLimitMb(),
): number | undefined {
let configuredBudget: number | undefined;
if (raw !== undefined && raw.trim() !== "") {
const value = Number(raw);
if (!Number.isFinite(value) || value <= 0) {
throw new Error(`${HDR_EXTRACTION_MAX_BYTES_ENV} must be a positive finite byte count`);
}
configuredBudget = Math.floor(value);
}
const cgroupBudget =
cgroupLimitMb !== null && Number.isFinite(cgroupLimitMb) && cgroupLimitMb > 0
? Math.floor(cgroupLimitMb * BYTES_PER_MIB * HDR_EXTRACTION_CGROUP_BUDGET_FRACTION)
: undefined;
if (configuredBudget === undefined) return cgroupBudget;
if (cgroupBudget === undefined) return configuredBudget;
return Math.min(configuredBudget, cgroupBudget);
}
export function resolveHdrExtractionActiveBudgetBytes(
configuredBudgetBytes: number | undefined,
freeBytes: number,
): number {
const diskBudgetBytes = Math.floor(freeBytes * HDR_EXTRACTION_HEADROOM_FRACTION);
return configuredBudgetBytes === undefined
? diskBudgetBytes
: Math.min(configuredBudgetBytes, diskBudgetBytes);
}
export function reserveHdrExtractionBytes(
estimatedBytes: number,
budgetBytes: number | undefined,
): () => void {
if (!Number.isFinite(estimatedBytes) || estimatedBytes < 0) {
throw new Error(`HDR extraction reservation must be a finite non-negative byte count`);
}
const aggregateBytes = aggregateHdrExtractionReservedBytes + estimatedBytes;
if (budgetBytes !== undefined && aggregateBytes > budgetBytes) {
throw new Error(
`Concurrent HDR pre-extractions need ~${(aggregateBytes / 1e9).toFixed(1)} GB of raw ` +
`16-bit frame scratch, exceeding the active ${(budgetBytes / 1e9).toFixed(1)} GB budget.`,
);
}
aggregateHdrExtractionReservedBytes = aggregateBytes;
let released = false;
return () => {
if (released) return;
released = true;
aggregateHdrExtractionReservedBytes = Math.max(
0,
aggregateHdrExtractionReservedBytes - estimatedBytes,
);
};
}
export function getHdrExtractionReservedBytes(): number {
return aggregateHdrExtractionReservedBytes;
}
export type HdrExtractionWindow = TimelineExtractionWindow;
export function resolveHdrExtractionWindow(
video: {
id: string;
start: number;
end: number;
mediaStart: number;
loop: boolean;
},
compositionDuration: number,
metadata: VideoMetadata,
): HdrExtractionWindow | null {
if (!Number.isFinite(compositionDuration) || compositionDuration <= 0) {
throw new Error(
`Cannot extract HDR video "${video.id}" with invalid composition duration ${String(compositionDuration)}`,
);
}
const window = resolveVideoExtractionWindow(video, metadata, compositionDuration);
if (!Number.isFinite(window.durationSeconds)) {
throw new Error(
`HDR video "${video.id}" has no finite interval inside the ${compositionDuration}s composition`,
);
}
if (window.durationSeconds <= 0) return null;
if (!Number.isFinite(window.mediaStart) || window.mediaStart < 0) {
throw new Error(`HDR video "${video.id}" has invalid mediaStart ${String(window.mediaStart)}`);
}
return window;
}
function cleanupHdrFrameDirectory(
frameDir: string,
rawPath: string | undefined,
log?: ProducerLogger,
): void {
if (process.env.KEEP_TEMP === "1") return;
try {
rmSync(frameDir, { recursive: true, force: true });
} catch (err) {
log?.warn("Failed to clean up HDR raw frame directory", {
frameDir,
rawPath,
error: err instanceof Error ? err.message : String(err),
});
}
}
/** Close the raw-frame descriptor and release its directory unless KEEP_TEMP=1. */
export function cleanupHdrVideoFrameSource(
source: HdrVideoFrameSource,
log?: ProducerLogger,
): void {
closeHdrVideoFrameSource(source, log);
cleanupHdrFrameDirectory(source.dir, source.rawPath, log);
}
/** /**
* Disk-headroom gate for raw rgb48le pre-extraction: throws if the planned * Disk-headroom gate for raw rgb48le pre-extraction: throws if the planned
@@ -202,18 +336,30 @@ function assertHdrExtractionDiskHeadroom(
plannedVideos: Array<{ durationSeconds: number; width: number; height: number }>, plannedVideos: Array<{ durationSeconds: number; width: number; height: number }>,
fps: number, fps: number,
log: ProducerLogger, log: ProducerLogger,
): void { ): { estimatedBytes: number; budgetBytes: number | undefined } {
const estimatedBytes = estimateHdrExtractionBytes(plannedVideos, fps); const estimatedBytes = estimateHdrExtractionBytes(plannedVideos, fps);
const configuredBudget = resolveHdrExtractionBudgetBytes(
process.env[HDR_EXTRACTION_MAX_BYTES_ENV],
);
const estimatedGb = (estimatedBytes / 1e9).toFixed(1);
if (configuredBudget !== undefined && estimatedBytes > configuredBudget) {
throw new Error(
`HDR pre-extraction needs ~${estimatedGb} GB of raw 16-bit frames, exceeding the ` +
`active scratch budget of ${(configuredBudget / 1e9).toFixed(1)} GB. ` +
`If the composition doesn't need HDR output, re-run with --sdr; otherwise reduce its ` +
`HDR duration/resolution or use a render container with a larger memory limit.`,
);
}
let freeBytes: number; let freeBytes: number;
try { try {
const stat = statfsSync(framesDir); const stat = statfsSync(framesDir);
freeBytes = stat.bavail * stat.bsize; freeBytes = stat.bavail * stat.bsize;
} catch { } catch {
// statfs unsupported on this platform/filesystem — skip the gate. // statfs unsupported on this platform/filesystem — skip the gate.
return; return { estimatedBytes, budgetBytes: configuredBudget };
} }
const estimatedGb = (estimatedBytes / 1e9).toFixed(1); const diskBudgetBytes = Math.floor(freeBytes * HDR_EXTRACTION_HEADROOM_FRACTION);
if (estimatedBytes > freeBytes * HDR_EXTRACTION_HEADROOM_FRACTION) { if (estimatedBytes > diskBudgetBytes) {
throw new Error( throw new Error(
`HDR pre-extraction needs ~${estimatedGb} GB of raw 16-bit frames but only ` + `HDR pre-extraction needs ~${estimatedGb} GB of raw 16-bit frames but only ` +
`${(freeBytes / 1e9).toFixed(1)} GB is free at ${framesDir}. ` + `${(freeBytes / 1e9).toFixed(1)} GB is free at ${framesDir}. ` +
@@ -228,13 +374,25 @@ function assertHdrExtractionDiskHeadroom(
{ estimatedBytes, freeBytes }, { estimatedBytes, freeBytes },
); );
} }
return {
estimatedBytes,
budgetBytes: resolveHdrExtractionActiveBudgetBytes(configuredBudget, freeBytes),
};
}
export interface HdrVideoExtractionResult {
sources: Map<string, HdrVideoFrameSource>;
estimatedBytes: number;
releaseReservation: () => void;
} }
/** /**
* Extract each HDR video into a raw rgb48le frame file via a single FFmpeg * Extract each HDR video into a raw rgb48le frame file via a single FFmpeg
* pass per video, and open a file descriptor for each. Returns a map keyed * pass per video, and open a file descriptor for each. The caller owns both
* by video id. Caller owns lifecycle teardown (closing fds + rm-rf). * source teardown and the aggregate scratch reservation, which intentionally
* remains held until the capture-stage finally block.
*/ */
// fallow-ignore-next-line complexity
export async function extractHdrVideoFrames(args: { export async function extractHdrVideoFrames(args: {
job: RenderJob; job: RenderJob;
log: ProducerLogger; log: ProducerLogger;
@@ -245,90 +403,142 @@ export async function extractHdrVideoFrames(args: {
height: number; height: number;
abortSignal: AbortSignal | undefined; abortSignal: AbortSignal | undefined;
hdrDiagnostics: HdrDiagnostics; hdrDiagnostics: HdrDiagnostics;
}): Promise<Map<string, HdrVideoFrameSource>> { runFfmpegImpl?: typeof runFfmpeg;
extractMediaMetadataImpl?: typeof extractMediaMetadata;
resolveFinalFrameExtractionWindowImpl?: typeof resolveFinalFrameExtractionWindow;
}): Promise<HdrVideoExtractionResult> {
const { job, log, framesDir, composition, prep, width, height, abortSignal, hdrDiagnostics } = const { job, log, framesDir, composition, prep, width, height, abortSignal, hdrDiagnostics } =
args; args;
const runFfmpegImpl = args.runFfmpegImpl ?? runFfmpeg;
const extractMediaMetadataImpl = args.extractMediaMetadataImpl ?? extractMediaMetadata;
const resolveFinalFrameExtractionWindowImpl =
args.resolveFinalFrameExtractionWindowImpl ?? resolveFinalFrameExtractionWindow;
const out = new Map<string, HdrVideoFrameSource>(); const out = new Map<string, HdrVideoFrameSource>();
mkdirSync(framesDir, { recursive: true }); mkdirSync(framesDir, { recursive: true });
const plannedVideos: Array<{ durationSeconds: number; width: number; height: number }> = []; const plannedVideos: Array<{ durationSeconds: number; width: number; height: number }> = [];
for (const [videoId] of prep.hdrVideoSrcPaths) { const extractionWindows = new Map<string, HdrExtractionWindow>();
for (const [videoId, srcPath] of prep.hdrVideoSrcPaths) {
const video = composition.videos.find((v) => v.id === videoId); const video = composition.videos.find((v) => v.id === videoId);
if (!video) continue; if (!video) continue;
const dims = prep.hdrExtractionDims.get(videoId) ?? { width, height }; const dims = prep.hdrExtractionDims.get(videoId) ?? { width, height };
const metadata = await extractMediaMetadataImpl(srcPath);
const initialWindow = resolveHdrExtractionWindow(video, composition.duration, metadata);
if (!initialWindow) continue;
const window = await resolveFinalFrameExtractionWindowImpl(
srcPath,
video,
metadata,
initialWindow,
abortSignal,
);
extractionWindows.set(videoId, window);
prep.hdrVideoStartTimes.set(videoId, window.compositionStart);
plannedVideos.push({ plannedVideos.push({
durationSeconds: video.end - video.start, durationSeconds: window.durationSeconds,
width: dims.width, width: dims.width,
height: dims.height, height: dims.height,
}); });
} }
assertHdrExtractionDiskHeadroom(framesDir, plannedVideos, fpsToNumber(job.config.fps), log); const { estimatedBytes, budgetBytes } = assertHdrExtractionDiskHeadroom(
for (const [videoId, srcPath] of prep.hdrVideoSrcPaths) { framesDir,
const video = composition.videos.find((v) => v.id === videoId); plannedVideos,
if (!video) continue; fpsToNumber(job.config.fps),
mkdirSync(framesDir, { recursive: true }); log,
const frameDir = mkdtempSync(join(framesDir, `hdr_${tempDirSafePrefix(videoId)}-`)); );
const duration = video.end - video.start; const releaseReservation = reserveHdrExtractionBytes(estimatedBytes, budgetBytes);
const dims = prep.hdrExtractionDims.get(videoId) ?? { width, height }; const createdFrameDirs = new Set<string>();
const rawPath = join(frameDir, "frames.rgb48le"); try {
const ffmpegArgs = [ for (const [videoId, srcPath] of prep.hdrVideoSrcPaths) {
"-ss", const video = composition.videos.find((v) => v.id === videoId);
String(video.mediaStart), const window = extractionWindows.get(videoId);
"-i", if (!video || !window) continue;
srcPath, mkdirSync(framesDir, { recursive: true });
"-t", const frameDir = mkdtempSync(join(framesDir, `hdr_${tempDirSafePrefix(videoId)}-`));
String(duration), createdFrameDirs.add(frameDir);
"-r", const dims = prep.hdrExtractionDims.get(videoId) ?? { width, height };
fpsToFfmpegArg(job.config.fps), const rawPath = join(frameDir, "frames.rgb48le");
"-vf", const extractionStart = String(window.extractionMediaStart ?? window.mediaStart);
`scale=${dims.width}:${dims.height}:force_original_aspect_ratio=increase,crop=${dims.width}:${dims.height}`, const ffmpegArgs: string[] = [];
"-pix_fmt", if (window.finalFrameOnly) {
"rgb48le", // Decode before seeking for the one-frame path. Input-side seeking can
"-f", // return zero frames for valid unindexed/negative-base transports.
"rawvideo", ffmpegArgs.push("-i", srcPath, "-ss", extractionStart, "-frames:v", "1");
"-y", } else {
rawPath, ffmpegArgs.push(
]; "-ss",
const result = await runFfmpeg(ffmpegArgs, { signal: abortSignal }); extractionStart,
if (!result.success) { "-i",
hdrDiagnostics.videoExtractionFailures += 1; srcPath,
log.error("HDR frame pre-extraction failed; aborting render", { "-t",
videoId, String(window.durationSeconds),
srcPath, );
stderr: result.stderr.slice(-400), }
}); if (!window.finalFrameOnly) {
throw new Error( ffmpegArgs.push("-r", fpsToFfmpegArg(job.config.fps));
`HDR frame extraction failed for video "${videoId}". ` + }
`Aborting render to avoid shipping black HDR layers.`, ffmpegArgs.push(
"-vf",
`scale=${dims.width}:${dims.height}:force_original_aspect_ratio=increase,crop=${dims.width}:${dims.height}`,
"-pix_fmt",
"rgb48le",
"-f",
"rawvideo",
"-y",
rawPath,
); );
} const result = await runFfmpegImpl(ffmpegArgs, { signal: abortSignal });
const frameSize = dims.width * dims.height * 6; if (!result.success) {
const fd = openSync(rawPath, constants.O_RDONLY | NO_FOLLOW_FLAG);
let handedOff = false;
try {
const frameCount = Math.floor(fstatSync(fd).size / frameSize);
if (frameCount < 1) {
hdrDiagnostics.videoExtractionFailures += 1; hdrDiagnostics.videoExtractionFailures += 1;
log.error("HDR frame pre-extraction failed; aborting render", {
videoId,
srcPath,
stderr: result.stderr.slice(-400),
});
throw new Error( throw new Error(
`HDR frame extraction produced no frames for video "${videoId}". ` + `HDR frame extraction failed for video "${videoId}". ` +
`Aborting render to avoid shipping black HDR layers.`, `Aborting render to avoid shipping black HDR layers.`,
); );
} }
out.set(videoId, { const frameSize = dims.width * dims.height * 6;
dir: frameDir, const fd = openSync(rawPath, constants.O_RDONLY | NO_FOLLOW_FLAG);
rawPath, let handedOff = false;
fd, try {
width: dims.width, const frameCount = Math.floor(fstatSync(fd).size / frameSize);
height: dims.height, if (frameCount < 1) {
frameSize, hdrDiagnostics.videoExtractionFailures += 1;
frameCount, throw new Error(
scratch: Buffer.allocUnsafe(frameSize), `HDR frame extraction produced no frames for video "${videoId}". ` +
}); `Aborting render to avoid shipping black HDR layers.`,
handedOff = true; );
} finally { }
if (!handedOff) closeSync(fd); out.set(videoId, {
dir: frameDir,
rawPath,
fd,
width: dims.width,
height: dims.height,
frameSize,
frameCount,
scratch: Buffer.allocUnsafe(frameSize),
loop: video.loop,
});
handedOff = true;
} finally {
if (!handedOff) closeSync(fd);
}
} }
return { sources: out, estimatedBytes, releaseReservation };
} catch (error) {
for (const source of out.values()) {
cleanupHdrVideoFrameSource(source, log);
createdFrameDirs.delete(source.dir);
}
for (const frameDir of createdFrameDirs) {
cleanupHdrFrameDirectory(frameDir, undefined, log);
}
releaseReservation();
throw error;
} }
return out;
} }
/** /**
@@ -60,7 +60,6 @@ import {
type HdrTransitionMeta, type HdrTransitionMeta,
type HdrVideoFrameSource, type HdrVideoFrameSource,
type TransitionRange, type TransitionRange,
closeHdrVideoFrameSource,
resolveCompositeTransfer, resolveCompositeTransfer,
} from "../../hdrCompositor.js"; } from "../../hdrCompositor.js";
import { type HdrPerfCollector, createHdrPerfCollector } from "../hdrPerf.js"; import { type HdrPerfCollector, createHdrPerfCollector } from "../hdrPerf.js";
@@ -68,6 +67,7 @@ import type { HdrDiagnostics, ProgressCallback, RenderJob } from "../../renderOr
import type { CompositionMetadata } from "../shared.js"; import type { CompositionMetadata } from "../shared.js";
import { import {
decodeHdrImageBuffers, decodeHdrImageBuffers,
cleanupHdrVideoFrameSource,
extractHdrVideoFrames, extractHdrVideoFrames,
planHdrResources, planHdrResources,
probeHdrExtractionDims, probeHdrExtractionDims,
@@ -129,6 +129,7 @@ export interface CaptureHdrStageResult {
warnings: CaptureWarning[]; warnings: CaptureWarning[];
} }
// fallow-ignore-next-line complexity
export async function runCaptureHdrStage( export async function runCaptureHdrStage(
input: CaptureHdrStageInput, input: CaptureHdrStageInput,
): Promise<CaptureHdrStageResult> { ): Promise<CaptureHdrStageResult> {
@@ -215,6 +216,7 @@ export async function runCaptureHdrStage(
let hdrEncoder: StreamingEncoder | null = null; let hdrEncoder: StreamingEncoder | null = null;
let hdrEncoderClosed = false; let hdrEncoderClosed = false;
let domSessionClosed = false; let domSessionClosed = false;
let releaseHdrExtractionReservation: (() => void) | null = null;
const hdrVideoFrameSources = new Map<string, HdrVideoFrameSource>(); const hdrVideoFrameSources = new Map<string, HdrVideoFrameSource>();
try { try {
await initializeSession(domSession); await initializeSession(domSession);
@@ -296,7 +298,8 @@ export async function runCaptureHdrStage(
abortSignal, abortSignal,
hdrDiagnostics, hdrDiagnostics,
}); });
for (const [id, source] of extracted) hdrVideoFrameSources.set(id, source); releaseHdrExtractionReservation = extracted.releaseReservation;
for (const [id, source] of extracted.sources) hdrVideoFrameSources.set(id, source);
const hdrImageBuffers = decodeHdrImageBuffers({ const hdrImageBuffers = decodeHdrImageBuffers({
log, log,
hdrImageSrcPaths, hdrImageSrcPaths,
@@ -454,9 +457,11 @@ export async function runCaptureHdrStage(
}); });
} }
for (const frameSource of hdrVideoFrameSources.values()) { for (const frameSource of hdrVideoFrameSources.values()) {
closeHdrVideoFrameSource(frameSource, log); cleanupHdrVideoFrameSource(frameSource, log);
} }
hdrVideoFrameSources.clear(); hdrVideoFrameSources.clear();
releaseHdrExtractionReservation?.();
releaseHdrExtractionReservation = null;
} }
return { return {
@@ -1,3 +1,4 @@
// fallow-ignore-file code-duplication
import { describe, expect, it, mock } from "bun:test"; import { describe, expect, it, mock } from "bun:test";
import { getCaptureStageBrowserConsole } from "../captureStageError.js"; import { getCaptureStageBrowserConsole } from "../captureStageError.js";
import { createCapturePlan } from "../capturePlan.js"; import { createCapturePlan } from "../capturePlan.js";
@@ -131,8 +132,13 @@ mock.module("../../hdrCompositor.js", () => ({
})); }));
mock.module("./captureHdrResources.js", () => ({ mock.module("./captureHdrResources.js", () => ({
cleanupHdrVideoFrameSource: () => {},
decodeHdrImageBuffers: () => new Map(), decodeHdrImageBuffers: () => new Map(),
extractHdrVideoFrames: async () => new Map(), extractHdrVideoFrames: async () => ({
sources: new Map(),
estimatedBytes: 0,
releaseReservation: () => {},
}),
planHdrResources: () => ({ planHdrResources: () => ({
hdrVideoStartTimes: new Map(), hdrVideoStartTimes: new Map(),
nativeHdrVideos: [], nativeHdrVideos: [],
@@ -0,0 +1,120 @@
import { resolveConfig, type ExtractionResult, type VideoElement } from "@hyperframes/engine";
import { describe, expect, it, vi } from "vitest";
const extractionCalls = vi.hoisted(
() => new Array<{ timelineEnd: number | undefined; durationSeconds: number }>(),
);
const fixtureState = vi.hoisted(() => ({ sourceDurationSeconds: 60 }));
vi.mock("@hyperframes/engine", async (importOriginal) => {
const real = await importOriginal<typeof import("@hyperframes/engine")>();
return {
...real,
extractAllVideoFrames: async (
videos: VideoElement[],
_baseDir: string,
options: { timelineEnd?: number },
): Promise<ExtractionResult> => {
const sourceDurationSeconds = fixtureState.sourceDurationSeconds;
const video = videos[0];
if (!video) throw new Error("timeline-bound fixture requires one video");
const requestedDuration = video.end - video.start;
const naturalDuration = sourceDurationSeconds - video.mediaStart;
const resolvedDuration =
Number.isFinite(requestedDuration) && requestedDuration > 0
? requestedDuration
: naturalDuration;
const durationSeconds =
options.timelineEnd === undefined
? resolvedDuration
: Math.min(resolvedDuration, Math.max(0, options.timelineEnd - video.start));
video.end = video.start + durationSeconds;
extractionCalls.push({ timelineEnd: options.timelineEnd, durationSeconds });
return {
success: true,
extracted: [],
errors: [],
totalFramesExtracted: 0,
durationMs: 0,
phaseBreakdown: {
resolveMs: 0,
cachePublishFailures: 0,
cacheGcEvictions: 0,
cacheGcBytesFreed: 0,
cacheAgedPartialsCleared: 0,
hdrProbeMs: 0,
hdrPreflightMs: 0,
hdrPreflightCount: 0,
vfrProbeMs: 0,
vfrPreflightMs: 0,
vfrPreflightCount: 0,
extractMs: 0,
cacheHits: 0,
cacheMisses: 0,
transientRetries: 0,
},
};
},
};
});
import { createRenderJob } from "../../renderOrchestrator.js";
import { runExtractVideosStage } from "./extractVideosStage.js";
async function runStage(compositionDuration: number, materializeSymlinks: boolean): Promise<void> {
const composition = {
duration: compositionDuration,
videos: [
{
id: "root-video",
src: "long.mp4",
start: 0,
end: Number.POSITIVE_INFINITY,
mediaStart: 0,
loop: false,
hasAudio: false,
},
],
audios: [],
images: [],
width: 1920,
height: 1080,
};
await runExtractVideosStage({
projectDir: "/tmp/hf-timeline-bound-project",
compiledDir: "/tmp/hf-timeline-bound-compiled",
job: createRenderJob({
fps: { num: 30, den: 1 },
quality: "standard",
hdrMode: "force-sdr",
}),
cfg: resolveConfig(),
composition,
abortSignal: undefined,
assertNotAborted: () => {},
materializeSymlinks,
});
}
describe.each([
["in-process", false],
["distributed plan", true],
] as const)("%s video extraction timeline bound", (_mode, materializeSymlinks) => {
it("caps an open 60-second source to a two-second composition", async () => {
extractionCalls.splice(0);
fixtureState.sourceDurationSeconds = 60;
await runStage(2, materializeSymlinks);
expect(extractionCalls).toEqual([{ timelineEnd: 2, durationSeconds: 2 }]);
});
it("keeps a two-second natural source inside a ten-second composition", async () => {
extractionCalls.splice(0);
fixtureState.sourceDurationSeconds = 2;
await runStage(10, materializeSymlinks);
expect(extractionCalls).toEqual([{ timelineEnd: 10, durationSeconds: 2 }]);
});
});
@@ -376,6 +376,7 @@ export async function runExtractVideosStage(
fps: fpsToNumber(job.config.fps), fps: fpsToNumber(job.config.fps),
outputDir: join(compiledDir, "__hyperframes_video_frames"), outputDir: join(compiledDir, "__hyperframes_video_frames"),
format: job.config.videoFrameFormat ?? "auto", format: job.config.videoFrameFormat ?? "auto",
timelineEnd: composition.duration,
maxTransientRetries: extractionPolicy.maxTransientRetries, maxTransientRetries: extractionPolicy.maxTransientRetries,
collectProbeFailures: extractionPolicy.failureMode === "enforce", collectProbeFailures: extractionPolicy.failureMode === "enforce",
}, },
@@ -37,14 +37,24 @@ function makeVideo(overrides: Partial<VideoElement> & { id: string }): VideoElem
function makeExtracted( function makeExtracted(
videoId: string, videoId: string,
delivered: number, delivered: number,
options: { fps?: number; durationSeconds?: number; isVFR?: boolean } = {}, options: {
fps?: number;
durationSeconds?: number;
videoStreamDurationSeconds?: number;
isVFR?: boolean;
} = {},
): ExtractedFrames { ): ExtractedFrames {
const { fps = 30, durationSeconds = Number.POSITIVE_INFINITY, isVFR = false } = options; const {
fps = 30,
durationSeconds = Number.POSITIVE_INFINITY,
videoStreamDurationSeconds = durationSeconds,
isVFR = false,
} = options;
const framePaths = new Map<number, string>(); const framePaths = new Map<number, string>();
for (let i = 0; i < delivered; i += 1) framePaths.set(i, `/tmp/${videoId}/${i}.jpg`); for (let i = 0; i < delivered; i += 1) framePaths.set(i, `/tmp/${videoId}/${i}.jpg`);
const metadata: VideoMetadata = { const metadata: VideoMetadata = {
durationSeconds, durationSeconds,
videoStreamDurationSeconds: durationSeconds, videoStreamDurationSeconds,
width: 1280, width: 1280,
height: 720, height: 720,
fps, fps,
@@ -219,6 +229,28 @@ describe("computeVideoFrameCoverage", () => {
expect(reports[0]).toMatchObject({ expectedFrames: 90, capturedFrames: 90, ratio: 1 }); expect(reports[0]).toMatchObject({ expectedFrames: 90, capturedFrames: 90, ratio: 1 });
}); });
it.each([
{ loop: false, label: "held tail" },
{ loop: true, label: "loop" },
])(
"credits the playable video stream instead of longer container audio for a $label",
({ loop }) => {
const videos = [makeVideo({ id: "long-audio-mux", start: 0, end: 60, loop })];
const extracted = [
makeExtracted("long-audio-mux", 90, {
durationSeconds: 60,
videoStreamDurationSeconds: 3,
}),
];
expect(computeVideoFrameCoverage(videos, extracted, 30)[0]).toMatchObject({
expectedFrames: 90,
capturedFrames: 90,
ratio: 1,
});
},
);
it("still fails when a looping clip's source extraction is truncated", () => { it("still fails when a looping clip's source extraction is truncated", () => {
// Fail-loud preserved for a genuinely-broken loop: only 60/90 source // Fail-loud preserved for a genuinely-broken loop: only 60/90 source
// frames arrived, so the delivered set does NOT cover every repeat. // frames arrived, so the delivered set does NOT cover every repeat.
@@ -45,7 +45,11 @@
*/ */
import { parseHTML } from "linkedom"; import { parseHTML } from "linkedom";
import type { ExtractedFrames, VideoElement } from "@hyperframes/engine"; import {
resolvePlayableVideoDuration,
type ExtractedFrames,
type VideoElement,
} from "@hyperframes/engine";
export interface VideoFrameCoverageReport { export interface VideoFrameCoverageReport {
videoId: string; videoId: string;
@@ -155,7 +159,7 @@ function expectedFramesForVideo(
// full source *has* been delivered — the same 90 unique source frames // full source *has* been delivered — the same 90 unique source frames
// cover the 300-frame slot — so coverage must measure source-source, not // cover the 300-frame slot — so coverage must measure source-source, not
// slot-source. // slot-source.
const sourceDuration = entry.metadata.durationSeconds - video.mediaStart; const sourceDuration = resolvePlayableVideoDuration(entry.metadata) - video.mediaStart;
if (!Number.isFinite(sourceDuration) || sourceDuration <= 0) return slotFrames; if (!Number.isFinite(sourceDuration) || sourceDuration <= 0) return slotFrames;
const sourceFrames = expectedFramesForClip(0, sourceDuration, fps, rounding); const sourceFrames = expectedFramesForClip(0, sourceDuration, fps, rounding);