mirror of
https://github.com/heygen-com/hyperframes.git
synced 2026-09-10 22:20:14 +00:00
fix: bound HDR and video extraction resources (#2955)
* fix: bound HDR and video extraction resources * fix: trim negative video extraction preroll * fix: skip invisible video extraction windows * fix: preserve negative-start loop and held tails * fix: cap finite video slots to source duration * fix: bound held-tail frame extraction * fix: plan from playable video duration * fix: preserve open-ended held video tails * fix: resolve held tails from decoded frames * fix: normalize final-frame probe timestamps * fix: handle unseekable final-frame sources * fix: dedupe final-frame probes per render * refactor: clarify output dynamic range contract
This commit is contained in:
James Russo
@@ -1,3 +1,4 @@
|
||||
// fallow-ignore-file code-duplication
|
||||
import { afterEach, beforeEach, describe, it, expect, vi } from "vitest";
|
||||
import {
|
||||
_resetCgroupLimitCacheForTests,
|
||||
@@ -154,6 +155,30 @@ describe("parseCgroupLimitMb", () => {
|
||||
});
|
||||
});
|
||||
|
||||
describe("getCgroupMemoryLimitMb", () => {
|
||||
it("returns only an actual cgroup limit and never host RAM", async () => {
|
||||
await withSystemMemoryMocks(
|
||||
{
|
||||
files: { [CGROUP_V2_MEMORY_MAX_PATH]: `${24576 * BYTES_PER_MIB}` },
|
||||
hostTotalMb: 65536,
|
||||
},
|
||||
({ getCgroupMemoryLimitMb }) => {
|
||||
expect(getCgroupMemoryLimitMb()).toBe(24576);
|
||||
},
|
||||
);
|
||||
|
||||
await withSystemMemoryMocks(
|
||||
{
|
||||
files: { [CGROUP_V2_MEMORY_MAX_PATH]: "max" },
|
||||
hostTotalMb: 65536,
|
||||
},
|
||||
({ getCgroupMemoryLimitMb }) => {
|
||||
expect(getCgroupMemoryLimitMb()).toBeNull();
|
||||
},
|
||||
);
|
||||
});
|
||||
});
|
||||
|
||||
describe("getSystemTotalMb", () => {
|
||||
it("caches cgroup probes until the test reset hook clears the cache", async () => {
|
||||
const readCalls: string[] = [];
|
||||
|
||||
@@ -80,7 +80,11 @@ export function _resetCgroupLimitCacheForTests(): void {
|
||||
_warnedCgroupReadFailure = false;
|
||||
}
|
||||
|
||||
function getCgroupLimitMb(): number | null {
|
||||
/**
|
||||
* Actual Linux cgroup memory ceiling in MiB, or null when the process is not
|
||||
* cgroup-limited. Unlike getSystemTotalMb this never falls back to host RAM.
|
||||
*/
|
||||
export function getCgroupMemoryLimitMb(): number | null {
|
||||
if (_cachedCgroupLimitMb !== undefined) return _cachedCgroupLimitMb;
|
||||
|
||||
if (process.platform !== "linux") {
|
||||
@@ -142,7 +146,7 @@ function warnCgroupReadFailure(path: string, error: unknown): void {
|
||||
/** Total physical RAM in MiB. */
|
||||
export function getSystemTotalMb(): number {
|
||||
const hostTotalMb = Math.floor(totalmem() / BYTES_PER_MIB);
|
||||
const cgroupLimitMb = getCgroupLimitMb();
|
||||
const cgroupLimitMb = getCgroupMemoryLimitMb();
|
||||
|
||||
return cgroupLimitMb === null ? hostTotalMb : Math.min(hostTotalMb, cgroupLimitMb);
|
||||
}
|
||||
|
||||
@@ -24,6 +24,8 @@ import {
|
||||
resolveFrameFormat,
|
||||
codecMayHaveAlpha,
|
||||
decoderForCodec,
|
||||
resolveVideoExtractionWindow,
|
||||
resolveVideoExtractionDuration,
|
||||
getFrameAtTime,
|
||||
analyzeClipMediaFit,
|
||||
classifyVideoExtractionError,
|
||||
@@ -33,9 +35,14 @@ import {
|
||||
type ExtractedFrames,
|
||||
type ExtractionResult,
|
||||
} from "./videoFrameExtractor.js";
|
||||
import { extractVideoMetadata, type VideoMetadata } from "../utils/ffprobe.js";
|
||||
import {
|
||||
extractFinalVideoFrameTimestamp,
|
||||
extractVideoMetadata,
|
||||
type VideoMetadata,
|
||||
} from "../utils/ffprobe.js";
|
||||
import { runFfmpeg } from "../utils/runFfmpeg.js";
|
||||
import { COMPLETE_SENTINEL, GC_MARKER, SCHEMA_PREFIX } from "./extractionCache.js";
|
||||
import { resolveRuntimeMediaClipDuration } from "../../../core/src/runtime/media.js";
|
||||
|
||||
// ffmpeg is not preinstalled on GitHub's ubuntu-24.04 runners. The producer
|
||||
// regression test at packages/producer/tests/vfr-screen-recording/ runs inside
|
||||
@@ -45,6 +52,261 @@ import { COMPLETE_SENTINEL, GC_MARKER, SCHEMA_PREFIX } from "./extractionCache.j
|
||||
// synthesized VFR fixture.
|
||||
const HAS_FFMPEG = spawnSync("ffmpeg", ["-version"]).status === 0;
|
||||
|
||||
describe("resolveVideoExtractionDuration", () => {
|
||||
const metadata = (
|
||||
durationSeconds: number,
|
||||
videoStreamDurationSeconds = durationSeconds,
|
||||
): VideoMetadata => ({
|
||||
durationSeconds,
|
||||
videoStreamDurationSeconds,
|
||||
width: 1920,
|
||||
height: 1080,
|
||||
fps: 30,
|
||||
videoCodec: "h264",
|
||||
hasAudio: false,
|
||||
isVFR: false,
|
||||
hasAlpha: false,
|
||||
colorSpace: null,
|
||||
});
|
||||
const video = (overrides: Partial<VideoElement> = {}): VideoElement => ({
|
||||
id: "root-video",
|
||||
src: "video.mp4",
|
||||
start: 0,
|
||||
end: Number.POSITIVE_INFINITY,
|
||||
mediaStart: 0,
|
||||
loop: false,
|
||||
hasAudio: false,
|
||||
...overrides,
|
||||
});
|
||||
|
||||
it("caps an open 60-second root source to a two-second composition", () => {
|
||||
expect(resolveVideoExtractionDuration(video(), metadata(60), 2)).toBe(2);
|
||||
});
|
||||
|
||||
it("keeps a shorter natural source duration inside a longer composition", () => {
|
||||
expect(resolveVideoExtractionDuration(video(), metadata(2), 10)).toBe(2);
|
||||
});
|
||||
|
||||
it("falls back to container duration when stream duration is unavailable", () => {
|
||||
expect(resolveVideoExtractionDuration(video(), metadata(2, 0), 10)).toBe(2);
|
||||
});
|
||||
|
||||
it("preserves explicit bounds and loop flags while applying the timeline ceiling", () => {
|
||||
const explicitLoop = video({ end: 8, loop: true });
|
||||
expect(resolveVideoExtractionDuration(explicitLoop, metadata(60), 10)).toBe(8);
|
||||
expect(explicitLoop.loop).toBe(true);
|
||||
});
|
||||
|
||||
it("trims materially negative preroll and advances the source offset", () => {
|
||||
const preroll = video({ start: -60, end: 120, mediaStart: 0 });
|
||||
expect(resolveVideoExtractionWindow(preroll, metadata(120), 2)).toEqual({
|
||||
compositionStart: 0,
|
||||
mediaStart: 60,
|
||||
durationSeconds: 2,
|
||||
});
|
||||
expect(resolveVideoExtractionDuration(preroll, metadata(120), 2)).toBe(2);
|
||||
});
|
||||
|
||||
it("returns an empty window for a clip entirely before composition time zero", () => {
|
||||
expect(resolveVideoExtractionWindow(video({ start: -60, end: -10 }), metadata(120), 2)).toEqual(
|
||||
{ compositionStart: 0, mediaStart: 60, durationSeconds: 0 },
|
||||
);
|
||||
});
|
||||
|
||||
it("preserves a short source cycle when negative preroll crosses a loop boundary", () => {
|
||||
expect(
|
||||
resolveVideoExtractionWindow(
|
||||
video({ start: -5, end: 10, mediaStart: 0, loop: true }),
|
||||
metadata(3),
|
||||
2,
|
||||
),
|
||||
).toEqual({
|
||||
compositionStart: -5,
|
||||
mediaStart: 0,
|
||||
durationSeconds: 3,
|
||||
preserveTimelinePhase: true,
|
||||
});
|
||||
});
|
||||
|
||||
it("marks an entirely held interval for exact final-frame resolution", () => {
|
||||
expect(
|
||||
resolveVideoExtractionWindow(
|
||||
video({ start: -5, end: 10, mediaStart: 0, loop: false }),
|
||||
metadata(3),
|
||||
2,
|
||||
),
|
||||
).toEqual({
|
||||
compositionStart: -2.000001,
|
||||
mediaStart: 2.999999,
|
||||
durationSeconds: 0.000001,
|
||||
preserveTimelineEnd: true,
|
||||
ensureFinalFrame: true,
|
||||
});
|
||||
});
|
||||
|
||||
it.each([
|
||||
{ loop: true, preservation: { preserveTimelinePhase: true }, label: "loop" },
|
||||
{
|
||||
loop: false,
|
||||
preservation: { preserveTimelineEnd: true, ensureFinalFrame: true },
|
||||
label: "held tail",
|
||||
},
|
||||
])(
|
||||
"caps a finite long slot to one short source range for $label playback",
|
||||
({ loop, preservation }) => {
|
||||
expect(resolveVideoExtractionWindow(video({ end: 60, loop }), metadata(3), 60)).toEqual({
|
||||
compositionStart: 0,
|
||||
mediaStart: 0,
|
||||
durationSeconds: 3,
|
||||
...preservation,
|
||||
});
|
||||
},
|
||||
);
|
||||
|
||||
it.each([
|
||||
{ loop: true, preservation: { preserveTimelinePhase: true }, label: "loop" },
|
||||
{
|
||||
loop: false,
|
||||
preservation: { preserveTimelineEnd: true, ensureFinalFrame: true },
|
||||
label: "held tail",
|
||||
},
|
||||
])(
|
||||
"uses the playable video-stream duration for a long-audio mux in $label playback",
|
||||
({ loop, preservation }) => {
|
||||
expect(resolveVideoExtractionWindow(video({ end: 60, loop }), metadata(60, 3), 60)).toEqual({
|
||||
compositionStart: 0,
|
||||
mediaStart: 0,
|
||||
durationSeconds: 3,
|
||||
...preservation,
|
||||
});
|
||||
},
|
||||
);
|
||||
|
||||
it("preserves authored timing when the visible interval partially crosses a held tail", () => {
|
||||
expect(
|
||||
resolveVideoExtractionWindow(
|
||||
video({ start: -2, end: 10, mediaStart: 0, loop: false }),
|
||||
metadata(3),
|
||||
2,
|
||||
),
|
||||
).toEqual({
|
||||
compositionStart: 0,
|
||||
mediaStart: 2,
|
||||
durationSeconds: 1,
|
||||
preserveTimelineEnd: true,
|
||||
ensureFinalFrame: true,
|
||||
});
|
||||
});
|
||||
|
||||
it.each([
|
||||
{ start: 0, expected: { compositionStart: 0, mediaStart: 0, durationSeconds: 3 } },
|
||||
{ start: -2, expected: { compositionStart: 0, mediaStart: 2, durationSeconds: 1 } },
|
||||
{ start: -5, expected: { compositionStart: 0, mediaStart: 5, durationSeconds: 0 } },
|
||||
])(
|
||||
"keeps an omitted-duration clip source-bounded like runtime (start=$start)",
|
||||
({ start, expected }) => {
|
||||
for (const loop of [false, true]) {
|
||||
const parsed = parseVideoElements(
|
||||
`<video id="natural" src="video.mp4"${loop ? " loop" : ""}></video>`,
|
||||
)[0]!;
|
||||
const planned = { ...parsed, start };
|
||||
const runtimeDuration = resolveRuntimeMediaClipDuration({
|
||||
isVideo: true,
|
||||
sourceDuration: 3,
|
||||
hostRemaining: 15 - start,
|
||||
explicitDuration: null,
|
||||
});
|
||||
expect(runtimeDuration).toBe(3);
|
||||
expect(parsed.end).toBe(Number.POSITIVE_INFINITY);
|
||||
expect(resolveVideoExtractionWindow(planned, metadata(3), 15)).toEqual(expected);
|
||||
}
|
||||
},
|
||||
);
|
||||
|
||||
it("bounds an entirely held long source to its final-frame sample", () => {
|
||||
expect(
|
||||
resolveVideoExtractionWindow(
|
||||
video({ start: -600, end: 10, mediaStart: 0, loop: false }),
|
||||
metadata(120),
|
||||
2,
|
||||
),
|
||||
).toEqual({
|
||||
compositionStart: -480.000001,
|
||||
mediaStart: 119.999999,
|
||||
durationSeconds: 0.000001,
|
||||
preserveTimelineEnd: true,
|
||||
ensureFinalFrame: true,
|
||||
});
|
||||
});
|
||||
|
||||
it("preserves a complete loop cycle when visibility ends exactly on a wrap boundary", () => {
|
||||
expect(
|
||||
resolveVideoExtractionWindow(
|
||||
video({ start: -1, end: 10, mediaStart: 0, loop: true }),
|
||||
metadata(3),
|
||||
2,
|
||||
),
|
||||
).toEqual({
|
||||
compositionStart: -1,
|
||||
mediaStart: 0,
|
||||
durationSeconds: 3,
|
||||
preserveTimelinePhase: true,
|
||||
});
|
||||
});
|
||||
|
||||
it("keeps an open-ended loop source-bounded instead of inventing a longer slot", () => {
|
||||
expect(resolveVideoExtractionWindow(video({ loop: true }), metadata(3), 10)).toEqual({
|
||||
compositionStart: 0,
|
||||
mediaStart: 0,
|
||||
durationSeconds: 3,
|
||||
});
|
||||
});
|
||||
|
||||
it("never plans more extraction than the playable source range", () => {
|
||||
for (const loop of [false, true]) {
|
||||
for (const sourceDuration of [0.5, 3, 120]) {
|
||||
for (const mediaStart of [0, sourceDuration / 3]) {
|
||||
for (const start of [-600, -5, -1, 0, 2]) {
|
||||
const window = resolveVideoExtractionWindow(
|
||||
video({ start, end: start + 60, mediaStart, loop }),
|
||||
metadata(sourceDuration),
|
||||
10,
|
||||
);
|
||||
expect(window.durationSeconds).toBeLessThanOrEqual(sourceDuration - mediaStart);
|
||||
expect(window.durationSeconds).toBeGreaterThanOrEqual(0);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
});
|
||||
|
||||
it("rejects a media start at source EOF before planning extraction", () => {
|
||||
expect(() =>
|
||||
resolveVideoExtractionWindow(video({ mediaStart: 3 }), metadata(3), 10),
|
||||
).toThrowError(expect.objectContaining({ kind: "media_start_out_of_range", retryable: false }));
|
||||
});
|
||||
|
||||
it("rejects a media start at video-stream EOF even when the container continues", () => {
|
||||
expect(() =>
|
||||
resolveVideoExtractionWindow(video({ mediaStart: 3 }), metadata(60, 3), 10),
|
||||
).toThrowError(expect.objectContaining({ kind: "media_start_out_of_range", retryable: false }));
|
||||
});
|
||||
|
||||
it("rebases a loop phase when the visible window stays within one cycle", () => {
|
||||
expect(
|
||||
resolveVideoExtractionWindow(
|
||||
video({ start: -5, end: 10, mediaStart: 0, loop: true }),
|
||||
metadata(3),
|
||||
0.5,
|
||||
),
|
||||
).toEqual({ compositionStart: 0, mediaStart: 2, durationSeconds: 0.5 });
|
||||
});
|
||||
|
||||
it("retains legacy behavior when no timeline end is supplied", () => {
|
||||
expect(resolveVideoExtractionDuration(video(), metadata(60))).toBe(60);
|
||||
});
|
||||
});
|
||||
|
||||
describe("video extraction failure taxonomy and bounded retry", () => {
|
||||
it("classifies missing and transient HTTP sources without exposing retry ambiguity", () => {
|
||||
expect(classifyVideoExtractionError(new Error("HTTP 404: Not Found"))).toMatchObject({
|
||||
@@ -507,6 +769,28 @@ describe("FrameLookupTable", () => {
|
||||
expect(table.getActiveFramePayloads(4.5).get("hero")?.frameIndex).toBe(15);
|
||||
});
|
||||
|
||||
it("wraps at video-stream EOF when a mux container has longer audio", () => {
|
||||
const extracted = fakeExtracted(6, 2);
|
||||
extracted.metadata.durationSeconds = 60;
|
||||
extracted.metadata.videoStreamDurationSeconds = 3;
|
||||
const table = createFrameLookupTable(
|
||||
[
|
||||
{
|
||||
id: "hero",
|
||||
src: "clip.webm",
|
||||
start: 0,
|
||||
end: 60,
|
||||
mediaStart: 0,
|
||||
loop: true,
|
||||
hasAudio: false,
|
||||
},
|
||||
],
|
||||
[extracted],
|
||||
);
|
||||
|
||||
expect(table.getActiveFramePayloads(4).get("hero")?.frameIndex).toBe(2);
|
||||
});
|
||||
|
||||
it("holds the last frame for a non-looping clip until its authored slot ends", () => {
|
||||
const table = createFrameLookupTable(
|
||||
[
|
||||
@@ -958,6 +1242,137 @@ describe.skipIf(!HAS_FFMPEG)("video frame extraction format", () => {
|
||||
}, 60_000);
|
||||
});
|
||||
|
||||
describe.skipIf(!HAS_FFMPEG)("held tails on sparse-timestamp sources", () => {
|
||||
const fixtureDir = mkdtempSync(join(tmpdir(), "hf-sparse-held-tail-"));
|
||||
const cfrFixture = join(fixtureDir, "sub-1fps-cfr.mp4");
|
||||
const vfrFixture = join(fixtureDir, "sparse-vfr.mp4");
|
||||
const nonZeroStartFixture = join(fixtureDir, "nonzero-start.mp4");
|
||||
const negativeStartTransportFixture = join(fixtureDir, "negative-start.ts");
|
||||
|
||||
beforeAll(async () => {
|
||||
const fixtures = [
|
||||
{
|
||||
path: cfrFixture,
|
||||
input: "testsrc2=s=64x64:d=10:rate=1/5",
|
||||
filters: [] as string[],
|
||||
},
|
||||
{
|
||||
path: vfrFixture,
|
||||
input: "testsrc2=s=64x64:d=10:rate=1/2",
|
||||
filters: ["-vf", "select='eq(n,0)+eq(n,2)'", "-vsync", "vfr"],
|
||||
},
|
||||
{
|
||||
path: nonZeroStartFixture,
|
||||
input: "testsrc2=s=64x64:d=3:rate=1",
|
||||
filters: ["-output_ts_offset", "5"],
|
||||
},
|
||||
{
|
||||
path: negativeStartTransportFixture,
|
||||
input: "testsrc2=s=64x64:d=3:rate=1",
|
||||
filters: [
|
||||
"-mpegts_copyts",
|
||||
"1",
|
||||
"-muxdelay",
|
||||
"0",
|
||||
"-avoid_negative_ts",
|
||||
"disabled",
|
||||
"-output_ts_offset",
|
||||
"-2",
|
||||
],
|
||||
},
|
||||
];
|
||||
for (const fixture of fixtures) {
|
||||
const result = await runFfmpeg([
|
||||
"-hide_banner",
|
||||
"-loglevel",
|
||||
"error",
|
||||
"-f",
|
||||
"lavfi",
|
||||
"-i",
|
||||
fixture.input,
|
||||
...fixture.filters,
|
||||
"-c:v",
|
||||
"libx264",
|
||||
"-pix_fmt",
|
||||
"yuv420p",
|
||||
"-y",
|
||||
fixture.path,
|
||||
]);
|
||||
if (!result.success) {
|
||||
throw new Error(`sparse fixture synthesis failed: ${result.stderr.slice(-400)}`);
|
||||
}
|
||||
}
|
||||
}, 30_000);
|
||||
|
||||
afterAll(() => {
|
||||
rmSync(fixtureDir, { recursive: true, force: true });
|
||||
});
|
||||
|
||||
it.each([
|
||||
{
|
||||
label: "sub-1fps CFR",
|
||||
src: cfrFixture,
|
||||
expectedVfr: false,
|
||||
finalTimestamp: 5,
|
||||
streamStart: 0,
|
||||
},
|
||||
{
|
||||
label: "sparse VFR",
|
||||
src: vfrFixture,
|
||||
expectedVfr: true,
|
||||
finalTimestamp: 4,
|
||||
streamStart: 0,
|
||||
},
|
||||
{
|
||||
label: "non-zero stream start",
|
||||
src: nonZeroStartFixture,
|
||||
expectedVfr: false,
|
||||
finalTimestamp: 2,
|
||||
streamStart: 5,
|
||||
},
|
||||
{
|
||||
label: "unindexed negative-base MPEG-TS",
|
||||
src: negativeStartTransportFixture,
|
||||
expectedVfr: false,
|
||||
finalTimestamp: 2,
|
||||
streamStart: -2,
|
||||
},
|
||||
])(
|
||||
"extracts one real final SDR frame for $label",
|
||||
async ({ src, expectedVfr, finalTimestamp, streamStart }) => {
|
||||
const metadata = await extractVideoMetadata(src);
|
||||
expect(metadata.fps).toBeLessThanOrEqual(1);
|
||||
expect(metadata.isVFR).toBe(expectedVfr);
|
||||
expect(metadata.videoStreamStartSeconds).toBeCloseTo(streamStart, 6);
|
||||
await expect(extractFinalVideoFrameTimestamp(src, metadata)).resolves.toBe(finalTimestamp);
|
||||
const outputDir = mkdtempSync(join(fixtureDir, "out-"));
|
||||
const video: VideoElement = {
|
||||
id: `held-${String(expectedVfr)}`,
|
||||
src,
|
||||
start: -15,
|
||||
end: 5,
|
||||
mediaStart: 0,
|
||||
loop: false,
|
||||
hasAudio: false,
|
||||
};
|
||||
|
||||
const result = await extractAllVideoFrames([video], fixtureDir, {
|
||||
fps: 30,
|
||||
format: "png",
|
||||
outputDir,
|
||||
timelineEnd: 2,
|
||||
});
|
||||
|
||||
expect(result.errors).toEqual([]);
|
||||
expect(result.extracted).toHaveLength(1);
|
||||
expect(result.extracted[0]?.totalFrames).toBe(1);
|
||||
expect(video).toMatchObject({ start: 0, end: 5, loop: false });
|
||||
expect(video.mediaStart).toBeCloseTo(metadata.videoStreamDurationSeconds - 0.000001, 7);
|
||||
},
|
||||
30_000,
|
||||
);
|
||||
});
|
||||
|
||||
// Regression test for the VFR (variable frame rate) freeze bug.
|
||||
// Screen recordings and phone videos often have irregular timestamps.
|
||||
// When such inputs hit `extractVideoFramesRange`'s `-ss <start> -i ... -t <dur>
|
||||
@@ -1017,6 +1432,90 @@ describe.skipIf(!HAS_FFMPEG)("extractAllVideoFrames on a VFR source", () => {
|
||||
if (existsSync(FIXTURE_DIR)) rmSync(FIXTURE_DIR, { recursive: true, force: true });
|
||||
});
|
||||
|
||||
it("skips a clip entirely before time zero without reporting an extraction error", async () => {
|
||||
const outputDir = join(FIXTURE_DIR, "out-before-timeline");
|
||||
mkdirSync(outputDir, { recursive: true });
|
||||
const video: VideoElement = {
|
||||
id: "before-timeline",
|
||||
src: VFR_FIXTURE,
|
||||
start: -2,
|
||||
end: -1,
|
||||
mediaStart: 0,
|
||||
loop: false,
|
||||
hasAudio: false,
|
||||
};
|
||||
|
||||
const result = await extractAllVideoFrames([video], FIXTURE_DIR, {
|
||||
fps: 1,
|
||||
outputDir,
|
||||
timelineEnd: 2,
|
||||
});
|
||||
|
||||
expect(result).toMatchObject({ success: true, extracted: [], errors: [] });
|
||||
});
|
||||
|
||||
it("preserves loop phase when negative preroll crosses the source boundary", async () => {
|
||||
const outputDir = join(FIXTURE_DIR, "out-negative-loop");
|
||||
mkdirSync(outputDir, { recursive: true });
|
||||
const video: VideoElement = {
|
||||
id: "negative-loop",
|
||||
src: VFR_FIXTURE,
|
||||
start: -19,
|
||||
end: 5,
|
||||
mediaStart: 0,
|
||||
loop: true,
|
||||
hasAudio: false,
|
||||
};
|
||||
|
||||
const result = await extractAllVideoFrames([video], FIXTURE_DIR, {
|
||||
fps: 1,
|
||||
outputDir,
|
||||
timelineEnd: 2,
|
||||
});
|
||||
|
||||
expect(result.errors).toEqual([]);
|
||||
const extracted = result.extracted[0];
|
||||
if (!extracted) throw new Error("expected loop source frames");
|
||||
const lookup = createFrameLookupTable([video], result.extracted);
|
||||
expect(video).toMatchObject({ start: -19, mediaStart: 0, loop: true });
|
||||
expect(lookup.getFrame("negative-loop", 0)).toBe(
|
||||
extracted.framePaths.get(extracted.totalFrames - 1),
|
||||
);
|
||||
}, 30_000);
|
||||
|
||||
it("preserves the held final frame after negative preroll exhausts a source", async () => {
|
||||
const outputDir = join(FIXTURE_DIR, "out-negative-held-tail");
|
||||
mkdirSync(outputDir, { recursive: true });
|
||||
const video: VideoElement = {
|
||||
id: "negative-held-tail",
|
||||
src: VFR_FIXTURE,
|
||||
start: -15,
|
||||
end: 5,
|
||||
mediaStart: 0,
|
||||
loop: false,
|
||||
hasAudio: false,
|
||||
};
|
||||
|
||||
const result = await extractAllVideoFrames([video], FIXTURE_DIR, {
|
||||
fps: 1,
|
||||
outputDir,
|
||||
timelineEnd: 2,
|
||||
});
|
||||
|
||||
expect(result.errors).toEqual([]);
|
||||
const extracted = result.extracted[0];
|
||||
if (!extracted) throw new Error("expected held-tail source frames");
|
||||
const lookup = createFrameLookupTable([video], result.extracted);
|
||||
// The authored slot remains active through end=5, but lookup is rebased to
|
||||
// one exact final frame instead of assuming the last second contains a
|
||||
// timestamp or materializing the full source.
|
||||
expect(video).toMatchObject({ start: 0, end: 5, mediaStart: 9.999999, loop: false });
|
||||
expect(extracted.totalFrames).toBe(1);
|
||||
expect(lookup.getFrame("negative-held-tail", 0)).toBe(
|
||||
extracted.framePaths.get(extracted.totalFrames - 1),
|
||||
);
|
||||
}, 30_000);
|
||||
|
||||
it("detects the synthesized fixture as VFR", async () => {
|
||||
const md = await extractVideoMetadata(VFR_FIXTURE);
|
||||
expect(md.isVFR).toBe(true);
|
||||
@@ -1236,6 +1735,52 @@ describe.skipIf(!HAS_FFMPEG)("extractAllVideoFrames on a VFR source", () => {
|
||||
rmSync(CACHE_DIR, { recursive: true, force: true });
|
||||
}, 60_000);
|
||||
|
||||
it("reuses one-cycle loop extraction across different authored starts", async () => {
|
||||
const cacheDir = mkdtempSync(join(tmpdir(), "hf-extract-loop-phase-cache-test-"));
|
||||
const src = await synthCfrClip("cache-loop-phase-src.mp4", 3);
|
||||
try {
|
||||
const firstOutputDir = join(FIXTURE_DIR, "out-cache-loop-phase-first");
|
||||
const secondOutputDir = join(FIXTURE_DIR, "out-cache-loop-phase-second");
|
||||
mkdirSync(firstOutputDir, { recursive: true });
|
||||
mkdirSync(secondOutputDir, { recursive: true });
|
||||
|
||||
const first = await extractAllVideoFrames(
|
||||
[
|
||||
{
|
||||
...cfrClipElement("loop-cache-first", src, 60),
|
||||
loop: true,
|
||||
},
|
||||
],
|
||||
FIXTURE_DIR,
|
||||
{ fps: 30, outputDir: firstOutputDir, timelineEnd: 60 },
|
||||
undefined,
|
||||
{ extractCacheDir: cacheDir },
|
||||
);
|
||||
expect(first.errors).toEqual([]);
|
||||
expect(first.phaseBreakdown.cacheMisses).toBe(1);
|
||||
|
||||
const second = await extractAllVideoFrames(
|
||||
[
|
||||
{
|
||||
...cfrClipElement("loop-cache-second", src, 66),
|
||||
start: -6,
|
||||
loop: true,
|
||||
},
|
||||
],
|
||||
FIXTURE_DIR,
|
||||
{ fps: 30, outputDir: secondOutputDir, timelineEnd: 60 },
|
||||
undefined,
|
||||
{ extractCacheDir: cacheDir },
|
||||
);
|
||||
expect(second.errors).toEqual([]);
|
||||
expect(second.phaseBreakdown.cacheHits).toBe(1);
|
||||
expect(second.phaseBreakdown.cacheMisses).toBe(0);
|
||||
expect(second.extracted[0]?.totalFrames).toBe(first.extracted[0]?.totalFrames);
|
||||
} finally {
|
||||
rmSync(cacheDir, { recursive: true, force: true });
|
||||
}
|
||||
}, 60_000);
|
||||
|
||||
it("updates the cache sentinel mtime on a hit", async () => {
|
||||
const CACHE_DIR = mkdtempSync(join(tmpdir(), "hf-extract-cache-touch-test-"));
|
||||
const SRC = await synthCfrClip("cache-touch-src.mp4", 1);
|
||||
|
||||
@@ -11,7 +11,11 @@ import { isAbsolute, join, posix, resolve, sep } from "path";
|
||||
import { parseHTML } from "linkedom";
|
||||
import { decodeUrlPathVariants, MEDIA_DURATION_CLAMP_EPSILON_SECONDS } from "@hyperframes/core";
|
||||
import { resolveReferencedStart, type RefResolverEl } from "./referenceResolver.js";
|
||||
import { extractMediaMetadata, type VideoMetadata } from "../utils/ffprobe.js";
|
||||
import {
|
||||
extractFinalVideoFrameTimestamp,
|
||||
extractMediaMetadata,
|
||||
type VideoMetadata,
|
||||
} from "../utils/ffprobe.js";
|
||||
import {
|
||||
analyzeCompositionHdr,
|
||||
isHdrColorSpace as isHdrColorSpaceUtil,
|
||||
@@ -83,6 +87,15 @@ export interface ExtractionOptions {
|
||||
quality?: number;
|
||||
format?: VideoFrameFormat;
|
||||
sdrToHdrTransfer?: HdrTransfer;
|
||||
/** Extract exactly one frame at `startTime`. Used only after ffprobe has
|
||||
* resolved the actual final decoded-frame timestamp for a held tail. */
|
||||
finalFrameOnly?: boolean;
|
||||
/**
|
||||
* Absolute composition/timeline end in seconds. Applied only after source
|
||||
* metadata resolves open-ended/natural-duration media. Invisible negative
|
||||
* preroll is trimmed while advancing mediaStart to preserve source alignment.
|
||||
*/
|
||||
timelineEnd?: number;
|
||||
/**
|
||||
* Bounded per-source FFmpeg retries. Default 0 preserves stable behavior;
|
||||
* the producer may canary at most one retry after observing typed failures.
|
||||
@@ -444,7 +457,10 @@ export function parseVideoElements(html: string): VideoElement[] {
|
||||
// reference; the resolver handles both.
|
||||
const start = startAttr ? resolveReferencedStart(document, el, startCache, visiting) : 0;
|
||||
// Derive end from data-end → data-start+data-duration → Infinity (natural duration).
|
||||
// The caller (htmlCompiler) clamps Infinity to the composition's absoluteEnd.
|
||||
// Static compilation cannot always clamp root media because GSAP may supply
|
||||
// the root duration at runtime. The producer passes the resolved timeline
|
||||
// end into frame extraction, which caps the source duration only after the
|
||||
// natural duration is known without rewriting authored timing metadata.
|
||||
let end = 0;
|
||||
if (endAttr) {
|
||||
end = parseFloat(endAttr);
|
||||
@@ -542,20 +558,21 @@ export async function extractVideoFramesRange(
|
||||
} catch (error) {
|
||||
throw classifyVideoExtractionError(error);
|
||||
}
|
||||
if (!(metadata.durationSeconds > 0)) {
|
||||
const playableDuration = resolvePlayableVideoDuration(metadata);
|
||||
if (!(playableDuration > 0)) {
|
||||
throw new VideoSourceExtractionError(
|
||||
"invalid_media",
|
||||
false,
|
||||
"Video source has no positive duration",
|
||||
`Video source duration is ${metadata.durationSeconds}s`,
|
||||
`Playable video stream duration is ${playableDuration}s`,
|
||||
);
|
||||
}
|
||||
if (startTime >= metadata.durationSeconds) {
|
||||
if (startTime >= playableDuration) {
|
||||
throw new VideoSourceExtractionError(
|
||||
"media_start_out_of_range",
|
||||
false,
|
||||
"Video media start is outside the source duration",
|
||||
`Video media start ${startTime}s is outside source duration ${metadata.durationSeconds}s`,
|
||||
`Video media start ${startTime}s is outside playable video duration ${playableDuration}s`,
|
||||
);
|
||||
}
|
||||
const format = resolveFrameFormat(metadata, options.format);
|
||||
@@ -585,14 +602,22 @@ export async function extractVideoFramesRange(
|
||||
if (codecMayHaveAlpha(metadata.videoCodec)) {
|
||||
args.push("-c:v", decoderForCodec(metadata.videoCodec));
|
||||
}
|
||||
args.push("-ss", String(startTime), "-i", videoPath, "-t", String(duration));
|
||||
if (options.finalFrameOnly) {
|
||||
// Output-side seek decodes from the start before selecting the final
|
||||
// sample. This is intentionally reserved for the one-frame path: input
|
||||
// seeking is faster, but valid unindexed transports (notably MPEG-TS with
|
||||
// a negative timestamp base) can seek to EOF and emit zero frames.
|
||||
args.push("-i", videoPath, "-ss", String(startTime), "-frames:v", "1");
|
||||
} else {
|
||||
args.push("-ss", String(startTime), "-i", videoPath, "-t", String(duration));
|
||||
}
|
||||
|
||||
const vfFilters: string[] = [];
|
||||
if (isHdr && isMacOS) {
|
||||
// VideoToolbox tone-maps during decode; force output to bt709 SDR format
|
||||
vfFilters.push("format=nv12");
|
||||
}
|
||||
if (!metadata.isVFR) {
|
||||
if (!options.finalFrameOnly && !metadata.isVFR) {
|
||||
vfFilters.push(`fps=${fps}`);
|
||||
}
|
||||
if (options.sdrToHdrTransfer) {
|
||||
@@ -606,7 +631,9 @@ export async function extractVideoFramesRange(
|
||||
vfFilters.push(SDR_TO_HDR_COLORSPACE_FILTER);
|
||||
}
|
||||
if (vfFilters.length > 0) args.push("-vf", vfFilters.join(","));
|
||||
if (metadata.isVFR) args.push("-fps_mode", "cfr", "-r", String(fps));
|
||||
if (!options.finalFrameOnly && metadata.isVFR) {
|
||||
args.push("-fps_mode", "cfr", "-r", String(fps));
|
||||
}
|
||||
|
||||
args.push("-q:v", format === "jpg" ? String(Math.ceil((100 - quality) / 3)) : "0");
|
||||
// Render-scoped temp frames are read once; level 1 measured 3-5x faster for ~14% larger files.
|
||||
@@ -718,11 +745,223 @@ export function classifyFfmpegSpawnError(error: unknown, stderr = ""): VideoSour
|
||||
function resolveSegmentDuration(
|
||||
requested: number,
|
||||
mediaStart: number,
|
||||
metadata: VideoMetadata,
|
||||
sourceDuration: number,
|
||||
): number {
|
||||
if (Number.isFinite(requested) && requested > 0) return requested;
|
||||
const sourceRemaining = metadata.durationSeconds - mediaStart;
|
||||
return sourceRemaining > 0 ? sourceRemaining : metadata.durationSeconds;
|
||||
const sourceRemaining = sourceDuration - mediaStart;
|
||||
return sourceRemaining > 0 ? sourceRemaining : sourceDuration;
|
||||
}
|
||||
|
||||
/**
|
||||
* Return the range that can actually produce video frames.
|
||||
*
|
||||
* Container duration may include a longer audio stream or mux padding. Using
|
||||
* it for video extraction planning can reserve raw-frame scratch for seconds
|
||||
* where no video frames exist. `extractMediaMetadata` already falls back to
|
||||
* the container duration when ffprobe omits the stream duration; keep the
|
||||
* explicit fallback here for callers supplying older/manual metadata.
|
||||
*/
|
||||
export function resolvePlayableVideoDuration(metadata: VideoMetadata): number {
|
||||
return Number.isFinite(metadata.videoStreamDurationSeconds) &&
|
||||
metadata.videoStreamDurationSeconds > 0
|
||||
? metadata.videoStreamDurationSeconds
|
||||
: metadata.durationSeconds;
|
||||
}
|
||||
|
||||
export interface TimelineExtractionWindow {
|
||||
compositionStart: number;
|
||||
mediaStart: number;
|
||||
durationSeconds: number;
|
||||
/**
|
||||
* Preserve the authored timeline origin and mediaStart for lookup. This is
|
||||
* required when a looped visible interval crosses a source boundary and
|
||||
* still needs modulo phase against the complete extracted source cycle.
|
||||
*/
|
||||
preserveTimelinePhase?: boolean;
|
||||
/**
|
||||
* Keep the authored end while rebasing start/mediaStart to the extracted
|
||||
* source suffix. Non-looping lookup then holds the suffix's final frame
|
||||
* through the remainder of the authored slot.
|
||||
*/
|
||||
preserveTimelineEnd?: boolean;
|
||||
/** This window reaches a held tail and must be checked against the actual
|
||||
* final decoded-frame timestamp before extraction. */
|
||||
ensureFinalFrame?: boolean;
|
||||
/** Source timestamp used by FFmpeg when it differs from the logical lookup
|
||||
* mediaStart (the one-frame held-tail representation). */
|
||||
extractionMediaStart?: number;
|
||||
/** FFmpeg emits one decoded frame; lookup then holds that frame. */
|
||||
finalFrameOnly?: boolean;
|
||||
}
|
||||
|
||||
type TimelineWindowVideo = Pick<VideoElement, "start" | "end" | "mediaStart"> &
|
||||
Partial<Pick<VideoElement, "loop">>;
|
||||
|
||||
// Logical duration assigned to a one-frame held-tail representation. This is
|
||||
// deliberately below any supported output frame interval: coverage expects
|
||||
// one frame, while FFmpeg seeks to the separately probed real frame timestamp.
|
||||
const FINAL_FRAME_LOGICAL_DURATION_SECONDS = 1e-6;
|
||||
|
||||
/**
|
||||
* Intersect an authored slot with the render timeline, then select the
|
||||
* smallest playable source range that preserves timeline lookup semantics.
|
||||
*
|
||||
* A finite authored slot can outlive the source. In that case FFmpeg should
|
||||
* still extract at most one source range: lookup either wraps that range for
|
||||
* loops or holds its final frame for non-looping video. Keeping the authored
|
||||
* timeline origin separate from the extracted range is what makes both
|
||||
* behaviours survive the source-duration cap.
|
||||
*/
|
||||
export function resolveTimelineExtractionWindow(
|
||||
video: TimelineWindowVideo,
|
||||
resolvedDuration: number,
|
||||
timelineEnd?: number,
|
||||
sourceDuration?: number,
|
||||
): TimelineExtractionWindow {
|
||||
if (timelineEnd === undefined) {
|
||||
return {
|
||||
compositionStart: video.start,
|
||||
mediaStart: video.mediaStart,
|
||||
durationSeconds: resolvedDuration,
|
||||
};
|
||||
}
|
||||
if (!Number.isFinite(timelineEnd)) {
|
||||
throw new Error(`Video extraction timelineEnd must be finite; got ${String(timelineEnd)}`);
|
||||
}
|
||||
const compositionStart = Math.max(0, video.start);
|
||||
const trimmedPreroll = compositionStart - video.start;
|
||||
const timelineDuration = Math.max(0, timelineEnd - compositionStart);
|
||||
// Infinity means "natural source duration", not an authored infinite slot.
|
||||
// Explicit finite slots may outlive the source (loop or held tail), while an
|
||||
// omitted duration remains source-bounded exactly like the browser runtime.
|
||||
const resolvedVisibleDuration = resolvedDuration - trimmedPreroll;
|
||||
const visibleDuration = Math.max(0, Math.min(resolvedVisibleDuration, timelineDuration));
|
||||
let mediaStart = video.mediaStart + trimmedPreroll;
|
||||
if (visibleDuration > 0 && sourceDuration !== undefined) {
|
||||
const sourceRemaining = Math.max(0, sourceDuration - video.mediaStart);
|
||||
if (sourceRemaining > 0 && video.loop && Number.isFinite(video.end)) {
|
||||
const phaseOffset = trimmedPreroll % sourceRemaining;
|
||||
const phaseRemaining = sourceRemaining - phaseOffset;
|
||||
// The element visibility contract includes its end boundary. Preserve a
|
||||
// complete cycle on equality as well, otherwise a rebased suffix would
|
||||
// wrap to its own first frame instead of the source cycle's first frame.
|
||||
if (visibleDuration >= phaseRemaining) {
|
||||
return {
|
||||
compositionStart: video.start,
|
||||
mediaStart: video.mediaStart,
|
||||
durationSeconds: sourceRemaining,
|
||||
preserveTimelinePhase: true,
|
||||
};
|
||||
}
|
||||
mediaStart = video.mediaStart + phaseOffset;
|
||||
} else if (sourceRemaining > 0) {
|
||||
const sourceVisibleAfterPreroll = Math.max(0, sourceRemaining - trimmedPreroll);
|
||||
if (visibleDuration <= sourceVisibleAfterPreroll) {
|
||||
return {
|
||||
compositionStart,
|
||||
mediaStart,
|
||||
durationSeconds: visibleDuration,
|
||||
};
|
||||
}
|
||||
|
||||
// The visible interval enters (or is entirely inside) the held tail.
|
||||
// Extract the visible source suffix. If preroll is already at/past the
|
||||
// final decoded timestamp, the async resolver below replaces this tiny
|
||||
// provisional suffix with one exact final frame.
|
||||
const extractionDuration = Math.min(
|
||||
sourceRemaining,
|
||||
Math.max(sourceVisibleAfterPreroll, FINAL_FRAME_LOGICAL_DURATION_SECONDS),
|
||||
);
|
||||
const extractionOffset = sourceRemaining - extractionDuration;
|
||||
return {
|
||||
compositionStart: video.start + extractionOffset,
|
||||
mediaStart: video.mediaStart + extractionOffset,
|
||||
durationSeconds: extractionDuration,
|
||||
preserveTimelineEnd: true,
|
||||
ensureFinalFrame: true,
|
||||
};
|
||||
}
|
||||
}
|
||||
return {
|
||||
compositionStart,
|
||||
mediaStart,
|
||||
durationSeconds: visibleDuration,
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Replace a held-tail suffix that starts at/after the final decoded timestamp
|
||||
* with one exact frame. This keeps raw HDR scratch O(one frame) without
|
||||
* assuming a one-second seek window contains a CFR/VFR timestamp.
|
||||
*/
|
||||
export async function resolveFinalFrameExtractionWindow(
|
||||
videoPath: string,
|
||||
video: TimelineWindowVideo,
|
||||
metadata: VideoMetadata,
|
||||
window: TimelineExtractionWindow,
|
||||
signal?: AbortSignal,
|
||||
): Promise<TimelineExtractionWindow> {
|
||||
if (!window.ensureFinalFrame) return window;
|
||||
const playableDuration = resolvePlayableVideoDuration(metadata);
|
||||
const finalFrameTimestamp = await extractFinalVideoFrameTimestamp(
|
||||
videoPath,
|
||||
{
|
||||
videoStreamDurationSeconds: playableDuration,
|
||||
videoStreamStartSeconds: metadata.videoStreamStartSeconds,
|
||||
},
|
||||
signal,
|
||||
);
|
||||
if (window.mediaStart < finalFrameTimestamp - 1e-9) return window;
|
||||
|
||||
const sourceRemaining = playableDuration - video.mediaStart;
|
||||
const logicalDuration = Math.min(sourceRemaining, FINAL_FRAME_LOGICAL_DURATION_SECONDS);
|
||||
return {
|
||||
compositionStart: Math.max(0, video.start),
|
||||
mediaStart: playableDuration - logicalDuration,
|
||||
extractionMediaStart: finalFrameTimestamp,
|
||||
durationSeconds: logicalDuration,
|
||||
preserveTimelineEnd: true,
|
||||
finalFrameOnly: true,
|
||||
};
|
||||
}
|
||||
|
||||
/** Resolve source duration first, then intersect it with the render timeline. */
|
||||
export function resolveVideoExtractionWindow(
|
||||
video: TimelineWindowVideo,
|
||||
metadata: VideoMetadata,
|
||||
timelineEnd?: number,
|
||||
): TimelineExtractionWindow {
|
||||
const playableDuration = resolvePlayableVideoDuration(metadata);
|
||||
if (!(playableDuration > 0)) {
|
||||
throw new VideoSourceExtractionError(
|
||||
"invalid_media",
|
||||
false,
|
||||
"Video source has no positive duration",
|
||||
`Playable video stream duration is ${playableDuration}s`,
|
||||
);
|
||||
}
|
||||
if (video.mediaStart >= playableDuration) {
|
||||
throw new VideoSourceExtractionError(
|
||||
"media_start_out_of_range",
|
||||
false,
|
||||
"Video media start is outside the source duration",
|
||||
`Video media start ${video.mediaStart}s is outside playable video duration ${playableDuration}s`,
|
||||
);
|
||||
}
|
||||
const resolvedDuration = resolveSegmentDuration(
|
||||
video.end - video.start,
|
||||
video.mediaStart,
|
||||
playableDuration,
|
||||
);
|
||||
return resolveTimelineExtractionWindow(video, resolvedDuration, timelineEnd, playableDuration);
|
||||
}
|
||||
|
||||
export function resolveVideoExtractionDuration(
|
||||
video: TimelineWindowVideo,
|
||||
metadata: VideoMetadata,
|
||||
timelineEnd?: number,
|
||||
): number {
|
||||
return resolveVideoExtractionWindow(video, metadata, timelineEnd).durationSeconds;
|
||||
}
|
||||
|
||||
/**
|
||||
@@ -761,6 +1000,8 @@ type PreparedExtraction = {
|
||||
index: number;
|
||||
metadata: VideoMetadata;
|
||||
videoDuration: number;
|
||||
extractionMediaStart: number;
|
||||
finalFrameOnly: boolean;
|
||||
format: CacheFrameFormat;
|
||||
sdrToHdrTransfer?: HdrTransfer;
|
||||
dedupeKey: string;
|
||||
@@ -831,7 +1072,13 @@ function linkOrCopyFrame(src: string, dest: string): void {
|
||||
}
|
||||
|
||||
function supersetGroupingKey(work: PreparedExtraction, fps: number): string {
|
||||
return [work.videoPath, String(fps), work.format, work.sdrToHdrTransfer ?? ""].join("\0");
|
||||
return [
|
||||
work.videoPath,
|
||||
String(fps),
|
||||
work.format,
|
||||
work.sdrToHdrTransfer ?? "",
|
||||
work.finalFrameOnly ? "final" : "range",
|
||||
].join("\0");
|
||||
}
|
||||
|
||||
function isIntegralFrameOffset(offsetSeconds: number, fps: number): boolean {
|
||||
@@ -854,6 +1101,7 @@ function buildSupersetGroup(
|
||||
fps: number,
|
||||
): SupersetGroupPlan | null {
|
||||
if (misses.length < 2) return null;
|
||||
if (misses.some(({ work }) => work.finalFrameOnly)) return null;
|
||||
const baseStart = Math.min(...misses.map(({ work }) => work.video.mediaStart));
|
||||
if (!misses.every(({ work }) => isIntegralFrameOffset(work.video.mediaStart - baseStart, fps))) {
|
||||
return null;
|
||||
@@ -1029,6 +1277,11 @@ export async function extractAllVideoFrames(
|
||||
>,
|
||||
compiledDir?: string,
|
||||
): Promise<ExtractionResult> {
|
||||
if (options.timelineEnd !== undefined && !Number.isFinite(options.timelineEnd)) {
|
||||
throw new Error(
|
||||
`Video extraction timelineEnd must be finite; got ${String(options.timelineEnd)}`,
|
||||
);
|
||||
}
|
||||
const startTime = Date.now();
|
||||
const extracted: ExtractedFrames[] = [];
|
||||
const errors: VideoExtractionFailure[] = [];
|
||||
@@ -1062,6 +1315,7 @@ export async function extractAllVideoFrames(
|
||||
const warnedSrcs = new Set<string>();
|
||||
for (const video of videos) {
|
||||
if (signal?.aborted) break;
|
||||
if (options.timelineEnd !== undefined && video.start >= options.timelineEnd) continue;
|
||||
try {
|
||||
let videoPath = video.src;
|
||||
if (!isHttpUrl(videoPath)) {
|
||||
@@ -1113,10 +1367,11 @@ export async function extractAllVideoFrames(
|
||||
breakdown.resolveMs = Date.now() - phase1Start;
|
||||
|
||||
// Snapshot the pre-preflight key inputs so the extraction cache keys on the
|
||||
// user-visible source (original path, original mediaStart, original segment
|
||||
// bounds) rather than the workDir-local normalized file produced by the
|
||||
// user-visible source path rather than the
|
||||
// workDir-local normalized file produced by the
|
||||
// HDR preflight. Without this, every render would write a new
|
||||
// normalized file with a fresh mtime → fresh cache key → perpetual misses.
|
||||
// Phase 3 updates mediaStart after trimming any invisible negative preroll.
|
||||
const cacheKeyInputs = resolvedVideos.map(({ video, videoPath }) => {
|
||||
const stat = readKeyStat(videoPath);
|
||||
// Missing files return null — skip the cache path for that entry. The
|
||||
@@ -1129,8 +1384,6 @@ export async function extractAllVideoFrames(
|
||||
mtimeMs: stat.mtimeMs,
|
||||
size: stat.size,
|
||||
mediaStart: video.mediaStart,
|
||||
start: video.start,
|
||||
end: video.end,
|
||||
};
|
||||
});
|
||||
|
||||
@@ -1217,12 +1470,13 @@ export async function extractAllVideoFrames(
|
||||
// Guard against mediaStart past EOF — FFmpeg's `-ss` silently produces
|
||||
// a 0-byte file when seeking beyond the source duration, and the
|
||||
// downstream extractor then points at a broken input.
|
||||
if (entry.video.mediaStart >= metadata.durationSeconds) {
|
||||
const playableDuration = resolvePlayableVideoDuration(metadata);
|
||||
if (entry.video.mediaStart >= playableDuration) {
|
||||
errors.push({
|
||||
videoId: entry.video.id,
|
||||
kind: "media_start_out_of_range",
|
||||
retryable: false,
|
||||
error: `SDR→HDR conversion skipped: mediaStart (${entry.video.mediaStart}s) ≥ source duration (${metadata.durationSeconds}s)`,
|
||||
error: `SDR→HDR conversion skipped: mediaStart (${entry.video.mediaStart}s) ≥ playable video duration (${playableDuration}s)`,
|
||||
});
|
||||
hdrSkippedIndices.add(i);
|
||||
continue;
|
||||
@@ -1289,12 +1543,20 @@ export async function extractAllVideoFrames(
|
||||
};
|
||||
}
|
||||
|
||||
type PreparedExtractionResult = { work: PreparedExtraction } | { error: VideoExtractionFailure };
|
||||
type PreparedExtractionResult =
|
||||
| { work: PreparedExtraction }
|
||||
| { error: VideoExtractionFailure }
|
||||
| { skipped: true };
|
||||
|
||||
type ExtractionOutcome = { result: ExtractedFrames } | { error: VideoExtractionFailure };
|
||||
|
||||
function scopedExtractionOptions(work: PreparedExtraction): ExtractionOptions {
|
||||
return { ...options, format: work.format, sdrToHdrTransfer: work.sdrToHdrTransfer };
|
||||
return {
|
||||
...options,
|
||||
format: work.format,
|
||||
sdrToHdrTransfer: work.sdrToHdrTransfer,
|
||||
finalFrameOnly: work.finalFrameOnly,
|
||||
};
|
||||
}
|
||||
|
||||
function rehydratePublishedCache(work: PreparedExtraction, target: CacheMissTarget) {
|
||||
@@ -1312,21 +1574,18 @@ export async function extractAllVideoFrames(
|
||||
if (!cacheRootDir) return { work };
|
||||
const keyInput = cacheKeyInputs[work.index];
|
||||
if (!keyInput) return { work };
|
||||
const transform = work.sdrToHdrTransfer
|
||||
? sdrToHdrTransformKey(work.sdrToHdrTransfer)
|
||||
: undefined;
|
||||
const transformParts = [
|
||||
work.sdrToHdrTransfer ? sdrToHdrTransformKey(work.sdrToHdrTransfer) : undefined,
|
||||
work.finalFrameOnly ? "final-frame" : undefined,
|
||||
].filter((part): part is string => part !== undefined);
|
||||
const transform = transformParts.length > 0 ? transformParts.join("+") : undefined;
|
||||
|
||||
const keyDuration = resolveSegmentDuration(
|
||||
keyInput.end - keyInput.start,
|
||||
keyInput.mediaStart,
|
||||
work.metadata,
|
||||
);
|
||||
const lookup = lookupCacheEntry(cacheRootDir, {
|
||||
videoPath: keyInput.videoPath,
|
||||
mtimeMs: keyInput.mtimeMs,
|
||||
size: keyInput.size,
|
||||
mediaStart: keyInput.mediaStart,
|
||||
duration: keyDuration,
|
||||
duration: work.videoDuration,
|
||||
fps: options.fps,
|
||||
format: work.format,
|
||||
transform,
|
||||
@@ -1356,7 +1615,7 @@ export async function extractAllVideoFrames(
|
||||
extractVideoFramesRange(
|
||||
work.videoPath,
|
||||
work.video.id,
|
||||
work.video.mediaStart,
|
||||
work.extractionMediaStart,
|
||||
work.videoDuration,
|
||||
scopedExtractionOptions(work),
|
||||
signal,
|
||||
@@ -1382,7 +1641,7 @@ export async function extractAllVideoFrames(
|
||||
extractVideoFramesRange(
|
||||
work.videoPath,
|
||||
work.video.id,
|
||||
work.video.mediaStart,
|
||||
work.extractionMediaStart,
|
||||
work.videoDuration,
|
||||
scopedExtractionOptions(work),
|
||||
signal,
|
||||
@@ -1516,18 +1775,33 @@ export async function extractAllVideoFrames(
|
||||
}
|
||||
try {
|
||||
const metadata = videoMetadata[index] ?? (await extractMediaMetadata(videoPath));
|
||||
const videoDuration = resolveSegmentDuration(
|
||||
video.end - video.start,
|
||||
video.mediaStart,
|
||||
const initialWindow = resolveVideoExtractionWindow(video, metadata, options.timelineEnd);
|
||||
const window = await resolveFinalFrameExtractionWindow(
|
||||
videoPath,
|
||||
video,
|
||||
metadata,
|
||||
initialWindow,
|
||||
signal,
|
||||
);
|
||||
if (video.end - video.start !== videoDuration) {
|
||||
video.end = video.start + videoDuration;
|
||||
const videoDuration = window.durationSeconds;
|
||||
if (videoDuration <= 0) {
|
||||
return { skipped: true };
|
||||
}
|
||||
if (!window.preserveTimelinePhase) {
|
||||
video.start = window.compositionStart;
|
||||
if (!window.preserveTimelineEnd) {
|
||||
video.end = window.compositionStart + videoDuration;
|
||||
}
|
||||
video.mediaStart = window.mediaStart;
|
||||
}
|
||||
const keyInput = cacheKeyInputs[index];
|
||||
const extractionMediaStart = window.extractionMediaStart ?? window.mediaStart;
|
||||
if (keyInput) keyInput.mediaStart = extractionMediaStart;
|
||||
|
||||
const format = resolveFrameFormat(metadata, options.format);
|
||||
const sdrToHdrTransfer = sdrToHdrTransfers[index];
|
||||
const dedupeKey = `${videoPath}\0${video.mediaStart}\0${videoDuration}\0${options.fps}\0${format}\0${sdrToHdrTransfer ?? ""}`;
|
||||
const finalFrameOnly = window.finalFrameOnly === true;
|
||||
const dedupeKey = `${videoPath}\0${extractionMediaStart}\0${videoDuration}\0${options.fps}\0${format}\0${sdrToHdrTransfer ?? ""}\0${finalFrameOnly ? "final" : "range"}`;
|
||||
|
||||
return {
|
||||
work: {
|
||||
@@ -1536,6 +1810,8 @@ export async function extractAllVideoFrames(
|
||||
index,
|
||||
metadata,
|
||||
videoDuration,
|
||||
extractionMediaStart,
|
||||
finalFrameOnly,
|
||||
format,
|
||||
sdrToHdrTransfer,
|
||||
dedupeKey,
|
||||
@@ -1581,11 +1857,20 @@ export async function extractAllVideoFrames(
|
||||
for (const [key, outcome] of groupOutcomes) uniqueOutcomes.set(key, outcome);
|
||||
}
|
||||
|
||||
const results: ExtractionOutcome[] = preparedExtractions.map((prepared) => {
|
||||
if ("error" in prepared) return prepared;
|
||||
const results: ExtractionOutcome[] = [];
|
||||
for (const prepared of preparedExtractions) {
|
||||
if ("skipped" in prepared) continue;
|
||||
if ("error" in prepared) {
|
||||
results.push(prepared);
|
||||
continue;
|
||||
}
|
||||
const outcome = uniqueOutcomes.get(prepared.work.dedupeKey);
|
||||
if (!outcome)
|
||||
return { error: extractionError(prepared.work.video.id, "missing extraction result") };
|
||||
if (!outcome) {
|
||||
results.push({
|
||||
error: extractionError(prepared.work.video.id, "missing extraction result"),
|
||||
});
|
||||
continue;
|
||||
}
|
||||
if ("error" in outcome) {
|
||||
// A shared (deduped/superset) failure fans out to every element with the
|
||||
// same key; annotate followers with the leader's videoId so N copies of
|
||||
@@ -1594,17 +1879,18 @@ export async function extractAllVideoFrames(
|
||||
const message = isFollower
|
||||
? `[shared extraction, leader ${outcome.error.videoId}] ${outcome.error.error}`
|
||||
: outcome.error.error;
|
||||
return {
|
||||
results.push({
|
||||
error: {
|
||||
videoId: prepared.work.video.id,
|
||||
kind: outcome.error.kind,
|
||||
retryable: outcome.error.retryable,
|
||||
error: message,
|
||||
},
|
||||
};
|
||||
});
|
||||
continue;
|
||||
}
|
||||
return { result: { ...outcome.result, videoId: prepared.work.video.id } };
|
||||
});
|
||||
results.push({ result: { ...outcome.result, videoId: prepared.work.video.id } });
|
||||
}
|
||||
|
||||
breakdown.extractMs = Date.now() - phase3Start;
|
||||
|
||||
@@ -1653,7 +1939,7 @@ function getFrameIndexAtTime(
|
||||
): number | null {
|
||||
let localTime = globalTime - videoStart;
|
||||
if (localTime < 0) return null;
|
||||
const loopDuration = Math.max(0, extracted.metadata.durationSeconds - mediaStart);
|
||||
const loopDuration = Math.max(0, resolvePlayableVideoDuration(extracted.metadata) - mediaStart);
|
||||
if (loop && loopDuration > 0 && localTime >= loopDuration) {
|
||||
localTime %= loopDuration;
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user