From 2339757377f1534900af5c84ad4025c6e576da9e Mon Sep 17 00:00:00 2001 From: James Russo Date: Sun, 2 Aug 2026 20:21:54 -0700 Subject: [PATCH] fix: bound HDR and video extraction resources (#2955) * fix: bound HDR and video extraction resources * fix: trim negative video extraction preroll * fix: skip invisible video extraction windows * fix: preserve negative-start loop and held tails * fix: cap finite video slots to source duration * fix: bound held-tail frame extraction * fix: plan from playable video duration * fix: preserve open-ended held video tails * fix: resolve held tails from decoded frames * fix: normalize final-frame probe timestamps * fix: handle unseekable final-frame sources * fix: dedupe final-frame probes per render * refactor: clarify output dynamic range contract --- packages/engine/src/index.ts | 8 + .../engine/src/services/systemMemory.test.ts | 25 + packages/engine/src/services/systemMemory.ts | 8 +- .../src/services/videoFrameExtractor.test.ts | 547 +++++++++++++- .../src/services/videoFrameExtractor.ts | 380 ++++++++-- packages/engine/src/utils/ffprobe.test.ts | 150 ++++ packages/engine/src/utils/ffprobe.ts | 116 ++- .../src/server.outputDynamicRange.test.ts | 89 +++ packages/producer/src/server.test.ts | 50 ++ packages/producer/src/server.ts | 46 ++ .../src/services/distributed/shared.ts | 7 + .../distributed/videoMetadata.test.ts | 20 + .../producer/src/services/hdrCompositor.ts | 34 +- .../render/stages/captureHdrFrameShared.ts | 15 +- .../render/stages/captureHdrResources.test.ts | 666 +++++++++++++++++- .../render/stages/captureHdrResources.ts | 350 +++++++-- .../services/render/stages/captureHdrStage.ts | 11 +- .../stages/captureStreamingStage.test.ts | 8 +- .../extractVideosStage.timelineBound.test.ts | 120 ++++ .../render/stages/extractVideosStage.ts | 1 + .../render/videoFrameCoverage.test.ts | 38 +- .../src/services/render/videoFrameCoverage.ts | 8 +- 22 files changed, 2541 insertions(+), 156 deletions(-) create mode 100644 packages/producer/src/server.outputDynamicRange.test.ts create mode 100644 packages/producer/src/services/render/stages/extractVideosStage.timelineBound.test.ts diff --git a/packages/engine/src/index.ts b/packages/engine/src/index.ts index b479920dc..e9c348143 100644 --- a/packages/engine/src/index.ts +++ b/packages/engine/src/index.ts @@ -66,6 +66,7 @@ export { normalizeVp9CpuUsed, } from "./services/vp9Options.js"; export { + getCgroupMemoryLimitMb, getSystemTotalMb, isLowMemorySystem, LOW_MEMORY_TOTAL_MB_THRESHOLD, @@ -179,6 +180,11 @@ export { parseImageElements, extractVideoFramesRange, extractAllVideoFrames, + resolveTimelineExtractionWindow, + resolveVideoExtractionWindow, + resolveFinalFrameExtractionWindow, + resolveVideoExtractionDuration, + resolvePlayableVideoDuration, resolveProjectRelativeSrc, getFrameAtTime, createFrameLookupTable, @@ -194,6 +200,7 @@ export { type ExtractionOptions, type ExtractionResult, type ExtractionPhaseBreakdown, + type TimelineExtractionWindow, type VideoExtractionFailure, type VideoExtractionFailureKind, type VideoFrameFormat, @@ -255,6 +262,7 @@ export { readWebGlVendorInfoFromCanvas } from "./utils/readWebGlVendorInfoFromCa export { extractMediaMetadata, extractVideoMetadata, + extractFinalVideoFrameTimestamp, extractAudioMetadata, analyzeKeyframeIntervals, type VideoMetadata, diff --git a/packages/engine/src/services/systemMemory.test.ts b/packages/engine/src/services/systemMemory.test.ts index 315f04d14..f8c91528f 100644 --- a/packages/engine/src/services/systemMemory.test.ts +++ b/packages/engine/src/services/systemMemory.test.ts @@ -1,3 +1,4 @@ +// fallow-ignore-file code-duplication import { afterEach, beforeEach, describe, it, expect, vi } from "vitest"; import { _resetCgroupLimitCacheForTests, @@ -154,6 +155,30 @@ describe("parseCgroupLimitMb", () => { }); }); +describe("getCgroupMemoryLimitMb", () => { + it("returns only an actual cgroup limit and never host RAM", async () => { + await withSystemMemoryMocks( + { + files: { [CGROUP_V2_MEMORY_MAX_PATH]: `${24576 * BYTES_PER_MIB}` }, + hostTotalMb: 65536, + }, + ({ getCgroupMemoryLimitMb }) => { + expect(getCgroupMemoryLimitMb()).toBe(24576); + }, + ); + + await withSystemMemoryMocks( + { + files: { [CGROUP_V2_MEMORY_MAX_PATH]: "max" }, + hostTotalMb: 65536, + }, + ({ getCgroupMemoryLimitMb }) => { + expect(getCgroupMemoryLimitMb()).toBeNull(); + }, + ); + }); +}); + describe("getSystemTotalMb", () => { it("caches cgroup probes until the test reset hook clears the cache", async () => { const readCalls: string[] = []; diff --git a/packages/engine/src/services/systemMemory.ts b/packages/engine/src/services/systemMemory.ts index f60c893ae..c37d62c74 100644 --- a/packages/engine/src/services/systemMemory.ts +++ b/packages/engine/src/services/systemMemory.ts @@ -80,7 +80,11 @@ export function _resetCgroupLimitCacheForTests(): void { _warnedCgroupReadFailure = false; } -function getCgroupLimitMb(): number | null { +/** + * Actual Linux cgroup memory ceiling in MiB, or null when the process is not + * cgroup-limited. Unlike getSystemTotalMb this never falls back to host RAM. + */ +export function getCgroupMemoryLimitMb(): number | null { if (_cachedCgroupLimitMb !== undefined) return _cachedCgroupLimitMb; if (process.platform !== "linux") { @@ -142,7 +146,7 @@ function warnCgroupReadFailure(path: string, error: unknown): void { /** Total physical RAM in MiB. */ export function getSystemTotalMb(): number { const hostTotalMb = Math.floor(totalmem() / BYTES_PER_MIB); - const cgroupLimitMb = getCgroupLimitMb(); + const cgroupLimitMb = getCgroupMemoryLimitMb(); return cgroupLimitMb === null ? hostTotalMb : Math.min(hostTotalMb, cgroupLimitMb); } diff --git a/packages/engine/src/services/videoFrameExtractor.test.ts b/packages/engine/src/services/videoFrameExtractor.test.ts index d494565db..437af5ffe 100644 --- a/packages/engine/src/services/videoFrameExtractor.test.ts +++ b/packages/engine/src/services/videoFrameExtractor.test.ts @@ -24,6 +24,8 @@ import { resolveFrameFormat, codecMayHaveAlpha, decoderForCodec, + resolveVideoExtractionWindow, + resolveVideoExtractionDuration, getFrameAtTime, analyzeClipMediaFit, classifyVideoExtractionError, @@ -33,9 +35,14 @@ import { type ExtractedFrames, type ExtractionResult, } from "./videoFrameExtractor.js"; -import { extractVideoMetadata, type VideoMetadata } from "../utils/ffprobe.js"; +import { + extractFinalVideoFrameTimestamp, + extractVideoMetadata, + type VideoMetadata, +} from "../utils/ffprobe.js"; import { runFfmpeg } from "../utils/runFfmpeg.js"; import { COMPLETE_SENTINEL, GC_MARKER, SCHEMA_PREFIX } from "./extractionCache.js"; +import { resolveRuntimeMediaClipDuration } from "../../../core/src/runtime/media.js"; // ffmpeg is not preinstalled on GitHub's ubuntu-24.04 runners. The producer // regression test at packages/producer/tests/vfr-screen-recording/ runs inside @@ -45,6 +52,261 @@ import { COMPLETE_SENTINEL, GC_MARKER, SCHEMA_PREFIX } from "./extractionCache.j // synthesized VFR fixture. const HAS_FFMPEG = spawnSync("ffmpeg", ["-version"]).status === 0; +describe("resolveVideoExtractionDuration", () => { + const metadata = ( + durationSeconds: number, + videoStreamDurationSeconds = durationSeconds, + ): VideoMetadata => ({ + durationSeconds, + videoStreamDurationSeconds, + width: 1920, + height: 1080, + fps: 30, + videoCodec: "h264", + hasAudio: false, + isVFR: false, + hasAlpha: false, + colorSpace: null, + }); + const video = (overrides: Partial = {}): VideoElement => ({ + id: "root-video", + src: "video.mp4", + start: 0, + end: Number.POSITIVE_INFINITY, + mediaStart: 0, + loop: false, + hasAudio: false, + ...overrides, + }); + + it("caps an open 60-second root source to a two-second composition", () => { + expect(resolveVideoExtractionDuration(video(), metadata(60), 2)).toBe(2); + }); + + it("keeps a shorter natural source duration inside a longer composition", () => { + expect(resolveVideoExtractionDuration(video(), metadata(2), 10)).toBe(2); + }); + + it("falls back to container duration when stream duration is unavailable", () => { + expect(resolveVideoExtractionDuration(video(), metadata(2, 0), 10)).toBe(2); + }); + + it("preserves explicit bounds and loop flags while applying the timeline ceiling", () => { + const explicitLoop = video({ end: 8, loop: true }); + expect(resolveVideoExtractionDuration(explicitLoop, metadata(60), 10)).toBe(8); + expect(explicitLoop.loop).toBe(true); + }); + + it("trims materially negative preroll and advances the source offset", () => { + const preroll = video({ start: -60, end: 120, mediaStart: 0 }); + expect(resolveVideoExtractionWindow(preroll, metadata(120), 2)).toEqual({ + compositionStart: 0, + mediaStart: 60, + durationSeconds: 2, + }); + expect(resolveVideoExtractionDuration(preroll, metadata(120), 2)).toBe(2); + }); + + it("returns an empty window for a clip entirely before composition time zero", () => { + expect(resolveVideoExtractionWindow(video({ start: -60, end: -10 }), metadata(120), 2)).toEqual( + { compositionStart: 0, mediaStart: 60, durationSeconds: 0 }, + ); + }); + + it("preserves a short source cycle when negative preroll crosses a loop boundary", () => { + expect( + resolveVideoExtractionWindow( + video({ start: -5, end: 10, mediaStart: 0, loop: true }), + metadata(3), + 2, + ), + ).toEqual({ + compositionStart: -5, + mediaStart: 0, + durationSeconds: 3, + preserveTimelinePhase: true, + }); + }); + + it("marks an entirely held interval for exact final-frame resolution", () => { + expect( + resolveVideoExtractionWindow( + video({ start: -5, end: 10, mediaStart: 0, loop: false }), + metadata(3), + 2, + ), + ).toEqual({ + compositionStart: -2.000001, + mediaStart: 2.999999, + durationSeconds: 0.000001, + preserveTimelineEnd: true, + ensureFinalFrame: true, + }); + }); + + it.each([ + { loop: true, preservation: { preserveTimelinePhase: true }, label: "loop" }, + { + loop: false, + preservation: { preserveTimelineEnd: true, ensureFinalFrame: true }, + label: "held tail", + }, + ])( + "caps a finite long slot to one short source range for $label playback", + ({ loop, preservation }) => { + expect(resolveVideoExtractionWindow(video({ end: 60, loop }), metadata(3), 60)).toEqual({ + compositionStart: 0, + mediaStart: 0, + durationSeconds: 3, + ...preservation, + }); + }, + ); + + it.each([ + { loop: true, preservation: { preserveTimelinePhase: true }, label: "loop" }, + { + loop: false, + preservation: { preserveTimelineEnd: true, ensureFinalFrame: true }, + label: "held tail", + }, + ])( + "uses the playable video-stream duration for a long-audio mux in $label playback", + ({ loop, preservation }) => { + expect(resolveVideoExtractionWindow(video({ end: 60, loop }), metadata(60, 3), 60)).toEqual({ + compositionStart: 0, + mediaStart: 0, + durationSeconds: 3, + ...preservation, + }); + }, + ); + + it("preserves authored timing when the visible interval partially crosses a held tail", () => { + expect( + resolveVideoExtractionWindow( + video({ start: -2, end: 10, mediaStart: 0, loop: false }), + metadata(3), + 2, + ), + ).toEqual({ + compositionStart: 0, + mediaStart: 2, + durationSeconds: 1, + preserveTimelineEnd: true, + ensureFinalFrame: true, + }); + }); + + it.each([ + { start: 0, expected: { compositionStart: 0, mediaStart: 0, durationSeconds: 3 } }, + { start: -2, expected: { compositionStart: 0, mediaStart: 2, durationSeconds: 1 } }, + { start: -5, expected: { compositionStart: 0, mediaStart: 5, durationSeconds: 0 } }, + ])( + "keeps an omitted-duration clip source-bounded like runtime (start=$start)", + ({ start, expected }) => { + for (const loop of [false, true]) { + const parsed = parseVideoElements( + ``, + )[0]!; + const planned = { ...parsed, start }; + const runtimeDuration = resolveRuntimeMediaClipDuration({ + isVideo: true, + sourceDuration: 3, + hostRemaining: 15 - start, + explicitDuration: null, + }); + expect(runtimeDuration).toBe(3); + expect(parsed.end).toBe(Number.POSITIVE_INFINITY); + expect(resolveVideoExtractionWindow(planned, metadata(3), 15)).toEqual(expected); + } + }, + ); + + it("bounds an entirely held long source to its final-frame sample", () => { + expect( + resolveVideoExtractionWindow( + video({ start: -600, end: 10, mediaStart: 0, loop: false }), + metadata(120), + 2, + ), + ).toEqual({ + compositionStart: -480.000001, + mediaStart: 119.999999, + durationSeconds: 0.000001, + preserveTimelineEnd: true, + ensureFinalFrame: true, + }); + }); + + it("preserves a complete loop cycle when visibility ends exactly on a wrap boundary", () => { + expect( + resolveVideoExtractionWindow( + video({ start: -1, end: 10, mediaStart: 0, loop: true }), + metadata(3), + 2, + ), + ).toEqual({ + compositionStart: -1, + mediaStart: 0, + durationSeconds: 3, + preserveTimelinePhase: true, + }); + }); + + it("keeps an open-ended loop source-bounded instead of inventing a longer slot", () => { + expect(resolveVideoExtractionWindow(video({ loop: true }), metadata(3), 10)).toEqual({ + compositionStart: 0, + mediaStart: 0, + durationSeconds: 3, + }); + }); + + it("never plans more extraction than the playable source range", () => { + for (const loop of [false, true]) { + for (const sourceDuration of [0.5, 3, 120]) { + for (const mediaStart of [0, sourceDuration / 3]) { + for (const start of [-600, -5, -1, 0, 2]) { + const window = resolveVideoExtractionWindow( + video({ start, end: start + 60, mediaStart, loop }), + metadata(sourceDuration), + 10, + ); + expect(window.durationSeconds).toBeLessThanOrEqual(sourceDuration - mediaStart); + expect(window.durationSeconds).toBeGreaterThanOrEqual(0); + } + } + } + } + }); + + it("rejects a media start at source EOF before planning extraction", () => { + expect(() => + resolveVideoExtractionWindow(video({ mediaStart: 3 }), metadata(3), 10), + ).toThrowError(expect.objectContaining({ kind: "media_start_out_of_range", retryable: false })); + }); + + it("rejects a media start at video-stream EOF even when the container continues", () => { + expect(() => + resolveVideoExtractionWindow(video({ mediaStart: 3 }), metadata(60, 3), 10), + ).toThrowError(expect.objectContaining({ kind: "media_start_out_of_range", retryable: false })); + }); + + it("rebases a loop phase when the visible window stays within one cycle", () => { + expect( + resolveVideoExtractionWindow( + video({ start: -5, end: 10, mediaStart: 0, loop: true }), + metadata(3), + 0.5, + ), + ).toEqual({ compositionStart: 0, mediaStart: 2, durationSeconds: 0.5 }); + }); + + it("retains legacy behavior when no timeline end is supplied", () => { + expect(resolveVideoExtractionDuration(video(), metadata(60))).toBe(60); + }); +}); + describe("video extraction failure taxonomy and bounded retry", () => { it("classifies missing and transient HTTP sources without exposing retry ambiguity", () => { expect(classifyVideoExtractionError(new Error("HTTP 404: Not Found"))).toMatchObject({ @@ -507,6 +769,28 @@ describe("FrameLookupTable", () => { expect(table.getActiveFramePayloads(4.5).get("hero")?.frameIndex).toBe(15); }); + it("wraps at video-stream EOF when a mux container has longer audio", () => { + const extracted = fakeExtracted(6, 2); + extracted.metadata.durationSeconds = 60; + extracted.metadata.videoStreamDurationSeconds = 3; + const table = createFrameLookupTable( + [ + { + id: "hero", + src: "clip.webm", + start: 0, + end: 60, + mediaStart: 0, + loop: true, + hasAudio: false, + }, + ], + [extracted], + ); + + expect(table.getActiveFramePayloads(4).get("hero")?.frameIndex).toBe(2); + }); + it("holds the last frame for a non-looping clip until its authored slot ends", () => { const table = createFrameLookupTable( [ @@ -958,6 +1242,137 @@ describe.skipIf(!HAS_FFMPEG)("video frame extraction format", () => { }, 60_000); }); +describe.skipIf(!HAS_FFMPEG)("held tails on sparse-timestamp sources", () => { + const fixtureDir = mkdtempSync(join(tmpdir(), "hf-sparse-held-tail-")); + const cfrFixture = join(fixtureDir, "sub-1fps-cfr.mp4"); + const vfrFixture = join(fixtureDir, "sparse-vfr.mp4"); + const nonZeroStartFixture = join(fixtureDir, "nonzero-start.mp4"); + const negativeStartTransportFixture = join(fixtureDir, "negative-start.ts"); + + beforeAll(async () => { + const fixtures = [ + { + path: cfrFixture, + input: "testsrc2=s=64x64:d=10:rate=1/5", + filters: [] as string[], + }, + { + path: vfrFixture, + input: "testsrc2=s=64x64:d=10:rate=1/2", + filters: ["-vf", "select='eq(n,0)+eq(n,2)'", "-vsync", "vfr"], + }, + { + path: nonZeroStartFixture, + input: "testsrc2=s=64x64:d=3:rate=1", + filters: ["-output_ts_offset", "5"], + }, + { + path: negativeStartTransportFixture, + input: "testsrc2=s=64x64:d=3:rate=1", + filters: [ + "-mpegts_copyts", + "1", + "-muxdelay", + "0", + "-avoid_negative_ts", + "disabled", + "-output_ts_offset", + "-2", + ], + }, + ]; + for (const fixture of fixtures) { + const result = await runFfmpeg([ + "-hide_banner", + "-loglevel", + "error", + "-f", + "lavfi", + "-i", + fixture.input, + ...fixture.filters, + "-c:v", + "libx264", + "-pix_fmt", + "yuv420p", + "-y", + fixture.path, + ]); + if (!result.success) { + throw new Error(`sparse fixture synthesis failed: ${result.stderr.slice(-400)}`); + } + } + }, 30_000); + + afterAll(() => { + rmSync(fixtureDir, { recursive: true, force: true }); + }); + + it.each([ + { + label: "sub-1fps CFR", + src: cfrFixture, + expectedVfr: false, + finalTimestamp: 5, + streamStart: 0, + }, + { + label: "sparse VFR", + src: vfrFixture, + expectedVfr: true, + finalTimestamp: 4, + streamStart: 0, + }, + { + label: "non-zero stream start", + src: nonZeroStartFixture, + expectedVfr: false, + finalTimestamp: 2, + streamStart: 5, + }, + { + label: "unindexed negative-base MPEG-TS", + src: negativeStartTransportFixture, + expectedVfr: false, + finalTimestamp: 2, + streamStart: -2, + }, + ])( + "extracts one real final SDR frame for $label", + async ({ src, expectedVfr, finalTimestamp, streamStart }) => { + const metadata = await extractVideoMetadata(src); + expect(metadata.fps).toBeLessThanOrEqual(1); + expect(metadata.isVFR).toBe(expectedVfr); + expect(metadata.videoStreamStartSeconds).toBeCloseTo(streamStart, 6); + await expect(extractFinalVideoFrameTimestamp(src, metadata)).resolves.toBe(finalTimestamp); + const outputDir = mkdtempSync(join(fixtureDir, "out-")); + const video: VideoElement = { + id: `held-${String(expectedVfr)}`, + src, + start: -15, + end: 5, + mediaStart: 0, + loop: false, + hasAudio: false, + }; + + const result = await extractAllVideoFrames([video], fixtureDir, { + fps: 30, + format: "png", + outputDir, + timelineEnd: 2, + }); + + expect(result.errors).toEqual([]); + expect(result.extracted).toHaveLength(1); + expect(result.extracted[0]?.totalFrames).toBe(1); + expect(video).toMatchObject({ start: 0, end: 5, loop: false }); + expect(video.mediaStart).toBeCloseTo(metadata.videoStreamDurationSeconds - 0.000001, 7); + }, + 30_000, + ); +}); + // Regression test for the VFR (variable frame rate) freeze bug. // Screen recordings and phone videos often have irregular timestamps. // When such inputs hit `extractVideoFramesRange`'s `-ss -i ... -t @@ -1017,6 +1432,90 @@ describe.skipIf(!HAS_FFMPEG)("extractAllVideoFrames on a VFR source", () => { if (existsSync(FIXTURE_DIR)) rmSync(FIXTURE_DIR, { recursive: true, force: true }); }); + it("skips a clip entirely before time zero without reporting an extraction error", async () => { + const outputDir = join(FIXTURE_DIR, "out-before-timeline"); + mkdirSync(outputDir, { recursive: true }); + const video: VideoElement = { + id: "before-timeline", + src: VFR_FIXTURE, + start: -2, + end: -1, + mediaStart: 0, + loop: false, + hasAudio: false, + }; + + const result = await extractAllVideoFrames([video], FIXTURE_DIR, { + fps: 1, + outputDir, + timelineEnd: 2, + }); + + expect(result).toMatchObject({ success: true, extracted: [], errors: [] }); + }); + + it("preserves loop phase when negative preroll crosses the source boundary", async () => { + const outputDir = join(FIXTURE_DIR, "out-negative-loop"); + mkdirSync(outputDir, { recursive: true }); + const video: VideoElement = { + id: "negative-loop", + src: VFR_FIXTURE, + start: -19, + end: 5, + mediaStart: 0, + loop: true, + hasAudio: false, + }; + + const result = await extractAllVideoFrames([video], FIXTURE_DIR, { + fps: 1, + outputDir, + timelineEnd: 2, + }); + + expect(result.errors).toEqual([]); + const extracted = result.extracted[0]; + if (!extracted) throw new Error("expected loop source frames"); + const lookup = createFrameLookupTable([video], result.extracted); + expect(video).toMatchObject({ start: -19, mediaStart: 0, loop: true }); + expect(lookup.getFrame("negative-loop", 0)).toBe( + extracted.framePaths.get(extracted.totalFrames - 1), + ); + }, 30_000); + + it("preserves the held final frame after negative preroll exhausts a source", async () => { + const outputDir = join(FIXTURE_DIR, "out-negative-held-tail"); + mkdirSync(outputDir, { recursive: true }); + const video: VideoElement = { + id: "negative-held-tail", + src: VFR_FIXTURE, + start: -15, + end: 5, + mediaStart: 0, + loop: false, + hasAudio: false, + }; + + const result = await extractAllVideoFrames([video], FIXTURE_DIR, { + fps: 1, + outputDir, + timelineEnd: 2, + }); + + expect(result.errors).toEqual([]); + const extracted = result.extracted[0]; + if (!extracted) throw new Error("expected held-tail source frames"); + const lookup = createFrameLookupTable([video], result.extracted); + // The authored slot remains active through end=5, but lookup is rebased to + // one exact final frame instead of assuming the last second contains a + // timestamp or materializing the full source. + expect(video).toMatchObject({ start: 0, end: 5, mediaStart: 9.999999, loop: false }); + expect(extracted.totalFrames).toBe(1); + expect(lookup.getFrame("negative-held-tail", 0)).toBe( + extracted.framePaths.get(extracted.totalFrames - 1), + ); + }, 30_000); + it("detects the synthesized fixture as VFR", async () => { const md = await extractVideoMetadata(VFR_FIXTURE); expect(md.isVFR).toBe(true); @@ -1236,6 +1735,52 @@ describe.skipIf(!HAS_FFMPEG)("extractAllVideoFrames on a VFR source", () => { rmSync(CACHE_DIR, { recursive: true, force: true }); }, 60_000); + it("reuses one-cycle loop extraction across different authored starts", async () => { + const cacheDir = mkdtempSync(join(tmpdir(), "hf-extract-loop-phase-cache-test-")); + const src = await synthCfrClip("cache-loop-phase-src.mp4", 3); + try { + const firstOutputDir = join(FIXTURE_DIR, "out-cache-loop-phase-first"); + const secondOutputDir = join(FIXTURE_DIR, "out-cache-loop-phase-second"); + mkdirSync(firstOutputDir, { recursive: true }); + mkdirSync(secondOutputDir, { recursive: true }); + + const first = await extractAllVideoFrames( + [ + { + ...cfrClipElement("loop-cache-first", src, 60), + loop: true, + }, + ], + FIXTURE_DIR, + { fps: 30, outputDir: firstOutputDir, timelineEnd: 60 }, + undefined, + { extractCacheDir: cacheDir }, + ); + expect(first.errors).toEqual([]); + expect(first.phaseBreakdown.cacheMisses).toBe(1); + + const second = await extractAllVideoFrames( + [ + { + ...cfrClipElement("loop-cache-second", src, 66), + start: -6, + loop: true, + }, + ], + FIXTURE_DIR, + { fps: 30, outputDir: secondOutputDir, timelineEnd: 60 }, + undefined, + { extractCacheDir: cacheDir }, + ); + expect(second.errors).toEqual([]); + expect(second.phaseBreakdown.cacheHits).toBe(1); + expect(second.phaseBreakdown.cacheMisses).toBe(0); + expect(second.extracted[0]?.totalFrames).toBe(first.extracted[0]?.totalFrames); + } finally { + rmSync(cacheDir, { recursive: true, force: true }); + } + }, 60_000); + it("updates the cache sentinel mtime on a hit", async () => { const CACHE_DIR = mkdtempSync(join(tmpdir(), "hf-extract-cache-touch-test-")); const SRC = await synthCfrClip("cache-touch-src.mp4", 1); diff --git a/packages/engine/src/services/videoFrameExtractor.ts b/packages/engine/src/services/videoFrameExtractor.ts index a3d7df869..6b7169d45 100644 --- a/packages/engine/src/services/videoFrameExtractor.ts +++ b/packages/engine/src/services/videoFrameExtractor.ts @@ -11,7 +11,11 @@ import { isAbsolute, join, posix, resolve, sep } from "path"; import { parseHTML } from "linkedom"; import { decodeUrlPathVariants, MEDIA_DURATION_CLAMP_EPSILON_SECONDS } from "@hyperframes/core"; import { resolveReferencedStart, type RefResolverEl } from "./referenceResolver.js"; -import { extractMediaMetadata, type VideoMetadata } from "../utils/ffprobe.js"; +import { + extractFinalVideoFrameTimestamp, + extractMediaMetadata, + type VideoMetadata, +} from "../utils/ffprobe.js"; import { analyzeCompositionHdr, isHdrColorSpace as isHdrColorSpaceUtil, @@ -83,6 +87,15 @@ export interface ExtractionOptions { quality?: number; format?: VideoFrameFormat; sdrToHdrTransfer?: HdrTransfer; + /** Extract exactly one frame at `startTime`. Used only after ffprobe has + * resolved the actual final decoded-frame timestamp for a held tail. */ + finalFrameOnly?: boolean; + /** + * Absolute composition/timeline end in seconds. Applied only after source + * metadata resolves open-ended/natural-duration media. Invisible negative + * preroll is trimmed while advancing mediaStart to preserve source alignment. + */ + timelineEnd?: number; /** * Bounded per-source FFmpeg retries. Default 0 preserves stable behavior; * the producer may canary at most one retry after observing typed failures. @@ -444,7 +457,10 @@ export function parseVideoElements(html: string): VideoElement[] { // reference; the resolver handles both. const start = startAttr ? resolveReferencedStart(document, el, startCache, visiting) : 0; // Derive end from data-end → data-start+data-duration → Infinity (natural duration). - // The caller (htmlCompiler) clamps Infinity to the composition's absoluteEnd. + // Static compilation cannot always clamp root media because GSAP may supply + // the root duration at runtime. The producer passes the resolved timeline + // end into frame extraction, which caps the source duration only after the + // natural duration is known without rewriting authored timing metadata. let end = 0; if (endAttr) { end = parseFloat(endAttr); @@ -542,20 +558,21 @@ export async function extractVideoFramesRange( } catch (error) { throw classifyVideoExtractionError(error); } - if (!(metadata.durationSeconds > 0)) { + const playableDuration = resolvePlayableVideoDuration(metadata); + if (!(playableDuration > 0)) { throw new VideoSourceExtractionError( "invalid_media", false, "Video source has no positive duration", - `Video source duration is ${metadata.durationSeconds}s`, + `Playable video stream duration is ${playableDuration}s`, ); } - if (startTime >= metadata.durationSeconds) { + if (startTime >= playableDuration) { throw new VideoSourceExtractionError( "media_start_out_of_range", false, "Video media start is outside the source duration", - `Video media start ${startTime}s is outside source duration ${metadata.durationSeconds}s`, + `Video media start ${startTime}s is outside playable video duration ${playableDuration}s`, ); } const format = resolveFrameFormat(metadata, options.format); @@ -585,14 +602,22 @@ export async function extractVideoFramesRange( if (codecMayHaveAlpha(metadata.videoCodec)) { args.push("-c:v", decoderForCodec(metadata.videoCodec)); } - args.push("-ss", String(startTime), "-i", videoPath, "-t", String(duration)); + if (options.finalFrameOnly) { + // Output-side seek decodes from the start before selecting the final + // sample. This is intentionally reserved for the one-frame path: input + // seeking is faster, but valid unindexed transports (notably MPEG-TS with + // a negative timestamp base) can seek to EOF and emit zero frames. + args.push("-i", videoPath, "-ss", String(startTime), "-frames:v", "1"); + } else { + args.push("-ss", String(startTime), "-i", videoPath, "-t", String(duration)); + } const vfFilters: string[] = []; if (isHdr && isMacOS) { // VideoToolbox tone-maps during decode; force output to bt709 SDR format vfFilters.push("format=nv12"); } - if (!metadata.isVFR) { + if (!options.finalFrameOnly && !metadata.isVFR) { vfFilters.push(`fps=${fps}`); } if (options.sdrToHdrTransfer) { @@ -606,7 +631,9 @@ export async function extractVideoFramesRange( vfFilters.push(SDR_TO_HDR_COLORSPACE_FILTER); } if (vfFilters.length > 0) args.push("-vf", vfFilters.join(",")); - if (metadata.isVFR) args.push("-fps_mode", "cfr", "-r", String(fps)); + if (!options.finalFrameOnly && metadata.isVFR) { + args.push("-fps_mode", "cfr", "-r", String(fps)); + } args.push("-q:v", format === "jpg" ? String(Math.ceil((100 - quality) / 3)) : "0"); // Render-scoped temp frames are read once; level 1 measured 3-5x faster for ~14% larger files. @@ -718,11 +745,223 @@ export function classifyFfmpegSpawnError(error: unknown, stderr = ""): VideoSour function resolveSegmentDuration( requested: number, mediaStart: number, - metadata: VideoMetadata, + sourceDuration: number, ): number { if (Number.isFinite(requested) && requested > 0) return requested; - const sourceRemaining = metadata.durationSeconds - mediaStart; - return sourceRemaining > 0 ? sourceRemaining : metadata.durationSeconds; + const sourceRemaining = sourceDuration - mediaStart; + return sourceRemaining > 0 ? sourceRemaining : sourceDuration; +} + +/** + * Return the range that can actually produce video frames. + * + * Container duration may include a longer audio stream or mux padding. Using + * it for video extraction planning can reserve raw-frame scratch for seconds + * where no video frames exist. `extractMediaMetadata` already falls back to + * the container duration when ffprobe omits the stream duration; keep the + * explicit fallback here for callers supplying older/manual metadata. + */ +export function resolvePlayableVideoDuration(metadata: VideoMetadata): number { + return Number.isFinite(metadata.videoStreamDurationSeconds) && + metadata.videoStreamDurationSeconds > 0 + ? metadata.videoStreamDurationSeconds + : metadata.durationSeconds; +} + +export interface TimelineExtractionWindow { + compositionStart: number; + mediaStart: number; + durationSeconds: number; + /** + * Preserve the authored timeline origin and mediaStart for lookup. This is + * required when a looped visible interval crosses a source boundary and + * still needs modulo phase against the complete extracted source cycle. + */ + preserveTimelinePhase?: boolean; + /** + * Keep the authored end while rebasing start/mediaStart to the extracted + * source suffix. Non-looping lookup then holds the suffix's final frame + * through the remainder of the authored slot. + */ + preserveTimelineEnd?: boolean; + /** This window reaches a held tail and must be checked against the actual + * final decoded-frame timestamp before extraction. */ + ensureFinalFrame?: boolean; + /** Source timestamp used by FFmpeg when it differs from the logical lookup + * mediaStart (the one-frame held-tail representation). */ + extractionMediaStart?: number; + /** FFmpeg emits one decoded frame; lookup then holds that frame. */ + finalFrameOnly?: boolean; +} + +type TimelineWindowVideo = Pick & + Partial>; + +// Logical duration assigned to a one-frame held-tail representation. This is +// deliberately below any supported output frame interval: coverage expects +// one frame, while FFmpeg seeks to the separately probed real frame timestamp. +const FINAL_FRAME_LOGICAL_DURATION_SECONDS = 1e-6; + +/** + * Intersect an authored slot with the render timeline, then select the + * smallest playable source range that preserves timeline lookup semantics. + * + * A finite authored slot can outlive the source. In that case FFmpeg should + * still extract at most one source range: lookup either wraps that range for + * loops or holds its final frame for non-looping video. Keeping the authored + * timeline origin separate from the extracted range is what makes both + * behaviours survive the source-duration cap. + */ +export function resolveTimelineExtractionWindow( + video: TimelineWindowVideo, + resolvedDuration: number, + timelineEnd?: number, + sourceDuration?: number, +): TimelineExtractionWindow { + if (timelineEnd === undefined) { + return { + compositionStart: video.start, + mediaStart: video.mediaStart, + durationSeconds: resolvedDuration, + }; + } + if (!Number.isFinite(timelineEnd)) { + throw new Error(`Video extraction timelineEnd must be finite; got ${String(timelineEnd)}`); + } + const compositionStart = Math.max(0, video.start); + const trimmedPreroll = compositionStart - video.start; + const timelineDuration = Math.max(0, timelineEnd - compositionStart); + // Infinity means "natural source duration", not an authored infinite slot. + // Explicit finite slots may outlive the source (loop or held tail), while an + // omitted duration remains source-bounded exactly like the browser runtime. + const resolvedVisibleDuration = resolvedDuration - trimmedPreroll; + const visibleDuration = Math.max(0, Math.min(resolvedVisibleDuration, timelineDuration)); + let mediaStart = video.mediaStart + trimmedPreroll; + if (visibleDuration > 0 && sourceDuration !== undefined) { + const sourceRemaining = Math.max(0, sourceDuration - video.mediaStart); + if (sourceRemaining > 0 && video.loop && Number.isFinite(video.end)) { + const phaseOffset = trimmedPreroll % sourceRemaining; + const phaseRemaining = sourceRemaining - phaseOffset; + // The element visibility contract includes its end boundary. Preserve a + // complete cycle on equality as well, otherwise a rebased suffix would + // wrap to its own first frame instead of the source cycle's first frame. + if (visibleDuration >= phaseRemaining) { + return { + compositionStart: video.start, + mediaStart: video.mediaStart, + durationSeconds: sourceRemaining, + preserveTimelinePhase: true, + }; + } + mediaStart = video.mediaStart + phaseOffset; + } else if (sourceRemaining > 0) { + const sourceVisibleAfterPreroll = Math.max(0, sourceRemaining - trimmedPreroll); + if (visibleDuration <= sourceVisibleAfterPreroll) { + return { + compositionStart, + mediaStart, + durationSeconds: visibleDuration, + }; + } + + // The visible interval enters (or is entirely inside) the held tail. + // Extract the visible source suffix. If preroll is already at/past the + // final decoded timestamp, the async resolver below replaces this tiny + // provisional suffix with one exact final frame. + const extractionDuration = Math.min( + sourceRemaining, + Math.max(sourceVisibleAfterPreroll, FINAL_FRAME_LOGICAL_DURATION_SECONDS), + ); + const extractionOffset = sourceRemaining - extractionDuration; + return { + compositionStart: video.start + extractionOffset, + mediaStart: video.mediaStart + extractionOffset, + durationSeconds: extractionDuration, + preserveTimelineEnd: true, + ensureFinalFrame: true, + }; + } + } + return { + compositionStart, + mediaStart, + durationSeconds: visibleDuration, + }; +} + +/** + * Replace a held-tail suffix that starts at/after the final decoded timestamp + * with one exact frame. This keeps raw HDR scratch O(one frame) without + * assuming a one-second seek window contains a CFR/VFR timestamp. + */ +export async function resolveFinalFrameExtractionWindow( + videoPath: string, + video: TimelineWindowVideo, + metadata: VideoMetadata, + window: TimelineExtractionWindow, + signal?: AbortSignal, +): Promise { + if (!window.ensureFinalFrame) return window; + const playableDuration = resolvePlayableVideoDuration(metadata); + const finalFrameTimestamp = await extractFinalVideoFrameTimestamp( + videoPath, + { + videoStreamDurationSeconds: playableDuration, + videoStreamStartSeconds: metadata.videoStreamStartSeconds, + }, + signal, + ); + if (window.mediaStart < finalFrameTimestamp - 1e-9) return window; + + const sourceRemaining = playableDuration - video.mediaStart; + const logicalDuration = Math.min(sourceRemaining, FINAL_FRAME_LOGICAL_DURATION_SECONDS); + return { + compositionStart: Math.max(0, video.start), + mediaStart: playableDuration - logicalDuration, + extractionMediaStart: finalFrameTimestamp, + durationSeconds: logicalDuration, + preserveTimelineEnd: true, + finalFrameOnly: true, + }; +} + +/** Resolve source duration first, then intersect it with the render timeline. */ +export function resolveVideoExtractionWindow( + video: TimelineWindowVideo, + metadata: VideoMetadata, + timelineEnd?: number, +): TimelineExtractionWindow { + const playableDuration = resolvePlayableVideoDuration(metadata); + if (!(playableDuration > 0)) { + throw new VideoSourceExtractionError( + "invalid_media", + false, + "Video source has no positive duration", + `Playable video stream duration is ${playableDuration}s`, + ); + } + if (video.mediaStart >= playableDuration) { + throw new VideoSourceExtractionError( + "media_start_out_of_range", + false, + "Video media start is outside the source duration", + `Video media start ${video.mediaStart}s is outside playable video duration ${playableDuration}s`, + ); + } + const resolvedDuration = resolveSegmentDuration( + video.end - video.start, + video.mediaStart, + playableDuration, + ); + return resolveTimelineExtractionWindow(video, resolvedDuration, timelineEnd, playableDuration); +} + +export function resolveVideoExtractionDuration( + video: TimelineWindowVideo, + metadata: VideoMetadata, + timelineEnd?: number, +): number { + return resolveVideoExtractionWindow(video, metadata, timelineEnd).durationSeconds; } /** @@ -761,6 +1000,8 @@ type PreparedExtraction = { index: number; metadata: VideoMetadata; videoDuration: number; + extractionMediaStart: number; + finalFrameOnly: boolean; format: CacheFrameFormat; sdrToHdrTransfer?: HdrTransfer; dedupeKey: string; @@ -831,7 +1072,13 @@ function linkOrCopyFrame(src: string, dest: string): void { } function supersetGroupingKey(work: PreparedExtraction, fps: number): string { - return [work.videoPath, String(fps), work.format, work.sdrToHdrTransfer ?? ""].join("\0"); + return [ + work.videoPath, + String(fps), + work.format, + work.sdrToHdrTransfer ?? "", + work.finalFrameOnly ? "final" : "range", + ].join("\0"); } function isIntegralFrameOffset(offsetSeconds: number, fps: number): boolean { @@ -854,6 +1101,7 @@ function buildSupersetGroup( fps: number, ): SupersetGroupPlan | null { if (misses.length < 2) return null; + if (misses.some(({ work }) => work.finalFrameOnly)) return null; const baseStart = Math.min(...misses.map(({ work }) => work.video.mediaStart)); if (!misses.every(({ work }) => isIntegralFrameOffset(work.video.mediaStart - baseStart, fps))) { return null; @@ -1029,6 +1277,11 @@ export async function extractAllVideoFrames( >, compiledDir?: string, ): Promise { + if (options.timelineEnd !== undefined && !Number.isFinite(options.timelineEnd)) { + throw new Error( + `Video extraction timelineEnd must be finite; got ${String(options.timelineEnd)}`, + ); + } const startTime = Date.now(); const extracted: ExtractedFrames[] = []; const errors: VideoExtractionFailure[] = []; @@ -1062,6 +1315,7 @@ export async function extractAllVideoFrames( const warnedSrcs = new Set(); for (const video of videos) { if (signal?.aborted) break; + if (options.timelineEnd !== undefined && video.start >= options.timelineEnd) continue; try { let videoPath = video.src; if (!isHttpUrl(videoPath)) { @@ -1113,10 +1367,11 @@ export async function extractAllVideoFrames( breakdown.resolveMs = Date.now() - phase1Start; // Snapshot the pre-preflight key inputs so the extraction cache keys on the - // user-visible source (original path, original mediaStart, original segment - // bounds) rather than the workDir-local normalized file produced by the + // user-visible source path rather than the + // workDir-local normalized file produced by the // HDR preflight. Without this, every render would write a new // normalized file with a fresh mtime → fresh cache key → perpetual misses. + // Phase 3 updates mediaStart after trimming any invisible negative preroll. const cacheKeyInputs = resolvedVideos.map(({ video, videoPath }) => { const stat = readKeyStat(videoPath); // Missing files return null — skip the cache path for that entry. The @@ -1129,8 +1384,6 @@ export async function extractAllVideoFrames( mtimeMs: stat.mtimeMs, size: stat.size, mediaStart: video.mediaStart, - start: video.start, - end: video.end, }; }); @@ -1217,12 +1470,13 @@ export async function extractAllVideoFrames( // Guard against mediaStart past EOF — FFmpeg's `-ss` silently produces // a 0-byte file when seeking beyond the source duration, and the // downstream extractor then points at a broken input. - if (entry.video.mediaStart >= metadata.durationSeconds) { + const playableDuration = resolvePlayableVideoDuration(metadata); + if (entry.video.mediaStart >= playableDuration) { errors.push({ videoId: entry.video.id, kind: "media_start_out_of_range", retryable: false, - error: `SDR→HDR conversion skipped: mediaStart (${entry.video.mediaStart}s) ≥ source duration (${metadata.durationSeconds}s)`, + error: `SDR→HDR conversion skipped: mediaStart (${entry.video.mediaStart}s) ≥ playable video duration (${playableDuration}s)`, }); hdrSkippedIndices.add(i); continue; @@ -1289,12 +1543,20 @@ export async function extractAllVideoFrames( }; } - type PreparedExtractionResult = { work: PreparedExtraction } | { error: VideoExtractionFailure }; + type PreparedExtractionResult = + | { work: PreparedExtraction } + | { error: VideoExtractionFailure } + | { skipped: true }; type ExtractionOutcome = { result: ExtractedFrames } | { error: VideoExtractionFailure }; function scopedExtractionOptions(work: PreparedExtraction): ExtractionOptions { - return { ...options, format: work.format, sdrToHdrTransfer: work.sdrToHdrTransfer }; + return { + ...options, + format: work.format, + sdrToHdrTransfer: work.sdrToHdrTransfer, + finalFrameOnly: work.finalFrameOnly, + }; } function rehydratePublishedCache(work: PreparedExtraction, target: CacheMissTarget) { @@ -1312,21 +1574,18 @@ export async function extractAllVideoFrames( if (!cacheRootDir) return { work }; const keyInput = cacheKeyInputs[work.index]; if (!keyInput) return { work }; - const transform = work.sdrToHdrTransfer - ? sdrToHdrTransformKey(work.sdrToHdrTransfer) - : undefined; + const transformParts = [ + work.sdrToHdrTransfer ? sdrToHdrTransformKey(work.sdrToHdrTransfer) : undefined, + work.finalFrameOnly ? "final-frame" : undefined, + ].filter((part): part is string => part !== undefined); + const transform = transformParts.length > 0 ? transformParts.join("+") : undefined; - const keyDuration = resolveSegmentDuration( - keyInput.end - keyInput.start, - keyInput.mediaStart, - work.metadata, - ); const lookup = lookupCacheEntry(cacheRootDir, { videoPath: keyInput.videoPath, mtimeMs: keyInput.mtimeMs, size: keyInput.size, mediaStart: keyInput.mediaStart, - duration: keyDuration, + duration: work.videoDuration, fps: options.fps, format: work.format, transform, @@ -1356,7 +1615,7 @@ export async function extractAllVideoFrames( extractVideoFramesRange( work.videoPath, work.video.id, - work.video.mediaStart, + work.extractionMediaStart, work.videoDuration, scopedExtractionOptions(work), signal, @@ -1382,7 +1641,7 @@ export async function extractAllVideoFrames( extractVideoFramesRange( work.videoPath, work.video.id, - work.video.mediaStart, + work.extractionMediaStart, work.videoDuration, scopedExtractionOptions(work), signal, @@ -1516,18 +1775,33 @@ export async function extractAllVideoFrames( } try { const metadata = videoMetadata[index] ?? (await extractMediaMetadata(videoPath)); - const videoDuration = resolveSegmentDuration( - video.end - video.start, - video.mediaStart, + const initialWindow = resolveVideoExtractionWindow(video, metadata, options.timelineEnd); + const window = await resolveFinalFrameExtractionWindow( + videoPath, + video, metadata, + initialWindow, + signal, ); - if (video.end - video.start !== videoDuration) { - video.end = video.start + videoDuration; + const videoDuration = window.durationSeconds; + if (videoDuration <= 0) { + return { skipped: true }; } + if (!window.preserveTimelinePhase) { + video.start = window.compositionStart; + if (!window.preserveTimelineEnd) { + video.end = window.compositionStart + videoDuration; + } + video.mediaStart = window.mediaStart; + } + const keyInput = cacheKeyInputs[index]; + const extractionMediaStart = window.extractionMediaStart ?? window.mediaStart; + if (keyInput) keyInput.mediaStart = extractionMediaStart; const format = resolveFrameFormat(metadata, options.format); const sdrToHdrTransfer = sdrToHdrTransfers[index]; - const dedupeKey = `${videoPath}\0${video.mediaStart}\0${videoDuration}\0${options.fps}\0${format}\0${sdrToHdrTransfer ?? ""}`; + const finalFrameOnly = window.finalFrameOnly === true; + const dedupeKey = `${videoPath}\0${extractionMediaStart}\0${videoDuration}\0${options.fps}\0${format}\0${sdrToHdrTransfer ?? ""}\0${finalFrameOnly ? "final" : "range"}`; return { work: { @@ -1536,6 +1810,8 @@ export async function extractAllVideoFrames( index, metadata, videoDuration, + extractionMediaStart, + finalFrameOnly, format, sdrToHdrTransfer, dedupeKey, @@ -1581,11 +1857,20 @@ export async function extractAllVideoFrames( for (const [key, outcome] of groupOutcomes) uniqueOutcomes.set(key, outcome); } - const results: ExtractionOutcome[] = preparedExtractions.map((prepared) => { - if ("error" in prepared) return prepared; + const results: ExtractionOutcome[] = []; + for (const prepared of preparedExtractions) { + if ("skipped" in prepared) continue; + if ("error" in prepared) { + results.push(prepared); + continue; + } const outcome = uniqueOutcomes.get(prepared.work.dedupeKey); - if (!outcome) - return { error: extractionError(prepared.work.video.id, "missing extraction result") }; + if (!outcome) { + results.push({ + error: extractionError(prepared.work.video.id, "missing extraction result"), + }); + continue; + } if ("error" in outcome) { // A shared (deduped/superset) failure fans out to every element with the // same key; annotate followers with the leader's videoId so N copies of @@ -1594,17 +1879,18 @@ export async function extractAllVideoFrames( const message = isFollower ? `[shared extraction, leader ${outcome.error.videoId}] ${outcome.error.error}` : outcome.error.error; - return { + results.push({ error: { videoId: prepared.work.video.id, kind: outcome.error.kind, retryable: outcome.error.retryable, error: message, }, - }; + }); + continue; } - return { result: { ...outcome.result, videoId: prepared.work.video.id } }; - }); + results.push({ result: { ...outcome.result, videoId: prepared.work.video.id } }); + } breakdown.extractMs = Date.now() - phase3Start; @@ -1653,7 +1939,7 @@ function getFrameIndexAtTime( ): number | null { let localTime = globalTime - videoStart; if (localTime < 0) return null; - const loopDuration = Math.max(0, extracted.metadata.durationSeconds - mediaStart); + const loopDuration = Math.max(0, resolvePlayableVideoDuration(extracted.metadata) - mediaStart); if (loop && loopDuration > 0 && localTime >= loopDuration) { localTime %= loopDuration; } diff --git a/packages/engine/src/utils/ffprobe.test.ts b/packages/engine/src/utils/ffprobe.test.ts index ad9727098..256cfa1a9 100644 --- a/packages/engine/src/utils/ffprobe.test.ts +++ b/packages/engine/src/utils/ffprobe.test.ts @@ -972,6 +972,156 @@ describe("AAC duration refinement must never fail or distort the call", () => { }); }); +describe("final video frame timestamp probes", () => { + afterEach(() => { + vi.resetModules(); + vi.doUnmock("child_process"); + }); + + it("normalizes absolute frame PTS by the selected video stream start", async () => { + const { spawn, calls } = createSpawnSpy([ + { + kind: "exit", + code: 0, + stdout: JSON.stringify({ + streams: [ + { + codec_type: "video", + codec_name: "h264", + width: 64, + height: 64, + duration: "3", + start_time: "5", + r_frame_rate: "1/1", + avg_frame_rate: "1/1", + }, + ], + format: { duration: "8" }, + }), + }, + { kind: "exit", code: 0, stdout: "5.000000,\n6.000000\n7.000000\n" }, + ]); + vi.resetModules(); + vi.doMock("child_process", () => ({ spawn })); + const { extractFinalVideoFrameTimestamp, extractMediaMetadata } = await import("./ffprobe.js"); + + const metadata = await extractMediaMetadata("/tmp/nonzero-start.mp4"); + expect(metadata.videoStreamDurationSeconds).toBe(3); + expect(metadata.videoStreamStartSeconds).toBe(5); + await expect(extractFinalVideoFrameTimestamp("/tmp/nonzero-start.mp4", metadata)).resolves.toBe( + 2, + ); + + const intervalIndex = calls[1]?.args.indexOf("-read_intervals") ?? -1; + expect(calls[1]?.args[intervalIndex + 1]).toBe("7%8"); + }); + + it("falls back to a bounded-output full scan when a transport cannot interval-seek", async () => { + const { spawn, calls } = createSpawnSpy([ + { kind: "exit", code: 0, stdout: "" }, + { kind: "exit", code: 0, stdout: "-2.000000,\n-1.000000\n0.000000\n" }, + ]); + vi.resetModules(); + vi.doMock("child_process", () => ({ spawn })); + const { extractFinalVideoFrameTimestamp } = await import("./ffprobe.js"); + + await expect( + extractFinalVideoFrameTimestamp("/tmp/unindexed-negative-base.ts", { + videoStreamDurationSeconds: 3, + videoStreamStartSeconds: -2, + }), + ).resolves.toBe(2); + + const intervalIndex = calls[0]?.args.indexOf("-read_intervals") ?? -1; + expect(calls[0]?.args[intervalIndex + 1]).toBe("0%1"); + expect(calls[1]?.args).not.toContain("-read_intervals"); + }); + + it("does not share a caller-cancellable probe across render consumers", async () => { + type KillableFakeProc = FakeProc & { kill: (signal?: NodeJS.Signals) => boolean }; + const processes: KillableFakeProc[] = []; + const spawn = () => { + const proc = new EventEmitter() as KillableFakeProc; + proc.stdout = new EventEmitter(); + proc.stderr = new EventEmitter(); + proc.kill = vi.fn(() => { + process.nextTick(() => proc.emit("close", null, "SIGTERM")); + return true; + }); + processes.push(proc); + process.nextTick(() => proc.emit("spawn")); + return proc; + }; + vi.resetModules(); + vi.doMock("child_process", () => ({ spawn })); + const { extractFinalVideoFrameTimestamp } = await import("./ffprobe.js"); + const metadata = { videoStreamDurationSeconds: 3, videoStreamStartSeconds: 0 }; + const firstController = new AbortController(); + const secondController = new AbortController(); + + const first = extractFinalVideoFrameTimestamp( + "/tmp/shared-source.mp4", + metadata, + firstController.signal, + ); + const second = extractFinalVideoFrameTimestamp( + "/tmp/shared-source.mp4", + metadata, + secondController.signal, + ); + expect(processes).toHaveLength(2); + + firstController.abort(); + await expect(first).rejects.toThrow(/ffprobe abort/); + expect(processes[0]?.kill).toHaveBeenCalledWith("SIGTERM"); + expect(processes[1]?.kill).not.toHaveBeenCalled(); + + processes[1]?.stdout.emit("data", Buffer.from("2.000000\n")); + processes[1]?.emit("close", 0, null); + await expect(second).resolves.toBe(2); + + expect(secondController.signal.aborted).toBe(false); + }); + + it("deduplicates the interval and fallback chain within one cancellation scope", async () => { + const { spawn, calls } = createSpawnSpy([ + { kind: "exit", code: 0, stdout: "" }, + { kind: "exit", code: 0, stdout: "-2.000000\n-1.000000\n0.000000\n" }, + ]); + vi.resetModules(); + vi.doMock("child_process", () => ({ spawn })); + const { extractFinalVideoFrameTimestamp } = await import("./ffprobe.js"); + const metadata = { videoStreamDurationSeconds: 3, videoStreamStartSeconds: -2 }; + const signal = new AbortController().signal; + + await expect( + Promise.all([ + extractFinalVideoFrameTimestamp("/tmp/repeated-held-tail.ts", metadata, signal), + extractFinalVideoFrameTimestamp("/tmp/repeated-held-tail.ts", metadata, signal), + ]), + ).resolves.toEqual([2, 2]); + + expect(calls).toHaveLength(2); + expect(calls[0]?.args).toContain("-read_intervals"); + expect(calls[1]?.args).not.toContain("-read_intervals"); + }); + + it("still deduplicates cancellation-independent probes", async () => { + const { spawn, calls } = createSpawnSpy([{ kind: "exit", code: 0, stdout: "2.000000\n" }]); + vi.resetModules(); + vi.doMock("child_process", () => ({ spawn })); + const { extractFinalVideoFrameTimestamp } = await import("./ffprobe.js"); + const metadata = { videoStreamDurationSeconds: 3, videoStreamStartSeconds: 0 }; + + await Promise.all([ + extractFinalVideoFrameTimestamp("/tmp/shared-source.mp4", metadata), + extractFinalVideoFrameTimestamp("/tmp/shared-source.mp4", metadata), + ]); + + expect(calls).toHaveLength(1); + }); +}); + describe("runFfprobe process and stream handling", () => { afterEach(() => { vi.resetModules(); diff --git a/packages/engine/src/utils/ffprobe.ts b/packages/engine/src/utils/ffprobe.ts index da10672fb..72a06f5b1 100644 --- a/packages/engine/src/utils/ffprobe.ts +++ b/packages/engine/src/utils/ffprobe.ts @@ -52,6 +52,7 @@ async function runFfprobe( filePath: string, argsWithoutInput: string[], signal?: AbortSignal, + stdoutOptions?: { retainTail?: boolean; maxChars?: number }, ): Promise { // `--` stops option parsing so a path like "-intro.mp4" is a filename, but // it does NOT cover a path of exactly "-": ffprobe rewrites that to `fd:` @@ -79,6 +80,7 @@ async function runFfprobe( const decoder = new StringDecoder("utf8"); let stdout = ""; let stdoutTruncated = false; + const stdoutMaxChars = stdoutOptions?.maxChars ?? FFPROBE_STDOUT_MAX_CHARS; proc.stdout.on("data", (data: Buffer) => { // stderr is capped by ManagedChildProcess; stdout had no bound at all, and // analyzeKeyframeIntervals emits one line per frame — an all-intra ProRes @@ -87,9 +89,13 @@ async function runFfprobe( stdout += decoder.write(data); // Checked AFTER appending: a single chunk can already exceed the bound, // so a pre-append check only ever stops the second one. - if (stdout.length > FFPROBE_STDOUT_MAX_CHARS) { - stdoutTruncated = true; - stdout = ""; + if (stdout.length > stdoutMaxChars) { + if (stdoutOptions?.retainTail) { + stdout = stdout.slice(-stdoutMaxChars); + } else { + stdoutTruncated = true; + stdout = ""; + } } }); const managed = new ManagedChildProcess(proc, { @@ -101,7 +107,7 @@ async function runFfprobe( stdout += decoder.end(); if (stdoutTruncated) { throw new Error( - `[FFmpeg] ffprobe output exceeded ${FFPROBE_STDOUT_MAX_CHARS} characters; refusing to parse a truncated result.`, + `[FFmpeg] ffprobe output exceeded ${stdoutMaxChars} characters; refusing to parse a truncated result.`, ); } if (outcome.reason === "spawn_error") { @@ -135,6 +141,11 @@ function parseProbeJson(stdout: string): FFProbeOutput { } const videoMetadataCache = new Map>(); +const finalVideoFrameTimestampCache = new Map>(); +const finalVideoFrameTimestampSignalCaches = new WeakMap< + AbortSignal, + Map> +>(); const audioMetadataCache = new Map>(); // FFmpeg's built-in AAC encoder emits AAC-LC, which has 1024 samples per packet. const AAC_LC_SAMPLES_PER_PACKET = 1024; @@ -151,6 +162,11 @@ export interface VideoColorSpace { export interface VideoMetadata { durationSeconds: number; videoStreamDurationSeconds: number; + /** Absolute presentation timestamp at which the selected video stream + * starts. FFmpeg input seeks are relative to this point, while ffprobe frame + * timestamps are absolute, so callers crossing those APIs must normalize by + * this value. Absent only in legacy/manually-constructed metadata. */ + videoStreamStartSeconds?: number; width: number; height: number; fps: number; @@ -185,6 +201,7 @@ interface FFProbeStream { width?: number; height?: number; duration?: string; + start_time?: string; nb_frames?: string; nb_read_packets?: string; pix_fmt?: string; @@ -478,6 +495,7 @@ export async function extractMediaMetadata(filePath: string): Promise 0 ? streamDuration : containerDuration, + videoStreamStartSeconds: streamStart, width: videoStream.width || stillImage()?.width || 0, height: videoStream.height || stillImage()?.height || 0, fps, @@ -552,6 +573,93 @@ export async function extractMediaMetadata(filePath: string): Promise, + signal?: AbortSignal, +): Promise { + const videoDurationSeconds = metadata.videoStreamDurationSeconds; + const candidateStreamStart = metadata.videoStreamStartSeconds ?? 0; + const videoStreamStartSeconds = Number.isFinite(candidateStreamStart) ? candidateStreamStart : 0; + const cacheKey = `${filePath}\0${String(videoStreamStartSeconds)}\0${String(videoDurationSeconds)}`; + // A caller-owned abort signal cannot safely own a globally shared process + // promise: aborting one render would fail unrelated consumers. Calls in the + // SAME cancellation scope should still share the expensive interval + + // fallback chain, though — duplicate held-tail elements in one render carry + // the same signal and otherwise fan out N full-file scans before extraction + // dedupe. Weakly key the cache by cancellation owner to preserve both + // aggregate work bounds and cross-render isolation. + let probeCache = finalVideoFrameTimestampCache; + if (signal) { + probeCache = finalVideoFrameTimestampSignalCaches.get(signal) ?? new Map(); + finalVideoFrameTimestampSignalCaches.set(signal, probeCache); + } + const cached = probeCache.get(cacheKey); + if (cached) return cached; + + const probePromise = (async () => { + if (!(videoDurationSeconds > 0) || !Number.isFinite(videoDurationSeconds)) { + throw new Error( + `[FFmpeg] Cannot locate final video frame for invalid duration ${String(videoDurationSeconds)}`, + ); + } + const streamEnd = videoStreamStartSeconds + videoDurationSeconds; + const intervalStart = Math.max(videoStreamStartSeconds, streamEnd - 1); + const parseFinalTimestamp = (stdout: string): number | undefined => + stdout + .split("\n") + .map((line) => line.trim().split(",")[0]?.trim() ?? "") + .filter((value) => value.length > 0) + .map((value) => Number(value)) + .filter((timestamp) => Number.isFinite(timestamp)) + .at(-1); + const probe = async (readInterval?: string): Promise => { + const args = [ + "-select_streams", + "v:0", + "-show_entries", + "frame=best_effort_timestamp_time", + "-of", + "csv=p=0", + ]; + if (readInterval) args.splice(2, 0, "-read_intervals", readInterval); + const stdout = await runFfprobe(filePath, args, signal, { + retainTail: true, + maxChars: 64 * 1024, + }); + return parseFinalTimestamp(stdout); + }; + const timestamp = + (await probe(`${intervalStart}%${streamEnd}`)) ?? (await probe(/* full scan */)); + if (timestamp === undefined) { + throw new Error("[FFmpeg] ffprobe found no decodable final video frame"); + } + return Math.min(Math.max(timestamp - videoStreamStartSeconds, 0), videoDurationSeconds); + })(); + + probeCache.set(cacheKey, probePromise); + probePromise.catch(() => { + if (probeCache.get(cacheKey) === probePromise) { + probeCache.delete(cacheKey); + } + }); + return probePromise; +} + /** * @deprecated Use `extractMediaMetadata` — this name is kept for backward * compatibility with consumers that imported the original video-only name diff --git a/packages/producer/src/server.outputDynamicRange.test.ts b/packages/producer/src/server.outputDynamicRange.test.ts new file mode 100644 index 000000000..469e8746b --- /dev/null +++ b/packages/producer/src/server.outputDynamicRange.test.ts @@ -0,0 +1,89 @@ +import { Hono } from "hono"; +import { beforeEach, describe, expect, it, vi } from "vitest"; + +const capturedRenderConfigs = vi.hoisted(() => new Array>()); + +vi.mock("./services/renderOrchestrator.js", () => { + class RenderCancelledError extends Error {} + + return { + RenderCancelledError, + createRenderJob: (config: Record) => { + capturedRenderConfigs.push(config); + return { + config, + progress: 0, + currentStage: "queued", + framesRendered: 0, + totalFrames: 0, + warnings: [], + }; + }, + executeRenderJob: async (job: Record) => { + job.outcome = "completed"; + job.currentStage = "complete"; + }, + }; +}); + +import { createRenderHandlers } from "./server.js"; + +function createInternalStreamingApp(): Hono { + const app = new Hono(); + const handlers = createRenderHandlers({ + getRequestId: () => "hdr-mode-test", + maxConcurrentRenders: 1, + }); + app.post("/v1/render-stream", handlers.renderStream); + return app; +} + +function requestRender(overrides: Record) { + return createInternalStreamingApp().request("/v1/render-stream", { + method: "POST", + headers: { "content-type": "application/json" }, + body: JSON.stringify({ html: "", ...overrides }), + }); +} + +describe("POST /v1/render-stream — outputDynamicRange", () => { + beforeEach(() => capturedRenderConfigs.splice(0)); + + it.each([ + ["auto", "auto"], + ["hdr", "force-hdr"], + ["sdr", "force-sdr"], + ] as const)( + "maps %s through createRenderRequest to internal hdrMode %s", + async (outputDynamicRange, hdrMode) => { + const response = await requestRender({ outputDynamicRange }); + + expect(response.status).toBe(200); + expect(await response.text()).toContain('"type":"complete"'); + expect(capturedRenderConfigs).toHaveLength(1); + expect(capturedRenderConfigs[0]?.hdrMode).toBe(hdrMode); + }, + ); + + it("rejects an invalid mode before creating a render job", async () => { + const response = await requestRender({ outputDynamicRange: "force-sdr" }); + + expect(response.status).toBe(200); + expect(await response.text()).toContain( + 'outputDynamicRange must be one of: \\"auto\\", \\"hdr\\", \\"sdr\\"', + ); + expect(capturedRenderConfigs).toHaveLength(0); + }); + + it("accepts the matching legacy field during rolling deployment", async () => { + const response = await requestRender({ + outputDynamicRange: "sdr", + hdrMode: "force-sdr", + }); + + expect(response.status).toBe(200); + expect(await response.text()).toContain('"type":"complete"'); + expect(capturedRenderConfigs).toHaveLength(1); + expect(capturedRenderConfigs[0]?.hdrMode).toBe("force-sdr"); + }); +}); diff --git a/packages/producer/src/server.test.ts b/packages/producer/src/server.test.ts index 813a2fba1..d6509d8cf 100644 --- a/packages/producer/src/server.test.ts +++ b/packages/producer/src/server.test.ts @@ -46,6 +46,33 @@ describe("parseRenderOptions — render strictness", () => { }); }); +describe("parseRenderOptions — outputDynamicRange", () => { + it.each(["auto", "hdr", "sdr"] as const)("forwards %s", (outputDynamicRange) => { + expect(parseRenderOptions({ outputDynamicRange }).outputDynamicRange).toBe(outputDynamicRange); + }); + + it("drops invalid values from the lenient parser", () => { + expect( + parseRenderOptions({ outputDynamicRange: "force-sdr" }).outputDynamicRange, + ).toBeUndefined(); + expect(parseRenderOptions({ outputDynamicRange: true }).outputDynamicRange).toBeUndefined(); + }); + + it.each([ + ["auto", "auto"], + ["force-hdr", "hdr"], + ["force-sdr", "sdr"], + ] as const)("maps legacy hdrMode %s to %s", (hdrMode, outputDynamicRange) => { + expect(parseRenderOptions({ hdrMode }).outputDynamicRange).toBe(outputDynamicRange); + }); + + it("prefers the canonical field when both equivalent fields are present", () => { + expect( + parseRenderOptions({ outputDynamicRange: "sdr", hdrMode: "force-sdr" }).outputDynamicRange, + ).toBe("sdr"); + }); +}); + describe("prepareRenderBody — validation", () => { it.each(["", " "])( "treats an empty projectDir as absent and uses inline HTML", @@ -67,6 +94,29 @@ describe("prepareRenderBody — validation", () => { expect((result as { error: string }).error).toContain("variables must be a JSON object"); }); + it("rejects an explicitly-supplied invalid outputDynamicRange", async () => { + const result = await prepareRenderBody({ + outputDynamicRange: "force-sdr", + html: "", + }); + expect(result).toHaveProperty("error"); + expect((result as { error: string }).error).toContain( + 'outputDynamicRange must be one of: "auto", "hdr", "sdr"', + ); + }); + + it("rejects conflicting canonical and legacy policies", async () => { + const result = await prepareRenderBody({ + outputDynamicRange: "sdr", + hdrMode: "force-hdr", + html: "", + }); + expect(result).toHaveProperty("error"); + expect((result as { error: string }).error).toContain( + "outputDynamicRange and legacy hdrMode must describe the same output policy", + ); + }); + it("rejects an explicitly-supplied invalid outputResolution", async () => { const result = await prepareRenderBody({ outputResolution: "8k", html: "" }); expect(result).toHaveProperty("error"); diff --git a/packages/producer/src/server.ts b/packages/producer/src/server.ts index 75611b262..6787f8a96 100644 --- a/packages/producer/src/server.ts +++ b/packages/producer/src/server.ts @@ -84,6 +84,7 @@ interface RenderInput { quality: "draft" | "standard" | "high"; format?: "mp4" | "webm" | "mov"; videoFrameFormat?: RenderConfig["videoFrameFormat"]; + outputDynamicRange?: "auto" | "hdr" | "sdr"; workers?: number; useGpu: boolean; debug: boolean; @@ -158,6 +159,28 @@ function parseServerFormat(value: unknown): RenderInput["format"] { return value === "mp4" || value === "webm" || value === "mov" ? value : undefined; } +function parseServerOutputDynamicRange(value: unknown): RenderInput["outputDynamicRange"] { + return value === "auto" || value === "hdr" || value === "sdr" ? value : undefined; +} + +function parseLegacyServerHdrMode(value: unknown): RenderConfig["hdrMode"] { + return value === "auto" || value === "force-hdr" || value === "force-sdr" ? value : undefined; +} + +function fromRenderHdrMode(hdrMode: RenderConfig["hdrMode"]): RenderInput["outputDynamicRange"] { + if (hdrMode === "force-hdr") return "hdr"; + if (hdrMode === "force-sdr") return "sdr"; + return hdrMode; +} + +function toRenderHdrMode( + outputDynamicRange: RenderInput["outputDynamicRange"], +): RenderConfig["hdrMode"] { + if (outputDynamicRange === "hdr") return "force-hdr"; + if (outputDynamicRange === "sdr") return "force-sdr"; + return outputDynamicRange; +} + export function parseRenderOptions(body: Record): Omit { // Accept either a JSON `number` (integer fps) or a JSON `string` (rational // like "30000/1001"). Falls back to 30 fps on parse failure to preserve the @@ -176,6 +199,9 @@ export function parseRenderOptions(body: Record): Omit): Omit): string | undefi if (body.variables !== undefined && !isPlainObject(body.variables)) { return 'variables must be a JSON object keyed by variable id (e.g. {"title":"Hello"})'; } + if ( + body.outputDynamicRange !== undefined && + parseServerOutputDynamicRange(body.outputDynamicRange) === undefined + ) { + return 'outputDynamicRange must be one of: "auto", "hdr", "sdr"'; + } + const legacyHdrMode = parseLegacyServerHdrMode(body.hdrMode); + if (body.hdrMode !== undefined && legacyHdrMode === undefined) { + return 'legacy hdrMode must be one of: "auto", "force-hdr", "force-sdr"'; + } + const outputDynamicRange = parseServerOutputDynamicRange(body.outputDynamicRange); + if ( + outputDynamicRange !== undefined && + legacyHdrMode !== undefined && + outputDynamicRange !== fromRenderHdrMode(legacyHdrMode) + ) { + return "outputDynamicRange and legacy hdrMode must describe the same output policy"; + } return validateOutputResolutionOverride(body); } diff --git a/packages/producer/src/services/distributed/shared.ts b/packages/producer/src/services/distributed/shared.ts index 8db989e26..063864a2c 100644 --- a/packages/producer/src/services/distributed/shared.ts +++ b/packages/producer/src/services/distributed/shared.ts @@ -153,6 +153,13 @@ function readVideoMetadata( record.videoStreamDurationSeconds, `${field}.videoStreamDurationSeconds`, ), + // Plans written before stream-start metadata existed implicitly used the + // overwhelmingly common start-at-zero domain. Preserve that compatibility + // while carrying non-zero edit-list/transport timestamps in new plans. + videoStreamStartSeconds: + record.videoStreamStartSeconds === undefined + ? 0 + : readFiniteNumber(record.videoStreamStartSeconds, `${field}.videoStreamStartSeconds`), width: readPositiveInteger(record.width, `${field}.width`), height: readPositiveInteger(record.height, `${field}.height`), fps: readFiniteNumber(record.fps, `${field}.fps`), diff --git a/packages/producer/src/services/distributed/videoMetadata.test.ts b/packages/producer/src/services/distributed/videoMetadata.test.ts index 26e09190c..194615728 100644 --- a/packages/producer/src/services/distributed/videoMetadata.test.ts +++ b/packages/producer/src/services/distributed/videoMetadata.test.ts @@ -94,6 +94,26 @@ describe("distributed video metadata", () => { expect(sourceDerived.videos[0]?.mediaStart).toBe(1); }); + it("round-trips a non-zero video stream start and defaults legacy plans to zero", () => { + const withStart = buildPlanVideosJson({ + videos: [video()], + extracted: [ + extractedMetadata({ + metadata: { + ...extractedMetadata().metadata, + videoStreamStartSeconds: 5, + }, + }), + ], + compositionEnd: 8, + }); + expect(parsePlanVideosJson(withStart).extracted[0]?.metadata.videoStreamStartSeconds).toBe(5); + + const legacy = structuredClone(withStart); + delete legacy.extracted[0]?.metadata.videoStreamStartSeconds; + expect(parsePlanVideosJson(legacy).extracted[0]?.metadata.videoStreamStartSeconds).toBe(0); + }); + it.each([Number.NaN, Number.POSITIVE_INFINITY, 0, 2])( "fails closed when no safe composition boundary can be derived (%s)", (compositionEnd) => { diff --git a/packages/producer/src/services/hdrCompositor.ts b/packages/producer/src/services/hdrCompositor.ts index 970cba39d..8ace0d41d 100644 --- a/packages/producer/src/services/hdrCompositor.ts +++ b/packages/producer/src/services/hdrCompositor.ts @@ -139,6 +139,8 @@ export interface HdrVideoFrameSource { frameSize: number; frameCount: number; scratch: Buffer; + /** The raw file contains one playable source cycle and must wrap at EOF. */ + loop?: boolean; } export function closeHdrVideoFrameSource(source: HdrVideoFrameSource, log?: ProducerLogger): void { @@ -152,6 +154,18 @@ export function closeHdrVideoFrameSource(source: HdrVideoFrameSource, log?: Prod } } +export function resolveHdrVideoFrameIndex( + time: number, + startTime: number, + fps: number, + frameCount: number, + loop = false, +): number | null { + const frameIndex = Math.round((time - startTime) * fps); + if (frameIndex < 0 || frameCount < 1) return null; + return loop ? frameIndex % frameCount : Math.min(frameIndex, frameCount - 1); +} + // fallow-ignore-next-line complexity export function blitHdrVideoLayer( canvas: Buffer, @@ -173,14 +187,18 @@ export function blitHdrVideoLayer( return; } - // Frame index within the video. Clamp to the extracted raw frame count so - // a composition that outlives the source clip freezes on the last frame, - // matching Chrome's