From db42e60223b0fdd075bb3c3433e84bd0615725fe Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Miguel=20=C3=81ngel?= Date: Sat, 26 Sep 2026 08:52:21 +0000 Subject: [PATCH] perf(studio): a clip's filmstrip decodes 9 frames instead of 272 and reaches the clip's end (#4526) * perf(studio): build a video clip's filmstrip from its keyframes * fix(studio): a filmstrip slot keeps its own time when its keyframe is far away * perf(studio): a WebM filmstrip downloads each cluster once Each keyframe lookup now runs just before its frame is decoded, from a generator the canvas sink pulls, instead of all lookups before the first decode. A WebM lookup loads its whole cluster, so resolving every lookup first pushed the early clusters out of the source cache and the decode pass fetched them again. A cancel during a lookup ends the generator quietly so the decoder still closes; the cancel is raised after the loop. * fix(studio): a trimmed video with no reported length fills its filmstrip When the file reports no duration (a fragmented MP4), the source was assumed to end at the trim length, so a trim that starts later collapsed to zero length and every tile showed the trim start. It now ends at the trim start plus the trim length. * fix(studio): a video filmstrip reaches the end of the clip The strip requests a power of two of frames spread over the whole clip, but tile i showed frame i, so fewer tiles than frames showed only the start of the clip and more tiles repeated the strip. Each tile now shows the frame nearest its own place along the clip. --- .../player/components/VideoThumbnail.test.tsx | 13 +++ .../src/player/components/VideoThumbnail.tsx | 2 +- .../player/lib/thumbnailVideoDecoder.test.ts | 108 +++++++++++++++++- .../src/player/lib/thumbnailVideoDecoder.ts | 25 +++- 4 files changed, 136 insertions(+), 12 deletions(-) diff --git a/packages/studio/src/player/components/VideoThumbnail.test.tsx b/packages/studio/src/player/components/VideoThumbnail.test.tsx index 5e11752f85..9f76ca8715 100644 --- a/packages/studio/src/player/components/VideoThumbnail.test.tsx +++ b/packages/studio/src/player/components/VideoThumbnail.test.tsx @@ -86,6 +86,19 @@ describe("VideoThumbnail", () => { expect(host.querySelectorAll("img").length).toBeGreaterThan(0); }); + it("spreads the frames across every tile so the strip reaches the clip's end", async () => { + const urls = Array.from({ length: 8 }, (_, index) => `blob:${index}`); + vi.mocked(decodeVideoThumbnail).mockResolvedValue({ + value: { kind: "filmstrip", urls, aspect: 16 / 9 }, + weight: 256, + }); + + await render(500); + + const tiles = [...host.querySelectorAll("img")].map((img) => img.getAttribute("src")); + expect(tiles).toEqual(["blob:0", "blob:2", "blob:4", "blob:5", "blob:7"]); + }); + it("issues a single decode job for a narrow clip", async () => { vi.mocked(decodeVideoThumbnail).mockResolvedValue({ value: { kind: "image", url: "blob:poster", aspect: 16 / 9 }, diff --git a/packages/studio/src/player/components/VideoThumbnail.tsx b/packages/studio/src/player/components/VideoThumbnail.tsx index 23c6835a47..6576dcdf75 100644 --- a/packages/studio/src/player/components/VideoThumbnail.tsx +++ b/packages/studio/src/player/components/VideoThumbnail.tsx @@ -146,7 +146,7 @@ export const VideoThumbnail = memo(function VideoThumbnail({ {urls.length > 0 && (
{Array.from({ length: frameCount }, (_, index) => { - const src = urls[index % urls.length]; + const src = urls[Math.round((index * (urls.length - 1)) / Math.max(1, frameCount - 1))]; return (
null as { timestamp: number } | null); const input = { getPrimaryVideoTrack: vi.fn(), dispose, @@ -22,10 +23,25 @@ vi.mock("mediabunny", () => ({ CanvasSink: class { canvasesAtTimestamps = canvasesAtTimestamps; }, + EncodedPacketSink: class { + getKeyPacket = getKeyPacket; + }, })); +function recordDecodes(decoded: number[][]): void { + canvasesAtTimestamps.mockImplementation(async function* (timestamps: AsyncIterable) { + const run: number[] = []; + decoded.push(run); + for await (const time of timestamps) { + run.push(time); + yield { canvas: document.createElement("canvas") }; + } + }); +} + beforeEach(() => { vi.clearAllMocks(); + getKeyPacket.mockImplementation(async () => null); vi.spyOn(URL, "createObjectURL").mockReturnValueOnce("blob:one").mockReturnValueOnce("blob:two"); vi.spyOn(URL, "revokeObjectURL").mockImplementation(() => {}); HTMLCanvasElement.prototype.toBlob = function toBlob(callback) { @@ -52,17 +68,14 @@ describe("videoThumbnailTimestamps", () => { describe("decodeVideoThumbnail", () => { it("extracts sparse frames, returns object URLs, and disposes once", async () => { - const canvas = document.createElement("canvas"); - canvasesAtTimestamps.mockImplementation(async function* (timestamps: number[]) { - expect(timestamps).toEqual([2, 8]); - yield { canvas, timestamp: 2, duration: 1 }; - yield { canvas, timestamp: 8, duration: 1 }; - }); + const decoded: number[][] = []; + recordDecodes(decoded); const result = await decodeVideoThumbnail( { source: "/clip.mp4", sourceStart: 2, sourceRangeDuration: 6, frameCount: 2 }, new AbortController().signal, ); + expect(decoded).toEqual([[2, 8]]); expect(result.value).toEqual({ kind: "filmstrip", urls: ["blob:one", "blob:two"], @@ -74,6 +87,89 @@ describe("decodeVideoThumbnail", () => { expect(dispose).toHaveBeenCalledTimes(1); }); + it("decodes each strip frame at its keyframe unless that keyframe is before the clip's range", async () => { + getKeyPacket.mockImplementation(async (time) => ({ timestamp: Math.floor(time / 4) * 4 })); + const decoded: number[][] = []; + recordDecodes(decoded); + await decodeVideoThumbnail( + { source: "/clip.mp4", sourceStart: 2, sourceRangeDuration: 8, frameCount: 3 }, + new AbortController().signal, + ); + expect(decoded).toEqual([[2, 4, 8]]); + }); + + it("keeps a slot's own time when its keyframe is more than half a slot earlier", async () => { + getKeyPacket.mockImplementation(async () => ({ timestamp: 0 })); + const decoded: number[][] = []; + recordDecodes(decoded); + await decodeVideoThumbnail( + { source: "/clip.mp4", sourceStart: 0, sourceRangeDuration: 10, frameCount: 3 }, + new AbortController().signal, + ); + await decodeVideoThumbnail( + { source: "/clip.mp4", sourceStart: 0, sourceRangeDuration: 10, frameCount: 1 }, + new AbortController().signal, + ); + expect(decoded).toEqual([[0, 5, 10], [5]]); + }); + + it("looks up each keyframe just before decoding it", async () => { + const events: string[] = []; + getKeyPacket.mockImplementation(async (time) => { + events.push(`key ${time}`); + return { timestamp: time }; + }); + canvasesAtTimestamps.mockImplementation(async function* (timestamps: AsyncIterable) { + for await (const time of timestamps) { + events.push(`frame ${time}`); + yield { canvas: document.createElement("canvas") }; + } + }); + await decodeVideoThumbnail( + { source: "/clip.mp4", sourceStart: 0, sourceRangeDuration: 10, frameCount: 3 }, + new AbortController().signal, + ); + expect(events).toEqual(["key 0", "frame 0", "key 5", "frame 5", "key 10", "frame 10"]); + }); + + it("ends the decode times without throwing when cancelled during a keyframe lookup", async () => { + const controller = new AbortController(); + getKeyPacket.mockImplementation(async () => (controller.abort(), null)); + const decoded: number[] = []; + let timesEnded = false; + canvasesAtTimestamps.mockImplementation(async function* (timestamps: AsyncIterable) { + for await (const time of timestamps) { + decoded.push(time); + yield { canvas: document.createElement("canvas") }; + } + timesEnded = true; + }); + await expect( + decodeVideoThumbnail( + { source: "/clip.mp4", sourceStart: 0, sourceRangeDuration: 10, frameCount: 3 }, + controller.signal, + ), + ).rejects.toThrow("Aborted"); + expect(getKeyPacket).toHaveBeenCalledTimes(1); + expect(decoded).toEqual([]); + expect(timesEnded).toBe(true); + }); + + it("spreads a trimmed strip across its range when the file reports no duration", async () => { + input.getPrimaryVideoTrack.mockResolvedValue({ + getDisplayWidth: vi.fn(async () => 1080), + getDisplayHeight: vi.fn(async () => 1920), + getDurationFromMetadata: vi.fn(async () => null), + }); + const decoded: number[][] = []; + recordDecodes(decoded); + await decodeVideoThumbnail( + { source: "/clip.mp4", sourceStart: 5, sourceRangeDuration: 5, frameCount: 3 }, + new AbortController().signal, + ); + expect(decoded).toEqual([[5, 7.5, 10]]); + }); + it("releases input and degrades when the source has no video track", async () => { input.getPrimaryVideoTrack.mockResolvedValue(null); await expect( diff --git a/packages/studio/src/player/lib/thumbnailVideoDecoder.ts b/packages/studio/src/player/lib/thumbnailVideoDecoder.ts index 75634621f9..b892c6cc94 100644 --- a/packages/studio/src/player/lib/thumbnailVideoDecoder.ts +++ b/packages/studio/src/player/lib/thumbnailVideoDecoder.ts @@ -44,7 +44,7 @@ interface DecodedResources { interface ThumbnailCanvasSink { canvasesAtTimestamps( - timestamps: number[], + timestamps: AsyncIterable, ): AsyncIterable<{ canvas: HTMLCanvasElement | OffscreenCanvas } | null>; } @@ -77,7 +77,7 @@ function targetDimensions( async function decodeFrames( sink: ThumbnailCanvasSink, - timestamps: number[], + timestamps: AsyncIterable, signal: AbortSignal, resources: DecodedResources, ): Promise { @@ -89,6 +89,7 @@ async function decodeFrames( throwIfAborted(signal); resources.urls.push(URL.createObjectURL(blob)); } + throwIfAborted(signal); } function loadedResult( @@ -138,8 +139,12 @@ export async function decodeVideoThumbnail( } const metadataDuration = await track.getDurationFromMetadata({ skipLiveWait: true }); throwIfAborted(signal); - const sourceDuration = Math.max(0, metadataDuration ?? request.sourceRangeDuration ?? 0); - const sourceStart = Math.min(Math.max(0, request.sourceStart ?? 0), sourceDuration); + const requestedStart = Math.max(0, request.sourceStart ?? 0); + const sourceDuration = Math.max( + 0, + metadataDuration ?? requestedStart + (request.sourceRangeDuration ?? 0), + ); + const sourceStart = Math.min(requestedStart, sourceDuration); const requestedDuration = request.sourceRangeDuration ?? Math.max(0, sourceDuration - sourceStart); const duration = Math.min( @@ -151,6 +156,16 @@ export async function decodeVideoThumbnail( duration, Math.min(request.frameCount, budgets.richPreviewFrameCount), ); + const keys = new mediabunny.EncodedPacketSink(track); + const maxKeyframeLead = duration / Math.max(2, timestamps.length - 1) / 2; + async function* decodeTimesAtNearbyKeyframes() { + for (const time of timestamps) { + const key = await keys.getKeyPacket(time, { metadataOnly: true }); + if (signal.aborted) return; + const near = key && key.timestamp >= sourceStart && time - key.timestamp <= maxKeyframeLead; + yield near ? key.timestamp : time; + } + } const aspect = displayWidth / displayHeight; const target = targetDimensions(aspect, budgets); const sink = new mediabunny.CanvasSink(track, { @@ -159,7 +174,7 @@ export async function decodeVideoThumbnail( fit: request.fit ?? "cover", poolSize: 1, }); - await decodeFrames(sink, timestamps, signal, resources); + await decodeFrames(sink, decodeTimesAtNearbyKeyframes(), signal, resources); return loadedResult(resources, aspect, target.width, target.height); } catch (error) { releaseDecodedResources(resources);