perf(studio): a clip's filmstrip decodes 9 frames instead of 272 and reaches the clip's end (#4526)

* perf(studio): build a video clip's filmstrip from its keyframes

* fix(studio): a filmstrip slot keeps its own time when its keyframe is far away

* perf(studio): a WebM filmstrip downloads each cluster once

Each keyframe lookup now runs just before its frame is decoded, from a
generator the canvas sink pulls, instead of all lookups before the first
decode. A WebM lookup loads its whole cluster, so resolving every lookup
first pushed the early clusters out of the source cache and the decode
pass fetched them again. A cancel during a lookup ends the generator
quietly so the decoder still closes; the cancel is raised after the loop.

* fix(studio): a trimmed video with no reported length fills its filmstrip

When the file reports no duration (a fragmented MP4), the source was
assumed to end at the trim length, so a trim that starts later collapsed
to zero length and every tile showed the trim start. It now ends at the
trim start plus the trim length.

* fix(studio): a video filmstrip reaches the end of the clip

The strip requests a power of two of frames spread over the whole clip,
but tile i showed frame i, so fewer tiles than frames showed only the
start of the clip and more tiles repeated the strip. Each tile now shows
the frame nearest its own place along the clip.

packages/studio/src/player/components/VideoThumbnail.test.tsx

@@ -86,6 +86,19 @@ describe("VideoThumbnail", () => {
     expect(host.querySelectorAll("img").length).toBeGreaterThan(0);
   });
 
+  it("spreads the frames across every tile so the strip reaches the clip's end", async () => {
+    const urls = Array.from({ length: 8 }, (_, index) => `blob:${index}`);
+    vi.mocked(decodeVideoThumbnail).mockResolvedValue({
+      value: { kind: "filmstrip", urls, aspect: 16 / 9 },
+      weight: 256,
+    });
+
+    await render(500);
+
+    const tiles = [...host.querySelectorAll("img")].map((img) => img.getAttribute("src"));
+    expect(tiles).toEqual(["blob:0", "blob:2", "blob:4", "blob:5", "blob:7"]);
+  });
+
   it("issues a single decode job for a narrow clip", async () => {
     vi.mocked(decodeVideoThumbnail).mockResolvedValue({
       value: { kind: "image", url: "blob:poster", aspect: 16 / 9 },

packages/studio/src/player/components/VideoThumbnail.tsx

@@ -146,7 +146,7 @@ export const VideoThumbnail = memo(function VideoThumbnail({
       {urls.length > 0 && (
         <div className="absolute inset-0 flex">
           {Array.from({ length: frameCount }, (_, index) => {
-            const src = urls[index % urls.length];
+            const src = urls[Math.round((index * (urls.length - 1)) / Math.max(1, frameCount - 1))];
             return (
               <div
                 key={index}

packages/studio/src/player/lib/thumbnailVideoDecoder.test.ts

@@ -5,6 +5,7 @@ import { decodeVideoThumbnail, videoThumbnailTimestamps } from "./thumbnailVideo
 
 const dispose = vi.fn();
 const canvasesAtTimestamps = vi.fn();
+const getKeyPacket = vi.fn(async (_time: number) => null as { timestamp: number } | null);
 const input = {
   getPrimaryVideoTrack: vi.fn(),
   dispose,
@@ -22,10 +23,25 @@ vi.mock("mediabunny", () => ({
   CanvasSink: class {
     canvasesAtTimestamps = canvasesAtTimestamps;
   },
+  EncodedPacketSink: class {
+    getKeyPacket = getKeyPacket;
+  },
 }));
 
+function recordDecodes(decoded: number[][]): void {
+  canvasesAtTimestamps.mockImplementation(async function* (timestamps: AsyncIterable<number>) {
+    const run: number[] = [];
+    decoded.push(run);
+    for await (const time of timestamps) {
+      run.push(time);
+      yield { canvas: document.createElement("canvas") };
+    }
+  });
+}
+
 beforeEach(() => {
   vi.clearAllMocks();
+  getKeyPacket.mockImplementation(async () => null);
   vi.spyOn(URL, "createObjectURL").mockReturnValueOnce("blob:one").mockReturnValueOnce("blob:two");
   vi.spyOn(URL, "revokeObjectURL").mockImplementation(() => {});
   HTMLCanvasElement.prototype.toBlob = function toBlob(callback) {
@@ -52,17 +68,14 @@ describe("videoThumbnailTimestamps", () => {
 
 describe("decodeVideoThumbnail", () => {
   it("extracts sparse frames, returns object URLs, and disposes once", async () => {
-    const canvas = document.createElement("canvas");
-    canvasesAtTimestamps.mockImplementation(async function* (timestamps: number[]) {
-      expect(timestamps).toEqual([2, 8]);
-      yield { canvas, timestamp: 2, duration: 1 };
-      yield { canvas, timestamp: 8, duration: 1 };
-    });
+    const decoded: number[][] = [];
+    recordDecodes(decoded);
     const result = await decodeVideoThumbnail(
       { source: "/clip.mp4", sourceStart: 2, sourceRangeDuration: 6, frameCount: 2 },
       new AbortController().signal,
     );
 
+    expect(decoded).toEqual([[2, 8]]);
     expect(result.value).toEqual({
       kind: "filmstrip",
       urls: ["blob:one", "blob:two"],
@@ -74,6 +87,89 @@ describe("decodeVideoThumbnail", () => {
     expect(dispose).toHaveBeenCalledTimes(1);
   });
 
+  it("decodes each strip frame at its keyframe unless that keyframe is before the clip's range", async () => {
+    getKeyPacket.mockImplementation(async (time) => ({ timestamp: Math.floor(time / 4) * 4 }));
+    const decoded: number[][] = [];
+    recordDecodes(decoded);
+    await decodeVideoThumbnail(
+      { source: "/clip.mp4", sourceStart: 2, sourceRangeDuration: 8, frameCount: 3 },
+      new AbortController().signal,
+    );
+    expect(decoded).toEqual([[2, 4, 8]]);
+  });
+
+  it("keeps a slot's own time when its keyframe is more than half a slot earlier", async () => {
+    getKeyPacket.mockImplementation(async () => ({ timestamp: 0 }));
+    const decoded: number[][] = [];
+    recordDecodes(decoded);
+    await decodeVideoThumbnail(
+      { source: "/clip.mp4", sourceStart: 0, sourceRangeDuration: 10, frameCount: 3 },
+      new AbortController().signal,
+    );
+    await decodeVideoThumbnail(
+      { source: "/clip.mp4", sourceStart: 0, sourceRangeDuration: 10, frameCount: 1 },
+      new AbortController().signal,
+    );
+    expect(decoded).toEqual([[0, 5, 10], [5]]);
+  });
+
+  it("looks up each keyframe just before decoding it", async () => {
+    const events: string[] = [];
+    getKeyPacket.mockImplementation(async (time) => {
+      events.push(`key ${time}`);
+      return { timestamp: time };
+    });
+    canvasesAtTimestamps.mockImplementation(async function* (timestamps: AsyncIterable<number>) {
+      for await (const time of timestamps) {
+        events.push(`frame ${time}`);
+        yield { canvas: document.createElement("canvas") };
+      }
+    });
+    await decodeVideoThumbnail(
+      { source: "/clip.mp4", sourceStart: 0, sourceRangeDuration: 10, frameCount: 3 },
+      new AbortController().signal,
+    );
+    expect(events).toEqual(["key 0", "frame 0", "key 5", "frame 5", "key 10", "frame 10"]);
+  });
+
+  it("ends the decode times without throwing when cancelled during a keyframe lookup", async () => {
+    const controller = new AbortController();
+    getKeyPacket.mockImplementation(async () => (controller.abort(), null));
+    const decoded: number[] = [];
+    let timesEnded = false;
+    canvasesAtTimestamps.mockImplementation(async function* (timestamps: AsyncIterable<number>) {
+      for await (const time of timestamps) {
+        decoded.push(time);
+        yield { canvas: document.createElement("canvas") };
+      }
+      timesEnded = true;
+    });
+    await expect(
+      decodeVideoThumbnail(
+        { source: "/clip.mp4", sourceStart: 0, sourceRangeDuration: 10, frameCount: 3 },
+        controller.signal,
+      ),
+    ).rejects.toThrow("Aborted");
+    expect(getKeyPacket).toHaveBeenCalledTimes(1);
+    expect(decoded).toEqual([]);
+    expect(timesEnded).toBe(true);
+  });
+
+  it("spreads a trimmed strip across its range when the file reports no duration", async () => {
+    input.getPrimaryVideoTrack.mockResolvedValue({
+      getDisplayWidth: vi.fn(async () => 1080),
+      getDisplayHeight: vi.fn(async () => 1920),
+      getDurationFromMetadata: vi.fn(async () => null),
+    });
+    const decoded: number[][] = [];
+    recordDecodes(decoded);
+    await decodeVideoThumbnail(
+      { source: "/clip.mp4", sourceStart: 5, sourceRangeDuration: 5, frameCount: 3 },
+      new AbortController().signal,
+    );
+    expect(decoded).toEqual([[5, 7.5, 10]]);
+  });
+
   it("releases input and degrades when the source has no video track", async () => {
     input.getPrimaryVideoTrack.mockResolvedValue(null);
     await expect(

packages/studio/src/player/lib/thumbnailVideoDecoder.ts

@@ -44,7 +44,7 @@ interface DecodedResources {
 
 interface ThumbnailCanvasSink {
   canvasesAtTimestamps(
-    timestamps: number[],
+    timestamps: AsyncIterable<number>,
   ): AsyncIterable<{ canvas: HTMLCanvasElement | OffscreenCanvas } | null>;
 }
 
@@ -77,7 +77,7 @@ function targetDimensions(
 
 async function decodeFrames(
   sink: ThumbnailCanvasSink,
-  timestamps: number[],
+  timestamps: AsyncIterable<number>,
   signal: AbortSignal,
   resources: DecodedResources,
 ): Promise<void> {
@@ -89,6 +89,7 @@ async function decodeFrames(
     throwIfAborted(signal);
     resources.urls.push(URL.createObjectURL(blob));
   }
+  throwIfAborted(signal);
 }
 
 function loadedResult(
@@ -138,8 +139,12 @@ export async function decodeVideoThumbnail(
     }
     const metadataDuration = await track.getDurationFromMetadata({ skipLiveWait: true });
     throwIfAborted(signal);
-    const sourceDuration = Math.max(0, metadataDuration ?? request.sourceRangeDuration ?? 0);
-    const sourceStart = Math.min(Math.max(0, request.sourceStart ?? 0), sourceDuration);
+    const requestedStart = Math.max(0, request.sourceStart ?? 0);
+    const sourceDuration = Math.max(
+      0,
+      metadataDuration ?? requestedStart + (request.sourceRangeDuration ?? 0),
+    );
+    const sourceStart = Math.min(requestedStart, sourceDuration);
     const requestedDuration =
       request.sourceRangeDuration ?? Math.max(0, sourceDuration - sourceStart);
     const duration = Math.min(
@@ -151,6 +156,16 @@ export async function decodeVideoThumbnail(
       duration,
       Math.min(request.frameCount, budgets.richPreviewFrameCount),
     );
+    const keys = new mediabunny.EncodedPacketSink(track);
+    const maxKeyframeLead = duration / Math.max(2, timestamps.length - 1) / 2;
+    async function* decodeTimesAtNearbyKeyframes() {
+      for (const time of timestamps) {
+        const key = await keys.getKeyPacket(time, { metadataOnly: true });
+        if (signal.aborted) return;
+        const near = key && key.timestamp >= sourceStart && time - key.timestamp <= maxKeyframeLead;
+        yield near ? key.timestamp : time;
+      }
+    }
     const aspect = displayWidth / displayHeight;
     const target = targetDimensions(aspect, budgets);
     const sink = new mediabunny.CanvasSink(track, {
@@ -159,7 +174,7 @@ export async function decodeVideoThumbnail(
       fit: request.fit ?? "cover",
       poolSize: 1,
     });
-    await decodeFrames(sink, timestamps, signal, resources);
+    await decodeFrames(sink, decodeTimesAtNearbyKeyframes(), signal, resources);
     return loadedResult(resources, aspect, target.width, target.height);
   } catch (error) {
     releaseDecodedResources(resources);