From f207f612f1fba902d5c1480ea98a90c1469752e5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Miguel=20=C3=81ngel?= Date: Sat, 14 Nov 2026 03:05:19 +0000 Subject: [PATCH] perf(studio): scale timeline strip frames off the main thread (#7651) --- .../src/player/lib/thumbnailFrameSink.test.ts | 92 +++++++++++++++++++ .../src/player/lib/thumbnailFrameSink.ts | 83 +++++++++++++++++ .../player/lib/thumbnailVideoDecoder.test.ts | 59 ++++++++---- .../src/player/lib/thumbnailVideoDecoder.ts | 12 +-- 4 files changed, 224 insertions(+), 22 deletions(-) create mode 100644 packages/studio/src/player/lib/thumbnailFrameSink.test.ts create mode 100644 packages/studio/src/player/lib/thumbnailFrameSink.ts diff --git a/packages/studio/src/player/lib/thumbnailFrameSink.test.ts b/packages/studio/src/player/lib/thumbnailFrameSink.test.ts new file mode 100644 index 0000000000..bc74cc2d00 --- /dev/null +++ b/packages/studio/src/player/lib/thumbnailFrameSink.test.ts @@ -0,0 +1,92 @@ +// @vitest-environment happy-dom + +import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; +import { scaledFrames } from "./thumbnailFrameSink"; + +const drawImage = vi.fn(); +const createImageBitmapMock = vi.fn(); +const bitmap = { close: vi.fn() }; + +function sample(over: Partial<{ rotation: number; flipped: boolean }> = {}) { + const frame = { displayWidth: 3456, displayHeight: 2234, close: vi.fn() }; + return { + frame, + rotation: 0, + flipped: false, + toVideoFrame: vi.fn(() => frame), + drawWithFit: vi.fn(), + close: vi.fn(), + ...over, + }; +} + +async function collect(samples: (ReturnType | null)[], fit: "cover" | "contain") { + const sink = { + async *samplesAtTimestamps() { + yield* samples; + }, + }; + const out: ({ canvas: HTMLCanvasElement } | null)[] = []; + const times = (async function* () {})(); + for await (const item of scaledFrames(sink, times, { width: 360, height: 240 }, fit)) + out.push(item as { canvas: HTMLCanvasElement } | null); + return out; +} + +describe("scaledFrames", () => { + beforeEach(() => { + vi.stubGlobal("createImageBitmap", createImageBitmapMock.mockResolvedValue(bitmap)); + vi.spyOn(HTMLCanvasElement.prototype, "getContext").mockReturnValue({ + drawImage, + } as unknown as RenderingContext); + }); + afterEach(() => { + vi.unstubAllGlobals(); + vi.restoreAllMocks(); + vi.clearAllMocks(); + }); + + it("crops and scales a straight frame in createImageBitmap, centred for cover", async () => { + const s = sample(); + const [out] = await collect([s], "cover"); + + expect(s.drawWithFit).not.toHaveBeenCalled(); + const [frame, x, y, w, h, options] = createImageBitmapMock.mock.calls[0]!; + expect(frame).toBe(s.frame); + expect(options).toMatchObject({ resizeWidth: 360, resizeHeight: 240 }); + expect(w / h).toBeCloseTo(360 / 240, 3); + expect(x + w / 2).toBeCloseTo(3456 / 2, 3); + expect(y + h / 2).toBeCloseTo(2234 / 2, 3); + expect(drawImage).toHaveBeenCalledWith(bitmap, 0, 0, 360, 240); + expect([out!.canvas.width, out!.canvas.height]).toEqual([360, 240]); + expect(s.frame.close).toHaveBeenCalled(); + expect(bitmap.close).toHaveBeenCalled(); + expect(s.close).toHaveBeenCalled(); + }); + + it("contain keeps the whole frame", async () => { + await collect([sample()], "contain"); + const [, x, y, w, h] = createImageBitmapMock.mock.calls[0]!; + expect([x, y, w, h]).toEqual([0, 0, 3456, 2234]); + }); + + it("leaves turned and flipped frames to the sample's own draw, which applies the turn", async () => { + const turned = sample({ rotation: 90 }); + const flipped = sample({ flipped: true }); + await collect([turned, flipped], "cover"); + + expect(createImageBitmapMock).not.toHaveBeenCalled(); + expect(turned.drawWithFit).toHaveBeenCalledWith(expect.anything(), { fit: "cover" }); + expect(flipped.drawWithFit).toHaveBeenCalled(); + }); + + it("passes a missing frame through and closes the sample even when the scale fails", async () => { + expect(await collect([null], "cover")).toEqual([null]); + + const s = sample(); + createImageBitmapMock.mockRejectedValueOnce(new Error("decode failed")); + await expect(collect([s], "cover")).rejects.toThrow("decode failed"); + expect(s.frame.close).toHaveBeenCalled(); + expect(s.close).toHaveBeenCalled(); + }); +}); diff --git a/packages/studio/src/player/lib/thumbnailFrameSink.ts b/packages/studio/src/player/lib/thumbnailFrameSink.ts new file mode 100644 index 0000000000..b03f4fab48 --- /dev/null +++ b/packages/studio/src/player/lib/thumbnailFrameSink.ts @@ -0,0 +1,83 @@ +/** The slice of a Mediabunny video sample the strip reads. */ +interface FrameSample { + rotation: number; + flipped?: boolean; + toVideoFrame(): VideoFrame; + drawWithFit( + context: CanvasRenderingContext2D | OffscreenCanvasRenderingContext2D, + options: { fit: "contain" | "cover" }, + ): void; + close(): void; +} + +interface FrameSampleSink { + samplesAtTimestamps(timestamps: AsyncIterable): AsyncIterable; +} + +type FrameCanvas = HTMLCanvasElement | OffscreenCanvas; +type Box = { width: number; height: number }; + +function newCanvas({ width, height }: Box): FrameCanvas { + if (typeof document === "undefined") return new OffscreenCanvas(width, height); + const canvas = document.createElement("canvas"); + canvas.width = width; + canvas.height = height; + return canvas; +} + +/** The part of a source that fills a box of the target's shape, centred. */ +function coverCrop(source: Box, target: Box) { + const scale = Math.max(target.width / source.width, target.height / source.height); + const width = Math.min(source.width, target.width / scale); + const height = Math.min(source.height, target.height / scale); + return { x: (source.width - width) / 2, y: (source.height - height) / 2, width, height }; +} + +async function drawScaled( + sample: FrameSample, + context: CanvasRenderingContext2D, + target: Box, + fit: "contain" | "cover", +): Promise { + const frame = sample.toVideoFrame(); + const source = { width: frame.displayWidth, height: frame.displayHeight }; + const crop = fit === "cover" ? coverCrop(source, target) : { x: 0, y: 0, ...source }; + const bitmap = await createImageBitmap(frame, crop.x, crop.y, crop.width, crop.height, { + resizeWidth: target.width, + resizeHeight: target.height, + resizeQuality: "medium", + }).finally(() => frame.close()); + try { + context.drawImage(bitmap, 0, 0, target.width, target.height); + } finally { + bitmap.close(); + } +} + +/** + * Strip frames cropped and scaled by the browser off the main thread: drawing a 4K frame onto a strip-sized canvas + * holds the main thread for 100 ms or more. Turned or flipped frames keep the sample's own draw, which turns them. + */ +export async function* scaledFrames( + sink: FrameSampleSink, + times: AsyncIterable, + target: Box, + fit: "contain" | "cover", +): AsyncGenerator<{ canvas: FrameCanvas } | null> { + for await (const sample of sink.samplesAtTimestamps(times)) { + if (!sample) { + yield null; + continue; + } + try { + const canvas = newCanvas(target); + const context = canvas.getContext("2d") as CanvasRenderingContext2D | null; + if (!context) throw new Error("Thumbnail canvas has no 2d context"); + if (sample.rotation === 0 && !sample.flipped) await drawScaled(sample, context, target, fit); + else sample.drawWithFit(context, { fit }); + yield { canvas }; + } finally { + sample.close(); + } + } +} diff --git a/packages/studio/src/player/lib/thumbnailVideoDecoder.test.ts b/packages/studio/src/player/lib/thumbnailVideoDecoder.test.ts index e0dee377dd..a5bd3358e0 100644 --- a/packages/studio/src/player/lib/thumbnailVideoDecoder.test.ts +++ b/packages/studio/src/player/lib/thumbnailVideoDecoder.test.ts @@ -11,6 +11,15 @@ const sourceRetryRules: string[] = []; const cancelledReads: ReturnType[] = []; const canvasesAtTimestamps = vi.fn(); const canvasSinks: unknown[] = []; +const drawImage = vi.fn(); +const createImageBitmapMock = vi.fn(); +const fourKSample = () => ({ + rotation: 0, + flipped: false, + toVideoFrame: () => ({ displayWidth: 3456, displayHeight: 2234, close: vi.fn() }), + drawWithFit: vi.fn(), + close: vi.fn(), +}); const getKeyPacket = vi.fn(async (_time: number) => null as { timestamp: number } | null); const input = { getPrimaryVideoTrack: vi.fn(), @@ -37,11 +46,16 @@ vi.mock("mediabunny", () => ({ getPrimaryVideoTrack = input.getPrimaryVideoTrack; dispose = input.dispose; }, - CanvasSink: class { + // Each frame the canvas seam yields becomes one straight (unturned) sample of a 4K source. + VideoSampleSink: class { constructor() { canvasSinks.push(this); } - canvasesAtTimestamps = canvasesAtTimestamps; + async *samplesAtTimestamps(timestamps: AsyncIterable) { + for await (const wrapped of canvasesAtTimestamps(timestamps)) { + yield wrapped === null ? null : fourKSample(); + } + } }, EncodedPacketSink: class { getKeyPacket = getKeyPacket; @@ -89,6 +103,11 @@ beforeEach(() => { HTMLCanvasElement.prototype.toBlob = function toBlob(callback) { callback(new Blob(["frame"], { type: "image/jpeg" })); }; + HTMLCanvasElement.prototype.getContext = (() => ({ drawImage })) as never; + vi.stubGlobal( + "createImageBitmap", + createImageBitmapMock.mockImplementation(async () => ({ close: vi.fn() })), + ); input.getPrimaryVideoTrack.mockResolvedValue({ getDisplayWidth: vi.fn(async () => 1080), getDisplayHeight: vi.fn(async () => 1920), @@ -96,6 +115,28 @@ beforeEach(() => { }); }); +describe("decodeVideoThumbnail frame scaling", () => { + it("never hands a full-size frame to a canvas draw: the browser crops and scales it to the strip's size", async () => { + recordDecodes([]); + await decodeVideoThumbnail( + { source, sourceStart: 0, sourceRangeDuration: 10, frameCount: 2 }, + new AbortController().signal, + ); + + expect(createImageBitmapMock).toHaveBeenCalled(); + for (const [, , , cropWidth, cropHeight, options] of createImageBitmapMock.mock.calls) { + expect(options.resizeWidth).toBeLessThan(cropWidth); + expect(options.resizeHeight).toBeLessThan(cropHeight); + } + // What lands on the canvas is the small bitmap, never a 4K frame. + for (const [image, , , width, height] of drawImage.mock.calls) { + expect(image).not.toHaveProperty("displayWidth"); + expect(width).toBeLessThan(1000); + expect(height).toBeLessThan(1000); + } + }); +}); + describe("videoThumbnailTimestamps", () => { it("uses the midpoint for a poster, and each slice's left edge then the end for a strip", () => { expect(videoThumbnailTimestamps(2, 6, 1)).toEqual([5]); @@ -706,20 +747,6 @@ describe("decodeVideoThumbnail's shared source", () => { expect(dispose).not.toHaveBeenCalled(); }); - it("keeps the reader when an offscreen frame will not encode", async () => { - const offscreen = { width: 1, height: 1, convertToBlob: () => Promise.reject(new Error("no")) }; - canvasesAtTimestamps.mockImplementationOnce(async function* ( - timestamps: AsyncIterable, - ) { - for await (const _time of timestamps) yield { canvas: offscreen }; - }); - await expect(decodeVideoThumbnail(clip(0), signal)).rejects.toThrow("encode failed"); - recordDecodes([]); - await decodeVideoThumbnail(clip(6), signal); - expect(input.getPrimaryVideoTrack).toHaveBeenCalledTimes(1); - expect(dispose).not.toHaveBeenCalled(); - }); - it("keeps the reader of a clip scrolled away from, for the next clip", async () => { vi.useFakeTimers(); try { diff --git a/packages/studio/src/player/lib/thumbnailVideoDecoder.ts b/packages/studio/src/player/lib/thumbnailVideoDecoder.ts index 8a26744f4a..e1dc928384 100644 --- a/packages/studio/src/player/lib/thumbnailVideoDecoder.ts +++ b/packages/studio/src/player/lib/thumbnailVideoDecoder.ts @@ -4,6 +4,7 @@ import { type ThumbnailLoadedResult, type ThumbnailValue, } from "./thumbnailScheduler"; +import { scaledFrames } from "./thumbnailFrameSink"; import { MediaFetchError, reportMediaFetchFailure } from "./mediaFetch"; import { acquireThumbnailSource, @@ -395,12 +396,11 @@ async function decodeMissingFrames( } } const target = targetDimensions(info.aspect, budgets); - const sink = new mediabunny.CanvasSink(track, { - width: target.width, - height: target.height, - fit: request.fit ?? "cover", - poolSize: 1, - }); + const frames = new mediabunny.VideoSampleSink(track); + const sink: ThumbnailCanvasSink = { + canvasesAtTimestamps: (timestamps) => + scaledFrames(frames, timestamps, target, request.fit ?? "cover"), + }; const times = decodeTimesAtNearbyKeyframes(); await decodeFrames(sink, times, slots, signal, resources, source.progress); }