perf(studio): scale timeline strip frames off the main thread (#7651)

This commit is contained in:
Miguel Ángel
2026-11-14 03:05:19 +00:00
committed by GitHub
parent cb072eb047
commit f207f612f1
4 changed files with 224 additions and 22 deletions
@@ -0,0 +1,92 @@
// @vitest-environment happy-dom
import { afterEach, beforeEach, describe, expect, it, vi } from "vitest";
import { scaledFrames } from "./thumbnailFrameSink";
const drawImage = vi.fn();
const createImageBitmapMock = vi.fn();
const bitmap = { close: vi.fn() };
function sample(over: Partial<{ rotation: number; flipped: boolean }> = {}) {
const frame = { displayWidth: 3456, displayHeight: 2234, close: vi.fn() };
return {
frame,
rotation: 0,
flipped: false,
toVideoFrame: vi.fn(() => frame),
drawWithFit: vi.fn(),
close: vi.fn(),
...over,
};
}
async function collect(samples: (ReturnType<typeof sample> | null)[], fit: "cover" | "contain") {
const sink = {
async *samplesAtTimestamps() {
yield* samples;
},
};
const out: ({ canvas: HTMLCanvasElement } | null)[] = [];
const times = (async function* () {})();
for await (const item of scaledFrames(sink, times, { width: 360, height: 240 }, fit))
out.push(item as { canvas: HTMLCanvasElement } | null);
return out;
}
describe("scaledFrames", () => {
beforeEach(() => {
vi.stubGlobal("createImageBitmap", createImageBitmapMock.mockResolvedValue(bitmap));
vi.spyOn(HTMLCanvasElement.prototype, "getContext").mockReturnValue({
drawImage,
} as unknown as RenderingContext);
});
afterEach(() => {
vi.unstubAllGlobals();
vi.restoreAllMocks();
vi.clearAllMocks();
});
it("crops and scales a straight frame in createImageBitmap, centred for cover", async () => {
const s = sample();
const [out] = await collect([s], "cover");
expect(s.drawWithFit).not.toHaveBeenCalled();
const [frame, x, y, w, h, options] = createImageBitmapMock.mock.calls[0]!;
expect(frame).toBe(s.frame);
expect(options).toMatchObject({ resizeWidth: 360, resizeHeight: 240 });
expect(w / h).toBeCloseTo(360 / 240, 3);
expect(x + w / 2).toBeCloseTo(3456 / 2, 3);
expect(y + h / 2).toBeCloseTo(2234 / 2, 3);
expect(drawImage).toHaveBeenCalledWith(bitmap, 0, 0, 360, 240);
expect([out!.canvas.width, out!.canvas.height]).toEqual([360, 240]);
expect(s.frame.close).toHaveBeenCalled();
expect(bitmap.close).toHaveBeenCalled();
expect(s.close).toHaveBeenCalled();
});
it("contain keeps the whole frame", async () => {
await collect([sample()], "contain");
const [, x, y, w, h] = createImageBitmapMock.mock.calls[0]!;
expect([x, y, w, h]).toEqual([0, 0, 3456, 2234]);
});
it("leaves turned and flipped frames to the sample's own draw, which applies the turn", async () => {
const turned = sample({ rotation: 90 });
const flipped = sample({ flipped: true });
await collect([turned, flipped], "cover");
expect(createImageBitmapMock).not.toHaveBeenCalled();
expect(turned.drawWithFit).toHaveBeenCalledWith(expect.anything(), { fit: "cover" });
expect(flipped.drawWithFit).toHaveBeenCalled();
});
it("passes a missing frame through and closes the sample even when the scale fails", async () => {
expect(await collect([null], "cover")).toEqual([null]);
const s = sample();
createImageBitmapMock.mockRejectedValueOnce(new Error("decode failed"));
await expect(collect([s], "cover")).rejects.toThrow("decode failed");
expect(s.frame.close).toHaveBeenCalled();
expect(s.close).toHaveBeenCalled();
});
});
@@ -0,0 +1,83 @@
/** The slice of a Mediabunny video sample the strip reads. */
interface FrameSample {
rotation: number;
flipped?: boolean;
toVideoFrame(): VideoFrame;
drawWithFit(
context: CanvasRenderingContext2D | OffscreenCanvasRenderingContext2D,
options: { fit: "contain" | "cover" },
): void;
close(): void;
}
interface FrameSampleSink {
samplesAtTimestamps(timestamps: AsyncIterable<number>): AsyncIterable<FrameSample | null>;
}
type FrameCanvas = HTMLCanvasElement | OffscreenCanvas;
type Box = { width: number; height: number };
function newCanvas({ width, height }: Box): FrameCanvas {
if (typeof document === "undefined") return new OffscreenCanvas(width, height);
const canvas = document.createElement("canvas");
canvas.width = width;
canvas.height = height;
return canvas;
}
/** The part of a source that fills a box of the target's shape, centred. */
function coverCrop(source: Box, target: Box) {
const scale = Math.max(target.width / source.width, target.height / source.height);
const width = Math.min(source.width, target.width / scale);
const height = Math.min(source.height, target.height / scale);
return { x: (source.width - width) / 2, y: (source.height - height) / 2, width, height };
}
async function drawScaled(
sample: FrameSample,
context: CanvasRenderingContext2D,
target: Box,
fit: "contain" | "cover",
): Promise<void> {
const frame = sample.toVideoFrame();
const source = { width: frame.displayWidth, height: frame.displayHeight };
const crop = fit === "cover" ? coverCrop(source, target) : { x: 0, y: 0, ...source };
const bitmap = await createImageBitmap(frame, crop.x, crop.y, crop.width, crop.height, {
resizeWidth: target.width,
resizeHeight: target.height,
resizeQuality: "medium",
}).finally(() => frame.close());
try {
context.drawImage(bitmap, 0, 0, target.width, target.height);
} finally {
bitmap.close();
}
}
/**
* Strip frames cropped and scaled by the browser off the main thread: drawing a 4K frame onto a strip-sized canvas
* holds the main thread for 100 ms or more. Turned or flipped frames keep the sample's own draw, which turns them.
*/
export async function* scaledFrames(
sink: FrameSampleSink,
times: AsyncIterable<number>,
target: Box,
fit: "contain" | "cover",
): AsyncGenerator<{ canvas: FrameCanvas } | null> {
for await (const sample of sink.samplesAtTimestamps(times)) {
if (!sample) {
yield null;
continue;
}
try {
const canvas = newCanvas(target);
const context = canvas.getContext("2d") as CanvasRenderingContext2D | null;
if (!context) throw new Error("Thumbnail canvas has no 2d context");
if (sample.rotation === 0 && !sample.flipped) await drawScaled(sample, context, target, fit);
else sample.drawWithFit(context, { fit });
yield { canvas };
} finally {
sample.close();
}
}
}
@@ -11,6 +11,15 @@ const sourceRetryRules: string[] = [];
const cancelledReads: ReturnType<typeof vi.fn>[] = [];
const canvasesAtTimestamps = vi.fn();
const canvasSinks: unknown[] = [];
const drawImage = vi.fn();
const createImageBitmapMock = vi.fn();
const fourKSample = () => ({
rotation: 0,
flipped: false,
toVideoFrame: () => ({ displayWidth: 3456, displayHeight: 2234, close: vi.fn() }),
drawWithFit: vi.fn(),
close: vi.fn(),
});
const getKeyPacket = vi.fn(async (_time: number) => null as { timestamp: number } | null);
const input = {
getPrimaryVideoTrack: vi.fn(),
@@ -37,11 +46,16 @@ vi.mock("mediabunny", () => ({
getPrimaryVideoTrack = input.getPrimaryVideoTrack;
dispose = input.dispose;
},
CanvasSink: class {
// Each frame the canvas seam yields becomes one straight (unturned) sample of a 4K source.
VideoSampleSink: class {
constructor() {
canvasSinks.push(this);
}
canvasesAtTimestamps = canvasesAtTimestamps;
async *samplesAtTimestamps(timestamps: AsyncIterable<number>) {
for await (const wrapped of canvasesAtTimestamps(timestamps)) {
yield wrapped === null ? null : fourKSample();
}
}
},
EncodedPacketSink: class {
getKeyPacket = getKeyPacket;
@@ -89,6 +103,11 @@ beforeEach(() => {
HTMLCanvasElement.prototype.toBlob = function toBlob(callback) {
callback(new Blob(["frame"], { type: "image/jpeg" }));
};
HTMLCanvasElement.prototype.getContext = (() => ({ drawImage })) as never;
vi.stubGlobal(
"createImageBitmap",
createImageBitmapMock.mockImplementation(async () => ({ close: vi.fn() })),
);
input.getPrimaryVideoTrack.mockResolvedValue({
getDisplayWidth: vi.fn(async () => 1080),
getDisplayHeight: vi.fn(async () => 1920),
@@ -96,6 +115,28 @@ beforeEach(() => {
});
});
describe("decodeVideoThumbnail frame scaling", () => {
it("never hands a full-size frame to a canvas draw: the browser crops and scales it to the strip's size", async () => {
recordDecodes([]);
await decodeVideoThumbnail(
{ source, sourceStart: 0, sourceRangeDuration: 10, frameCount: 2 },
new AbortController().signal,
);
expect(createImageBitmapMock).toHaveBeenCalled();
for (const [, , , cropWidth, cropHeight, options] of createImageBitmapMock.mock.calls) {
expect(options.resizeWidth).toBeLessThan(cropWidth);
expect(options.resizeHeight).toBeLessThan(cropHeight);
}
// What lands on the canvas is the small bitmap, never a 4K frame.
for (const [image, , , width, height] of drawImage.mock.calls) {
expect(image).not.toHaveProperty("displayWidth");
expect(width).toBeLessThan(1000);
expect(height).toBeLessThan(1000);
}
});
});
describe("videoThumbnailTimestamps", () => {
it("uses the midpoint for a poster, and each slice's left edge then the end for a strip", () => {
expect(videoThumbnailTimestamps(2, 6, 1)).toEqual([5]);
@@ -706,20 +747,6 @@ describe("decodeVideoThumbnail's shared source", () => {
expect(dispose).not.toHaveBeenCalled();
});
it("keeps the reader when an offscreen frame will not encode", async () => {
const offscreen = { width: 1, height: 1, convertToBlob: () => Promise.reject(new Error("no")) };
canvasesAtTimestamps.mockImplementationOnce(async function* (
timestamps: AsyncIterable<number>,
) {
for await (const _time of timestamps) yield { canvas: offscreen };
});
await expect(decodeVideoThumbnail(clip(0), signal)).rejects.toThrow("encode failed");
recordDecodes([]);
await decodeVideoThumbnail(clip(6), signal);
expect(input.getPrimaryVideoTrack).toHaveBeenCalledTimes(1);
expect(dispose).not.toHaveBeenCalled();
});
it("keeps the reader of a clip scrolled away from, for the next clip", async () => {
vi.useFakeTimers();
try {
@@ -4,6 +4,7 @@ import {
type ThumbnailLoadedResult,
type ThumbnailValue,
} from "./thumbnailScheduler";
import { scaledFrames } from "./thumbnailFrameSink";
import { MediaFetchError, reportMediaFetchFailure } from "./mediaFetch";
import {
acquireThumbnailSource,
@@ -395,12 +396,11 @@ async function decodeMissingFrames(
}
}
const target = targetDimensions(info.aspect, budgets);
const sink = new mediabunny.CanvasSink(track, {
width: target.width,
height: target.height,
fit: request.fit ?? "cover",
poolSize: 1,
});
const frames = new mediabunny.VideoSampleSink(track);
const sink: ThumbnailCanvasSink = {
canvasesAtTimestamps: (timestamps) =>
scaledFrames(frames, timestamps, target, request.fit ?? "cover"),
};
const times = decodeTimesAtNearbyKeyframes();
await decodeFrames(sink, times, slots, signal, resources, source.progress);
}