mirror of
https://github.com/heygen-com/hyperframes.git
synced 2026-11-19 00:18:27 +00:00
perf(studio): scale timeline strip frames off the main thread (#7651)
This commit is contained in:
@@ -0,0 +1,92 @@
|
||||
// @vitest-environment happy-dom
|
||||
|
||||
import { afterEach, beforeEach, describe, expect, it, vi } from "vitest";
|
||||
import { scaledFrames } from "./thumbnailFrameSink";
|
||||
|
||||
const drawImage = vi.fn();
|
||||
const createImageBitmapMock = vi.fn();
|
||||
const bitmap = { close: vi.fn() };
|
||||
|
||||
function sample(over: Partial<{ rotation: number; flipped: boolean }> = {}) {
|
||||
const frame = { displayWidth: 3456, displayHeight: 2234, close: vi.fn() };
|
||||
return {
|
||||
frame,
|
||||
rotation: 0,
|
||||
flipped: false,
|
||||
toVideoFrame: vi.fn(() => frame),
|
||||
drawWithFit: vi.fn(),
|
||||
close: vi.fn(),
|
||||
...over,
|
||||
};
|
||||
}
|
||||
|
||||
async function collect(samples: (ReturnType<typeof sample> | null)[], fit: "cover" | "contain") {
|
||||
const sink = {
|
||||
async *samplesAtTimestamps() {
|
||||
yield* samples;
|
||||
},
|
||||
};
|
||||
const out: ({ canvas: HTMLCanvasElement } | null)[] = [];
|
||||
const times = (async function* () {})();
|
||||
for await (const item of scaledFrames(sink, times, { width: 360, height: 240 }, fit))
|
||||
out.push(item as { canvas: HTMLCanvasElement } | null);
|
||||
return out;
|
||||
}
|
||||
|
||||
describe("scaledFrames", () => {
|
||||
beforeEach(() => {
|
||||
vi.stubGlobal("createImageBitmap", createImageBitmapMock.mockResolvedValue(bitmap));
|
||||
vi.spyOn(HTMLCanvasElement.prototype, "getContext").mockReturnValue({
|
||||
drawImage,
|
||||
} as unknown as RenderingContext);
|
||||
});
|
||||
afterEach(() => {
|
||||
vi.unstubAllGlobals();
|
||||
vi.restoreAllMocks();
|
||||
vi.clearAllMocks();
|
||||
});
|
||||
|
||||
it("crops and scales a straight frame in createImageBitmap, centred for cover", async () => {
|
||||
const s = sample();
|
||||
const [out] = await collect([s], "cover");
|
||||
|
||||
expect(s.drawWithFit).not.toHaveBeenCalled();
|
||||
const [frame, x, y, w, h, options] = createImageBitmapMock.mock.calls[0]!;
|
||||
expect(frame).toBe(s.frame);
|
||||
expect(options).toMatchObject({ resizeWidth: 360, resizeHeight: 240 });
|
||||
expect(w / h).toBeCloseTo(360 / 240, 3);
|
||||
expect(x + w / 2).toBeCloseTo(3456 / 2, 3);
|
||||
expect(y + h / 2).toBeCloseTo(2234 / 2, 3);
|
||||
expect(drawImage).toHaveBeenCalledWith(bitmap, 0, 0, 360, 240);
|
||||
expect([out!.canvas.width, out!.canvas.height]).toEqual([360, 240]);
|
||||
expect(s.frame.close).toHaveBeenCalled();
|
||||
expect(bitmap.close).toHaveBeenCalled();
|
||||
expect(s.close).toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it("contain keeps the whole frame", async () => {
|
||||
await collect([sample()], "contain");
|
||||
const [, x, y, w, h] = createImageBitmapMock.mock.calls[0]!;
|
||||
expect([x, y, w, h]).toEqual([0, 0, 3456, 2234]);
|
||||
});
|
||||
|
||||
it("leaves turned and flipped frames to the sample's own draw, which applies the turn", async () => {
|
||||
const turned = sample({ rotation: 90 });
|
||||
const flipped = sample({ flipped: true });
|
||||
await collect([turned, flipped], "cover");
|
||||
|
||||
expect(createImageBitmapMock).not.toHaveBeenCalled();
|
||||
expect(turned.drawWithFit).toHaveBeenCalledWith(expect.anything(), { fit: "cover" });
|
||||
expect(flipped.drawWithFit).toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it("passes a missing frame through and closes the sample even when the scale fails", async () => {
|
||||
expect(await collect([null], "cover")).toEqual([null]);
|
||||
|
||||
const s = sample();
|
||||
createImageBitmapMock.mockRejectedValueOnce(new Error("decode failed"));
|
||||
await expect(collect([s], "cover")).rejects.toThrow("decode failed");
|
||||
expect(s.frame.close).toHaveBeenCalled();
|
||||
expect(s.close).toHaveBeenCalled();
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,83 @@
|
||||
/** The slice of a Mediabunny video sample the strip reads. */
|
||||
interface FrameSample {
|
||||
rotation: number;
|
||||
flipped?: boolean;
|
||||
toVideoFrame(): VideoFrame;
|
||||
drawWithFit(
|
||||
context: CanvasRenderingContext2D | OffscreenCanvasRenderingContext2D,
|
||||
options: { fit: "contain" | "cover" },
|
||||
): void;
|
||||
close(): void;
|
||||
}
|
||||
|
||||
interface FrameSampleSink {
|
||||
samplesAtTimestamps(timestamps: AsyncIterable<number>): AsyncIterable<FrameSample | null>;
|
||||
}
|
||||
|
||||
type FrameCanvas = HTMLCanvasElement | OffscreenCanvas;
|
||||
type Box = { width: number; height: number };
|
||||
|
||||
function newCanvas({ width, height }: Box): FrameCanvas {
|
||||
if (typeof document === "undefined") return new OffscreenCanvas(width, height);
|
||||
const canvas = document.createElement("canvas");
|
||||
canvas.width = width;
|
||||
canvas.height = height;
|
||||
return canvas;
|
||||
}
|
||||
|
||||
/** The part of a source that fills a box of the target's shape, centred. */
|
||||
function coverCrop(source: Box, target: Box) {
|
||||
const scale = Math.max(target.width / source.width, target.height / source.height);
|
||||
const width = Math.min(source.width, target.width / scale);
|
||||
const height = Math.min(source.height, target.height / scale);
|
||||
return { x: (source.width - width) / 2, y: (source.height - height) / 2, width, height };
|
||||
}
|
||||
|
||||
async function drawScaled(
|
||||
sample: FrameSample,
|
||||
context: CanvasRenderingContext2D,
|
||||
target: Box,
|
||||
fit: "contain" | "cover",
|
||||
): Promise<void> {
|
||||
const frame = sample.toVideoFrame();
|
||||
const source = { width: frame.displayWidth, height: frame.displayHeight };
|
||||
const crop = fit === "cover" ? coverCrop(source, target) : { x: 0, y: 0, ...source };
|
||||
const bitmap = await createImageBitmap(frame, crop.x, crop.y, crop.width, crop.height, {
|
||||
resizeWidth: target.width,
|
||||
resizeHeight: target.height,
|
||||
resizeQuality: "medium",
|
||||
}).finally(() => frame.close());
|
||||
try {
|
||||
context.drawImage(bitmap, 0, 0, target.width, target.height);
|
||||
} finally {
|
||||
bitmap.close();
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Strip frames cropped and scaled by the browser off the main thread: drawing a 4K frame onto a strip-sized canvas
|
||||
* holds the main thread for 100 ms or more. Turned or flipped frames keep the sample's own draw, which turns them.
|
||||
*/
|
||||
export async function* scaledFrames(
|
||||
sink: FrameSampleSink,
|
||||
times: AsyncIterable<number>,
|
||||
target: Box,
|
||||
fit: "contain" | "cover",
|
||||
): AsyncGenerator<{ canvas: FrameCanvas } | null> {
|
||||
for await (const sample of sink.samplesAtTimestamps(times)) {
|
||||
if (!sample) {
|
||||
yield null;
|
||||
continue;
|
||||
}
|
||||
try {
|
||||
const canvas = newCanvas(target);
|
||||
const context = canvas.getContext("2d") as CanvasRenderingContext2D | null;
|
||||
if (!context) throw new Error("Thumbnail canvas has no 2d context");
|
||||
if (sample.rotation === 0 && !sample.flipped) await drawScaled(sample, context, target, fit);
|
||||
else sample.drawWithFit(context, { fit });
|
||||
yield { canvas };
|
||||
} finally {
|
||||
sample.close();
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -11,6 +11,15 @@ const sourceRetryRules: string[] = [];
|
||||
const cancelledReads: ReturnType<typeof vi.fn>[] = [];
|
||||
const canvasesAtTimestamps = vi.fn();
|
||||
const canvasSinks: unknown[] = [];
|
||||
const drawImage = vi.fn();
|
||||
const createImageBitmapMock = vi.fn();
|
||||
const fourKSample = () => ({
|
||||
rotation: 0,
|
||||
flipped: false,
|
||||
toVideoFrame: () => ({ displayWidth: 3456, displayHeight: 2234, close: vi.fn() }),
|
||||
drawWithFit: vi.fn(),
|
||||
close: vi.fn(),
|
||||
});
|
||||
const getKeyPacket = vi.fn(async (_time: number) => null as { timestamp: number } | null);
|
||||
const input = {
|
||||
getPrimaryVideoTrack: vi.fn(),
|
||||
@@ -37,11 +46,16 @@ vi.mock("mediabunny", () => ({
|
||||
getPrimaryVideoTrack = input.getPrimaryVideoTrack;
|
||||
dispose = input.dispose;
|
||||
},
|
||||
CanvasSink: class {
|
||||
// Each frame the canvas seam yields becomes one straight (unturned) sample of a 4K source.
|
||||
VideoSampleSink: class {
|
||||
constructor() {
|
||||
canvasSinks.push(this);
|
||||
}
|
||||
canvasesAtTimestamps = canvasesAtTimestamps;
|
||||
async *samplesAtTimestamps(timestamps: AsyncIterable<number>) {
|
||||
for await (const wrapped of canvasesAtTimestamps(timestamps)) {
|
||||
yield wrapped === null ? null : fourKSample();
|
||||
}
|
||||
}
|
||||
},
|
||||
EncodedPacketSink: class {
|
||||
getKeyPacket = getKeyPacket;
|
||||
@@ -89,6 +103,11 @@ beforeEach(() => {
|
||||
HTMLCanvasElement.prototype.toBlob = function toBlob(callback) {
|
||||
callback(new Blob(["frame"], { type: "image/jpeg" }));
|
||||
};
|
||||
HTMLCanvasElement.prototype.getContext = (() => ({ drawImage })) as never;
|
||||
vi.stubGlobal(
|
||||
"createImageBitmap",
|
||||
createImageBitmapMock.mockImplementation(async () => ({ close: vi.fn() })),
|
||||
);
|
||||
input.getPrimaryVideoTrack.mockResolvedValue({
|
||||
getDisplayWidth: vi.fn(async () => 1080),
|
||||
getDisplayHeight: vi.fn(async () => 1920),
|
||||
@@ -96,6 +115,28 @@ beforeEach(() => {
|
||||
});
|
||||
});
|
||||
|
||||
describe("decodeVideoThumbnail frame scaling", () => {
|
||||
it("never hands a full-size frame to a canvas draw: the browser crops and scales it to the strip's size", async () => {
|
||||
recordDecodes([]);
|
||||
await decodeVideoThumbnail(
|
||||
{ source, sourceStart: 0, sourceRangeDuration: 10, frameCount: 2 },
|
||||
new AbortController().signal,
|
||||
);
|
||||
|
||||
expect(createImageBitmapMock).toHaveBeenCalled();
|
||||
for (const [, , , cropWidth, cropHeight, options] of createImageBitmapMock.mock.calls) {
|
||||
expect(options.resizeWidth).toBeLessThan(cropWidth);
|
||||
expect(options.resizeHeight).toBeLessThan(cropHeight);
|
||||
}
|
||||
// What lands on the canvas is the small bitmap, never a 4K frame.
|
||||
for (const [image, , , width, height] of drawImage.mock.calls) {
|
||||
expect(image).not.toHaveProperty("displayWidth");
|
||||
expect(width).toBeLessThan(1000);
|
||||
expect(height).toBeLessThan(1000);
|
||||
}
|
||||
});
|
||||
});
|
||||
|
||||
describe("videoThumbnailTimestamps", () => {
|
||||
it("uses the midpoint for a poster, and each slice's left edge then the end for a strip", () => {
|
||||
expect(videoThumbnailTimestamps(2, 6, 1)).toEqual([5]);
|
||||
@@ -706,20 +747,6 @@ describe("decodeVideoThumbnail's shared source", () => {
|
||||
expect(dispose).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it("keeps the reader when an offscreen frame will not encode", async () => {
|
||||
const offscreen = { width: 1, height: 1, convertToBlob: () => Promise.reject(new Error("no")) };
|
||||
canvasesAtTimestamps.mockImplementationOnce(async function* (
|
||||
timestamps: AsyncIterable<number>,
|
||||
) {
|
||||
for await (const _time of timestamps) yield { canvas: offscreen };
|
||||
});
|
||||
await expect(decodeVideoThumbnail(clip(0), signal)).rejects.toThrow("encode failed");
|
||||
recordDecodes([]);
|
||||
await decodeVideoThumbnail(clip(6), signal);
|
||||
expect(input.getPrimaryVideoTrack).toHaveBeenCalledTimes(1);
|
||||
expect(dispose).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it("keeps the reader of a clip scrolled away from, for the next clip", async () => {
|
||||
vi.useFakeTimers();
|
||||
try {
|
||||
|
||||
@@ -4,6 +4,7 @@ import {
|
||||
type ThumbnailLoadedResult,
|
||||
type ThumbnailValue,
|
||||
} from "./thumbnailScheduler";
|
||||
import { scaledFrames } from "./thumbnailFrameSink";
|
||||
import { MediaFetchError, reportMediaFetchFailure } from "./mediaFetch";
|
||||
import {
|
||||
acquireThumbnailSource,
|
||||
@@ -395,12 +396,11 @@ async function decodeMissingFrames(
|
||||
}
|
||||
}
|
||||
const target = targetDimensions(info.aspect, budgets);
|
||||
const sink = new mediabunny.CanvasSink(track, {
|
||||
width: target.width,
|
||||
height: target.height,
|
||||
fit: request.fit ?? "cover",
|
||||
poolSize: 1,
|
||||
});
|
||||
const frames = new mediabunny.VideoSampleSink(track);
|
||||
const sink: ThumbnailCanvasSink = {
|
||||
canvasesAtTimestamps: (timestamps) =>
|
||||
scaledFrames(frames, timestamps, target, request.fit ?? "cover"),
|
||||
};
|
||||
const times = decodeTimesAtNearbyKeyframes();
|
||||
await decodeFrames(sink, times, slots, signal, resources, source.progress);
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user