mirror of
https://github.com/heygen-com/hyperframes.git
synced 2026-11-18 16:08:56 +00:00
* perf(studio): reuse filmstrip frames when a zoom changes a strip's width A zoom that changed a strip's frame count decoded the whole strip again on the main thread, and showed the clip's single poster frame until it landed. Frames are now sampled at each slice's left edge, so a strip twice as long holds every frame of the shorter one; decoded frames are shared per source and time, so zooming out decodes nothing and zooming in decodes only the new half. The strip on screen stays until the new width is ready. * perf(studio): redraw only the waveforms on or near the screen after a zoom A zoom resizes every mounted clip, and every mounted waveform redrew in that same frame, on screen or not. A waveform now draws only while it is on or near the screen (one shared observer, a screen of margin each side) and draws when it comes near if a zoom changed it meanwhile. * perf(studio): never start a beat-analysis stage while the timeline zooms Analyzing a long music track runs for over a second on the main thread, and when it started late it landed inside a zoom. The analysis now awaits an optional pause before each long stage; Studio's waits for the timeline to rest and the browser to go idle. Results are unchanged. The tempo detector's stage is one call, so a zoom that starts inside it still waits for it. * fix(studio): reuse a filmstrip frame only where its own strip would show it A shared frame was keyed by the time a strip asked for, not the time it was decoded at, so a coarse strip's keyframe-early frame could land in a finer strip and run its frames backwards. A frame now carries its decoded time and is reused only when the strip would have accepted that keyframe itself. The frame-count ladder now tops out at a power of two (32), so its top step shares frames with the one below instead of none. * feat(studio): show the clip's last frame at the end of every filmstrip Left-edge sampling never showed a clip's final slice, so an end card or a fade-out was missing from its strip. Each strip now also decodes the clip's last frame (at its own time, never an earlier keyframe), shared across zoom steps like the others, and the last tile shows it. The strip on screen is now held per clip and range, not per props: a clip crossing between on and off screen keeps it, and a clip narrowed to one frame lets it go, so widening again never re-decodes a dropped width. * docs(studio): keep the filmstrip files at or under main's comment share The comment ratchet allows no file a higher share of comment lines than on main; the added explanations move to the PR description. * fix(studio): keep a held filmstrip leased so a full cache cannot evict it mid-zoom The hold leased the strip on screen only once a new width was requested, so for one commit it had no lease; with the cache over budget it was evicted, the poster showed, and the old width decoded again. The strip on screen is now leased for as long as it is the clip's, and shown only while held. A frame that fails to decode takes the frame before it, so every slot keeps its time and the last tile still shows the end. * refactor(studio): one owner for which clip a held filmstrip belongs to The hold checked the clip twice, in its lease and again before showing it; the lease alone decides now, and the cross-clip test pins it. A test now also pins that a filled frame is not charged to the cache. * fix(core): hold the tempo detector until idle after a slow import * fix(studio): scope shared video frames to timeline sessions * chore(core): keep beat analysis comments within the ratchet
329 lines
11 KiB
TypeScript
329 lines
11 KiB
TypeScript
import { TIMELINE_VIEWPORT_BUDGETS, type TimelineViewportBudgets } from "./timelineViewportBudgets";
|
|
import {
|
|
createThumbnailKey,
|
|
type ThumbnailLoadedResult,
|
|
type ThumbnailValue,
|
|
} from "./thumbnailScheduler";
|
|
|
|
export interface VideoThumbnailDecodeRequest {
|
|
source: string;
|
|
contentVersion?: string;
|
|
sourceStart?: number;
|
|
sourceRangeDuration?: number;
|
|
frameCount: number;
|
|
fit?: "contain" | "cover";
|
|
}
|
|
|
|
const LAST_FRAME_INSET_S = 0.001;
|
|
|
|
export function videoThumbnailTimestamps(
|
|
start: number,
|
|
duration: number,
|
|
frameCount: number,
|
|
): number[] {
|
|
const safeStart = Math.max(0, Number.isFinite(start) ? start : 0);
|
|
const safeDuration = Math.max(0, Number.isFinite(duration) ? duration : 0);
|
|
const count = Math.max(1, Number.isFinite(frameCount) ? Math.floor(frameCount) : 1);
|
|
if (count === 1) return [safeStart + safeDuration / 2];
|
|
const edges = Array.from(
|
|
{ length: count },
|
|
(_, index) => safeStart + (safeDuration * index) / count,
|
|
);
|
|
return [...edges, safeStart + Math.max(0, safeDuration - LAST_FRAME_INSET_S)];
|
|
}
|
|
|
|
interface SharedFrame {
|
|
url: string;
|
|
decodedAt: number;
|
|
users: number;
|
|
}
|
|
|
|
interface HeldFrame {
|
|
key: string;
|
|
frame: SharedFrame;
|
|
}
|
|
|
|
interface FrameWindow {
|
|
sourceStart: number;
|
|
lead: number;
|
|
end: number;
|
|
}
|
|
|
|
const sharedFrames = new Map<string, SharedFrame[]>();
|
|
const sourceInfos = new Map<string, { version: string | undefined; info: SourceInfo }>();
|
|
|
|
const accepts = (window: FrameWindow, time: number, decodedAt: number) =>
|
|
decodedAt >= window.sourceStart && time - decodedAt <= (time >= window.end ? 0 : window.lead);
|
|
|
|
function takeSharedFrame(key: string, time: number, window: FrameWindow): SharedFrame | undefined {
|
|
const frame = sharedFrames.get(key)?.find((shared) => accepts(window, time, shared.decodedAt));
|
|
if (frame) frame.users += 1;
|
|
return frame;
|
|
}
|
|
|
|
function shareFrame(key: string, decodedAt: number, url: string): SharedFrame {
|
|
const frames = sharedFrames.get(key) ?? [];
|
|
const existing = frames.find((shared) => shared.decodedAt === decodedAt);
|
|
if (existing) {
|
|
existing.users += 1;
|
|
URL.revokeObjectURL(url);
|
|
return existing;
|
|
}
|
|
const frame = { url, decodedAt, users: 1 };
|
|
sharedFrames.set(key, [...frames, frame]);
|
|
return frame;
|
|
}
|
|
|
|
function releaseSharedFrames(held: HeldFrame[]): void {
|
|
for (const { key, frame } of held.splice(0)) {
|
|
if (--frame.users > 0) continue;
|
|
const rest = sharedFrames.get(key)?.filter((shared) => shared !== frame) ?? [];
|
|
if (rest.length > 0) sharedFrames.set(key, rest);
|
|
else sharedFrames.delete(key);
|
|
URL.revokeObjectURL(frame.url);
|
|
}
|
|
}
|
|
|
|
async function canvasToBlob(canvas: HTMLCanvasElement | OffscreenCanvas): Promise<Blob> {
|
|
if (canvas instanceof HTMLCanvasElement) {
|
|
return new Promise((resolve, reject) => {
|
|
canvas.toBlob(
|
|
(blob) => (blob ? resolve(blob) : reject(new Error("Video thumbnail encode failed"))),
|
|
"image/jpeg",
|
|
0.72,
|
|
);
|
|
});
|
|
}
|
|
return canvas.convertToBlob({ type: "image/jpeg", quality: 0.72 });
|
|
}
|
|
|
|
interface DecodedResources {
|
|
urls: (string | undefined)[];
|
|
held: HeldFrame[];
|
|
canvases: Set<HTMLCanvasElement | OffscreenCanvas>;
|
|
}
|
|
|
|
interface ThumbnailCanvasSink {
|
|
canvasesAtTimestamps(
|
|
timestamps: AsyncIterable<number>,
|
|
): AsyncIterable<{ canvas: HTMLCanvasElement | OffscreenCanvas } | null>;
|
|
}
|
|
|
|
interface SourceInfo {
|
|
aspect: number;
|
|
metadataDuration: number | null;
|
|
}
|
|
|
|
function throwIfAborted(signal: AbortSignal): void {
|
|
if (signal.aborted) throw new DOMException("Aborted", "AbortError");
|
|
}
|
|
|
|
function releaseDecodedResources(resources: DecodedResources): void {
|
|
resources.urls.length = 0;
|
|
releaseSharedFrames(resources.held);
|
|
for (const canvas of resources.canvases) {
|
|
canvas.width = 0;
|
|
canvas.height = 0;
|
|
}
|
|
resources.canvases.clear();
|
|
}
|
|
|
|
function targetDimensions(
|
|
aspect: number,
|
|
budgets: Readonly<TimelineViewportBudgets>,
|
|
): { width: number; height: number } {
|
|
const width = Math.max(
|
|
1,
|
|
Math.min(budgets.posterMaxPhysicalWidth, Math.round(budgets.posterMaxPhysicalHeight * aspect)),
|
|
);
|
|
return {
|
|
width,
|
|
height: Math.max(1, Math.min(budgets.posterMaxPhysicalHeight, Math.round(width / aspect))),
|
|
};
|
|
}
|
|
|
|
function stripTimes(
|
|
request: VideoThumbnailDecodeRequest,
|
|
info: SourceInfo,
|
|
budgets: Readonly<TimelineViewportBudgets>,
|
|
) {
|
|
const requestedStart = Math.max(0, request.sourceStart ?? 0);
|
|
const sourceDuration = Math.max(
|
|
0,
|
|
info.metadataDuration ?? requestedStart + (request.sourceRangeDuration ?? 0),
|
|
);
|
|
const sourceStart = Math.min(requestedStart, sourceDuration);
|
|
const requestedDuration =
|
|
request.sourceRangeDuration ?? Math.max(0, sourceDuration - sourceStart);
|
|
const duration = Math.min(
|
|
Math.max(0, requestedDuration),
|
|
Math.max(0, sourceDuration - sourceStart),
|
|
);
|
|
const timestamps = videoThumbnailTimestamps(
|
|
sourceStart,
|
|
duration,
|
|
Math.min(request.frameCount, budgets.richPreviewFrameCount),
|
|
);
|
|
const slices = Math.max(1, timestamps.length - 1);
|
|
const window: FrameWindow = {
|
|
sourceStart,
|
|
lead: duration / Math.max(2, slices) / 2,
|
|
end: timestamps.length > 1 ? timestamps.at(-1)! : Infinity,
|
|
};
|
|
return { window, timestamps };
|
|
}
|
|
|
|
function takeDecodedFrames(
|
|
{ timestamps, window }: ReturnType<typeof stripTimes>,
|
|
keyOf: (time: number) => string,
|
|
resources: DecodedResources,
|
|
): number[] {
|
|
const missing: number[] = [];
|
|
timestamps.forEach((time, slot) => {
|
|
const key = keyOf(time);
|
|
const frame = takeSharedFrame(key, time, window);
|
|
if (!frame) return void missing.push(slot);
|
|
resources.urls[slot] = frame.url;
|
|
resources.held.push({ key, frame });
|
|
});
|
|
return missing;
|
|
}
|
|
|
|
async function decodeFrames(
|
|
sink: ThumbnailCanvasSink,
|
|
times: AsyncIterable<number>,
|
|
slots: { slot: number; key: string; decodedAt?: number }[],
|
|
signal: AbortSignal,
|
|
resources: DecodedResources,
|
|
): Promise<void> {
|
|
let next = 0;
|
|
for await (const wrapped of sink.canvasesAtTimestamps(times)) {
|
|
throwIfAborted(signal);
|
|
const target = slots[next++];
|
|
if (!wrapped || target?.decodedAt === undefined) continue;
|
|
resources.canvases.add(wrapped.canvas);
|
|
const blob = await canvasToBlob(wrapped.canvas);
|
|
throwIfAborted(signal);
|
|
const frame = shareFrame(target.key, target.decodedAt, URL.createObjectURL(blob));
|
|
resources.urls[target.slot] = frame.url;
|
|
resources.held.push({ key: target.key, frame });
|
|
}
|
|
throwIfAborted(signal);
|
|
}
|
|
|
|
function loadedResult(
|
|
resources: DecodedResources,
|
|
aspect: number,
|
|
budgets: Readonly<TimelineViewportBudgets>,
|
|
frameCount = resources.urls.length,
|
|
): ThumbnailLoadedResult {
|
|
const decoded = resources.urls.filter((url): url is string => url !== undefined);
|
|
const firstUrl = decoded[0];
|
|
if (!firstUrl) throw new Error("Video source returned no thumbnail frames");
|
|
let nearest = firstUrl;
|
|
const urls = Array.from(
|
|
{ length: frameCount },
|
|
(_, slot) => (nearest = resources.urls[slot] ?? nearest),
|
|
);
|
|
const { width, height } = targetDimensions(aspect, budgets);
|
|
const value: ThumbnailValue =
|
|
urls.length === 1
|
|
? { kind: "image", url: firstUrl, aspect }
|
|
: { kind: "filmstrip", urls, aspect };
|
|
return {
|
|
value,
|
|
weight: width * height * 4 * decoded.length,
|
|
dispose: () => releaseDecodedResources(resources),
|
|
};
|
|
}
|
|
|
|
/** Sparse Mediabunny extraction with one pooled canvas and one cleanup owner. */
|
|
export async function decodeVideoThumbnail(
|
|
request: VideoThumbnailDecodeRequest,
|
|
signal: AbortSignal,
|
|
budgets: Readonly<TimelineViewportBudgets> = TIMELINE_VIEWPORT_BUDGETS,
|
|
): Promise<ThumbnailLoadedResult> {
|
|
const fit = request.fit ?? "cover";
|
|
const sourceKey = createThumbnailKey({ source: request.source, version: request.contentVersion });
|
|
const keyOf = (time: number) => `${sourceKey}\u0000${fit}\u0000${time}`;
|
|
const resources: DecodedResources = { urls: [], held: [], canvases: new Set() };
|
|
const entry = sourceInfos.get(request.source);
|
|
const known = entry?.version === request.contentVersion ? entry?.info : undefined;
|
|
try {
|
|
if (known) {
|
|
if (takeDecodedFrames(stripTimes(request, known, budgets), keyOf, resources).length === 0) {
|
|
throwIfAborted(signal);
|
|
return loadedResult(resources, known.aspect, budgets);
|
|
}
|
|
releaseDecodedResources(resources);
|
|
}
|
|
return await decodeMissingFrames(request, signal, budgets, keyOf, resources);
|
|
} catch (error) {
|
|
releaseDecodedResources(resources);
|
|
throw error;
|
|
}
|
|
}
|
|
|
|
async function decodeMissingFrames(
|
|
request: VideoThumbnailDecodeRequest,
|
|
signal: AbortSignal,
|
|
budgets: Readonly<TimelineViewportBudgets>,
|
|
keyOf: (time: number) => string,
|
|
resources: DecodedResources,
|
|
): Promise<ThumbnailLoadedResult> {
|
|
const mediabunny = await import("mediabunny");
|
|
throwIfAborted(signal);
|
|
|
|
const input = new mediabunny.Input({
|
|
source: new mediabunny.UrlSource(request.source),
|
|
formats: mediabunny.ALL_FORMATS,
|
|
});
|
|
try {
|
|
const track = await input.getPrimaryVideoTrack();
|
|
throwIfAborted(signal);
|
|
if (!track) throw new Error("Video source has no decodable video track");
|
|
const [displayWidth, displayHeight] = await Promise.all([
|
|
track.getDisplayWidth(),
|
|
track.getDisplayHeight(),
|
|
]);
|
|
throwIfAborted(signal);
|
|
if (!(displayWidth > 0 && displayHeight > 0)) {
|
|
throw new Error("Video source has invalid dimensions");
|
|
}
|
|
const metadataDuration = await track.getDurationFromMetadata({ skipLiveWait: true });
|
|
throwIfAborted(signal);
|
|
const info: SourceInfo = { aspect: displayWidth / displayHeight, metadataDuration };
|
|
sourceInfos.set(request.source, { version: request.contentVersion, info });
|
|
const strip = stripTimes(request, info, budgets);
|
|
const { timestamps, window } = strip;
|
|
const slots: { slot: number; key: string; decodedAt?: number }[] = takeDecodedFrames(
|
|
strip,
|
|
keyOf,
|
|
resources,
|
|
).map((slot) => ({ slot, key: keyOf(timestamps[slot]!) }));
|
|
if (slots.length > 0) {
|
|
const keys = new mediabunny.EncodedPacketSink(track);
|
|
async function* decodeTimesAtNearbyKeyframes() {
|
|
for (const target of slots) {
|
|
const time = timestamps[target.slot]!;
|
|
const key = await keys.getKeyPacket(time, { metadataOnly: true });
|
|
if (signal.aborted) return;
|
|
target.decodedAt = key && accepts(window, time, key.timestamp) ? key.timestamp : time;
|
|
yield target.decodedAt;
|
|
}
|
|
}
|
|
const target = targetDimensions(info.aspect, budgets);
|
|
const sink = new mediabunny.CanvasSink(track, {
|
|
width: target.width,
|
|
height: target.height,
|
|
fit: request.fit ?? "cover",
|
|
poolSize: 1,
|
|
});
|
|
await decodeFrames(sink, decodeTimesAtNearbyKeyframes(), slots, signal, resources);
|
|
}
|
|
return loadedResult(resources, info.aspect, budgets, timestamps.length);
|
|
} finally {
|
|
input.dispose();
|
|
}
|
|
}
|