import { TIMELINE_VIEWPORT_BUDGETS, type TimelineViewportBudgets } from "./timelineViewportBudgets"; import { createThumbnailKey, type ThumbnailLoadedResult, type ThumbnailValue, } from "./thumbnailScheduler"; import { MediaFetchError, reportMediaFetchFailure } from "./mediaFetch"; import { acquireThumbnailSource, timedOut, type SourceInfo, type ThumbnailSource, } from "./thumbnailSourcePool"; import { releaseSharedFrames, shareFrame, takeSharedFrame, type HeldFrame, } from "./thumbnailFrameCache"; import { readStoredFrame, readStoredInfo, rememberVersion, storedVersion, storeFrame, storeInfo, type StoredSource, } from "./thumbnailDiskCache"; export interface VideoThumbnailDecodeRequest { source: string; contentVersion?: string; sourceStart?: number; sourceRangeDuration?: number; frameCount: number; fit?: "contain" | "cover"; } const LAST_FRAME_INSET_S = 0.001; export function videoThumbnailTimestamps( start: number, duration: number, frameCount: number, ): number[] { const safeStart = Math.max(0, Number.isFinite(start) ? start : 0); const safeDuration = Math.max(0, Number.isFinite(duration) ? duration : 0); const count = Math.max(1, Number.isFinite(frameCount) ? Math.floor(frameCount) : 1); if (count === 1) return [safeStart + safeDuration / 2]; const edges = Array.from( { length: count }, (_, index) => safeStart + (safeDuration * index) / count, ); return [...edges, safeStart + Math.max(0, safeDuration - LAST_FRAME_INSET_S)]; } interface FrameWindow { sourceStart: number; lead: number; end: number; } const sourceInfos = new Map(); const storedVersions = new Map>(); const storedInfos = new Set(); const openOf = (request: VideoThumbnailDecodeRequest) => `${request.source}\u0000${request.contentVersion ?? ""}`; function storedVersionOf(request: VideoThumbnailDecodeRequest): Promise { let pending = storedVersions.get(openOf(request)); if (!pending) storedVersions.set(openOf(request), (pending = storedVersion(request.source))); return pending; } const accepts = (window: FrameWindow, time: number, decodedAt: number) => decodedAt >= window.sourceStart && time - decodedAt <= (time >= window.end ? 0 : window.lead); class ClipResultError extends Error {} async function canvasToBlob(canvas: HTMLCanvasElement | OffscreenCanvas): Promise { if (canvas instanceof HTMLCanvasElement) { return new Promise((resolve, reject) => { canvas.toBlob( (blob) => blob ? resolve(blob) : reject(new ClipResultError("Video thumbnail encode failed")), "image/jpeg", 0.72, ); }); } return canvas.convertToBlob({ type: "image/jpeg", quality: 0.72 }).catch(() => { throw new ClipResultError("Video thumbnail encode failed"); }); } interface DecodedResources { urls: (string | undefined)[]; held: HeldFrame[]; canvases: Set; releasedFrameBytes: number; store?: (slot: number, decodedAt: number, blob: Blob) => void; } interface ThumbnailCanvasSink { canvasesAtTimestamps( timestamps: AsyncIterable, ): AsyncIterable<{ canvas: HTMLCanvasElement | OffscreenCanvas } | null>; } const timeoutWithoutProgress = () => new MediaFetchError("thumbnail", null, "timeout"); const isAbortError = (error: unknown) => (error as { name?: unknown } | null)?.name === "AbortError"; const isSourceFailure = (error: unknown) => !isAbortError(error) && !(error instanceof ClipResultError); function throwIfAborted(signal: AbortSignal): void { if (signal.aborted) throw new DOMException("Aborted", "AbortError"); } function releaseDecodedResources(resources: DecodedResources): void { resources.urls.length = 0; releaseSharedFrames(resources.held, resources.releasedFrameBytes); for (const canvas of resources.canvases) { canvas.width = 0; canvas.height = 0; } resources.canvases.clear(); } function targetDimensions( aspect: number, budgets: Readonly, ): { width: number; height: number } { const width = Math.max( 1, Math.min(budgets.posterMaxPhysicalWidth, Math.round(budgets.posterMaxPhysicalHeight * aspect)), ); return { width, height: Math.max(1, Math.min(budgets.posterMaxPhysicalHeight, Math.round(width / aspect))), }; } function stripTimes( request: VideoThumbnailDecodeRequest, info: SourceInfo, budgets: Readonly, ) { const requestedStart = Math.max(0, request.sourceStart ?? 0); const sourceDuration = Math.max( 0, info.metadataDuration ?? requestedStart + (request.sourceRangeDuration ?? 0), ); const sourceStart = Math.min(requestedStart, sourceDuration); const requestedDuration = request.sourceRangeDuration ?? Math.max(0, sourceDuration - sourceStart); const duration = Math.min( Math.max(0, requestedDuration), Math.max(0, sourceDuration - sourceStart), ); const timestamps = videoThumbnailTimestamps( sourceStart, duration, Math.min(request.frameCount, budgets.richPreviewFrameCount), ); const slices = Math.max(1, timestamps.length - 1); const window: FrameWindow = { sourceStart, lead: duration / Math.max(2, slices) / 2, end: timestamps.length > 1 ? timestamps.at(-1)! : Infinity, }; return { window, timestamps }; } function takeDecodedFrames( { timestamps, window }: ReturnType, keyOf: (time: number) => string, resources: DecodedResources, ): number[] { const missing: number[] = []; timestamps.forEach((time, slot) => { const key = keyOf(time); const frame = takeSharedFrame(key, (decodedAt) => accepts(window, time, decodedAt)); if (!frame) return void missing.push(slot); resources.urls[slot] = frame.url; resources.held.push({ key, frame }); }); return missing; } async function decodeFrames( sink: ThumbnailCanvasSink, times: AsyncIterable, slots: { slot: number; key: string; decodedAt?: number }[], signal: AbortSignal, resources: DecodedResources, progress: ThumbnailSource["progress"], ): Promise { let next = 0; for await (const wrapped of sink.canvasesAtTimestamps(times)) { throwIfAborted(signal); const target = slots[next++]; if (!wrapped || target?.decodedAt === undefined) continue; resources.canvases.add(wrapped.canvas); const blob = await canvasToBlob(wrapped.canvas); throwIfAborted(signal); const frame = shareFrame(target.key, target.decodedAt, URL.createObjectURL(blob), blob.size); progress.framesFromAnyJob += 1; resources.store?.(target.slot, target.decodedAt, blob); resources.urls[target.slot] = frame.url; resources.held.push({ key: target.key, frame }); } throwIfAborted(signal); } function loadedResult( resources: DecodedResources, aspect: number, budgets: Readonly, frameCount = resources.urls.length, ): ThumbnailLoadedResult { const decoded = resources.urls.filter((url): url is string => url !== undefined); const firstUrl = decoded[0]; if (!firstUrl) throw new ClipResultError("Video source returned no thumbnail frames"); let nearest = firstUrl; const urls = Array.from( { length: frameCount }, (_, slot) => (nearest = resources.urls[slot] ?? nearest), ); const { width, height } = targetDimensions(aspect, budgets); const value: ThumbnailValue = urls.length === 1 ? { kind: "image", url: firstUrl, aspect } : { kind: "filmstrip", urls, aspect }; return { value, weight: width * height * 4 * decoded.length, dispose: () => releaseDecodedResources(resources), }; } const storedFrameName = (fit: string, info: SourceInfo, budgets: Readonly) => (time: number) => { const { width, height } = targetDimensions(info.aspect, budgets); return `${fit} ${width}x${height} ${time}`; }; async function takeStoredFrames( { timestamps, window }: ReturnType, slots: number[], keyOf: (time: number) => string, nameOf: (time: number) => string, resources: DecodedResources, stored: StoredSource, ): Promise { const found = await Promise.all( slots.map((slot) => readStoredFrame(stored, nameOf(timestamps[slot]!))), ); return slots.filter((slot, i) => { const time = timestamps[slot]!; const hit = found[i]; if (!hit || !accepts(window, time, hit.decodedAt)) return true; const key = keyOf(time); const frame = shareFrame(key, hit.decodedAt, URL.createObjectURL(hit.blob), hit.blob.size); resources.urls[slot] = frame.url; resources.held.push({ key, frame }); return false; }); } function versionRead( request: VideoThumbnailDecodeRequest, source: ThumbnailSource, ): StoredSource | undefined { if (!source.etag) return undefined; const version = { source: request.source, etag: source.etag }; storedVersions.set(openOf(request), Promise.resolve(version)); return version; } function storeForNextSession( request: VideoThumbnailDecodeRequest, info: SourceInfo, budgets: Readonly, resources: DecodedResources, version: StoredSource | undefined, ): void { if (!version) return; const id = `${version.source}\u0000${version.etag}`; if (!storedInfos.has(id)) { storedInfos.add(id); void rememberVersion(version).then(() => storeInfo(version, info)); } const nameOf = storedFrameName(request.fit ?? "cover", info, budgets); const { timestamps } = stripTimes(request, info, budgets); resources.store = (slot, decodedAt, blob) => void storeFrame(version, nameOf(timestamps[slot]!), decodedAt, blob); } /** Sparse Mediabunny extraction with one pooled canvas and one cleanup owner. */ export async function decodeVideoThumbnail( request: VideoThumbnailDecodeRequest, signal: AbortSignal, budgets: Readonly = TIMELINE_VIEWPORT_BUDGETS, ): Promise { const fit = request.fit ?? "cover"; const sourceKey = createThumbnailKey({ source: request.source, version: request.contentVersion }); const keyOf = (time: number) => `${sourceKey}\u0000${fit}\u0000${time}`; const resources: DecodedResources = { urls: [], held: [], canvases: new Set(), releasedFrameBytes: budgets.sharedFrameCacheBytes, }; const entry = sourceInfos.get(request.source); try { const stored = await storedVersionOf(request); const known = entry && entry.version === request.contentVersion ? entry.info : stored && (await readStoredInfo(stored)); if (known) { sourceInfos.set(request.source, { version: request.contentVersion, info: known }); const strip = stripTimes(request, known, budgets); const missing = takeDecodedFrames(strip, keyOf, resources); const nameOf = storedFrameName(fit, known, budgets); const left = stored && missing.length > 0 ? await takeStoredFrames(strip, missing, keyOf, nameOf, resources, stored) : missing; if (left.length === 0) { throwIfAborted(signal); return loadedResult(resources, known.aspect, budgets); } releaseDecodedResources(resources); } return await decodeMissingFrames(request, signal, budgets, keyOf, resources, stored); } catch (error) { releaseDecodedResources(resources); reportMediaFetchFailure(error); throw error; } } async function decodeMissingFrames( request: VideoThumbnailDecodeRequest, signal: AbortSignal, budgets: Readonly, keyOf: (time: number) => string, resources: DecodedResources, stored: StoredSource | undefined, ): Promise { let source: ThumbnailSource; try { source = await acquireThumbnailSource( request.source, request.contentVersion, signal, budgets.thumbnailSourceIdleMs, ); } catch (error) { if (timedOut(signal)) reportMediaFetchFailure(timeoutWithoutProgress()); throw error; } const framesBefore = source.progress.framesFromAnyJob; const releaseOnAbort = () => { const madeProgress = source.waitedForOpen || source.progress.framesFromAnyJob > framesBefore; const failed = timedOut(signal) && !madeProgress; if (failed) reportMediaFetchFailure(timeoutWithoutProgress()); source.release(failed); }; signal.addEventListener("abort", releaseOnAbort, { once: true }); try { throwIfAborted(signal); const { mediabunny, track, info } = source; sourceInfos.set(request.source, { version: request.contentVersion, info }); const strip = stripTimes(request, info, budgets); const { timestamps, window } = strip; storeForNextSession(request, info, budgets, resources, stored ?? versionRead(request, source)); const slots: { slot: number; key: string; decodedAt?: number }[] = takeDecodedFrames( strip, keyOf, resources, ).map((slot) => ({ slot, key: keyOf(timestamps[slot]!) })); if (slots.length > 0) { const keys = new mediabunny.EncodedPacketSink(track); async function* decodeTimesAtNearbyKeyframes() { for (const target of slots) { const time = timestamps[target.slot]!; const key = await keys.getKeyPacket(time, { metadataOnly: true }); if (signal.aborted) return; target.decodedAt = key && accepts(window, time, key.timestamp) ? key.timestamp : time; yield target.decodedAt; } } const target = targetDimensions(info.aspect, budgets); const sink = new mediabunny.CanvasSink(track, { width: target.width, height: target.height, fit: request.fit ?? "cover", poolSize: 1, }); const times = decodeTimesAtNearbyKeyframes(); await decodeFrames(sink, times, slots, signal, resources, source.progress); } const result = loadedResult(resources, info.aspect, budgets, timestamps.length); source.release(false); return result; } catch (error) { source.release(isSourceFailure(error)); throw error; } finally { signal.removeEventListener("abort", releaseOnAbort); } }