perf(studio-server): coordinate cancelable thumbnail generation (#2720)

* perf(studio): schedule adaptive timeline thumbnails

* perf(studio): bound thumbnail decoding resources

* perf(studio): virtualize timeline thumbnail media

* perf(studio): prioritize timeline thumbnail work

* perf(studio-server): coordinate cancelable thumbnail generation

---------

Co-authored-by: Codex <codex@local>
This commit is contained in:
Miguel Ángel
2026-08-05 20:41:35 -07:00
committed by GitHub
co-authored by Codex
parent 27d113e56c
commit 96861cbafc
59 changed files with 3537 additions and 879 deletions
@@ -0,0 +1,53 @@
// @vitest-environment happy-dom
import React, { act } from "react";
import { createRoot } from "react-dom/client";
import { afterEach, describe, expect, it, vi } from "vitest";
const { leaseSpy } = vi.hoisted(() => ({
leaseSpy: vi.fn((_request: unknown) => ({ status: "loading" as const })),
}));
vi.mock("../../hooks/useThumbnailLease", () => ({
useThumbnailLease: leaseSpy,
}));
import { AudioWaveform } from "./AudioWaveform";
(globalThis as unknown as { IS_REACT_ACT_ENVIRONMENT: boolean }).IS_REACT_ACT_ENVIRONMENT = true;
afterEach(() => {
leaseSpy.mockClear();
document.body.innerHTML = "";
});
describe("AudioWaveform", () => {
it("leases waveform decoding with the clip's project, session, and viewport priority", () => {
const host = document.createElement("div");
document.body.append(host);
const root = createRoot(host);
act(() => {
root.render(
<AudioWaveform
audioUrl="/media/voice.wav"
label=""
labelColor="#fff"
projectId="project-a"
sessionEpoch={9}
priority="interaction"
/>,
);
});
expect(leaseSpy).toHaveBeenCalled();
expect(leaseSpy.mock.calls.at(-1)?.[0]).toMatchObject({
projectId: "project-a",
sessionEpoch: 9,
kind: "waveform",
priority: "interaction",
rich: false,
});
act(() => root.unmount());
});
});
@@ -1,74 +1,105 @@
import { memo, useRef, useState, useCallback, useEffect } from "react";
import { memo, useCallback, useMemo, useRef } from "react";
import { useMountEffect } from "../../hooks/useMountEffect";
import { useThumbnailLease } from "../../hooks/useThumbnailLease";
import { createThumbnailKey, type ThumbnailPriority } from "../lib/thumbnailScheduler";
interface AudioWaveformProps {
audioUrl: string;
waveformUrl?: string;
label: string;
labelColor: string;
/**
* Fraction (01) of the source the clip starts at, after the media-start
* trim. Defaults to 0 (no front trim).
*/
trimStartFraction?: number;
/**
* Fraction (01) of the source the clip ends at. Defaults to 1 (no tail
* trim). Together these window the rendered peaks to the trimmed slice so the
* waveform tracks the clip edges instead of squeezing the whole file in.
*/
trimEndFraction?: number;
projectId: string;
sessionEpoch: number;
priority: ThumbnailPriority;
}
const BAR_W = 2;
const GAP = 1;
const STEP = BAR_W + GAP;
const BAR_WIDTH = 2;
const BAR_STEP = 3;
/** Downsample PCM channel data into peak amplitudes (01). */
function extractPeaks(channelData: Float32Array, barCount: number): number[] {
const peaks: number[] = [];
const samplesPerBar = Math.floor(channelData.length / barCount);
if (samplesPerBar === 0) return Array(barCount).fill(0);
for (let i = 0; i < barCount; i++) {
for (let index = 0; index < barCount; index++) {
let max = 0;
const start = i * samplesPerBar;
const start = index * samplesPerBar;
const end = Math.min(start + samplesPerBar, channelData.length);
for (let j = start; j < end; j++) {
// fallow-ignore-next-line code-duplication
const abs = Math.abs(channelData[j] ?? 0);
if (abs > max) max = abs;
for (let sample = start; sample < end; sample++) {
max = Math.max(max, Math.abs(channelData[sample] ?? 0));
}
peaks.push(max);
}
const maxPeak = Math.max(...peaks, 0.001);
return peaks.map((p) => p / maxPeak);
return peaks.map((peak) => peak / maxPeak);
}
/** Deterministic fake waveform as fallback (matches demo app). */
function fakePeaks(url: string, count: number): number[] {
let seed = 0;
for (let i = 0; i < url.length; i++) seed = ((seed << 5) - seed + url.charCodeAt(i)) | 0;
for (let index = 0; index < url.length; index++) {
seed = ((seed << 5) - seed + url.charCodeAt(index)) | 0;
}
seed = Math.abs(seed) || 42;
const rand = () => {
const random = () => {
seed = (seed * 16807) % 2147483647;
return (seed & 0x7fffffff) / 2147483647;
};
const peaks: number[] = [];
for (let i = 0; i < count; i++) {
const t = i / count;
const envelope = 0.3 + 0.3 * Math.sin(t * Math.PI * 3.2) + 0.2 * Math.sin(t * Math.PI * 7.1);
peaks.push(Math.max(0.05, Math.min(1, envelope * (0.4 + 0.6 * rand()))));
}
return peaks;
return Array.from({ length: count }, (_, index) => {
const time = index / count;
const envelope =
0.3 + 0.3 * Math.sin(time * Math.PI * 3.2) + 0.2 * Math.sin(time * Math.PI * 7.1);
return Math.max(0.05, Math.min(1, envelope * (0.4 + 0.6 * random())));
});
}
// Module-level cache so decoded audio persists across re-renders and re-mounts
const peaksCache = new Map<string, number[]>();
const decodeInFlight = new Map<string, Promise<number[]>>();
async function loadWaveform(
audioUrl: string,
waveformUrl: string | undefined,
signal: AbortSignal,
): Promise<number[]> {
try {
return waveformUrl
? await fetchWaveformPeaks(waveformUrl, signal)
: await decodeWaveformPeaks(audioUrl, signal);
} catch (error) {
if (signal.aborted) throw error;
return fakePeaks(waveformUrl ?? audioUrl, 4000);
}
}
/**
* Audio waveform rendered from real PCM data via Web Audio API.
* Falls back to a deterministic fake pattern if decoding fails.
* Bars grow from bottom to top, rendered as CSS divs for zoom resilience.
*/
async function fetchWaveformPeaks(url: string, signal: AbortSignal): Promise<number[]> {
const response = await fetch(url, { signal });
if (!response.ok) throw new Error(`Waveform request failed (${response.status})`);
const data: unknown = await response.json();
if (
typeof data !== "object" ||
data === null ||
!("peaks" in data) ||
!Array.isArray(data.peaks) ||
!data.peaks.every((peak) => typeof peak === "number")
) {
throw new Error("Invalid waveform response");
}
return data.peaks;
}
async function decodeWaveformPeaks(url: string, signal: AbortSignal): Promise<number[]> {
const response = await fetch(url, { signal });
if (!response.ok) throw new Error(`Audio request failed (${response.status})`);
const buffer = await response.arrayBuffer();
if (signal.aborted) throw new DOMException("Aborted", "AbortError");
const context = new AudioContext();
try {
const decoded = await context.decodeAudioData(buffer);
if (signal.aborted) throw new DOMException("Aborted", "AbortError");
return extractPeaks(decoded.getChannelData(0), 4000);
} finally {
await context.close();
}
}
/** Bounded waveform subscriber; cache, cancellation and dedupe live in one scheduler. */
export const AudioWaveform = memo(function AudioWaveform({
audioUrl,
waveformUrl,
@@ -76,130 +107,96 @@ export const AudioWaveform = memo(function AudioWaveform({
labelColor,
trimStartFraction,
trimEndFraction,
projectId,
sessionEpoch,
priority,
}: AudioWaveformProps) {
const containerRef = useRef<HTMLDivElement | null>(null);
const barsRef = useRef<HTMLDivElement | null>(null);
const roRef = useRef<ResizeObserver | null>(null);
const canvasRef = useRef<HTMLCanvasElement | null>(null);
const observerRef = useRef<ResizeObserver | null>(null);
const cacheKey = waveformUrl ?? audioUrl;
const [peaks, setPeaks] = useState<number[] | null>(peaksCache.get(cacheKey) ?? null);
const request = useMemo(
() => ({
key: createThumbnailKey({ kind: "waveform", source: cacheKey }),
projectId,
sessionEpoch,
kind: "waveform" as const,
priority,
rich: false,
load: async (signal: AbortSignal) => {
const peaks = await loadWaveform(audioUrl, waveformUrl, signal);
return {
value: { kind: "waveform" as const, peaks },
weight: peaks.length * Float64Array.BYTES_PER_ELEMENT,
};
},
}),
[audioUrl, cacheKey, priority, projectId, sessionEpoch, waveformUrl],
);
const snapshot = useThumbnailLease(cacheKey ? request : null);
const peaks =
snapshot.status === "ready" && snapshot.value.kind === "waveform" ? snapshot.value.peaks : null;
useEffect(() => {
if (peaks || !cacheKey) return;
let cancelled = false;
let promise = decodeInFlight.get(cacheKey);
if (!promise) {
promise = (
waveformUrl
? fetch(waveformUrl)
.then((r) => r.json())
.then((d: { peaks?: number[] }) => {
if (!Array.isArray(d.peaks)) throw new Error("bad response");
return d.peaks;
})
: fetch(audioUrl)
.then((r) => r.arrayBuffer())
.then((buf) => {
const ctx = new AudioContext();
return ctx.decodeAudioData(buf).finally(() => ctx.close());
})
.then((decoded) => extractPeaks(decoded.getChannelData(0), 4000))
)
.catch(() => fakePeaks(cacheKey, 4000))
.then((p) => {
peaksCache.set(cacheKey, p);
return p;
})
.finally(() => decodeInFlight.delete(cacheKey));
decodeInFlight.set(cacheKey, promise);
}
promise.then((p) => {
if (!cancelled) setPeaks(p);
});
return () => {
cancelled = true;
};
}, [audioUrl, waveformUrl, cacheKey, peaks]);
// Draw bars into the container using innerHTML (fast, zoom-resilient)
const draw = useCallback(() => {
const container = containerRef.current;
const barsEl = barsRef.current;
if (!container || !barsEl || !peaks) return;
// Window the peaks to the trimmed slice [start, end) of the source so the
// bars track the clip edges. Clamp to a valid, non-empty range.
const winStart = Math.max(0, Math.min(1, trimStartFraction ?? 0));
const winEnd = Math.max(winStart, Math.min(1, trimEndFraction ?? 1));
const lo = Math.floor(winStart * peaks.length);
const hi = Math.max(lo + 1, Math.ceil(winEnd * peaks.length));
const span = hi - lo;
// Fill the full (possibly zoomed) clip width with STEP-spaced bars, resampling
// the windowed peaks across them — upsampling (repeating peaks) when the clip
// is wider than the slice has samples, so the waveform stretches with zoom
// instead of stopping partway across.
const w = container.clientWidth || 400;
const barCount = Math.max(0, Math.floor(w / STEP));
let html = "";
for (let i = 0; i < barCount; i++) {
// Map bar index to peak index within the windowed range (resample)
const peakIdx = lo + Math.min(span - 1, Math.floor((i / barCount) * span));
const amp = peaks[peakIdx] ?? 0;
const pct = Math.max(3, Math.round(amp * 100));
const opacity = (0.45 + amp * 0.4).toFixed(2);
html += `<div style="position:absolute;bottom:0;left:${i * STEP}px;width:${BAR_W}px;height:${pct}%;background:rgba(75,163,210,${opacity})"></div>`;
const canvas = canvasRef.current;
if (!canvas || !peaks) return;
const width = Math.max(1, canvas.clientWidth);
const height = Math.max(1, canvas.clientHeight);
const scale = window.devicePixelRatio || 1;
canvas.width = Math.ceil(width * scale);
canvas.height = Math.ceil(height * scale);
const context = canvas.getContext("2d");
if (!context) return;
context.scale(scale, scale);
context.clearRect(0, 0, width, height);
const startFraction = Math.max(0, Math.min(1, trimStartFraction ?? 0));
const endFraction = Math.max(startFraction, Math.min(1, trimEndFraction ?? 1));
const start = Math.floor(startFraction * peaks.length);
const end = Math.max(start + 1, Math.ceil(endFraction * peaks.length));
const span = end - start;
const barCount = Math.floor(width / BAR_STEP);
for (let index = 0; index < barCount; index++) {
const peakIndex = start + Math.min(span - 1, Math.floor((index / barCount) * span));
const amplitude = peaks[peakIndex] ?? 0;
const barHeight = Math.max(2, amplitude * height);
context.fillStyle = `rgba(75,163,210,${(0.45 + amplitude * 0.4).toFixed(2)})`;
context.fillRect(index * BAR_STEP, height - barHeight, BAR_WIDTH, barHeight);
}
barsEl.innerHTML = html;
}, [peaks, trimStartFraction, trimEndFraction]);
}, [peaks, trimEndFraction, trimStartFraction]);
// Observe container size and redraw
const setContainerRef = useCallback(
(el: HTMLDivElement | null) => {
roRef.current?.disconnect();
containerRef.current = el;
if (!el) return;
const setCanvasRef = useCallback(
(canvas: HTMLCanvasElement | null) => {
observerRef.current?.disconnect();
canvasRef.current = canvas;
if (!canvas) return;
draw();
roRef.current = new ResizeObserver(() => draw());
roRef.current.observe(el);
observerRef.current = new ResizeObserver(draw);
observerRef.current.observe(canvas);
},
[draw],
);
// Redraw when peaks arrive
useEffect(() => {
draw();
}, [draw]);
useEffect(
() => () => {
roRef.current?.disconnect();
},
[],
);
useMountEffect(() => () => observerRef.current?.disconnect());
return (
<div ref={setContainerRef} className="absolute inset-0 overflow-hidden">
<div ref={barsRef} className="absolute left-0 right-0 bottom-0" style={{ top: 16 }} />
{/* Shimmer while decoding */}
{!peaks && (
<div className="absolute inset-0 overflow-hidden">
<canvas
ref={setCanvasRef}
className="absolute inset-x-0 bottom-0 w-full"
style={{ top: 16 }}
/>
{snapshot.status === "loading" && (
<div
className="absolute left-0 right-0 bottom-0 animate-pulse"
className="absolute inset-x-0 bottom-0 top-4 animate-pulse"
style={{
top: 16,
background:
"linear-gradient(90deg, rgba(255,255,255,0.02) 0%, rgba(255,255,255,0.05) 50%, rgba(255,255,255,0.02) 100%)",
}}
/>
)}
{label && (
<div className="absolute top-0 left-0 right-0 px-1.5 py-0.5 z-10">
<div className="absolute inset-x-0 top-0 z-10 px-1.5 py-0.5">
<span
className="text-[9px] font-semibold truncate block leading-tight"
className="block truncate text-[9px] font-semibold leading-tight"
style={{ color: labelColor, textShadow: "0 1px 3px rgba(0,0,0,0.9)" }}
>
{label}
@@ -2,7 +2,8 @@
import React, { act } from "react";
import { createRoot, type Root } from "react-dom/client";
import { afterEach, beforeEach, describe, expect, it } from "vitest";
import { afterEach, beforeEach, describe, expect, it, vi } from "vitest";
import { thumbnailScheduler } from "../lib/thumbnailScheduler";
import { buildCompositionThumbnailUrl, CompositionThumbnail } from "./CompositionThumbnail";
Object.defineProperty(globalThis, "IS_REACT_ACT_ENVIRONMENT", {
@@ -31,12 +32,18 @@ class MockImage {
const originalResizeObserver = globalThis.ResizeObserver;
const originalImage = globalThis.Image;
const originalFetch = globalThis.fetch;
const originalCreateObjectURL = URL.createObjectURL;
const originalRevokeObjectURL = URL.revokeObjectURL;
let host: HTMLDivElement;
let root: Root | null = null;
beforeEach(() => {
globalThis.ResizeObserver = MockResizeObserver as unknown as typeof ResizeObserver;
globalThis.Image = MockImage as unknown as typeof Image;
globalThis.fetch = vi.fn(async () => new Response(new Blob(["thumbnail"]), { status: 200 }));
URL.createObjectURL = vi.fn(() => "blob:composition-thumbnail");
URL.revokeObjectURL = vi.fn();
MockImage.instances = [];
host = document.createElement("div");
document.body.append(host);
@@ -45,8 +52,12 @@ beforeEach(() => {
afterEach(() => {
act(() => root?.unmount());
root = null;
thumbnailScheduler.invalidateProject("/api/projects/demo/preview");
globalThis.ResizeObserver = originalResizeObserver;
globalThis.Image = originalImage;
globalThis.fetch = originalFetch;
URL.createObjectURL = originalCreateObjectURL;
URL.revokeObjectURL = originalRevokeObjectURL;
document.body.replaceChildren();
});
@@ -68,9 +79,9 @@ describe("buildCompositionThumbnailUrl", () => {
});
describe("CompositionThumbnail", () => {
function renderThumbnail(): MockImage {
async function renderThumbnail(): Promise<MockImage> {
root = createRoot(host);
act(() => {
await act(async () => {
root!.render(
React.createElement(CompositionThumbnail, {
previewUrl: "/api/projects/demo/preview",
@@ -78,19 +89,27 @@ describe("CompositionThumbnail", () => {
labelColor: "#fff",
}),
);
await new Promise((resolve) => setTimeout(resolve, 0));
});
const probe = MockImage.instances[0];
if (!probe) throw new Error("Expected an image probe");
return probe;
}
it("renders visible tiles after the off-DOM probe loads", () => {
const probe = renderThumbnail();
it("renders visible tiles after the scheduled off-DOM probe loads", async () => {
const probe = await renderThumbnail();
act(() => {
expect(globalThis.fetch).toHaveBeenCalledWith(
expect.stringContaining("/api/projects/demo/thumbnail/index.html"),
expect.objectContaining({ signal: expect.any(AbortSignal) }),
);
expect(probe.src).toBe("blob:composition-thumbnail");
await act(async () => {
probe.naturalWidth = 1920;
probe.naturalHeight = 1080;
probe.onload?.();
await new Promise((resolve) => setTimeout(resolve, 0));
});
const tiles = [...host.querySelectorAll("img")];
@@ -98,16 +117,20 @@ describe("CompositionThumbnail", () => {
expect(tiles.every((tile) => !tile.classList.contains("hidden"))).toBe(true);
});
it("aborts its off-DOM image probe when unmounted", () => {
const probe = renderThumbnail();
it("aborts its scheduled off-DOM image probe when unmounted", async () => {
const probe = await renderThumbnail();
expect(host.querySelector("img")).toBeNull();
expect(probe.src).toContain("/api/projects/demo/thumbnail/index.html");
expect(probe.src).toBe("blob:composition-thumbnail");
act(() => root?.unmount());
await act(async () => {
root?.unmount();
await new Promise((resolve) => setTimeout(resolve, 0));
});
root = null;
expect(probe.onload).toBeNull();
expect(probe.onerror).toBeNull();
expect(probe.src).toBe("");
expect(URL.revokeObjectURL).toHaveBeenCalledWith("blob:composition-thumbnail");
});
});
@@ -1,5 +1,9 @@
import { memo, useCallback, useEffect, useState, useRef } from "react";
import { memo, useCallback, useMemo, useRef, useState } from "react";
import { useMountEffect } from "../../hooks/useMountEffect";
import { useThumbnailLease } from "../../hooks/useThumbnailLease";
import { createThumbnailKey, type ThumbnailPriority } from "../lib/thumbnailScheduler";
import { TIMELINE_VIEWPORT_BUDGETS } from "../lib/timelineViewportBudgets";
import { computeThumbnailStrip, probeImageAspect } from "./thumbnailUtils";
interface CompositionThumbnailProps {
previewUrl: string;
@@ -11,6 +15,10 @@ interface CompositionThumbnailProps {
duration?: number;
width?: number;
height?: number;
projectId?: string;
sessionEpoch?: number;
priority?: ThumbnailPriority;
rich?: boolean;
}
const CLIP_HEIGHT = 66;
@@ -34,9 +42,8 @@ export function buildCompositionThumbnailUrl({
const thumbnailBase = previewUrl
.replace("/preview/comp/", "/thumbnail/")
.replace(/\/preview$/, "/thumbnail/index.html");
const midTime = seekTime + duration / 2;
const thumbnailUrl = new URL(thumbnailBase, origin);
thumbnailUrl.searchParams.set("t", midTime.toFixed(2));
thumbnailUrl.searchParams.set("t", (seekTime + duration / 2).toFixed(2));
thumbnailUrl.searchParams.set("v", THUMBNAIL_URL_VERSION);
if (selector) {
thumbnailUrl.searchParams.set("selector", selector);
@@ -47,6 +54,29 @@ export function buildCompositionThumbnailUrl({
return thumbnailUrl.toString();
}
async function loadCompositionImage(url: string, signal: AbortSignal) {
const response = await fetch(url, { signal });
if (!response.ok) throw new Error(`Composition thumbnail failed (${response.status})`);
const blob = await response.blob();
if (signal.aborted) throw new DOMException("Aborted", "AbortError");
const objectUrl = URL.createObjectURL(blob);
try {
const aspect = await probeImageAspect(objectUrl, signal);
return {
value: { kind: "image" as const, url: objectUrl, aspect },
weight:
TIMELINE_VIEWPORT_BUDGETS.posterMaxPhysicalWidth *
TIMELINE_VIEWPORT_BUDGETS.posterMaxPhysicalHeight *
4,
dispose: () => URL.revokeObjectURL(objectUrl),
};
} catch (error) {
URL.revokeObjectURL(objectUrl);
throw error;
}
}
/** Server-rendered composition poster, deduplicated and budgeted by project/session. */
export const CompositionThumbnail = memo(function CompositionThumbnail({
previewUrl,
label,
@@ -55,31 +85,12 @@ export const CompositionThumbnail = memo(function CompositionThumbnail({
selectorIndex,
seekTime = 2,
duration = 5,
projectId = previewUrl,
sessionEpoch = 0,
priority = "visible",
}: CompositionThumbnailProps) {
const [containerWidth, setContainerWidth] = useState(0);
const [loaded, setLoaded] = useState(false);
const [aspect, setAspect] = useState(16 / 9);
const roRef = useRef<ResizeObserver | null>(null);
const setContainerRef = useCallback((el: HTMLDivElement | null) => {
roRef.current?.disconnect();
if (!el) return;
const measured = el.parentElement?.clientWidth || el.clientWidth;
// fallow-ignore-next-line code-duplication
setContainerWidth(measured);
const target = el.parentElement || el;
roRef.current = new ResizeObserver(([entry]) => {
setContainerWidth(entry.contentRect.width);
});
roRef.current.observe(target);
}, []);
useMountEffect(() => () => {
roRef.current?.disconnect();
});
const observerRef = useRef<ResizeObserver | null>(null);
const url = buildCompositionThumbnailUrl({
previewUrl,
seekTime,
@@ -88,52 +99,56 @@ export const CompositionThumbnail = memo(function CompositionThumbnail({
selectorIndex,
origin: window.location.origin,
});
const request = useMemo(
() => ({
key: createThumbnailKey({ kind: "composition", url }),
projectId,
sessionEpoch,
kind: "composition" as const,
priority,
rich: true,
load: (signal: AbortSignal) => loadCompositionImage(url, signal),
}),
[priority, projectId, sessionEpoch, url],
);
const snapshot = useThumbnailLease(request);
const value =
snapshot.status === "ready" && snapshot.value.kind === "image" ? snapshot.value : null;
const { frameW, frameCount } = computeThumbnailStrip(
containerWidth,
value?.aspect ?? 16 / 9,
CLIP_HEIGHT,
48,
);
// Probe outside React's DOM and explicitly abort on URL changes/unmount. A
// hidden React <img> keeps its Fiber reachable from Blink's pending-activity
// queue while a request is unresolved.
// eslint-disable-next-line no-restricted-syntax
useEffect(() => {
let cancelled = false;
setLoaded(false);
const probe = new Image();
probe.onload = () => {
if (cancelled) return;
if (probe.naturalWidth > 0 && probe.naturalHeight > 0) {
setAspect(probe.naturalWidth / probe.naturalHeight);
}
setLoaded(true);
};
probe.onerror = () => {
if (!cancelled) setLoaded(false);
};
probe.src = url;
return () => {
cancelled = true;
probe.onload = null;
probe.onerror = null;
probe.src = "";
};
}, [url]);
const setContainerRef = useCallback((element: HTMLDivElement | null) => {
observerRef.current?.disconnect();
if (!element) return;
const target = element.parentElement ?? element;
setContainerWidth(target.clientWidth);
observerRef.current = new ResizeObserver(([entry]) =>
setContainerWidth(entry.contentRect.width),
);
observerRef.current.observe(target);
}, []);
const frameW = Math.max(48, Math.round(CLIP_HEIGHT * aspect));
const frameCount = containerWidth > 0 ? Math.max(1, Math.ceil(containerWidth / frameW)) : 1;
useMountEffect(() => () => observerRef.current?.disconnect());
return (
<div ref={setContainerRef} className="absolute inset-0 overflow-hidden">
{loaded && (
{value && (
<div
className="absolute inset-0 flex"
style={{ animation: "hf-thumb-fade 200ms ease-out", mixBlendMode: "lighten" }}
>
{Array.from({ length: frameCount }).map((_, i) => (
{Array.from({ length: frameCount }, (_, index) => (
<div
key={i}
key={index}
className="relative h-full flex-shrink-0 overflow-hidden"
style={{ width: frameW }}
>
<img
src={url}
src={value.url}
alt=""
draggable={false}
className="absolute inset-0 h-full w-full object-cover"
@@ -143,14 +158,16 @@ export const CompositionThumbnail = memo(function CompositionThumbnail({
))}
</div>
)}
{snapshot.status === "loading" && (
<div className="absolute inset-0 animate-pulse bg-white/[0.035]" />
)}
{label && (
<div className="absolute left-3 top-0 bottom-0 flex items-center" style={{ zIndex: 10 }}>
<div className="absolute inset-y-0 left-3 z-10 flex items-center">
<span
className="block max-w-full truncate text-[10px] font-semibold leading-none"
style={{
color: labelColor,
textShadow: loaded ? "0 1px 4px rgba(0,0,0,0.9), 0 0 8px rgba(0,0,0,0.6)" : "none",
textShadow: value ? "0 1px 4px rgba(0,0,0,0.9), 0 0 8px rgba(0,0,0,0.6)" : "none",
}}
>
{label}
@@ -3,6 +3,7 @@
import React, { act } from "react";
import { createRoot, type Root } from "react-dom/client";
import { afterEach, beforeEach, describe, expect, it } from "vitest";
import { thumbnailScheduler } from "../lib/thumbnailScheduler";
import { ImageThumbnail } from "./ImageThumbnail";
Object.defineProperty(globalThis, "IS_REACT_ACT_ENVIRONMENT", {
@@ -68,6 +69,7 @@ beforeEach(() => {
afterEach(() => {
act(() => root?.unmount());
root = null;
thumbnailScheduler.invalidateProject("p");
globalThis.IntersectionObserver = originalIO;
globalThis.ResizeObserver = originalRO;
globalThis.Image = originalImage;
@@ -82,6 +84,10 @@ function render(props: { imageSrc: string; label?: string; labelColor?: string }
imageSrc={props.imageSrc}
label={props.label ?? ""}
labelColor={props.labelColor ?? "#fff"}
projectId="p"
sessionEpoch={1}
priority="visible"
rich={false}
/>,
);
});
@@ -93,6 +99,13 @@ function lastProbe(): MockImage {
return probe!;
}
async function resolveProbe(update: (probe: MockImage) => void): Promise<void> {
await act(async () => {
update(lastProbe());
await new Promise((resolve) => setTimeout(resolve, 0));
});
}
/** Assert at least one tile rendered and the first tile serves `expectedSrc`. */
function expectFirstTileSrc(expectedSrc: string): void {
const imgs = [...host.querySelectorAll("img")];
@@ -107,15 +120,15 @@ describe("ImageThumbnail", () => {
expect(host.querySelectorAll("img").length).toBe(0);
});
it("probes the resolved src and renders repeated object-cover tiles on load", () => {
it("probes the resolved src and renders repeated object-cover tiles on load", async () => {
render({ imageSrc: "/api/projects/p/preview/assets/pic.png" });
const probe = lastProbe();
expect(probe.src).toBe("/api/projects/p/preview/assets/pic.png");
act(() => {
probe.naturalWidth = 1920;
probe.naturalHeight = 1080;
probe.onload?.();
await resolveProbe((current) => {
current.naturalWidth = 1920;
current.naturalHeight = 1080;
current.onload?.();
});
const imgs = [...host.querySelectorAll("img")];
@@ -127,18 +140,18 @@ describe("ImageThumbnail", () => {
expect(host.querySelector(".animate-pulse")).toBeNull();
});
it("drops the shimmer and renders no tiles when a raster image fails to load", () => {
it("drops the shimmer and renders no tiles when a raster image fails to load", async () => {
render({ imageSrc: "/api/projects/p/preview/assets/missing.png" });
act(() => lastProbe().onerror?.());
await resolveProbe((probe) => probe.onerror?.());
expect(host.querySelectorAll("img").length).toBe(0);
expect(host.querySelector(".animate-pulse")).toBeNull();
});
it("renders tiles at 16:9 when an SVG has no intrinsic dimensions (naturalWidth=0)", () => {
it("renders tiles at 16:9 when an SVG has no intrinsic dimensions (naturalWidth=0)", async () => {
render({ imageSrc: "/api/projects/p/preview/assets/logo.svg" });
const probe = lastProbe();
act(() => {
await resolveProbe(() => {
// naturalWidth stays 0 — SVG with no width/height attribute
probe.onload?.();
});
@@ -147,21 +160,20 @@ describe("ImageThumbnail", () => {
expect(host.querySelector(".animate-pulse")).toBeNull();
});
it("renders SVG tiles at 16:9 fallback even when the probe fires onerror", () => {
it("renders SVG tiles at 16:9 fallback even when the probe fires onerror", async () => {
// Some browser/sandbox environments fire onerror for SVGs even though the
// <img> element itself can render the file — we must not blank the strip.
render({ imageSrc: "/api/projects/p/preview/assets/icon.svg" });
act(() => lastProbe().onerror?.());
await resolveProbe((probe) => probe.onerror?.());
expectFirstTileSrc("/api/projects/p/preview/assets/icon.svg");
expect(host.querySelector(".animate-pulse")).toBeNull();
});
it("renders the label above the strip when provided", () => {
it("renders the label above the strip when provided", async () => {
render({ imageSrc: "/x.png", label: "hero", labelColor: "#abc" });
act(() => {
const probe = lastProbe();
await resolveProbe((probe) => {
probe.naturalWidth = 100;
probe.naturalHeight = 100;
probe.onload?.();
@@ -1,135 +1,93 @@
import { memo, useRef, useState, useCallback, useEffect } from "react";
import { memo, useCallback, useMemo, useRef, useState } from "react";
import { useMountEffect } from "../../hooks/useMountEffect";
import { computeThumbnailStrip } from "./thumbnailUtils";
import { useThumbnailLease } from "../../hooks/useThumbnailLease";
import { createThumbnailKey, type ThumbnailPriority } from "../lib/thumbnailScheduler";
import { TIMELINE_VIEWPORT_BUDGETS } from "../lib/timelineViewportBudgets";
import { computeThumbnailStrip, probeImageAspect } from "./thumbnailUtils";
interface ImageThumbnailProps {
imageSrc: string;
label: string;
labelColor: string;
projectId?: string;
sessionEpoch?: number;
priority?: ThumbnailPriority;
rich?: boolean;
}
/**
* Renders a film-strip of a still image for a timeline clip. The image is a
* fixed-width tile (sized by its natural aspect ratio) repeated to fill the
* clip width — matching VideoThumbnail's visual pattern. Loading is lazy
* (IntersectionObserver) with the same shimmer fallback while decoding.
*/
/** A scheduler-backed still-image strip. Mounting is the sole work trigger. */
export const ImageThumbnail = memo(function ImageThumbnail({
imageSrc,
label,
labelColor,
projectId = imageSrc,
sessionEpoch = 0,
priority = "visible",
rich = false,
}: ImageThumbnailProps) {
const [containerWidth, setContainerWidth] = useState(0);
const [visible, setVisible] = useState(false);
const [status, setStatus] = useState<"loading" | "loaded" | "error">("loading");
const [aspect, setAspect] = useState(16 / 9);
const ioRef = useRef<IntersectionObserver | null>(null);
const roRef = useRef<ResizeObserver | null>(null);
const setContainerRef = useCallback((el: HTMLDivElement | null) => {
ioRef.current?.disconnect();
roRef.current?.disconnect();
if (!el) return;
const measured = el.parentElement?.clientWidth || el.clientWidth;
setContainerWidth(measured);
ioRef.current = new IntersectionObserver(
([entry]) => {
if (entry.isIntersecting) {
setVisible(true);
ioRef.current?.disconnect();
}
const observerRef = useRef<ResizeObserver | null>(null);
const request = useMemo(
() => ({
key: createThumbnailKey({ kind: "image", source: imageSrc, rich: Number(rich) }),
projectId,
sessionEpoch,
kind: "image" as const,
priority,
rich,
load: async (signal: AbortSignal) => {
const aspect = await probeImageAspect(imageSrc, signal, true);
return {
value: { kind: "image" as const, url: imageSrc, aspect },
weight:
TIMELINE_VIEWPORT_BUDGETS.posterMaxPhysicalWidth *
TIMELINE_VIEWPORT_BUDGETS.posterMaxPhysicalHeight *
4,
};
},
{ rootMargin: "200px" },
);
// fallow-ignore-next-line code-duplication
ioRef.current.observe(el);
}),
[imageSrc, priority, projectId, rich, sessionEpoch],
);
const snapshot = useThumbnailLease(request);
const value = snapshot.status === "ready" ? snapshot.value : null;
const aspect = value?.kind === "image" ? value.aspect : 16 / 9;
const { frameW, frameCount } = computeThumbnailStrip(containerWidth, aspect);
const target = el.parentElement || el;
roRef.current = new ResizeObserver(([entry]) => {
setContainerWidth(entry.contentRect.width);
});
roRef.current.observe(target);
const setContainerRef = useCallback((element: HTMLDivElement | null) => {
observerRef.current?.disconnect();
if (!element) return;
const target = element.parentElement ?? element;
setContainerWidth(target.clientWidth);
observerRef.current = new ResizeObserver(([entry]) =>
setContainerWidth(entry.contentRect.width),
);
observerRef.current.observe(target);
}, []);
useMountEffect(() => () => {
ioRef.current?.disconnect();
roRef.current?.disconnect();
});
// Probe the image once visible — measures the natural aspect ratio so the
// tile width matches, and flips to the error state (plain clip background)
// if the src can't load. The browser cache makes the tile <img>s free.
//
// SVG handling: SVGs without intrinsic width/height report naturalWidth=0 on
// load (treat as success with the 16:9 default aspect) and may fire onerror
// in some environments even though the file is valid and can be displayed —
// fall back to loaded-at-16:9 rather than hiding the strip entirely.
// eslint-disable-next-line no-restricted-syntax
useEffect(() => {
if (!visible) return;
let cancelled = false;
setStatus("loading");
const isSvg = /\.svg($|\?)/i.test(imageSrc);
const probe = new Image();
probe.onload = () => {
if (cancelled) return;
if (probe.naturalWidth > 0 && probe.naturalHeight > 0) {
setAspect(probe.naturalWidth / probe.naturalHeight);
}
// naturalWidth===0 (e.g. SVG with no intrinsic dimensions) falls through
// to "loaded" with the default 16:9 aspect already set in state.
setStatus("loaded");
};
probe.onerror = () => {
if (cancelled) return;
// SVGs can fail the probe in certain browser/sandbox environments even
// though the <img> tiles themselves render fine (different security
// context). Show the strip at the 16:9 fallback rather than blanking.
if (isSvg) {
setStatus("loaded");
} else {
setStatus("error");
}
};
probe.src = imageSrc;
return () => {
cancelled = true;
probe.onload = null;
probe.onerror = null;
probe.src = "";
};
}, [visible, imageSrc]);
const { frameW, frameCount } = computeThumbnailStrip(containerWidth, aspect);
useMountEffect(() => () => observerRef.current?.disconnect());
return (
<div ref={setContainerRef} className="absolute inset-0 overflow-hidden">
{visible && status === "loaded" && (
{value?.kind === "image" && (
<div className="absolute inset-0 flex">
{Array.from({ length: frameCount }).map((_, i) => (
{Array.from({ length: frameCount }, (_, index) => (
<div
key={i}
className="flex-shrink-0 h-full relative overflow-hidden bg-neutral-900"
key={index}
className="relative h-full flex-shrink-0 overflow-hidden bg-neutral-900"
style={{ width: frameW }}
>
<img
src={imageSrc}
src={value.url}
alt=""
draggable={false}
loading="lazy"
className="absolute inset-0 w-full h-full object-cover"
className="absolute inset-0 h-full w-full object-cover"
/>
</div>
))}
</div>
)}
{visible && status === "loading" && (
{snapshot.status === "loading" && (
<div
className="absolute inset-0 animate-pulse"
style={{
@@ -138,17 +96,16 @@ export const ImageThumbnail = memo(function ImageThumbnail({
}}
/>
)}
{label && (
<div
className="absolute bottom-0 left-0 right-0 z-10 px-1.5 pb-0.5 pt-3"
className="absolute inset-x-0 bottom-0 z-10 px-1.5 pb-0.5 pt-3"
style={{
background:
"linear-gradient(to top, rgba(0,0,0,0.85) 0%, rgba(0,0,0,0.4) 60%, transparent 100%)",
}}
>
<span
className="text-[9px] font-semibold truncate block leading-tight"
className="block truncate text-[9px] font-semibold leading-tight"
style={{ color: labelColor, textShadow: "0 1px 2px rgba(0,0,0,0.9)" }}
>
{label}
@@ -64,7 +64,6 @@ export {
getTimelineScrollTopForGeometryChange,
getTimelineVisibleTimeRange,
} from "./timelineViewportGeometry";
export const Timeline = memo(function Timeline({
onSeek,
onDrillDown,
@@ -317,24 +316,25 @@ export const Timeline = memo(function Timeline({
toggleSelectedKeyframe,
});
const { clipIndex, renderTimeRange, pinnedClipIdentities } = useTimelineClipRenderWindow({
tracks,
viewport,
pixelsPerSecond: pps,
contentOrigin,
duration: displayDuration,
selectedElementId: selectedElementId ?? undefined,
draggedElementId: draggedClip ? getTimelineElementIdentity(draggedClip.element) : undefined,
resizingElementIds,
focusedElementId: timelineFocus.pinnedElementId,
focusedEaseElementId: focusedEaseSegment?.elementId,
clipContextMenuElementId: clipContextMenu
? getTimelineElementIdentity(clipContextMenu.element)
: undefined,
keyframeContextMenuElementId: kfContextMenu
? getTimelineElementIdentity(kfContextMenu.element)
: undefined,
});
const { clipIndex, renderTimeRange, visibleTimeRange, pinnedClipIdentities } =
useTimelineClipRenderWindow({
tracks,
viewport,
pixelsPerSecond: pps,
contentOrigin,
duration: displayDuration,
selectedElementId: selectedElementId ?? undefined,
draggedElementId: draggedClip ? getTimelineElementIdentity(draggedClip.element) : undefined,
resizingElementIds,
focusedElementId: timelineFocus.pinnedElementId,
focusedEaseElementId: focusedEaseSegment?.elementId,
clipContextMenuElementId: clipContextMenu
? getTimelineElementIdentity(clipContextMenu.element)
: undefined,
keyframeContextMenuElementId: kfContextMenu
? getTimelineElementIdentity(kfContextMenu.element)
: undefined,
});
useTimelineActiveClips({
scrollRef,
currentTime,
@@ -504,6 +504,7 @@ export const Timeline = memo(function Timeline({
rowsVirtualized={timelineFocus.rowVirtualizationActive}
clipIndex={clipIndex}
renderTimeRange={renderTimeRange}
visibleTimeRange={visibleTimeRange}
pinnedClipIdentities={pinnedClipIdentities}
trackOrder={trackOrder}
tracks={tracks}
@@ -9,6 +9,7 @@ import { getTimelineDragOverlayPosition } from "./timelineClipDragPreview";
import type { DraggedClipState } from "./timelineClipDragTypes";
import type { TrackVisualStyle } from "./timelineIcons";
import { isTimelineClipActive } from "./useTimelineActiveClips";
import type { TimelineClipRenderContext } from "./TimelineTypes";
interface TimelineGestureOverlayProps {
drag: DraggedClipState | null;
@@ -22,6 +23,7 @@ interface TimelineGestureOverlayProps {
renderClipContent?: (
element: TimelineElement,
style: { clip: string; label: string },
context: TimelineClipRenderContext,
) => ReactNode;
renderClipOverlay?: (element: TimelineElement) => ReactNode;
}
@@ -87,6 +89,7 @@ export const TimelineGestureOverlay = memo(function TimelineGestureOverlay({
getTrackStyle(element.tag),
renderClipContent,
renderClipOverlay,
{ priority: "interaction", rich: true },
)}
</TimelineClip>
</div>
@@ -76,10 +76,14 @@ function renderLanes(options: RenderLanesOptions = {}): {
host: HTMLDivElement;
root: Root;
rerender: (next: RenderLanesOptions) => void;
setSelectedElementId: ReturnType<typeof vi.fn>;
onSelectElement: ReturnType<typeof vi.fn>;
} {
const host = document.createElement("div");
document.body.append(host);
const root = createRoot(host);
const setSelectedElementId = vi.fn();
const onSelectElement = vi.fn();
const render = (next: RenderLanesOptions) => {
const elements = next.elements ?? [element("clip-a", TRACK_A)];
const gsapAnimations = next.animations ?? new Map<string, GsapAnimation[]>();
@@ -118,6 +122,7 @@ function renderLanes(options: RenderLanesOptions = {}): {
})}
clipIndex={createTimelineClipIndex(tracks)}
renderTimeRange={{ start: 0, end: Number.POSITIVE_INFINITY }}
visibleTimeRange={{ start: 0, end: Number.POSITIVE_INFINITY }}
pinnedClipIdentities={new Set()}
trackOrder={displayTrackOrder}
tracks={tracks}
@@ -137,7 +142,7 @@ function renderLanes(options: RenderLanesOptions = {}): {
setRangeSelection={vi.fn()}
setResizingClip={vi.fn()}
setDraggedClip={vi.fn()}
setSelectedElementId={vi.fn()}
setSelectedElementId={setSelectedElementId}
shiftClickClipRef={createRef()}
getPreviewElement={(el) => el}
getTrackStyle={getTrackStyle}
@@ -149,6 +154,7 @@ function renderLanes(options: RenderLanesOptions = {}): {
onTogglePropertyGroupKeyframe={vi.fn()}
onResizeElement={vi.fn()}
onMoveElement={vi.fn()}
onSelectElement={onSelectElement}
onRazorSplit={vi.fn()}
onRazorSplitAll={vi.fn()}
/>,
@@ -156,7 +162,7 @@ function renderLanes(options: RenderLanesOptions = {}): {
});
};
render(options);
return { host, root, rerender: render };
return { host, root, rerender: render, setSelectedElementId, onSelectElement };
}
function visibilityLabels(host: HTMLElement): (string | null)[] {
@@ -347,3 +353,19 @@ describe("TimelineLanes disclosure target", () => {
act(() => view.root.unmount());
});
});
describe("TimelineLanes selection", () => {
it("keeps a selected clip selected when it is clicked again", () => {
const selected = element("clip-a", TRACK_A);
const view = renderLanes({
elements: [selected],
selectedElementIds: new Set([selected.id]),
});
act(() => view.host.querySelector<HTMLButtonElement>('[data-el-id="clip-a"]')?.click());
expect(view.setSelectedElementId).toHaveBeenCalledWith(selected.id);
expect(view.onSelectElement).toHaveBeenCalledWith(selected);
act(() => view.root.unmount());
});
});
@@ -22,7 +22,7 @@ import type { TimelineEditCallbacks } from "./timelineCallbacks";
import { trackStudioKeyframeLaneExpand } from "../../telemetry/events";
import { SPLIT_BOUNDARY_EPSILON_S } from "../../utils/timelineElementSplit";
import { isAudioTimelineElement, isMusicTrack } from "../../utils/timelineInspector";
import { renderClipChildren } from "./timelineClipChildren";
import { renderClipChildren, resolveClipRenderContext } from "./timelineClipChildren";
import { TimelineTrackRow } from "./TimelineTrackRow";
import { isTimelineClipActive } from "./useTimelineActiveClips";
import { queryTimelineClipIndex } from "../lib/timelineClipIndex";
@@ -57,6 +57,7 @@ export function TimelineLanes({
rowsVirtualized,
clipIndex,
renderTimeRange,
visibleTimeRange,
pinnedClipIdentities,
trackOrder,
tracks,
@@ -323,8 +324,7 @@ export function TimelineLanes({
const isSelected =
selectedElementId === elementKey || selectedElementIds.has(elementKey);
const isComposition = !!el.compositionSrc;
// The element identity is already unique per clip. Never fold in the map
// index, or a splice/reorder remounts every clip at/after the change.
// Element identity stays stable across clip splices and reorders.
const clipKey = elementKey;
const isDraggingClip =
draggedClip?.started === true &&
@@ -332,6 +332,11 @@ export function TimelineLanes({
getTimelineElementIdentity(draggedElement) === elementKey;
if (isDraggingClip) return null;
const previewElement = getPreviewElement(el);
const renderContext = resolveClipRenderContext(
previewElement,
visibleTimeRange,
isSelected || hoveredClip === clipKey || pinnedClipIdentities.has(clipKey),
);
// Passenger of a live multi-drag: preserve the formation without changing
// the passenger's timeline data until the owning drag commits.
const isPassenger =
@@ -475,15 +480,9 @@ export function TimelineLanes({
}
return;
}
// Plain click single-selects: drop any marquee multi-selection.
// Only a click on the PRIMARY selection toggles it off — a click
// on a marquee-selected clip narrows the selection to that clip.
const hadMultiSelection = selectedElementIds.size > 0;
usePlayerStore.getState().clearSelectedElementIds();
const nextElement =
selectedElementId === elementKey && !hadMultiSelection ? null : el;
setSelectedElementId(nextElement ? elementKey : null);
onSelectElement?.(nextElement);
// Clip selection is idempotent; empty timeline space owns deselection.
setSelectedElementId(elementKey);
onSelectElement?.(el);
}
}
onDoubleClick={(e) => {
@@ -497,6 +496,7 @@ export function TimelineLanes({
clipStyle,
renderClipContent,
renderClipOverlay,
renderContext,
)}
</TimelineClip>
);
@@ -4,6 +4,11 @@ import type { TimelineDropCallbacks } from "./timelineCallbacks";
import type { TimelineTheme } from "./timelineTheme";
import type { TimelineEditOverrides } from "./useResolvedTimelineEditCallbacks";
export interface TimelineClipRenderContext {
priority: "overscan" | "visible" | "interaction";
rich: boolean;
}
export interface TimelineProps extends TimelineDropCallbacks, TimelineEditOverrides {
/** Project-scoped reset boundary; soft source refreshes retain the same epoch. */
sessionEpoch?: number;
@@ -12,6 +17,7 @@ export interface TimelineProps extends TimelineDropCallbacks, TimelineEditOverri
renderClipContent?: (
element: TimelineElement,
style: { clip: string; label: string },
context: TimelineClipRenderContext,
) => ReactNode;
renderClipOverlay?: (element: TimelineElement) => ReactNode;
onDeleteElement?: (element: TimelineElement) => Promise<void> | void;
@@ -2,65 +2,28 @@
import React, { act } from "react";
import { createRoot, type Root } from "react-dom/client";
import { afterEach, beforeEach, describe, expect, it, vi } from "vitest";
import { thumbnailScheduler } from "../lib/thumbnailScheduler";
import { decodeVideoThumbnail } from "../lib/thumbnailVideoDecoder";
import { VideoThumbnail } from "./VideoThumbnail";
vi.mock("../lib/thumbnailVideoDecoder", () => ({ decodeVideoThumbnail: vi.fn() }));
Object.defineProperty(globalThis, "IS_REACT_ACT_ENVIRONMENT", {
configurable: true,
value: true,
});
// Fire "intersecting" immediately on observe so the extraction effect runs.
class MockIntersectionObserver {
private cb: IntersectionObserverCallback;
constructor(cb: IntersectionObserverCallback) {
this.cb = cb;
}
observe() {
this.cb(
[{ isIntersecting: true } as IntersectionObserverEntry],
this as unknown as IntersectionObserver,
);
}
disconnect() {}
unobserve() {}
takeRecords(): IntersectionObserverEntry[] {
return [];
}
}
class MockResizeObserver {
observe() {}
disconnect() {}
unobserve() {}
}
const originalIO = globalThis.IntersectionObserver;
const originalRO = globalThis.ResizeObserver;
let host: HTMLDivElement;
let root: Root | null = null;
let createdVideos: HTMLVideoElement[];
beforeEach(() => {
globalThis.IntersectionObserver =
MockIntersectionObserver as unknown as typeof IntersectionObserver;
globalThis.ResizeObserver = MockResizeObserver as unknown as typeof ResizeObserver;
createdVideos = [];
const origCreate = document.createElement.bind(document);
vi.spyOn(document, "createElement").mockImplementation((tag: string) => {
const el = origCreate(tag);
if (tag === "video") createdVideos.push(el as HTMLVideoElement);
return el;
});
// happy-dom's <video>/<canvas> don't decode media; stub the seam the
// extractor depends on so the effect can run deterministically.
vi.spyOn(HTMLMediaElement.prototype, "load").mockImplementation(() => {});
vi.spyOn(HTMLCanvasElement.prototype, "getContext").mockReturnValue({
drawImage: () => {},
} as unknown as CanvasRenderingContext2D);
host = document.createElement("div");
document.body.append(host);
});
@@ -68,85 +31,68 @@ beforeEach(() => {
afterEach(() => {
act(() => root?.unmount());
root = null;
vi.restoreAllMocks();
globalThis.IntersectionObserver = originalIO;
globalThis.ResizeObserver = originalRO;
thumbnailScheduler.invalidateProject("p");
vi.clearAllMocks();
document.body.innerHTML = "";
});
function render(videoSrc: string) {
async function render(rich = false) {
root = createRoot(host);
act(() => {
root!.render(React.createElement(VideoThumbnail, { videoSrc, label: "", labelColor: "#fff" }));
});
}
function lastVideo(): HTMLVideoElement {
const v = createdVideos.at(-1);
expect(v).toBeDefined();
return v!;
}
describe("VideoThumbnail — tainted-canvas fallback", () => {
it("stops the extractor and drops the shimmer when toDataURL throws a SecurityError", () => {
vi.spyOn(HTMLCanvasElement.prototype, "toDataURL").mockImplementation(() => {
throw new DOMException("Tainted canvases may not be exported.", "SecurityError");
});
render("https://cdn.example.com/no-cors.mp4");
// The effect ran once visible → a hidden <video> was created.
const video = lastVideo();
act(() => {
video.dispatchEvent(new Event("loadedmetadata"));
});
act(() => {
video.dispatchEvent(new Event("seeked"));
});
// No frame captured, and crucially the shimmer is gone (not spinning
// forever) — the clip falls back to its plain background.
expect(host.querySelectorAll("img").length).toBe(0);
expect(host.querySelector(".animate-pulse")).toBeNull();
});
it("drops the shimmer when a no-CORS load fires the video error event (0 frames) (#2214)", () => {
render("https://cdn.example.com/no-cors.mp4");
// Before the load resolves, the shimmer placeholder is up.
expect(host.querySelector(".animate-pulse")).not.toBeNull();
const video = lastVideo();
// crossOrigin="anonymous" against a CORS-less server fails the load outright —
// the error listener fires instead of loadedmetadata/seeked, so no frame is
// ever captured. The shimmer must stop rather than spin forever.
act(() => {
video.dispatchEvent(new Event("error"));
});
expect(host.querySelectorAll("img").length).toBe(0);
expect(host.querySelector(".animate-pulse")).toBeNull();
});
it("keeps streaming frames while the shimmer is up until a frame arrives", () => {
vi.spyOn(HTMLCanvasElement.prototype, "toDataURL").mockReturnValue(
"data:image/jpeg;base64,AAAA",
await act(async () => {
root!.render(
<VideoThumbnail
videoSrc="/api/projects/p/preview/assets/clip.mp4"
label=""
labelColor="#fff"
projectId="p"
sessionEpoch={1}
priority="visible"
rich={rich}
/>,
);
await Promise.resolve();
});
}
render("/api/projects/p/preview/assets/clip.mp4");
// Before any seek resolves, the shimmer placeholder is shown.
expect(host.querySelector(".animate-pulse")).not.toBeNull();
const video = lastVideo();
act(() => {
video.dispatchEvent(new Event("loadedmetadata"));
});
act(() => {
video.dispatchEvent(new Event("seeked"));
describe("VideoThumbnail", () => {
it("renders a scheduler-provided sparse poster", async () => {
vi.mocked(decodeVideoThumbnail).mockResolvedValue({
value: { kind: "image", url: "blob:poster", aspect: 16 / 9 },
weight: 128,
});
// A frame was captured, so tiles render and the shimmer clears.
expect(host.querySelectorAll("img").length).toBeGreaterThanOrEqual(1);
await render();
expect(decodeVideoThumbnail).toHaveBeenCalledWith(
expect.objectContaining({ frameCount: 1 }),
expect.any(AbortSignal),
);
expect(host.querySelector('img[src="blob:poster"]')).not.toBeNull();
expect(host.querySelector(".animate-pulse")).toBeNull();
});
it("requests a rich filmstrip only for interaction actors", async () => {
vi.mocked(decodeVideoThumbnail).mockResolvedValue({
value: { kind: "filmstrip", urls: ["blob:a", "blob:b"], aspect: 16 / 9 },
weight: 256,
});
await render(true);
expect(decodeVideoThumbnail).toHaveBeenCalledWith(
expect.objectContaining({ frameCount: 6 }),
expect.any(AbortSignal),
);
expect(host.querySelectorAll("img").length).toBeGreaterThan(0);
});
it("clears the loading shimmer when the scheduled decode fails", async () => {
vi.mocked(decodeVideoThumbnail).mockRejectedValue(new Error("decode failed"));
await render();
await vi.waitFor(() => expect(thumbnailScheduler.getDiagnostics().active).toBe(0));
expect(host.querySelector(".animate-pulse")).toBeNull();
expect(host.querySelector("img")).toBeNull();
});
});
@@ -1,5 +1,8 @@
import { memo, useRef, useState, useCallback, useEffect } from "react";
import { memo, useCallback, useMemo, useRef, useState } from "react";
import { useMountEffect } from "../../hooks/useMountEffect";
import { useThumbnailLease } from "../../hooks/useThumbnailLease";
import { createThumbnailKey, type ThumbnailPriority } from "../lib/thumbnailScheduler";
import { decodeVideoThumbnail } from "../lib/thumbnailVideoDecoder";
import { computeThumbnailStrip, THUMBNAIL_CLIP_HEIGHT } from "./thumbnailUtils";
interface VideoThumbnailProps {
@@ -7,189 +10,105 @@ interface VideoThumbnailProps {
label: string;
labelColor: string;
duration?: number;
sourceStart?: number;
sourceRangeDuration?: number;
projectId?: string;
sessionEpoch?: number;
priority?: ThumbnailPriority;
rich?: boolean;
}
const CLIP_HEIGHT = THUMBNAIL_CLIP_HEIGHT;
const MAX_UNIQUE_FRAMES: number = 6;
/**
* Renders a film-strip of video frames extracted client-side via a hidden
* <video> + <canvas>. Each frame is a fixed-width tile; frames repeat to
* fill the clip width matching ClipThumbnail's visual pattern.
*/
/** Sparse, bounded video frames supplied by the shared thumbnail scheduler. */
export const VideoThumbnail = memo(function VideoThumbnail({
videoSrc,
label,
labelColor,
duration = 5,
sourceStart,
sourceRangeDuration,
projectId = videoSrc,
sessionEpoch = 0,
priority = "visible",
rich = false,
}: VideoThumbnailProps) {
const [containerWidth, setContainerWidth] = useState(0);
const [visible, setVisible] = useState(false);
const [frames, setFrames] = useState<string[]>([]);
const [failed, setFailed] = useState(false);
const [aspect, setAspect] = useState(16 / 9);
const ioRef = useRef<IntersectionObserver | null>(null);
const roRef = useRef<ResizeObserver | null>(null);
const extractingRef = useRef(false);
const observerRef = useRef<ResizeObserver | null>(null);
const request = useMemo(
() => ({
key: createThumbnailKey({
kind: "video",
source: videoSrc,
start: sourceStart,
duration: sourceRangeDuration ?? duration,
frames: rich ? 6 : 1,
}),
projectId,
sessionEpoch,
kind: "video" as const,
priority,
rich,
load: (signal: AbortSignal) =>
decodeVideoThumbnail(
{
source: videoSrc,
sourceStart,
sourceRangeDuration: sourceRangeDuration ?? duration,
frameCount: rich ? 6 : 1,
fit: "cover",
},
signal,
),
}),
[duration, priority, projectId, rich, sessionEpoch, sourceRangeDuration, sourceStart, videoSrc],
);
const snapshot = useThumbnailLease(request);
const value = snapshot.status === "ready" ? snapshot.value : null;
const urls =
value?.kind === "filmstrip" ? value.urls : value?.kind === "image" ? [value.url] : [];
const aspect = value?.kind === "image" || value?.kind === "filmstrip" ? value.aspect : 16 / 9;
const { frameW, frameCount } = computeThumbnailStrip(
containerWidth,
aspect,
THUMBNAIL_CLIP_HEIGHT,
);
const setContainerRef = useCallback((el: HTMLDivElement | null) => {
ioRef.current?.disconnect();
roRef.current?.disconnect();
if (!el) return;
const measured = el.parentElement?.clientWidth || el.clientWidth;
setContainerWidth(measured);
ioRef.current = new IntersectionObserver(
([entry]) => {
if (entry.isIntersecting) {
setVisible(true);
ioRef.current?.disconnect();
}
},
{ rootMargin: "200px" },
const setContainerRef = useCallback((element: HTMLDivElement | null) => {
observerRef.current?.disconnect();
if (!element) return;
const target = element.parentElement ?? element;
setContainerWidth(target.clientWidth);
observerRef.current = new ResizeObserver(([entry]) =>
setContainerWidth(entry.contentRect.width),
);
// fallow-ignore-next-line code-duplication
ioRef.current.observe(el);
const target = el.parentElement || el;
roRef.current = new ResizeObserver(([entry]) => {
setContainerWidth(entry.contentRect.width);
});
roRef.current.observe(target);
observerRef.current.observe(target);
}, []);
useMountEffect(() => () => {
ioRef.current?.disconnect();
roRef.current?.disconnect();
});
// Extract frames progressively — each frame appears as soon as it's ready.
// Note: useEffect with deps is acceptable — syncs with external video element API,
// requires cleanup (cancel extraction, revoke URLs) when inputs change.
// eslint-disable-next-line no-restricted-syntax
useEffect(() => {
if (!visible || extractingRef.current) return;
extractingRef.current = true;
const video = document.createElement("video");
video.crossOrigin = "anonymous";
video.muted = true;
video.preload = "auto";
const canvas = document.createElement("canvas");
const ctx = canvas.getContext("2d");
if (!ctx) {
extractingRef.current = false;
return;
}
const timestamps: number[] = [];
const minSeek = Math.min(0.4, duration * 0.05);
for (let i = 0; i < MAX_UNIQUE_FRAMES; i++) {
const raw =
MAX_UNIQUE_FRAMES === 1 ? duration * 0.15 : (i / (MAX_UNIQUE_FRAMES - 1)) * duration;
timestamps.push(Math.max(raw, minSeek));
}
let idx = 0;
let cancelled = false;
const extractNext = () => {
if (cancelled || idx >= timestamps.length) {
if (!cancelled) {
video.src = "";
video.load();
}
return;
}
video.currentTime = timestamps[idx];
};
video.addEventListener("loadedmetadata", () => {
if (video.videoWidth > 0 && video.videoHeight > 0) {
setAspect(video.videoWidth / video.videoHeight);
const h = CLIP_HEIGHT * 2;
const w = Math.round(h * (video.videoWidth / video.videoHeight));
canvas.width = w;
canvas.height = h;
}
extractNext();
});
video.addEventListener("seeked", () => {
if (cancelled) return;
let dataUrl: string;
try {
ctx.drawImage(video, 0, 0, canvas.width, canvas.height);
dataUrl = canvas.toDataURL("image/jpeg", 0.6);
} catch {
// An external http(s) video served without CORS headers taints the
// canvas, so toDataURL throws a SecurityError. Stop the extractor
// cleanly and fall back to the no-thumbnail rendering (plain clip
// background), matching ImageThumbnail's error path — otherwise the
// shimmer placeholder would spin forever.
cancelled = true;
setFailed(true);
video.src = "";
video.load();
return;
}
// Stream each frame immediately
setFrames((prev) => [...prev, dataUrl]);
idx++;
extractNext();
});
video.addEventListener("error", () => {
// A no-CORS load fails outright (crossOrigin="anonymous" rejects a video
// served without CORS headers), firing this instead of the taint path in
// "seeked" — so 0 frames are ever extracted. Keep whatever frames we have,
// but mark failed so the shimmer placeholder stops spinning forever and we
// fall back to the plain clip background (#2214).
setFailed(true);
});
video.src = videoSrc;
video.load();
return () => {
cancelled = true;
extractingRef.current = false;
setFrames([]);
setFailed(false);
video.src = "";
video.load();
};
}, [visible, videoSrc, duration]);
const { frameW, frameCount } = computeThumbnailStrip(containerWidth, aspect, CLIP_HEIGHT);
useMountEffect(() => () => observerRef.current?.disconnect());
return (
<div ref={setContainerRef} className="absolute inset-0 overflow-hidden">
{visible && frames.length > 0 && (
{urls.length > 0 && (
<div className="absolute inset-0 flex">
{Array.from({ length: frameCount }).map((_, i) => {
const src = frames[i % frames.length];
{Array.from({ length: frameCount }, (_, index) => {
const src = urls[index % urls.length];
return (
<div
key={i}
className="flex-shrink-0 h-full relative overflow-hidden bg-neutral-900"
key={index}
className="relative h-full flex-shrink-0 overflow-hidden bg-neutral-900"
style={{ width: frameW }}
>
<img
src={src}
alt=""
draggable={false}
className="absolute inset-0 w-full h-full object-cover"
className="absolute inset-0 h-full w-full object-cover"
/>
</div>
);
})}
</div>
)}
{visible && frames.length === 0 && !failed && (
{snapshot.status === "loading" && urls.length === 0 && (
<div
className="absolute inset-0 animate-pulse"
style={{
@@ -198,17 +117,16 @@ export const VideoThumbnail = memo(function VideoThumbnail({
}}
/>
)}
{label && (
<div
className="absolute bottom-0 left-0 right-0 z-10 px-1.5 pb-0.5 pt-3"
className="absolute inset-x-0 bottom-0 z-10 px-1.5 pb-0.5 pt-3"
style={{
background:
"linear-gradient(to top, rgba(0,0,0,0.85) 0%, rgba(0,0,0,0.4) 60%, transparent 100%)",
}}
>
<span
className="text-[9px] font-semibold truncate block leading-tight"
className="block truncate text-[9px] font-semibold leading-tight"
style={{ color: labelColor, textShadow: "0 1px 2px rgba(0,0,0,0.9)" }}
>
{label}
@@ -41,6 +41,13 @@ describe("computeThumbnailStrip", () => {
it("honors a custom clip height", () => {
expect(computeThumbnailStrip(300, 2, 40).frameW).toBe(80);
});
it("keeps narrow tiles above a caller-owned minimum", () => {
expect(computeThumbnailStrip(300, 0.25, 40, 48)).toEqual({
frameW: 48,
frameCount: 7,
});
});
});
describe("resolveMediaPreviewUrl", () => {
@@ -8,6 +8,52 @@ export interface ThumbnailStripLayout {
frameCount: number;
}
/**
* Measure an image without mounting it in React's DOM. The scheduler owns the
* abort signal, so an unmounted clip cannot leave Blink retaining a pending
* image request and its former React tree.
*/
export function probeImageAspect(
imageSrc: string,
signal: AbortSignal,
tolerateSvgError = false,
): Promise<number> {
return new Promise((resolve, reject) => {
const image = new Image();
const cleanup = () => {
image.onload = null;
image.onerror = null;
signal.removeEventListener("abort", onAbort);
};
const onAbort = () => {
cleanup();
image.src = "";
reject(new DOMException("Aborted", "AbortError"));
};
if (signal.aborted) {
onAbort();
return;
}
image.onload = () => {
cleanup();
resolve(
image.naturalWidth > 0 && image.naturalHeight > 0
? image.naturalWidth / image.naturalHeight
: 16 / 9,
);
};
image.onerror = () => {
cleanup();
if (tolerateSvgError && /\.svg($|\?)/i.test(imageSrc)) resolve(16 / 9);
else reject(new Error("Image thumbnail failed to load"));
};
signal.addEventListener("abort", onAbort, { once: true });
image.src = imageSrc;
});
}
/**
* Compute the film-strip tile layout for a clip thumbnail: fixed-height tiles
* sized by the media's aspect ratio, repeated to fill the clip width.
@@ -17,9 +63,10 @@ export function computeThumbnailStrip(
containerWidth: number,
aspect: number,
clipHeight: number = THUMBNAIL_CLIP_HEIGHT,
minFrameWidth = 1,
): ThumbnailStripLayout {
const safeAspect = Number.isFinite(aspect) && aspect > 0 ? aspect : 16 / 9;
const frameW = Math.max(1, Math.round(clipHeight * safeAspect));
const frameW = Math.max(minFrameWidth, Math.round(clipHeight * safeAspect));
const frameCount = containerWidth > 0 ? Math.max(1, Math.ceil(containerWidth / frameW)) : 1;
return { frameW, frameCount };
}
@@ -0,0 +1,31 @@
import { describe, expect, it } from "vitest";
import { resolveClipRenderContext } from "./timelineClipChildren";
import type { TimelineElement } from "../store/playerStore";
const clip: TimelineElement = {
id: "clip",
tag: "video",
start: 10,
duration: 5,
track: 0,
};
describe("resolveClipRenderContext", () => {
it("prioritizes interactive clips and enables rich thumbnails", () => {
expect(resolveClipRenderContext(clip, { start: 0, end: 1 }, true)).toEqual({
priority: "interaction",
rich: true,
});
});
it("distinguishes visible clips from overscan clips", () => {
expect(resolveClipRenderContext(clip, { start: 14, end: 16 }, false)).toEqual({
priority: "visible",
rich: false,
});
expect(resolveClipRenderContext(clip, { start: 15, end: 20 }, false)).toEqual({
priority: "overscan",
rich: false,
});
});
});
@@ -1,6 +1,20 @@
import { type ReactNode } from "react";
import { usePlayerStore, type TimelineElement } from "../store/playerStore";
import type { TimelineTimeRange } from "../lib/timelineClipIndex";
import type { TrackVisualStyle } from "./timelineIcons";
import type { TimelineClipRenderContext } from "./TimelineTypes";
export function resolveClipRenderContext(
element: TimelineElement,
visibleTimeRange: TimelineTimeRange,
interactive: boolean,
): TimelineClipRenderContext {
if (interactive) return { priority: "interaction", rich: true };
const visible =
element.start < visibleTimeRange.end &&
element.start + element.duration > visibleTimeRange.start;
return { priority: visible ? "visible" : "overscan", rich: false };
}
function ClipLintDot({ element }: { element: TimelineElement }) {
const lint = usePlayerStore((s) => s.lintFindingsByElement.get(element.key ?? element.id));
@@ -18,9 +32,14 @@ export function renderClipChildren(
element: TimelineElement,
clipStyle: TrackVisualStyle,
renderClipContent:
| ((element: TimelineElement, style: { clip: string; label: string }) => ReactNode)
| ((
element: TimelineElement,
style: { clip: string; label: string },
context: TimelineClipRenderContext,
) => ReactNode)
| undefined,
renderClipOverlay: ((element: TimelineElement) => ReactNode) | undefined,
context: TimelineClipRenderContext = { priority: "visible", rich: false },
): ReactNode {
return (
<>
@@ -31,7 +50,7 @@ export function renderClipChildren(
// diamonds hang outside its bounds), so the thumbnail layer must clip
// itself to the clip's rounded corners or sharp corners poke out.
<div className="absolute inset-0 overflow-hidden" style={{ borderRadius: "inherit" }}>
{renderClipContent(element, clipStyle)}
{renderClipContent(element, clipStyle, context)}
</div>
)}
</>
@@ -10,6 +10,7 @@ import type { TimelineClipIndex, TimelineTimeRange } from "../lib/timelineClipIn
import type { TimelineRowGeometry } from "./timelineLayout";
import type { TimelineVirtualRow } from "./useTimelineVirtualRows";
import type { TimelineLogicalRow } from "./timelineKeyboardNavigation";
import type { TimelineClipRenderContext } from "./TimelineTypes";
/**
* Props shared by the scroll container ({@link import("./TimelineCanvas")}) and
@@ -33,6 +34,7 @@ export interface TimelineLaneBaseProps {
rowsVirtualized: boolean;
clipIndex: TimelineClipIndex;
renderTimeRange: TimelineTimeRange;
visibleTimeRange: TimelineTimeRange;
pinnedClipIdentities: ReadonlySet<string>;
trackOrder: number[];
tracks: [number, TimelineElement[]][];
@@ -48,6 +50,7 @@ export interface TimelineLaneBaseProps {
renderClipContent?: (
element: TimelineElement,
style: { clip: string; label: string },
context: TimelineClipRenderContext,
) => ReactNode;
renderClipOverlay?: (element: TimelineElement) => ReactNode;
onDrillDown?: (element: TimelineElement) => void;
@@ -1,6 +1,9 @@
import { useMemo } from "react";
import { createTimelineClipIndex } from "../lib/timelineClipIndex";
import { getTimelineRenderTimeRange } from "./timelineViewportGeometry";
import {
getTimelineRenderTimeRange,
getTimelineVisibleTimeRange,
} from "./timelineViewportGeometry";
import type { TimelineScrollViewportSnapshot } from "./useTimelineScrollViewport";
interface UseTimelineClipRenderWindowInput {
@@ -37,6 +40,10 @@ export function useTimelineClipRenderWindow({
() => getTimelineRenderTimeRange(viewport, pixelsPerSecond, contentOrigin, duration),
[contentOrigin, duration, pixelsPerSecond, viewport],
);
const visibleTimeRange = useMemo(
() => getTimelineVisibleTimeRange(viewport, pixelsPerSecond, contentOrigin, duration),
[contentOrigin, duration, pixelsPerSecond, viewport],
);
const pinnedClipIdentities = useMemo(
() =>
new Set(
@@ -60,5 +67,5 @@ export function useTimelineClipRenderWindow({
selectedElementId,
],
);
return { clipIndex, renderTimeRange, pinnedClipIdentities };
return { clipIndex, renderTimeRange, visibleTimeRange, pinnedClipIdentities };
}
@@ -0,0 +1,81 @@
// @vitest-environment happy-dom
import { afterEach, beforeEach, describe, expect, it, vi } from "vitest";
import { getMediaProbeDiagnostics, probeMediaUrl, resetMediaProbeRegistry } from "./mediaProbe";
const dispose = vi.fn();
const getDurationFromMetadata = vi.fn(async () => 5);
vi.mock("mediabunny", () => ({
ALL_FORMATS: {},
UrlSource: class {
constructor(readonly url: string) {}
},
Input: class {
getDurationFromMetadata = getDurationFromMetadata;
getPrimaryVideoTrack = vi.fn(async () => ({ displayWidth: 640, displayHeight: 360 }));
getAudioTracks = vi.fn(async () => []);
dispose = dispose;
},
}));
beforeEach(() => {
resetMediaProbeRegistry();
vi.clearAllMocks();
getDurationFromMetadata.mockResolvedValue(5);
});
afterEach(() => {
vi.useRealTimers();
});
describe("media probe registry", () => {
it("deduplicates and caches successful probes", async () => {
const [first, second] = await Promise.all([
probeMediaUrl("/video.mp4"),
probeMediaUrl("/video.mp4"),
]);
expect(first).toEqual(second);
expect(getDurationFromMetadata).toHaveBeenCalledTimes(1);
expect(dispose).toHaveBeenCalledTimes(1);
expect(getMediaProbeDiagnostics()).toEqual({ cached: 1, failed: 0, inflight: 0 });
});
it("bounds retained successes to the configured registry count", async () => {
for (let index = 0; index < 513; index++) {
await probeMediaUrl(`/video-${index}.mp4`);
}
expect(getMediaProbeDiagnostics().cached).toBe(512);
});
it("limits concurrent metadata probes and drains the queue", async () => {
const resolvers: Array<(duration: number) => void> = [];
getDurationFromMetadata.mockImplementation(
() => new Promise<number>((resolve) => resolvers.push(resolve)),
);
const probes = Array.from({ length: 5 }, (_, index) => probeMediaUrl(`/queued-${index}.mp4`));
await Promise.resolve();
await Promise.resolve();
expect(getDurationFromMetadata).toHaveBeenCalledTimes(4);
resolvers[0]?.(5);
await vi.waitFor(() => expect(getDurationFromMetadata).toHaveBeenCalledTimes(5));
for (const resolve of resolvers.slice(1)) resolve(5);
await expect(Promise.all(probes)).resolves.toHaveLength(5);
expect(getMediaProbeDiagnostics()).toEqual({ cached: 5, failed: 0, inflight: 0 });
});
it("retries failures only after the failure TTL", async () => {
vi.useFakeTimers();
getDurationFromMetadata.mockRejectedValue(new Error("bad source"));
await expect(probeMediaUrl("/bad.mp4")).resolves.toBeNull();
await expect(probeMediaUrl("/bad.mp4")).resolves.toBeNull();
expect(getDurationFromMetadata).toHaveBeenCalledTimes(1);
vi.advanceTimersByTime(30_001);
await expect(probeMediaUrl("/bad.mp4")).resolves.toBeNull();
expect(getDurationFromMetadata).toHaveBeenCalledTimes(2);
});
});
+94 -14
View File
@@ -1,4 +1,6 @@
interface MediaProbeResult {
import { TIMELINE_VIEWPORT_BUDGETS } from "./timelineViewportBudgets";
export interface MediaProbeResult {
duration: number;
width?: number;
height?: number;
@@ -6,11 +8,24 @@ interface MediaProbeResult {
hasAudio: boolean;
}
const cache = new Map<string, MediaProbeResult>();
interface CachedProbe {
result: MediaProbeResult;
lastAccess: number;
}
const cache = new Map<string, CachedProbe>();
const inflight = new Map<string, Promise<MediaProbeResult | null>>();
// URLs whose probe failed (CORS, 404, non-media). Remembered so the rAF-driven
// timeline re-derive doesn't re-fetch them every frame and flood the console.
const failed = new Set<string>();
const failed = new Map<string, { failedAt: number; lastAccess: number }>();
let accessSequence = 0;
let activeProbes = 0;
let registryEpoch = 0;
const probeQueue: Array<{
key: string;
epoch: number;
resolve: (result: MediaProbeResult | null) => void;
}> = [];
let mediabunnyModule: typeof import("mediabunny") | null | false = null;
@@ -65,7 +80,22 @@ async function probeOne(url: string): Promise<MediaProbeResult | null> {
}
function getCachedProbe(url: string): MediaProbeResult | undefined {
return cache.get(normalizeUrl(url));
const cached = cache.get(normalizeUrl(url));
if (cached) cached.lastAccess = ++accessSequence;
return cached?.result;
}
function evictMetadataOverflow(): void {
const overflow = cache.size + failed.size - TIMELINE_VIEWPORT_BUDGETS.metadataRegistryEntries;
if (overflow <= 0) return;
const entries = [
...Array.from(cache, ([key, value]) => ({ key, at: value.lastAccess, failed: false })),
...Array.from(failed, ([key, value]) => ({ key, at: value.lastAccess, failed: true })),
].sort((left, right) => left.at - right.at);
for (const entry of entries.slice(0, overflow)) {
if (entry.failed) failed.delete(entry.key);
else cache.delete(entry.key);
}
}
/**
@@ -101,32 +131,82 @@ export async function probeMissingSourceDurations<
el.sourceDuration == null &&
["video", "audio"].includes(el.tag.toLowerCase()) &&
!getCachedProbe(el.src) &&
!failed.has(normalizeUrl(el.src)),
!hasFreshFailure(normalizeUrl(el.src)),
);
if (needs.length === 0) return;
await Promise.allSettled(
needs.map(async (el) => {
const result = await probeMediaUrl(el.src!);
const source = el.src;
if (!source) return;
const result = await probeMediaUrl(source);
if (result) apply(el.key ?? el.id, result.duration);
}),
);
}
async function probeMediaUrl(url: string): Promise<MediaProbeResult | null> {
function hasFreshFailure(key: string): boolean {
const failedAt = failed.get(key);
if (failedAt === undefined) return false;
if (Date.now() - failedAt.failedAt < TIMELINE_VIEWPORT_BUDGETS.metadataFailureTtlMs) {
failedAt.lastAccess = ++accessSequence;
return true;
}
failed.delete(key);
return false;
}
export async function probeMediaUrl(url: string): Promise<MediaProbeResult | null> {
const key = normalizeUrl(url);
const cached = cache.get(key);
const cached = getCachedProbe(key);
if (cached) return cached;
if (failed.has(key)) return null;
if (hasFreshFailure(key)) return null;
let pending = inflight.get(key);
if (pending) return pending;
pending = probeOne(key).then((result) => {
inflight.delete(key);
if (result) cache.set(key, result);
else failed.add(key);
return result;
pending = new Promise<MediaProbeResult | null>((resolve) => {
probeQueue.push({ key, epoch: registryEpoch, resolve });
pumpProbeQueue();
});
inflight.set(key, pending);
return pending;
}
function pumpProbeQueue(): void {
while (activeProbes < TIMELINE_VIEWPORT_BUDGETS.concurrentMetadataJobs) {
const queued = probeQueue.shift();
if (!queued) return;
if (queued.epoch !== registryEpoch) {
queued.resolve(null);
continue;
}
activeProbes++;
void probeOne(queued.key)
.then((result) => {
if (queued.epoch !== registryEpoch) return null;
inflight.delete(queued.key);
if (result) cache.set(queued.key, { result, lastAccess: ++accessSequence });
else failed.set(queued.key, { failedAt: Date.now(), lastAccess: ++accessSequence });
evictMetadataOverflow();
return result;
})
.then(queued.resolve)
.finally(() => {
activeProbes--;
pumpProbeQueue();
});
}
}
export function getMediaProbeDiagnostics() {
return { cached: cache.size, failed: failed.size, inflight: inflight.size };
}
export function resetMediaProbeRegistry(): void {
registryEpoch++;
for (const queued of probeQueue.splice(0)) queued.resolve(null);
cache.clear();
failed.clear();
inflight.clear();
accessSequence = 0;
}
@@ -0,0 +1,14 @@
import { describe, expect, it } from "vitest";
import { defaultThumbnailMode, effectiveThumbnailMode } from "./thumbnailPolicy";
describe("thumbnail runtime policy", () => {
it("defaults missing preferences adaptively after activation", () => {
expect(defaultThumbnailMode(undefined, "follow-preference")).toBe("adaptive");
expect(defaultThumbnailMode(undefined, "legacy-default")).toBe("hidden");
});
it("forces the safe renderer without overwriting user intent", () => {
expect(effectiveThumbnailMode("adaptive", "force-hidden")).toBe("hidden");
expect(effectiveThumbnailMode("adaptive", "follow-preference")).toBe("adaptive");
});
});
@@ -0,0 +1,24 @@
export type ThumbnailMode = "adaptive" | "hidden";
export type ThumbnailRuntimePolicy = "follow-preference" | "force-hidden" | "legacy-default";
// "Adaptive" currently means the scheduler pauses rich work while scrolling.
// The mode name leaves room for finer-grained runtime heuristics later.
const rawPolicy = import.meta.env.VITE_STUDIO_TIMELINE_THUMBNAIL_POLICY;
const studioThumbnailRuntimePolicy: ThumbnailRuntimePolicy =
rawPolicy === "force-hidden" || rawPolicy === "legacy-default" ? rawPolicy : "follow-preference";
export function defaultThumbnailMode(
storedMode: ThumbnailMode | undefined,
policy: ThumbnailRuntimePolicy = studioThumbnailRuntimePolicy,
): ThumbnailMode {
return storedMode ?? (policy === "legacy-default" ? "hidden" : "adaptive");
}
export function effectiveThumbnailMode(
preferredMode: ThumbnailMode,
policy: ThumbnailRuntimePolicy = studioThumbnailRuntimePolicy,
): ThumbnailMode {
return policy === "force-hidden" ? "hidden" : preferredMode;
}
@@ -0,0 +1,388 @@
import { describe, expect, it, vi } from "vitest";
import { resolveTimelineViewportBudgets } from "./timelineViewportBudgets";
import {
createThumbnailKey,
createThumbnailRequestIdentity,
ThumbnailScheduler,
type ThumbnailLoadedResult,
type ThumbnailPriority,
type ThumbnailRequest,
} from "./thumbnailScheduler";
function deferred<T>() {
let resolve!: (value: T) => void;
let reject!: (reason?: unknown) => void;
const promise = new Promise<T>((accept, fail) => {
resolve = accept;
reject = fail;
});
return { promise, resolve, reject };
}
function result(name: string, weight = 1, dispose = vi.fn()): ThumbnailLoadedResult {
return { value: { kind: "image", url: name, aspect: 1 }, weight, dispose };
}
function request(
key: string,
load: ThumbnailRequest["load"],
priority: ThumbnailPriority = "visible",
overrides: Partial<ThumbnailRequest> = {},
): ThumbnailRequest {
return {
key,
projectId: "project-a",
sessionEpoch: 1,
kind: "image",
priority,
load,
...overrides,
};
}
async function flush(): Promise<void> {
await Promise.resolve();
await Promise.resolve();
}
describe("ThumbnailScheduler", () => {
it("deduplicates identical requests and releases subscribers independently", async () => {
const scheduler = new ThumbnailScheduler();
const pending = deferred<ThumbnailLoadedResult>();
const load = vi.fn(() => pending.promise);
const listenerA = vi.fn();
const listenerB = vi.fn();
const leaseA = scheduler.acquire(request("same", load), listenerA);
const leaseB = scheduler.acquire(request("same", load), listenerB);
expect(load).toHaveBeenCalledTimes(1);
leaseA.release();
listenerA.mockClear();
pending.resolve(result("poster"));
await flush();
expect(scheduler.getSnapshot(request("same", load))).toMatchObject({
status: "ready",
value: { url: "poster" },
});
expect(listenerA).not.toHaveBeenCalled();
expect(listenerB).toHaveBeenCalled();
leaseB.release();
expect(scheduler.getDiagnostics().leases).toBe(0);
});
it("keeps snapshots referentially stable and supports independent leases sharing a callback", async () => {
const scheduler = new ThumbnailScheduler();
const listener = vi.fn();
const shared = request("stable", async () => result("stable"));
const first = scheduler.acquire(shared, listener);
const second = scheduler.acquire(shared, listener);
await flush();
expect(scheduler.getSnapshot(shared)).toBe(scheduler.getSnapshot(shared));
listener.mockClear();
scheduler.invalidateProject("project-a");
expect(listener).not.toHaveBeenCalled();
expect(scheduler.getDiagnostics().leases).toBe(2);
first.release();
second.release();
scheduler.invalidateProject("project-a");
expect(scheduler.getSnapshot(shared).status).toBe("idle");
});
it("scopes the same caller key by work shape", async () => {
const scheduler = new ThumbnailScheduler();
const poster = request("shared", async () => result("poster"));
const strip = request("shared", async () => result("strip"), "visible", {
kind: "video",
rich: true,
});
scheduler.acquire(poster, vi.fn());
scheduler.acquire(strip, vi.fn());
await flush();
expect(scheduler.getSnapshot(poster)).toMatchObject({
status: "ready",
value: { url: "poster" },
});
expect(scheduler.getSnapshot(strip)).toMatchObject({
status: "ready",
value: { url: "strip" },
});
});
it("orders interaction before visible before overscan when a slot opens", async () => {
const scheduler = new ThumbnailScheduler(
resolveTimelineViewportBudgets({ concurrentMetadataJobs: 1 }),
);
const first = deferred<ThumbnailLoadedResult>();
const starts: string[] = [];
const load = (name: string, pending?: ReturnType<typeof deferred<ThumbnailLoadedResult>>) =>
vi.fn(() => {
starts.push(name);
return pending?.promise ?? Promise.resolve(result(name));
});
scheduler.acquire(request("first", load("first", first)), vi.fn());
scheduler.acquire(request("overscan", load("overscan"), "overscan"), vi.fn());
scheduler.acquire(request("visible", load("visible"), "visible"), vi.fn());
scheduler.acquire(request("interaction", load("interaction"), "interaction"), vi.fn());
expect(starts).toEqual(["first"]);
first.resolve(result("first"));
await vi.waitFor(() => {
expect(starts).toEqual(["first", "interaction", "visible", "overscan"]);
});
});
it("suspends new rich work while scrolling but lets posters proceed", async () => {
const scheduler = new ThumbnailScheduler();
const richLoad = vi.fn(async () => result("rich"));
const posterLoad = vi.fn(async () => result("poster"));
scheduler.setScrolling(true);
scheduler.acquire(request("rich", richLoad, "interaction", { rich: true }), vi.fn());
scheduler.acquire(request("poster", posterLoad), vi.fn());
await flush();
expect(richLoad).not.toHaveBeenCalled();
expect(posterLoad).toHaveBeenCalledTimes(1);
scheduler.setScrolling(false);
await flush();
expect(richLoad).toHaveBeenCalledTimes(1);
});
it("aborts queued and active jobs after the final release", async () => {
const scheduler = new ThumbnailScheduler(
resolveTimelineViewportBudgets({ concurrentVideoDecodes: 1 }),
);
const active = deferred<ThumbnailLoadedResult>();
let activeSignal: AbortSignal | undefined;
const activeLease = scheduler.acquire(
request(
"active",
(signal) => {
activeSignal = signal;
return active.promise;
},
"visible",
{ kind: "video" },
),
vi.fn(),
);
const queuedLoad = vi.fn(async () => result("queued"));
const queuedLease = scheduler.acquire(
request("queued", queuedLoad, "visible", { kind: "video" }),
vi.fn(),
);
queuedLease.release();
activeLease.release();
expect(activeSignal?.aborted).toBe(true);
expect(queuedLoad).not.toHaveBeenCalled();
active.reject(new DOMException("aborted", "AbortError"));
await flush();
expect(scheduler.getDiagnostics()).toMatchObject({ queued: 0, active: 0, leases: 0 });
});
it("disposes a late result exactly once after its final lease releases", async () => {
const scheduler = new ThumbnailScheduler();
const pending = deferred<ThumbnailLoadedResult>();
const dispose = vi.fn();
const lease = scheduler.acquire(
request("late", () => pending.promise),
vi.fn(),
);
lease.release();
pending.resolve(result("late", 1, dispose));
await flush();
expect(dispose).toHaveBeenCalledTimes(1);
expect(scheduler.getSnapshot(request("late", () => pending.promise))).toEqual({
status: "idle",
});
});
it("preserves a synchronous re-acquire when an expired failure is replaced", async () => {
vi.useFakeTimers();
const scheduler = new ThumbnailScheduler(
resolveTimelineViewportBudgets({ metadataFailureTtlMs: 10 }),
);
const load = vi
.fn<ThumbnailRequest["load"]>()
.mockRejectedValueOnce(new Error("temporary"))
.mockResolvedValue(result("recovered"));
const failed = request("retry", load);
let reacquireOnNotify = false;
let nestedLease: ReturnType<ThumbnailScheduler["acquire"]> | undefined;
const firstLease = scheduler.acquire(failed, () => {
if (!reacquireOnNotify) return;
reacquireOnNotify = false;
nestedLease = scheduler.acquire(failed, vi.fn());
});
await flush();
vi.advanceTimersByTime(11);
reacquireOnNotify = true;
const outerLease = scheduler.acquire(failed, vi.fn());
await flush();
expect(load).toHaveBeenCalledTimes(2);
expect(scheduler.getSnapshot(failed)).toMatchObject({
status: "ready",
value: { url: "recovered" },
});
expect(scheduler.getDiagnostics().leases).toBe(2);
firstLease.release();
nestedLease?.release();
outerLease.release();
vi.useRealTimers();
});
it("times out a hung loader, frees its bucket, and disposes a late result", async () => {
vi.useFakeTimers();
const scheduler = new ThumbnailScheduler(
resolveTimelineViewportBudgets({
concurrentVideoDecodes: 1,
thumbnailLoadTimeoutMs: 10,
}),
);
const hung = deferred<ThumbnailLoadedResult>();
const dispose = vi.fn();
let signal: AbortSignal | undefined;
const hungRequest = request(
"hung",
(activeSignal) => {
signal = activeSignal;
return hung.promise;
},
"visible",
{ kind: "video" },
);
const nextLoad = vi.fn(async () => result("next"));
const nextRequest = request("next", nextLoad, "visible", { kind: "video" });
const hungLease = scheduler.acquire(hungRequest, vi.fn());
const nextLease = scheduler.acquire(nextRequest, vi.fn());
await vi.advanceTimersByTimeAsync(11);
await flush();
expect(signal?.aborted).toBe(true);
expect(nextLoad).toHaveBeenCalledTimes(1);
expect(scheduler.getSnapshot(hungRequest).status).toBe("error");
hung.resolve(result("late", 1, dispose));
await flush();
expect(dispose).toHaveBeenCalledTimes(1);
hungLease.release();
nextLease.release();
vi.useRealTimers();
});
it("isolates late results by session epoch", async () => {
const scheduler = new ThumbnailScheduler();
const old = deferred<ThumbnailLoadedResult>();
const oldRequest = request("poster", () => old.promise, "visible", { sessionEpoch: 1 });
const nextRequest = request("poster", async () => result("next"), "visible", {
sessionEpoch: 2,
});
scheduler.acquire(oldRequest, vi.fn());
scheduler.acquire(nextRequest, vi.fn());
await flush();
old.resolve(result("old"));
await flush();
expect(scheduler.getSnapshot(nextRequest)).toMatchObject({
status: "ready",
value: { url: "next" },
});
});
it("turns synchronous loader errors into bounded failure snapshots", async () => {
const scheduler = new ThumbnailScheduler();
const bad = request("throws", () => {
throw new Error("sync failure");
});
const lease = scheduler.acquire(bad, vi.fn());
await flush();
expect(scheduler.getSnapshot(bad)).toMatchObject({ status: "error" });
expect(scheduler.getDiagnostics().active).toBe(0);
lease.release();
});
it("disposes uncached results after the final lease and tolerates broken disposers", async () => {
const scheduler = new ThumbnailScheduler(
resolveTimelineViewportBudgets({ thumbnailCacheBytes: 1 }),
);
const dispose = vi.fn(() => {
throw new Error("cleanup failure");
});
const oversized = request("oversized", async () => result("large", 2, dispose));
const lease = scheduler.acquire(oversized, vi.fn());
await flush();
expect(scheduler.getSnapshot(oversized).status).toBe("ready");
expect(() => lease.release()).not.toThrow();
expect(dispose).toHaveBeenCalledTimes(1);
expect(scheduler.getSnapshot(oversized).status).toBe("idle");
});
it("evicts least-recently-used unleased values by bytes, count, and project count", async () => {
const scheduler = new ThumbnailScheduler(
resolveTimelineViewportBudgets({
thumbnailCacheBytes: 6,
thumbnailCacheEntries: 2,
thumbnailCacheEntriesPerProject: 2,
}),
);
const disposals = [vi.fn(), vi.fn(), vi.fn()];
for (let index = 0; index < 3; index++) {
const lease = scheduler.acquire(
request(`key-${index}`, async () => result(`url-${index}`, 3, disposals[index])),
vi.fn(),
);
await flush();
lease.release();
}
expect(scheduler.getDiagnostics()).toMatchObject({ cacheEntries: 2, cacheBytes: 6 });
expect(disposals[0]).toHaveBeenCalledTimes(1);
expect(disposals[1]).not.toHaveBeenCalled();
expect(disposals[2]).not.toHaveBeenCalled();
});
it("keeps failed requests quiet until the TTL expires", async () => {
vi.useFakeTimers();
const scheduler = new ThumbnailScheduler(
resolveTimelineViewportBudgets({ metadataFailureTtlMs: 100 }),
);
const load = vi.fn(async () => {
throw new Error("unsupported");
});
const first = scheduler.acquire(request("failed", load), vi.fn());
await flush();
first.release();
const second = scheduler.acquire(request("failed", load), vi.fn());
expect(load).toHaveBeenCalledTimes(1);
second.release();
vi.advanceTimersByTime(101);
scheduler.acquire(request("failed", load), vi.fn());
await flush();
expect(load).toHaveBeenCalledTimes(2);
vi.useRealTimers();
});
});
describe("createThumbnailKey", () => {
it("normalizes field order and preserves sentinel zero values", () => {
expect(createThumbnailKey({ source: "clip a", time: 0, missing: undefined })).toBe(
createThumbnailKey({ time: 0, source: "clip a" }),
);
});
it("includes the work shape in request identity", () => {
const poster = request("shared", vi.fn(), "visible", { kind: "video" });
const strip = { ...poster, rich: true };
expect(createThumbnailRequestIdentity(poster)).not.toBe(createThumbnailRequestIdentity(strip));
});
});
@@ -0,0 +1,483 @@
import { TIMELINE_VIEWPORT_BUDGETS, type TimelineViewportBudgets } from "./timelineViewportBudgets";
export type ThumbnailPriority = "overscan" | "visible" | "interaction";
export type ThumbnailJobKind = "video" | "image" | "composition" | "waveform";
export type ThumbnailValue =
| { kind: "image"; url: string; aspect: number }
| { kind: "filmstrip"; urls: readonly string[]; aspect: number }
| { kind: "waveform"; peaks: readonly number[] };
export interface ThumbnailLoadedResult {
value: ThumbnailValue;
/** Estimated decoded/object-URL bytes retained by this result. */
weight: number;
dispose?: () => void;
}
export interface ThumbnailRequest {
key: string;
projectId: string;
sessionEpoch: number;
kind: ThumbnailJobKind;
priority: ThumbnailPriority;
/** Rich work is paused while the timeline is fast-scrolling. */
rich?: boolean;
load: (signal: AbortSignal) => Promise<ThumbnailLoadedResult>;
}
export type ThumbnailSnapshot =
| { status: "idle" | "queued" | "loading" }
| { status: "ready"; value: ThumbnailValue }
| { status: "error"; error: Error };
export interface ThumbnailLease {
updatePriority(priority: ThumbnailPriority): void;
release(): void;
}
export interface ThumbnailSchedulerDiagnostics {
queued: number;
active: number;
leases: number;
cacheEntries: number;
cacheBytes: number;
waveformCacheEntries: number;
waveformCacheBytes: number;
activeByKind: Readonly<Record<ThumbnailJobKind, number>>;
}
type EntryState = "queued" | "loading" | "ready" | "error";
interface ThumbnailEntry {
request: ThumbnailRequest;
scopedKey: string;
state: EntryState;
leases: Map<number, ThumbnailPriority>;
listeners: Map<number, () => void>;
controller: AbortController | null;
value?: ThumbnailValue;
error?: Error;
failedAt?: number;
weight: number;
dispose?: () => void;
disposed: boolean;
cached: boolean;
lastAccess: number;
snapshot: ThumbnailSnapshot;
}
const PRIORITY_SCORE: Readonly<Record<ThumbnailPriority, number>> = {
overscan: 0,
visible: 1,
interaction: 2,
};
const EMPTY_SNAPSHOT: ThumbnailSnapshot = Object.freeze({ status: "idle" });
function concurrencyBucket(kind: ThumbnailJobKind): "video" | "composition" | "general" {
if (kind === "video") return "video";
if (kind === "composition") return "composition";
return "general";
}
function errorFrom(reason: unknown): Error {
return reason instanceof Error ? reason : new Error(String(reason));
}
export function createThumbnailKey(parts: Readonly<Record<string, string | number | undefined>>) {
return Object.entries(parts)
.filter((entry): entry is [string, string | number] => entry[1] !== undefined)
.sort(([left], [right]) => left.localeCompare(right))
.map(([name, value]) => `${encodeURIComponent(name)}=${encodeURIComponent(String(value))}`)
.join("&");
}
export function createThumbnailRequestIdentity(
request: Pick<ThumbnailRequest, "key" | "projectId" | "sessionEpoch" | "kind" | "rich">,
) {
return createThumbnailKey({
project: request.projectId,
session: request.sessionEpoch,
request: request.key,
kind: request.kind,
rich: request.rich ? 1 : 0,
});
}
/** Sole client owner for thumbnail work, cached resources, and cleanup. */
export class ThumbnailScheduler {
private readonly entries = new Map<string, ThumbnailEntry>();
private readonly budgets: Readonly<TimelineViewportBudgets>;
private nextLeaseId = 1;
private nextSequence = 1;
private scrolling = false;
private cacheBytes = 0;
private waveformCacheBytes = 0;
private readonly activeByBucket = { video: 0, composition: 0, general: 0 };
private readonly activeByKind: Record<ThumbnailJobKind, number> = {
video: 0,
image: 0,
composition: 0,
waveform: 0,
};
private readonly now: () => number;
constructor(
budgets: Readonly<TimelineViewportBudgets> = TIMELINE_VIEWPORT_BUDGETS,
now: () => number = Date.now,
) {
this.budgets = budgets;
this.now = now;
}
acquire(request: ThumbnailRequest, listener: () => void): ThumbnailLease {
const scopedKey = createThumbnailRequestIdentity(request);
let entry = this.entries.get(scopedKey);
if (
entry?.state === "error" &&
entry.failedAt !== undefined &&
this.now() - entry.failedAt >= this.budgets.metadataFailureTtlMs
) {
this.deleteEntry(scopedKey, entry);
entry = this.entries.get(scopedKey);
}
if (!entry) {
entry = {
request,
scopedKey,
state: "queued",
leases: new Map(),
listeners: new Map(),
controller: null,
weight: 0,
disposed: false,
cached: false,
lastAccess: this.nextSequence++,
snapshot: Object.freeze({ status: "queued" }),
};
this.entries.set(scopedKey, entry);
}
const leaseId = this.nextLeaseId++;
entry.leases.set(leaseId, request.priority);
entry.listeners.set(leaseId, listener);
entry.lastAccess = this.nextSequence++;
this.pump();
let released = false;
return {
updatePriority: (priority) => {
const current = this.entries.get(scopedKey);
if (!current || !current.leases.has(leaseId)) return;
current.leases.set(leaseId, priority);
current.lastAccess = this.nextSequence++;
this.pump();
},
release: () => {
if (released) return;
released = true;
const current = this.entries.get(scopedKey);
if (!current) return;
current.leases.delete(leaseId);
current.listeners.delete(leaseId);
if (current.leases.size === 0) {
if (current.state === "queued") {
this.deleteEntry(scopedKey, current);
} else if (current.state === "loading") {
current.controller?.abort();
this.deleteEntry(scopedKey, current);
} else if (current.state === "ready" && !current.cached) {
this.deleteEntry(scopedKey, current);
}
}
this.evict();
},
};
}
getSnapshot(
request: Pick<ThumbnailRequest, "key" | "projectId" | "sessionEpoch" | "kind" | "rich">,
): ThumbnailSnapshot {
const entry = this.entries.get(createThumbnailRequestIdentity(request));
if (!entry) return EMPTY_SNAPSHOT;
return entry.snapshot;
}
setScrolling(scrolling: boolean): void {
if (this.scrolling === scrolling) return;
this.scrolling = scrolling;
if (!scrolling) this.pump();
}
/** Evict project cache entries that are no longer owned by mounted consumers. */
invalidateProject(projectId: string): void {
for (const [key, entry] of this.entries) {
if (entry.request.projectId !== projectId) continue;
if (entry.leases.size > 0) continue;
entry.controller?.abort();
this.deleteEntry(key, entry);
}
}
getDiagnostics(): ThumbnailSchedulerDiagnostics {
let queued = 0;
let active = 0;
let leases = 0;
let cacheEntries = 0;
let waveformCacheEntries = 0;
for (const entry of this.entries.values()) {
if (entry.state === "queued") queued++;
if (entry.state === "loading") active++;
if (entry.cached) {
if (entry.request.kind === "waveform") waveformCacheEntries++;
else cacheEntries++;
}
leases += entry.leases.size;
}
return {
queued,
active,
leases,
cacheEntries,
cacheBytes: this.cacheBytes,
waveformCacheEntries,
waveformCacheBytes: this.waveformCacheBytes,
activeByKind: { ...this.activeByKind },
};
}
private pump(): void {
const queued = Array.from(this.entries.values())
.filter((entry) => entry.state === "queued" && entry.leases.size > 0)
.sort((left, right) => {
const priorityDelta = this.entryPriority(right) - this.entryPriority(left);
return priorityDelta || left.lastAccess - right.lastAccess;
});
for (const entry of queued) {
if (this.scrolling && entry.request.rich) continue;
const bucket = concurrencyBucket(entry.request.kind);
if (this.activeByBucket[bucket] >= this.bucketLimit(bucket)) continue;
this.start(entry, bucket);
}
}
private start(entry: ThumbnailEntry, bucket: "video" | "composition" | "general"): void {
entry.state = "loading";
entry.snapshot = Object.freeze({ status: "loading" });
const controller = new AbortController();
entry.controller = controller;
this.activeByBucket[bucket]++;
this.activeByKind[entry.request.kind]++;
this.notify(entry);
const pending = this.loadWithTimeout(entry, controller);
void pending
.then((result) => this.acceptResult(entry, controller, result))
.catch((reason: unknown) => {
if (this.entries.get(entry.scopedKey) !== entry) return;
if (controller.signal.aborted && entry.leases.size === 0) {
this.deleteEntry(entry.scopedKey, entry);
return;
}
entry.state = "error";
entry.error = errorFrom(reason);
entry.failedAt = this.now();
entry.snapshot = Object.freeze({ status: "error", error: entry.error });
this.notify(entry);
this.evictFailures();
})
.finally(() => {
if (entry.controller === controller) entry.controller = null;
this.activeByBucket[bucket]--;
this.activeByKind[entry.request.kind]--;
this.pump();
});
}
private acceptResult(
entry: ThumbnailEntry,
controller: AbortController,
result: ThumbnailLoadedResult,
): void {
if (controller.signal.aborted || this.entries.get(entry.scopedKey) !== entry) {
this.safeDispose(result.dispose);
return;
}
this.validateResult(result);
entry.state = "ready";
entry.value = result.value;
entry.weight = result.weight;
entry.dispose = result.dispose;
this.cacheResult(entry);
entry.snapshot = Object.freeze({ status: "ready", value: result.value });
this.notify(entry);
this.evict();
if (!entry.cached && entry.leases.size === 0) this.deleteEntry(entry.scopedKey, entry);
}
private validateResult(result: ThumbnailLoadedResult): void {
if (Number.isFinite(result.weight) && result.weight >= 0) return;
this.safeDispose(result.dispose);
throw new RangeError("Thumbnail result weight must be finite and non-negative");
}
private cacheResult(entry: ThumbnailEntry): void {
const isWaveform = entry.request.kind === "waveform";
const byteBudget = isWaveform
? this.budgets.waveformCacheBytes
: this.budgets.thumbnailCacheBytes;
entry.cached = entry.weight <= byteBudget;
if (!entry.cached) return;
if (isWaveform) this.waveformCacheBytes += entry.weight;
else this.cacheBytes += entry.weight;
}
private entryPriority(entry: ThumbnailEntry): number {
let score = -1;
for (const priority of entry.leases.values()) {
score = Math.max(score, PRIORITY_SCORE[priority]);
}
return score;
}
private bucketLimit(bucket: "video" | "composition" | "general"): number {
if (bucket === "video") return this.budgets.concurrentVideoDecodes;
if (bucket === "composition") return this.budgets.concurrentCompositionFetches;
return this.budgets.concurrentMetadataJobs;
}
private evict(): void {
this.evictPartition(false);
this.evictPartition(true);
}
private evictPartition(waveform: boolean): void {
const cached = Array.from(this.entries.values()).filter(
(entry) => entry.cached && (entry.request.kind === "waveform") === waveform,
);
const perProject = new Map<string, number>();
for (const entry of cached) {
perProject.set(entry.request.projectId, (perProject.get(entry.request.projectId) ?? 0) + 1);
}
cached.sort((left, right) => left.lastAccess - right.lastAccess);
let cacheEntries = cached.length;
for (const entry of cached) {
const projectEntries = perProject.get(entry.request.projectId) ?? 0;
if (!this.isCacheOverBudget(waveform, cacheEntries, projectEntries)) continue;
this.uncache(entry);
cacheEntries--;
perProject.set(entry.request.projectId, projectEntries - 1);
if (entry.leases.size === 0) this.deleteEntry(entry.scopedKey, entry);
}
}
private isCacheOverBudget(
waveform: boolean,
cacheEntries: number,
projectEntries: number,
): boolean {
const entryBudget = waveform
? this.budgets.waveformCacheEntries
: this.budgets.thumbnailCacheEntries;
const byteBudget = waveform
? this.budgets.waveformCacheBytes
: this.budgets.thumbnailCacheBytes;
const cacheBytes = waveform ? this.waveformCacheBytes : this.cacheBytes;
const projectOverBudget =
!waveform && projectEntries > this.budgets.thumbnailCacheEntriesPerProject;
return cacheEntries > entryBudget || cacheBytes > byteBudget || projectOverBudget;
}
private uncache(entry: ThumbnailEntry): void {
entry.cached = false;
if (entry.request.kind === "waveform") this.waveformCacheBytes -= entry.weight;
else this.cacheBytes -= entry.weight;
}
private notify(entry: ThumbnailEntry): void {
for (const listener of new Set(entry.listeners.values())) listener();
}
private loadWithTimeout(
entry: ThumbnailEntry,
controller: AbortController,
): Promise<ThumbnailLoadedResult> {
let load: Promise<ThumbnailLoadedResult>;
try {
load = entry.request.load(controller.signal);
} catch (reason) {
load = Promise.reject(reason);
}
return new Promise((resolve, reject) => {
let settled = false;
const timeout = setTimeout(() => {
if (settled) return;
settled = true;
controller.abort();
reject(
new Error(`Thumbnail load timed out after ${this.budgets.thumbnailLoadTimeoutMs}ms`),
);
}, this.budgets.thumbnailLoadTimeoutMs);
load.then(
(result) => {
if (settled) {
this.safeDispose(result.dispose);
return;
}
settled = true;
clearTimeout(timeout);
resolve(result);
},
(reason: unknown) => {
if (settled) return;
settled = true;
clearTimeout(timeout);
reject(reason);
},
);
});
}
private deleteEntry(key: string, entry: ThumbnailEntry): void {
if (this.entries.get(key) !== entry) return;
this.entries.delete(key);
entry.snapshot = EMPTY_SNAPSHOT;
this.notify(entry);
if (entry.cached) {
entry.cached = false;
if (entry.request.kind === "waveform") this.waveformCacheBytes -= entry.weight;
else this.cacheBytes -= entry.weight;
}
if (!entry.disposed) {
entry.disposed = true;
this.safeDispose(entry.dispose);
}
entry.listeners.clear();
entry.leases.clear();
}
private evictFailures(): void {
const failed = Array.from(this.entries.values())
.filter((entry) => entry.state === "error" && entry.leases.size === 0)
.sort((left, right) => left.lastAccess - right.lastAccess);
const overflow = failed.length - this.budgets.thumbnailCacheEntries;
for (const entry of failed.slice(0, Math.max(0, overflow))) {
this.deleteEntry(entry.scopedKey, entry);
}
}
private safeDispose(dispose: (() => void) | undefined): void {
try {
dispose?.();
} catch {
// Cleanup is best-effort; one broken resource must not block the remaining cleanup.
}
}
}
export const thumbnailScheduler = new ThumbnailScheduler();
@@ -0,0 +1,127 @@
// @vitest-environment happy-dom
import { beforeEach, describe, expect, it, vi } from "vitest";
import { decodeVideoThumbnail, videoThumbnailTimestamps } from "./thumbnailVideoDecoder";
const dispose = vi.fn();
const canvasesAtTimestamps = vi.fn();
const input = {
getPrimaryVideoTrack: vi.fn(),
dispose,
};
vi.mock("mediabunny", () => ({
ALL_FORMATS: {},
UrlSource: class {
constructor(readonly url: string) {}
},
Input: class {
getPrimaryVideoTrack = input.getPrimaryVideoTrack;
dispose = input.dispose;
},
CanvasSink: class {
canvasesAtTimestamps = canvasesAtTimestamps;
},
}));
beforeEach(() => {
vi.clearAllMocks();
vi.spyOn(URL, "createObjectURL").mockReturnValueOnce("blob:one").mockReturnValueOnce("blob:two");
vi.spyOn(URL, "revokeObjectURL").mockImplementation(() => {});
HTMLCanvasElement.prototype.toBlob = function toBlob(callback) {
callback(new Blob(["frame"], { type: "image/jpeg" }));
};
input.getPrimaryVideoTrack.mockResolvedValue({
getDisplayWidth: vi.fn(async () => 1080),
getDisplayHeight: vi.fn(async () => 1920),
getDurationFromMetadata: vi.fn(async () => 10),
});
});
describe("videoThumbnailTimestamps", () => {
it("uses the midpoint for a poster and sorted sparse points for a strip", () => {
expect(videoThumbnailTimestamps(2, 6, 1)).toEqual([5]);
expect(videoThumbnailTimestamps(2, 6, 4)).toEqual([2, 4, 6, 8]);
});
it("clamps invalid source ranges", () => {
expect(videoThumbnailTimestamps(-2, Number.NaN, 0)).toEqual([0]);
expect(videoThumbnailTimestamps(2, 8, Number.NaN)).toEqual([6]);
});
});
describe("decodeVideoThumbnail", () => {
it("extracts sparse frames, returns object URLs, and disposes once", async () => {
const canvas = document.createElement("canvas");
canvasesAtTimestamps.mockImplementation(async function* (timestamps: number[]) {
expect(timestamps).toEqual([2, 8]);
yield { canvas, timestamp: 2, duration: 1 };
yield { canvas, timestamp: 8, duration: 1 };
});
const result = await decodeVideoThumbnail(
{ source: "/clip.mp4", sourceStart: 2, sourceRangeDuration: 6, frameCount: 2 },
new AbortController().signal,
);
expect(result.value).toEqual({
kind: "filmstrip",
urls: ["blob:one", "blob:two"],
aspect: 9 / 16,
});
result.dispose?.();
result.dispose?.();
expect(URL.revokeObjectURL).toHaveBeenCalledTimes(2);
expect(dispose).toHaveBeenCalledTimes(1);
});
it("releases input and degrades when the source has no video track", async () => {
input.getPrimaryVideoTrack.mockResolvedValue(null);
await expect(
decodeVideoThumbnail({ source: "/audio.mp3", frameCount: 1 }, new AbortController().signal),
).rejects.toThrow("no decodable video track");
expect(dispose).toHaveBeenCalledTimes(1);
});
it("revokes partial results when cancellation lands during extraction", async () => {
const controller = new AbortController();
const canvas = document.createElement("canvas");
canvasesAtTimestamps.mockImplementation(async function* () {
yield { canvas, timestamp: 1, duration: 1 };
controller.abort();
yield { canvas, timestamp: 2, duration: 1 };
});
await expect(
decodeVideoThumbnail({ source: "/clip.mp4", frameCount: 2 }, controller.signal),
).rejects.toMatchObject({ name: "AbortError" });
expect(URL.revokeObjectURL).toHaveBeenCalledTimes(1);
expect(dispose).toHaveBeenCalledTimes(1);
});
it("stops after metadata cancellation before occupying the decoder", async () => {
const controller = new AbortController();
let resolveWidth!: (width: number) => void;
const width = new Promise<number>((resolve) => {
resolveWidth = resolve;
});
const getDisplayWidth = vi.fn(() => width);
const getDurationFromMetadata = vi.fn(async () => 10);
input.getPrimaryVideoTrack.mockResolvedValue({
getDisplayWidth,
getDisplayHeight: vi.fn(async () => 1920),
getDurationFromMetadata,
});
const decoding = decodeVideoThumbnail(
{ source: "/clip.mp4", frameCount: 1 },
controller.signal,
);
await vi.waitFor(() => expect(getDisplayWidth).toHaveBeenCalledOnce());
controller.abort();
resolveWidth(1080);
await expect(decoding).rejects.toMatchObject({ name: "AbortError" });
expect(getDurationFromMetadata).not.toHaveBeenCalled();
expect(canvasesAtTimestamps).not.toHaveBeenCalled();
expect(dispose).toHaveBeenCalledTimes(1);
});
});
@@ -0,0 +1,170 @@
import { TIMELINE_VIEWPORT_BUDGETS, type TimelineViewportBudgets } from "./timelineViewportBudgets";
import type { ThumbnailLoadedResult, ThumbnailValue } from "./thumbnailScheduler";
export interface VideoThumbnailDecodeRequest {
source: string;
sourceStart?: number;
sourceRangeDuration?: number;
frameCount: number;
fit?: "contain" | "cover";
}
export function videoThumbnailTimestamps(
start: number,
duration: number,
frameCount: number,
): number[] {
const safeStart = Math.max(0, Number.isFinite(start) ? start : 0);
const safeDuration = Math.max(0, Number.isFinite(duration) ? duration : 0);
const count = Math.max(1, Number.isFinite(frameCount) ? Math.floor(frameCount) : 1);
if (count === 1) return [safeStart + safeDuration / 2];
return Array.from(
{ length: count },
(_, index) => safeStart + (safeDuration * index) / (count - 1),
);
}
async function canvasToBlob(canvas: HTMLCanvasElement | OffscreenCanvas): Promise<Blob> {
if (canvas instanceof HTMLCanvasElement) {
return new Promise((resolve, reject) => {
canvas.toBlob(
(blob) => (blob ? resolve(blob) : reject(new Error("Video thumbnail encode failed"))),
"image/jpeg",
0.72,
);
});
}
return canvas.convertToBlob({ type: "image/jpeg", quality: 0.72 });
}
interface DecodedResources {
urls: string[];
canvases: Set<HTMLCanvasElement | OffscreenCanvas>;
}
interface ThumbnailCanvasSink {
canvasesAtTimestamps(
timestamps: number[],
): AsyncIterable<{ canvas: HTMLCanvasElement | OffscreenCanvas } | null>;
}
function throwIfAborted(signal: AbortSignal): void {
if (signal.aborted) throw new DOMException("Aborted", "AbortError");
}
function releaseDecodedResources(resources: DecodedResources): void {
for (const url of resources.urls.splice(0)) URL.revokeObjectURL(url);
for (const canvas of resources.canvases) {
canvas.width = 0;
canvas.height = 0;
}
resources.canvases.clear();
}
function targetDimensions(
aspect: number,
budgets: Readonly<TimelineViewportBudgets>,
): { width: number; height: number } {
const width = Math.max(
1,
Math.min(budgets.posterMaxPhysicalWidth, Math.round(budgets.posterMaxPhysicalHeight * aspect)),
);
return {
width,
height: Math.max(1, Math.min(budgets.posterMaxPhysicalHeight, Math.round(width / aspect))),
};
}
async function decodeFrames(
sink: ThumbnailCanvasSink,
timestamps: number[],
signal: AbortSignal,
resources: DecodedResources,
): Promise<void> {
for await (const wrapped of sink.canvasesAtTimestamps(timestamps)) {
throwIfAborted(signal);
if (!wrapped) continue;
resources.canvases.add(wrapped.canvas);
const blob = await canvasToBlob(wrapped.canvas);
throwIfAborted(signal);
resources.urls.push(URL.createObjectURL(blob));
}
}
function loadedResult(
resources: DecodedResources,
aspect: number,
width: number,
height: number,
): ThumbnailLoadedResult {
const firstUrl = resources.urls[0];
if (!firstUrl) throw new Error("Video source returned no thumbnail frames");
const value: ThumbnailValue =
resources.urls.length === 1
? { kind: "image", url: firstUrl, aspect }
: { kind: "filmstrip", urls: [...resources.urls], aspect };
return {
value,
weight: width * height * 4 * resources.urls.length,
dispose: () => releaseDecodedResources(resources),
};
}
/** Sparse Mediabunny extraction with one pooled canvas and one cleanup owner. */
export async function decodeVideoThumbnail(
request: VideoThumbnailDecodeRequest,
signal: AbortSignal,
budgets: Readonly<TimelineViewportBudgets> = TIMELINE_VIEWPORT_BUDGETS,
): Promise<ThumbnailLoadedResult> {
const mediabunny = await import("mediabunny");
throwIfAborted(signal);
const input = new mediabunny.Input({
source: new mediabunny.UrlSource(request.source),
formats: mediabunny.ALL_FORMATS,
});
const resources: DecodedResources = { urls: [], canvases: new Set() };
try {
const track = await input.getPrimaryVideoTrack();
throwIfAborted(signal);
if (!track) throw new Error("Video source has no decodable video track");
const [displayWidth, displayHeight] = await Promise.all([
track.getDisplayWidth(),
track.getDisplayHeight(),
]);
throwIfAborted(signal);
if (!(displayWidth > 0 && displayHeight > 0)) {
throw new Error("Video source has invalid dimensions");
}
const metadataDuration = await track.getDurationFromMetadata({ skipLiveWait: true });
throwIfAborted(signal);
const sourceDuration = Math.max(0, metadataDuration ?? request.sourceRangeDuration ?? 0);
const sourceStart = Math.min(Math.max(0, request.sourceStart ?? 0), sourceDuration);
const requestedDuration =
request.sourceRangeDuration ?? Math.max(0, sourceDuration - sourceStart);
const duration = Math.min(
Math.max(0, requestedDuration),
Math.max(0, sourceDuration - sourceStart),
);
const timestamps = videoThumbnailTimestamps(
sourceStart,
duration,
Math.min(request.frameCount, budgets.richPreviewFrameCount),
);
const aspect = displayWidth / displayHeight;
const target = targetDimensions(aspect, budgets);
const sink = new mediabunny.CanvasSink(track, {
width: target.width,
height: target.height,
fit: request.fit ?? "cover",
poolSize: 1,
});
await decodeFrames(sink, timestamps, signal, resources);
return loadedResult(resources, aspect, target.width, target.height);
} catch (error) {
releaseDecodedResources(resources);
throw error;
} finally {
input.dispose();
}
}
@@ -14,6 +14,7 @@ export interface TimelineViewportBudgets {
concurrentMetadataJobs: number;
concurrentCompositionFetches: number;
concurrentServerPages: number;
thumbnailLoadTimeoutMs: number;
thumbnailCacheBytes: number;
thumbnailCacheEntries: number;
thumbnailCacheEntriesPerProject: number;
@@ -70,6 +71,7 @@ export const TIMELINE_VIEWPORT_BUDGETS: Readonly<TimelineViewportBudgets> = Obje
concurrentMetadataJobs: 4,
concurrentCompositionFetches: 2,
concurrentServerPages: 1,
thumbnailLoadTimeoutMs: 30_000,
thumbnailCacheBytes: 64 * MEBIBYTE,
thumbnailCacheEntries: 256,
thumbnailCacheEntriesPerProject: 96,
@@ -11,6 +11,7 @@ import {
import { clampTimelineZoomPercent, computePinnedZoomPercent } from "../components/timelineZoom";
import { createKeyframeSlice, type KeyframeCacheEntry, type KeyframeSlice } from "./keyframeSlice";
import { createTimelineFocusRequest, type TimelineFocusRequest } from "./timelineFocusState";
import { createThumbnailSlice, type ThumbnailSlice } from "./thumbnailSlice";
export type { KeyframeCacheEntry } from "./keyframeSlice";
export { liveTime } from "./liveTime";
@@ -103,7 +104,7 @@ function resolveElementSelection(
};
}
interface PlayerState extends KeyframeSlice {
interface PlayerState extends KeyframeSlice, ThumbnailSlice {
isPlaying: boolean;
currentTime: number;
duration: number;
@@ -340,6 +341,7 @@ export const usePlayerStore = create<PlayerState>((set, get) => ({
timelineProjectId: get().timelineProjectId,
timelineSessionEpoch: get().timelineSessionEpoch,
})),
...createThumbnailSlice(set),
activeKeyframePct: null,
setActiveKeyframePct: (pct) => set({ activeKeyframePct: pct }),
@@ -0,0 +1,18 @@
import type { StoreApi } from "zustand";
import { readStudioUiPreferences, writeStudioUiPreferences } from "../../utils/studioUiPreferences";
import { defaultThumbnailMode, type ThumbnailMode } from "../lib/thumbnailPolicy";
export interface ThumbnailSlice {
thumbnailMode: ThumbnailMode;
setThumbnailMode: (mode: ThumbnailMode) => void;
}
export function createThumbnailSlice(set: StoreApi<ThumbnailSlice>["setState"]): ThumbnailSlice {
return {
thumbnailMode: defaultThumbnailMode(readStudioUiPreferences().thumbnailMode),
setThumbnailMode: (mode) => {
writeStudioUiPreferences({ thumbnailMode: mode });
set({ thumbnailMode: mode });
},
};
}