mirror of
https://github.com/heygen-com/hyperframes.git
synced 2026-09-12 15:20:13 +00:00
perf(studio-server): coordinate cancelable thumbnail generation (#2720)
* perf(studio): schedule adaptive timeline thumbnails * perf(studio): bound thumbnail decoding resources * perf(studio): virtualize timeline thumbnail media * perf(studio): prioritize timeline thumbnail work * perf(studio-server): coordinate cancelable thumbnail generation --------- Co-authored-by: Codex <codex@local>
This commit is contained in:
Miguel Ángel
@@ -0,0 +1,53 @@
|
||||
// @vitest-environment happy-dom
|
||||
|
||||
import React, { act } from "react";
|
||||
import { createRoot } from "react-dom/client";
|
||||
import { afterEach, describe, expect, it, vi } from "vitest";
|
||||
const { leaseSpy } = vi.hoisted(() => ({
|
||||
leaseSpy: vi.fn((_request: unknown) => ({ status: "loading" as const })),
|
||||
}));
|
||||
|
||||
vi.mock("../../hooks/useThumbnailLease", () => ({
|
||||
useThumbnailLease: leaseSpy,
|
||||
}));
|
||||
|
||||
import { AudioWaveform } from "./AudioWaveform";
|
||||
|
||||
(globalThis as unknown as { IS_REACT_ACT_ENVIRONMENT: boolean }).IS_REACT_ACT_ENVIRONMENT = true;
|
||||
|
||||
afterEach(() => {
|
||||
leaseSpy.mockClear();
|
||||
document.body.innerHTML = "";
|
||||
});
|
||||
|
||||
describe("AudioWaveform", () => {
|
||||
it("leases waveform decoding with the clip's project, session, and viewport priority", () => {
|
||||
const host = document.createElement("div");
|
||||
document.body.append(host);
|
||||
const root = createRoot(host);
|
||||
|
||||
act(() => {
|
||||
root.render(
|
||||
<AudioWaveform
|
||||
audioUrl="/media/voice.wav"
|
||||
label=""
|
||||
labelColor="#fff"
|
||||
projectId="project-a"
|
||||
sessionEpoch={9}
|
||||
priority="interaction"
|
||||
/>,
|
||||
);
|
||||
});
|
||||
|
||||
expect(leaseSpy).toHaveBeenCalled();
|
||||
expect(leaseSpy.mock.calls.at(-1)?.[0]).toMatchObject({
|
||||
projectId: "project-a",
|
||||
sessionEpoch: 9,
|
||||
kind: "waveform",
|
||||
priority: "interaction",
|
||||
rich: false,
|
||||
});
|
||||
|
||||
act(() => root.unmount());
|
||||
});
|
||||
});
|
||||
@@ -1,74 +1,105 @@
|
||||
import { memo, useRef, useState, useCallback, useEffect } from "react";
|
||||
import { memo, useCallback, useMemo, useRef } from "react";
|
||||
import { useMountEffect } from "../../hooks/useMountEffect";
|
||||
import { useThumbnailLease } from "../../hooks/useThumbnailLease";
|
||||
import { createThumbnailKey, type ThumbnailPriority } from "../lib/thumbnailScheduler";
|
||||
|
||||
interface AudioWaveformProps {
|
||||
audioUrl: string;
|
||||
waveformUrl?: string;
|
||||
label: string;
|
||||
labelColor: string;
|
||||
/**
|
||||
* Fraction (0–1) of the source the clip starts at, after the media-start
|
||||
* trim. Defaults to 0 (no front trim).
|
||||
*/
|
||||
trimStartFraction?: number;
|
||||
/**
|
||||
* Fraction (0–1) of the source the clip ends at. Defaults to 1 (no tail
|
||||
* trim). Together these window the rendered peaks to the trimmed slice so the
|
||||
* waveform tracks the clip edges instead of squeezing the whole file in.
|
||||
*/
|
||||
trimEndFraction?: number;
|
||||
projectId: string;
|
||||
sessionEpoch: number;
|
||||
priority: ThumbnailPriority;
|
||||
}
|
||||
|
||||
const BAR_W = 2;
|
||||
const GAP = 1;
|
||||
const STEP = BAR_W + GAP;
|
||||
const BAR_WIDTH = 2;
|
||||
const BAR_STEP = 3;
|
||||
|
||||
/** Downsample PCM channel data into peak amplitudes (0–1). */
|
||||
function extractPeaks(channelData: Float32Array, barCount: number): number[] {
|
||||
const peaks: number[] = [];
|
||||
const samplesPerBar = Math.floor(channelData.length / barCount);
|
||||
if (samplesPerBar === 0) return Array(barCount).fill(0);
|
||||
for (let i = 0; i < barCount; i++) {
|
||||
for (let index = 0; index < barCount; index++) {
|
||||
let max = 0;
|
||||
const start = i * samplesPerBar;
|
||||
const start = index * samplesPerBar;
|
||||
const end = Math.min(start + samplesPerBar, channelData.length);
|
||||
for (let j = start; j < end; j++) {
|
||||
// fallow-ignore-next-line code-duplication
|
||||
const abs = Math.abs(channelData[j] ?? 0);
|
||||
if (abs > max) max = abs;
|
||||
for (let sample = start; sample < end; sample++) {
|
||||
max = Math.max(max, Math.abs(channelData[sample] ?? 0));
|
||||
}
|
||||
peaks.push(max);
|
||||
}
|
||||
const maxPeak = Math.max(...peaks, 0.001);
|
||||
return peaks.map((p) => p / maxPeak);
|
||||
return peaks.map((peak) => peak / maxPeak);
|
||||
}
|
||||
|
||||
/** Deterministic fake waveform as fallback (matches demo app). */
|
||||
function fakePeaks(url: string, count: number): number[] {
|
||||
let seed = 0;
|
||||
for (let i = 0; i < url.length; i++) seed = ((seed << 5) - seed + url.charCodeAt(i)) | 0;
|
||||
for (let index = 0; index < url.length; index++) {
|
||||
seed = ((seed << 5) - seed + url.charCodeAt(index)) | 0;
|
||||
}
|
||||
seed = Math.abs(seed) || 42;
|
||||
const rand = () => {
|
||||
const random = () => {
|
||||
seed = (seed * 16807) % 2147483647;
|
||||
return (seed & 0x7fffffff) / 2147483647;
|
||||
};
|
||||
const peaks: number[] = [];
|
||||
for (let i = 0; i < count; i++) {
|
||||
const t = i / count;
|
||||
const envelope = 0.3 + 0.3 * Math.sin(t * Math.PI * 3.2) + 0.2 * Math.sin(t * Math.PI * 7.1);
|
||||
peaks.push(Math.max(0.05, Math.min(1, envelope * (0.4 + 0.6 * rand()))));
|
||||
}
|
||||
return peaks;
|
||||
return Array.from({ length: count }, (_, index) => {
|
||||
const time = index / count;
|
||||
const envelope =
|
||||
0.3 + 0.3 * Math.sin(time * Math.PI * 3.2) + 0.2 * Math.sin(time * Math.PI * 7.1);
|
||||
return Math.max(0.05, Math.min(1, envelope * (0.4 + 0.6 * random())));
|
||||
});
|
||||
}
|
||||
|
||||
// Module-level cache so decoded audio persists across re-renders and re-mounts
|
||||
const peaksCache = new Map<string, number[]>();
|
||||
const decodeInFlight = new Map<string, Promise<number[]>>();
|
||||
async function loadWaveform(
|
||||
audioUrl: string,
|
||||
waveformUrl: string | undefined,
|
||||
signal: AbortSignal,
|
||||
): Promise<number[]> {
|
||||
try {
|
||||
return waveformUrl
|
||||
? await fetchWaveformPeaks(waveformUrl, signal)
|
||||
: await decodeWaveformPeaks(audioUrl, signal);
|
||||
} catch (error) {
|
||||
if (signal.aborted) throw error;
|
||||
return fakePeaks(waveformUrl ?? audioUrl, 4000);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Audio waveform rendered from real PCM data via Web Audio API.
|
||||
* Falls back to a deterministic fake pattern if decoding fails.
|
||||
* Bars grow from bottom to top, rendered as CSS divs for zoom resilience.
|
||||
*/
|
||||
async function fetchWaveformPeaks(url: string, signal: AbortSignal): Promise<number[]> {
|
||||
const response = await fetch(url, { signal });
|
||||
if (!response.ok) throw new Error(`Waveform request failed (${response.status})`);
|
||||
const data: unknown = await response.json();
|
||||
if (
|
||||
typeof data !== "object" ||
|
||||
data === null ||
|
||||
!("peaks" in data) ||
|
||||
!Array.isArray(data.peaks) ||
|
||||
!data.peaks.every((peak) => typeof peak === "number")
|
||||
) {
|
||||
throw new Error("Invalid waveform response");
|
||||
}
|
||||
return data.peaks;
|
||||
}
|
||||
|
||||
async function decodeWaveformPeaks(url: string, signal: AbortSignal): Promise<number[]> {
|
||||
const response = await fetch(url, { signal });
|
||||
if (!response.ok) throw new Error(`Audio request failed (${response.status})`);
|
||||
const buffer = await response.arrayBuffer();
|
||||
if (signal.aborted) throw new DOMException("Aborted", "AbortError");
|
||||
const context = new AudioContext();
|
||||
try {
|
||||
const decoded = await context.decodeAudioData(buffer);
|
||||
if (signal.aborted) throw new DOMException("Aborted", "AbortError");
|
||||
return extractPeaks(decoded.getChannelData(0), 4000);
|
||||
} finally {
|
||||
await context.close();
|
||||
}
|
||||
}
|
||||
|
||||
/** Bounded waveform subscriber; cache, cancellation and dedupe live in one scheduler. */
|
||||
export const AudioWaveform = memo(function AudioWaveform({
|
||||
audioUrl,
|
||||
waveformUrl,
|
||||
@@ -76,130 +107,96 @@ export const AudioWaveform = memo(function AudioWaveform({
|
||||
labelColor,
|
||||
trimStartFraction,
|
||||
trimEndFraction,
|
||||
projectId,
|
||||
sessionEpoch,
|
||||
priority,
|
||||
}: AudioWaveformProps) {
|
||||
const containerRef = useRef<HTMLDivElement | null>(null);
|
||||
const barsRef = useRef<HTMLDivElement | null>(null);
|
||||
const roRef = useRef<ResizeObserver | null>(null);
|
||||
const canvasRef = useRef<HTMLCanvasElement | null>(null);
|
||||
const observerRef = useRef<ResizeObserver | null>(null);
|
||||
const cacheKey = waveformUrl ?? audioUrl;
|
||||
const [peaks, setPeaks] = useState<number[] | null>(peaksCache.get(cacheKey) ?? null);
|
||||
const request = useMemo(
|
||||
() => ({
|
||||
key: createThumbnailKey({ kind: "waveform", source: cacheKey }),
|
||||
projectId,
|
||||
sessionEpoch,
|
||||
kind: "waveform" as const,
|
||||
priority,
|
||||
rich: false,
|
||||
load: async (signal: AbortSignal) => {
|
||||
const peaks = await loadWaveform(audioUrl, waveformUrl, signal);
|
||||
return {
|
||||
value: { kind: "waveform" as const, peaks },
|
||||
weight: peaks.length * Float64Array.BYTES_PER_ELEMENT,
|
||||
};
|
||||
},
|
||||
}),
|
||||
[audioUrl, cacheKey, priority, projectId, sessionEpoch, waveformUrl],
|
||||
);
|
||||
const snapshot = useThumbnailLease(cacheKey ? request : null);
|
||||
const peaks =
|
||||
snapshot.status === "ready" && snapshot.value.kind === "waveform" ? snapshot.value.peaks : null;
|
||||
|
||||
useEffect(() => {
|
||||
if (peaks || !cacheKey) return;
|
||||
|
||||
let cancelled = false;
|
||||
|
||||
let promise = decodeInFlight.get(cacheKey);
|
||||
if (!promise) {
|
||||
promise = (
|
||||
waveformUrl
|
||||
? fetch(waveformUrl)
|
||||
.then((r) => r.json())
|
||||
.then((d: { peaks?: number[] }) => {
|
||||
if (!Array.isArray(d.peaks)) throw new Error("bad response");
|
||||
return d.peaks;
|
||||
})
|
||||
: fetch(audioUrl)
|
||||
.then((r) => r.arrayBuffer())
|
||||
.then((buf) => {
|
||||
const ctx = new AudioContext();
|
||||
return ctx.decodeAudioData(buf).finally(() => ctx.close());
|
||||
})
|
||||
.then((decoded) => extractPeaks(decoded.getChannelData(0), 4000))
|
||||
)
|
||||
.catch(() => fakePeaks(cacheKey, 4000))
|
||||
.then((p) => {
|
||||
peaksCache.set(cacheKey, p);
|
||||
return p;
|
||||
})
|
||||
.finally(() => decodeInFlight.delete(cacheKey));
|
||||
|
||||
decodeInFlight.set(cacheKey, promise);
|
||||
}
|
||||
|
||||
promise.then((p) => {
|
||||
if (!cancelled) setPeaks(p);
|
||||
});
|
||||
return () => {
|
||||
cancelled = true;
|
||||
};
|
||||
}, [audioUrl, waveformUrl, cacheKey, peaks]);
|
||||
|
||||
// Draw bars into the container using innerHTML (fast, zoom-resilient)
|
||||
const draw = useCallback(() => {
|
||||
const container = containerRef.current;
|
||||
const barsEl = barsRef.current;
|
||||
if (!container || !barsEl || !peaks) return;
|
||||
|
||||
// Window the peaks to the trimmed slice [start, end) of the source so the
|
||||
// bars track the clip edges. Clamp to a valid, non-empty range.
|
||||
const winStart = Math.max(0, Math.min(1, trimStartFraction ?? 0));
|
||||
const winEnd = Math.max(winStart, Math.min(1, trimEndFraction ?? 1));
|
||||
const lo = Math.floor(winStart * peaks.length);
|
||||
const hi = Math.max(lo + 1, Math.ceil(winEnd * peaks.length));
|
||||
const span = hi - lo;
|
||||
|
||||
// Fill the full (possibly zoomed) clip width with STEP-spaced bars, resampling
|
||||
// the windowed peaks across them — upsampling (repeating peaks) when the clip
|
||||
// is wider than the slice has samples, so the waveform stretches with zoom
|
||||
// instead of stopping partway across.
|
||||
const w = container.clientWidth || 400;
|
||||
const barCount = Math.max(0, Math.floor(w / STEP));
|
||||
|
||||
let html = "";
|
||||
for (let i = 0; i < barCount; i++) {
|
||||
// Map bar index to peak index within the windowed range (resample)
|
||||
const peakIdx = lo + Math.min(span - 1, Math.floor((i / barCount) * span));
|
||||
const amp = peaks[peakIdx] ?? 0;
|
||||
const pct = Math.max(3, Math.round(amp * 100));
|
||||
const opacity = (0.45 + amp * 0.4).toFixed(2);
|
||||
html += `<div style="position:absolute;bottom:0;left:${i * STEP}px;width:${BAR_W}px;height:${pct}%;background:rgba(75,163,210,${opacity})"></div>`;
|
||||
const canvas = canvasRef.current;
|
||||
if (!canvas || !peaks) return;
|
||||
const width = Math.max(1, canvas.clientWidth);
|
||||
const height = Math.max(1, canvas.clientHeight);
|
||||
const scale = window.devicePixelRatio || 1;
|
||||
canvas.width = Math.ceil(width * scale);
|
||||
canvas.height = Math.ceil(height * scale);
|
||||
const context = canvas.getContext("2d");
|
||||
if (!context) return;
|
||||
context.scale(scale, scale);
|
||||
context.clearRect(0, 0, width, height);
|
||||
const startFraction = Math.max(0, Math.min(1, trimStartFraction ?? 0));
|
||||
const endFraction = Math.max(startFraction, Math.min(1, trimEndFraction ?? 1));
|
||||
const start = Math.floor(startFraction * peaks.length);
|
||||
const end = Math.max(start + 1, Math.ceil(endFraction * peaks.length));
|
||||
const span = end - start;
|
||||
const barCount = Math.floor(width / BAR_STEP);
|
||||
for (let index = 0; index < barCount; index++) {
|
||||
const peakIndex = start + Math.min(span - 1, Math.floor((index / barCount) * span));
|
||||
const amplitude = peaks[peakIndex] ?? 0;
|
||||
const barHeight = Math.max(2, amplitude * height);
|
||||
context.fillStyle = `rgba(75,163,210,${(0.45 + amplitude * 0.4).toFixed(2)})`;
|
||||
context.fillRect(index * BAR_STEP, height - barHeight, BAR_WIDTH, barHeight);
|
||||
}
|
||||
barsEl.innerHTML = html;
|
||||
}, [peaks, trimStartFraction, trimEndFraction]);
|
||||
}, [peaks, trimEndFraction, trimStartFraction]);
|
||||
|
||||
// Observe container size and redraw
|
||||
const setContainerRef = useCallback(
|
||||
(el: HTMLDivElement | null) => {
|
||||
roRef.current?.disconnect();
|
||||
containerRef.current = el;
|
||||
if (!el) return;
|
||||
const setCanvasRef = useCallback(
|
||||
(canvas: HTMLCanvasElement | null) => {
|
||||
observerRef.current?.disconnect();
|
||||
canvasRef.current = canvas;
|
||||
if (!canvas) return;
|
||||
draw();
|
||||
roRef.current = new ResizeObserver(() => draw());
|
||||
roRef.current.observe(el);
|
||||
observerRef.current = new ResizeObserver(draw);
|
||||
observerRef.current.observe(canvas);
|
||||
},
|
||||
[draw],
|
||||
);
|
||||
|
||||
// Redraw when peaks arrive
|
||||
useEffect(() => {
|
||||
draw();
|
||||
}, [draw]);
|
||||
|
||||
useEffect(
|
||||
() => () => {
|
||||
roRef.current?.disconnect();
|
||||
},
|
||||
[],
|
||||
);
|
||||
useMountEffect(() => () => observerRef.current?.disconnect());
|
||||
|
||||
return (
|
||||
<div ref={setContainerRef} className="absolute inset-0 overflow-hidden">
|
||||
<div ref={barsRef} className="absolute left-0 right-0 bottom-0" style={{ top: 16 }} />
|
||||
{/* Shimmer while decoding */}
|
||||
{!peaks && (
|
||||
<div className="absolute inset-0 overflow-hidden">
|
||||
<canvas
|
||||
ref={setCanvasRef}
|
||||
className="absolute inset-x-0 bottom-0 w-full"
|
||||
style={{ top: 16 }}
|
||||
/>
|
||||
{snapshot.status === "loading" && (
|
||||
<div
|
||||
className="absolute left-0 right-0 bottom-0 animate-pulse"
|
||||
className="absolute inset-x-0 bottom-0 top-4 animate-pulse"
|
||||
style={{
|
||||
top: 16,
|
||||
background:
|
||||
"linear-gradient(90deg, rgba(255,255,255,0.02) 0%, rgba(255,255,255,0.05) 50%, rgba(255,255,255,0.02) 100%)",
|
||||
}}
|
||||
/>
|
||||
)}
|
||||
{label && (
|
||||
<div className="absolute top-0 left-0 right-0 px-1.5 py-0.5 z-10">
|
||||
<div className="absolute inset-x-0 top-0 z-10 px-1.5 py-0.5">
|
||||
<span
|
||||
className="text-[9px] font-semibold truncate block leading-tight"
|
||||
className="block truncate text-[9px] font-semibold leading-tight"
|
||||
style={{ color: labelColor, textShadow: "0 1px 3px rgba(0,0,0,0.9)" }}
|
||||
>
|
||||
{label}
|
||||
|
||||
@@ -2,7 +2,8 @@
|
||||
|
||||
import React, { act } from "react";
|
||||
import { createRoot, type Root } from "react-dom/client";
|
||||
import { afterEach, beforeEach, describe, expect, it } from "vitest";
|
||||
import { afterEach, beforeEach, describe, expect, it, vi } from "vitest";
|
||||
import { thumbnailScheduler } from "../lib/thumbnailScheduler";
|
||||
import { buildCompositionThumbnailUrl, CompositionThumbnail } from "./CompositionThumbnail";
|
||||
|
||||
Object.defineProperty(globalThis, "IS_REACT_ACT_ENVIRONMENT", {
|
||||
@@ -31,12 +32,18 @@ class MockImage {
|
||||
|
||||
const originalResizeObserver = globalThis.ResizeObserver;
|
||||
const originalImage = globalThis.Image;
|
||||
const originalFetch = globalThis.fetch;
|
||||
const originalCreateObjectURL = URL.createObjectURL;
|
||||
const originalRevokeObjectURL = URL.revokeObjectURL;
|
||||
let host: HTMLDivElement;
|
||||
let root: Root | null = null;
|
||||
|
||||
beforeEach(() => {
|
||||
globalThis.ResizeObserver = MockResizeObserver as unknown as typeof ResizeObserver;
|
||||
globalThis.Image = MockImage as unknown as typeof Image;
|
||||
globalThis.fetch = vi.fn(async () => new Response(new Blob(["thumbnail"]), { status: 200 }));
|
||||
URL.createObjectURL = vi.fn(() => "blob:composition-thumbnail");
|
||||
URL.revokeObjectURL = vi.fn();
|
||||
MockImage.instances = [];
|
||||
host = document.createElement("div");
|
||||
document.body.append(host);
|
||||
@@ -45,8 +52,12 @@ beforeEach(() => {
|
||||
afterEach(() => {
|
||||
act(() => root?.unmount());
|
||||
root = null;
|
||||
thumbnailScheduler.invalidateProject("/api/projects/demo/preview");
|
||||
globalThis.ResizeObserver = originalResizeObserver;
|
||||
globalThis.Image = originalImage;
|
||||
globalThis.fetch = originalFetch;
|
||||
URL.createObjectURL = originalCreateObjectURL;
|
||||
URL.revokeObjectURL = originalRevokeObjectURL;
|
||||
document.body.replaceChildren();
|
||||
});
|
||||
|
||||
@@ -68,9 +79,9 @@ describe("buildCompositionThumbnailUrl", () => {
|
||||
});
|
||||
|
||||
describe("CompositionThumbnail", () => {
|
||||
function renderThumbnail(): MockImage {
|
||||
async function renderThumbnail(): Promise<MockImage> {
|
||||
root = createRoot(host);
|
||||
act(() => {
|
||||
await act(async () => {
|
||||
root!.render(
|
||||
React.createElement(CompositionThumbnail, {
|
||||
previewUrl: "/api/projects/demo/preview",
|
||||
@@ -78,19 +89,27 @@ describe("CompositionThumbnail", () => {
|
||||
labelColor: "#fff",
|
||||
}),
|
||||
);
|
||||
await new Promise((resolve) => setTimeout(resolve, 0));
|
||||
});
|
||||
const probe = MockImage.instances[0];
|
||||
if (!probe) throw new Error("Expected an image probe");
|
||||
return probe;
|
||||
}
|
||||
|
||||
it("renders visible tiles after the off-DOM probe loads", () => {
|
||||
const probe = renderThumbnail();
|
||||
it("renders visible tiles after the scheduled off-DOM probe loads", async () => {
|
||||
const probe = await renderThumbnail();
|
||||
|
||||
act(() => {
|
||||
expect(globalThis.fetch).toHaveBeenCalledWith(
|
||||
expect.stringContaining("/api/projects/demo/thumbnail/index.html"),
|
||||
expect.objectContaining({ signal: expect.any(AbortSignal) }),
|
||||
);
|
||||
expect(probe.src).toBe("blob:composition-thumbnail");
|
||||
|
||||
await act(async () => {
|
||||
probe.naturalWidth = 1920;
|
||||
probe.naturalHeight = 1080;
|
||||
probe.onload?.();
|
||||
await new Promise((resolve) => setTimeout(resolve, 0));
|
||||
});
|
||||
|
||||
const tiles = [...host.querySelectorAll("img")];
|
||||
@@ -98,16 +117,20 @@ describe("CompositionThumbnail", () => {
|
||||
expect(tiles.every((tile) => !tile.classList.contains("hidden"))).toBe(true);
|
||||
});
|
||||
|
||||
it("aborts its off-DOM image probe when unmounted", () => {
|
||||
const probe = renderThumbnail();
|
||||
it("aborts its scheduled off-DOM image probe when unmounted", async () => {
|
||||
const probe = await renderThumbnail();
|
||||
expect(host.querySelector("img")).toBeNull();
|
||||
expect(probe.src).toContain("/api/projects/demo/thumbnail/index.html");
|
||||
expect(probe.src).toBe("blob:composition-thumbnail");
|
||||
|
||||
act(() => root?.unmount());
|
||||
await act(async () => {
|
||||
root?.unmount();
|
||||
await new Promise((resolve) => setTimeout(resolve, 0));
|
||||
});
|
||||
root = null;
|
||||
|
||||
expect(probe.onload).toBeNull();
|
||||
expect(probe.onerror).toBeNull();
|
||||
expect(probe.src).toBe("");
|
||||
expect(URL.revokeObjectURL).toHaveBeenCalledWith("blob:composition-thumbnail");
|
||||
});
|
||||
});
|
||||
|
||||
@@ -1,5 +1,9 @@
|
||||
import { memo, useCallback, useEffect, useState, useRef } from "react";
|
||||
import { memo, useCallback, useMemo, useRef, useState } from "react";
|
||||
import { useMountEffect } from "../../hooks/useMountEffect";
|
||||
import { useThumbnailLease } from "../../hooks/useThumbnailLease";
|
||||
import { createThumbnailKey, type ThumbnailPriority } from "../lib/thumbnailScheduler";
|
||||
import { TIMELINE_VIEWPORT_BUDGETS } from "../lib/timelineViewportBudgets";
|
||||
import { computeThumbnailStrip, probeImageAspect } from "./thumbnailUtils";
|
||||
|
||||
interface CompositionThumbnailProps {
|
||||
previewUrl: string;
|
||||
@@ -11,6 +15,10 @@ interface CompositionThumbnailProps {
|
||||
duration?: number;
|
||||
width?: number;
|
||||
height?: number;
|
||||
projectId?: string;
|
||||
sessionEpoch?: number;
|
||||
priority?: ThumbnailPriority;
|
||||
rich?: boolean;
|
||||
}
|
||||
|
||||
const CLIP_HEIGHT = 66;
|
||||
@@ -34,9 +42,8 @@ export function buildCompositionThumbnailUrl({
|
||||
const thumbnailBase = previewUrl
|
||||
.replace("/preview/comp/", "/thumbnail/")
|
||||
.replace(/\/preview$/, "/thumbnail/index.html");
|
||||
const midTime = seekTime + duration / 2;
|
||||
const thumbnailUrl = new URL(thumbnailBase, origin);
|
||||
thumbnailUrl.searchParams.set("t", midTime.toFixed(2));
|
||||
thumbnailUrl.searchParams.set("t", (seekTime + duration / 2).toFixed(2));
|
||||
thumbnailUrl.searchParams.set("v", THUMBNAIL_URL_VERSION);
|
||||
if (selector) {
|
||||
thumbnailUrl.searchParams.set("selector", selector);
|
||||
@@ -47,6 +54,29 @@ export function buildCompositionThumbnailUrl({
|
||||
return thumbnailUrl.toString();
|
||||
}
|
||||
|
||||
async function loadCompositionImage(url: string, signal: AbortSignal) {
|
||||
const response = await fetch(url, { signal });
|
||||
if (!response.ok) throw new Error(`Composition thumbnail failed (${response.status})`);
|
||||
const blob = await response.blob();
|
||||
if (signal.aborted) throw new DOMException("Aborted", "AbortError");
|
||||
const objectUrl = URL.createObjectURL(blob);
|
||||
try {
|
||||
const aspect = await probeImageAspect(objectUrl, signal);
|
||||
return {
|
||||
value: { kind: "image" as const, url: objectUrl, aspect },
|
||||
weight:
|
||||
TIMELINE_VIEWPORT_BUDGETS.posterMaxPhysicalWidth *
|
||||
TIMELINE_VIEWPORT_BUDGETS.posterMaxPhysicalHeight *
|
||||
4,
|
||||
dispose: () => URL.revokeObjectURL(objectUrl),
|
||||
};
|
||||
} catch (error) {
|
||||
URL.revokeObjectURL(objectUrl);
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
|
||||
/** Server-rendered composition poster, deduplicated and budgeted by project/session. */
|
||||
export const CompositionThumbnail = memo(function CompositionThumbnail({
|
||||
previewUrl,
|
||||
label,
|
||||
@@ -55,31 +85,12 @@ export const CompositionThumbnail = memo(function CompositionThumbnail({
|
||||
selectorIndex,
|
||||
seekTime = 2,
|
||||
duration = 5,
|
||||
projectId = previewUrl,
|
||||
sessionEpoch = 0,
|
||||
priority = "visible",
|
||||
}: CompositionThumbnailProps) {
|
||||
const [containerWidth, setContainerWidth] = useState(0);
|
||||
const [loaded, setLoaded] = useState(false);
|
||||
const [aspect, setAspect] = useState(16 / 9);
|
||||
const roRef = useRef<ResizeObserver | null>(null);
|
||||
|
||||
const setContainerRef = useCallback((el: HTMLDivElement | null) => {
|
||||
roRef.current?.disconnect();
|
||||
if (!el) return;
|
||||
|
||||
const measured = el.parentElement?.clientWidth || el.clientWidth;
|
||||
// fallow-ignore-next-line code-duplication
|
||||
setContainerWidth(measured);
|
||||
|
||||
const target = el.parentElement || el;
|
||||
roRef.current = new ResizeObserver(([entry]) => {
|
||||
setContainerWidth(entry.contentRect.width);
|
||||
});
|
||||
roRef.current.observe(target);
|
||||
}, []);
|
||||
|
||||
useMountEffect(() => () => {
|
||||
roRef.current?.disconnect();
|
||||
});
|
||||
|
||||
const observerRef = useRef<ResizeObserver | null>(null);
|
||||
const url = buildCompositionThumbnailUrl({
|
||||
previewUrl,
|
||||
seekTime,
|
||||
@@ -88,52 +99,56 @@ export const CompositionThumbnail = memo(function CompositionThumbnail({
|
||||
selectorIndex,
|
||||
origin: window.location.origin,
|
||||
});
|
||||
const request = useMemo(
|
||||
() => ({
|
||||
key: createThumbnailKey({ kind: "composition", url }),
|
||||
projectId,
|
||||
sessionEpoch,
|
||||
kind: "composition" as const,
|
||||
priority,
|
||||
rich: true,
|
||||
load: (signal: AbortSignal) => loadCompositionImage(url, signal),
|
||||
}),
|
||||
[priority, projectId, sessionEpoch, url],
|
||||
);
|
||||
const snapshot = useThumbnailLease(request);
|
||||
const value =
|
||||
snapshot.status === "ready" && snapshot.value.kind === "image" ? snapshot.value : null;
|
||||
const { frameW, frameCount } = computeThumbnailStrip(
|
||||
containerWidth,
|
||||
value?.aspect ?? 16 / 9,
|
||||
CLIP_HEIGHT,
|
||||
48,
|
||||
);
|
||||
|
||||
// Probe outside React's DOM and explicitly abort on URL changes/unmount. A
|
||||
// hidden React <img> keeps its Fiber reachable from Blink's pending-activity
|
||||
// queue while a request is unresolved.
|
||||
// eslint-disable-next-line no-restricted-syntax
|
||||
useEffect(() => {
|
||||
let cancelled = false;
|
||||
setLoaded(false);
|
||||
const probe = new Image();
|
||||
probe.onload = () => {
|
||||
if (cancelled) return;
|
||||
if (probe.naturalWidth > 0 && probe.naturalHeight > 0) {
|
||||
setAspect(probe.naturalWidth / probe.naturalHeight);
|
||||
}
|
||||
setLoaded(true);
|
||||
};
|
||||
probe.onerror = () => {
|
||||
if (!cancelled) setLoaded(false);
|
||||
};
|
||||
probe.src = url;
|
||||
return () => {
|
||||
cancelled = true;
|
||||
probe.onload = null;
|
||||
probe.onerror = null;
|
||||
probe.src = "";
|
||||
};
|
||||
}, [url]);
|
||||
const setContainerRef = useCallback((element: HTMLDivElement | null) => {
|
||||
observerRef.current?.disconnect();
|
||||
if (!element) return;
|
||||
const target = element.parentElement ?? element;
|
||||
setContainerWidth(target.clientWidth);
|
||||
observerRef.current = new ResizeObserver(([entry]) =>
|
||||
setContainerWidth(entry.contentRect.width),
|
||||
);
|
||||
observerRef.current.observe(target);
|
||||
}, []);
|
||||
|
||||
const frameW = Math.max(48, Math.round(CLIP_HEIGHT * aspect));
|
||||
const frameCount = containerWidth > 0 ? Math.max(1, Math.ceil(containerWidth / frameW)) : 1;
|
||||
useMountEffect(() => () => observerRef.current?.disconnect());
|
||||
|
||||
return (
|
||||
<div ref={setContainerRef} className="absolute inset-0 overflow-hidden">
|
||||
{loaded && (
|
||||
{value && (
|
||||
<div
|
||||
className="absolute inset-0 flex"
|
||||
style={{ animation: "hf-thumb-fade 200ms ease-out", mixBlendMode: "lighten" }}
|
||||
>
|
||||
{Array.from({ length: frameCount }).map((_, i) => (
|
||||
{Array.from({ length: frameCount }, (_, index) => (
|
||||
<div
|
||||
key={i}
|
||||
key={index}
|
||||
className="relative h-full flex-shrink-0 overflow-hidden"
|
||||
style={{ width: frameW }}
|
||||
>
|
||||
<img
|
||||
src={url}
|
||||
src={value.url}
|
||||
alt=""
|
||||
draggable={false}
|
||||
className="absolute inset-0 h-full w-full object-cover"
|
||||
@@ -143,14 +158,16 @@ export const CompositionThumbnail = memo(function CompositionThumbnail({
|
||||
))}
|
||||
</div>
|
||||
)}
|
||||
|
||||
{snapshot.status === "loading" && (
|
||||
<div className="absolute inset-0 animate-pulse bg-white/[0.035]" />
|
||||
)}
|
||||
{label && (
|
||||
<div className="absolute left-3 top-0 bottom-0 flex items-center" style={{ zIndex: 10 }}>
|
||||
<div className="absolute inset-y-0 left-3 z-10 flex items-center">
|
||||
<span
|
||||
className="block max-w-full truncate text-[10px] font-semibold leading-none"
|
||||
style={{
|
||||
color: labelColor,
|
||||
textShadow: loaded ? "0 1px 4px rgba(0,0,0,0.9), 0 0 8px rgba(0,0,0,0.6)" : "none",
|
||||
textShadow: value ? "0 1px 4px rgba(0,0,0,0.9), 0 0 8px rgba(0,0,0,0.6)" : "none",
|
||||
}}
|
||||
>
|
||||
{label}
|
||||
|
||||
@@ -3,6 +3,7 @@
|
||||
import React, { act } from "react";
|
||||
import { createRoot, type Root } from "react-dom/client";
|
||||
import { afterEach, beforeEach, describe, expect, it } from "vitest";
|
||||
import { thumbnailScheduler } from "../lib/thumbnailScheduler";
|
||||
import { ImageThumbnail } from "./ImageThumbnail";
|
||||
|
||||
Object.defineProperty(globalThis, "IS_REACT_ACT_ENVIRONMENT", {
|
||||
@@ -68,6 +69,7 @@ beforeEach(() => {
|
||||
afterEach(() => {
|
||||
act(() => root?.unmount());
|
||||
root = null;
|
||||
thumbnailScheduler.invalidateProject("p");
|
||||
globalThis.IntersectionObserver = originalIO;
|
||||
globalThis.ResizeObserver = originalRO;
|
||||
globalThis.Image = originalImage;
|
||||
@@ -82,6 +84,10 @@ function render(props: { imageSrc: string; label?: string; labelColor?: string }
|
||||
imageSrc={props.imageSrc}
|
||||
label={props.label ?? ""}
|
||||
labelColor={props.labelColor ?? "#fff"}
|
||||
projectId="p"
|
||||
sessionEpoch={1}
|
||||
priority="visible"
|
||||
rich={false}
|
||||
/>,
|
||||
);
|
||||
});
|
||||
@@ -93,6 +99,13 @@ function lastProbe(): MockImage {
|
||||
return probe!;
|
||||
}
|
||||
|
||||
async function resolveProbe(update: (probe: MockImage) => void): Promise<void> {
|
||||
await act(async () => {
|
||||
update(lastProbe());
|
||||
await new Promise((resolve) => setTimeout(resolve, 0));
|
||||
});
|
||||
}
|
||||
|
||||
/** Assert at least one tile rendered and the first tile serves `expectedSrc`. */
|
||||
function expectFirstTileSrc(expectedSrc: string): void {
|
||||
const imgs = [...host.querySelectorAll("img")];
|
||||
@@ -107,15 +120,15 @@ describe("ImageThumbnail", () => {
|
||||
expect(host.querySelectorAll("img").length).toBe(0);
|
||||
});
|
||||
|
||||
it("probes the resolved src and renders repeated object-cover tiles on load", () => {
|
||||
it("probes the resolved src and renders repeated object-cover tiles on load", async () => {
|
||||
render({ imageSrc: "/api/projects/p/preview/assets/pic.png" });
|
||||
const probe = lastProbe();
|
||||
expect(probe.src).toBe("/api/projects/p/preview/assets/pic.png");
|
||||
|
||||
act(() => {
|
||||
probe.naturalWidth = 1920;
|
||||
probe.naturalHeight = 1080;
|
||||
probe.onload?.();
|
||||
await resolveProbe((current) => {
|
||||
current.naturalWidth = 1920;
|
||||
current.naturalHeight = 1080;
|
||||
current.onload?.();
|
||||
});
|
||||
|
||||
const imgs = [...host.querySelectorAll("img")];
|
||||
@@ -127,18 +140,18 @@ describe("ImageThumbnail", () => {
|
||||
expect(host.querySelector(".animate-pulse")).toBeNull();
|
||||
});
|
||||
|
||||
it("drops the shimmer and renders no tiles when a raster image fails to load", () => {
|
||||
it("drops the shimmer and renders no tiles when a raster image fails to load", async () => {
|
||||
render({ imageSrc: "/api/projects/p/preview/assets/missing.png" });
|
||||
act(() => lastProbe().onerror?.());
|
||||
await resolveProbe((probe) => probe.onerror?.());
|
||||
expect(host.querySelectorAll("img").length).toBe(0);
|
||||
expect(host.querySelector(".animate-pulse")).toBeNull();
|
||||
});
|
||||
|
||||
it("renders tiles at 16:9 when an SVG has no intrinsic dimensions (naturalWidth=0)", () => {
|
||||
it("renders tiles at 16:9 when an SVG has no intrinsic dimensions (naturalWidth=0)", async () => {
|
||||
render({ imageSrc: "/api/projects/p/preview/assets/logo.svg" });
|
||||
const probe = lastProbe();
|
||||
|
||||
act(() => {
|
||||
await resolveProbe(() => {
|
||||
// naturalWidth stays 0 — SVG with no width/height attribute
|
||||
probe.onload?.();
|
||||
});
|
||||
@@ -147,21 +160,20 @@ describe("ImageThumbnail", () => {
|
||||
expect(host.querySelector(".animate-pulse")).toBeNull();
|
||||
});
|
||||
|
||||
it("renders SVG tiles at 16:9 fallback even when the probe fires onerror", () => {
|
||||
it("renders SVG tiles at 16:9 fallback even when the probe fires onerror", async () => {
|
||||
// Some browser/sandbox environments fire onerror for SVGs even though the
|
||||
// <img> element itself can render the file — we must not blank the strip.
|
||||
render({ imageSrc: "/api/projects/p/preview/assets/icon.svg" });
|
||||
|
||||
act(() => lastProbe().onerror?.());
|
||||
await resolveProbe((probe) => probe.onerror?.());
|
||||
|
||||
expectFirstTileSrc("/api/projects/p/preview/assets/icon.svg");
|
||||
expect(host.querySelector(".animate-pulse")).toBeNull();
|
||||
});
|
||||
|
||||
it("renders the label above the strip when provided", () => {
|
||||
it("renders the label above the strip when provided", async () => {
|
||||
render({ imageSrc: "/x.png", label: "hero", labelColor: "#abc" });
|
||||
act(() => {
|
||||
const probe = lastProbe();
|
||||
await resolveProbe((probe) => {
|
||||
probe.naturalWidth = 100;
|
||||
probe.naturalHeight = 100;
|
||||
probe.onload?.();
|
||||
|
||||
@@ -1,135 +1,93 @@
|
||||
import { memo, useRef, useState, useCallback, useEffect } from "react";
|
||||
import { memo, useCallback, useMemo, useRef, useState } from "react";
|
||||
import { useMountEffect } from "../../hooks/useMountEffect";
|
||||
import { computeThumbnailStrip } from "./thumbnailUtils";
|
||||
import { useThumbnailLease } from "../../hooks/useThumbnailLease";
|
||||
import { createThumbnailKey, type ThumbnailPriority } from "../lib/thumbnailScheduler";
|
||||
import { TIMELINE_VIEWPORT_BUDGETS } from "../lib/timelineViewportBudgets";
|
||||
import { computeThumbnailStrip, probeImageAspect } from "./thumbnailUtils";
|
||||
|
||||
interface ImageThumbnailProps {
|
||||
imageSrc: string;
|
||||
label: string;
|
||||
labelColor: string;
|
||||
projectId?: string;
|
||||
sessionEpoch?: number;
|
||||
priority?: ThumbnailPriority;
|
||||
rich?: boolean;
|
||||
}
|
||||
|
||||
/**
|
||||
* Renders a film-strip of a still image for a timeline clip. The image is a
|
||||
* fixed-width tile (sized by its natural aspect ratio) repeated to fill the
|
||||
* clip width — matching VideoThumbnail's visual pattern. Loading is lazy
|
||||
* (IntersectionObserver) with the same shimmer fallback while decoding.
|
||||
*/
|
||||
/** A scheduler-backed still-image strip. Mounting is the sole work trigger. */
|
||||
export const ImageThumbnail = memo(function ImageThumbnail({
|
||||
imageSrc,
|
||||
label,
|
||||
labelColor,
|
||||
projectId = imageSrc,
|
||||
sessionEpoch = 0,
|
||||
priority = "visible",
|
||||
rich = false,
|
||||
}: ImageThumbnailProps) {
|
||||
const [containerWidth, setContainerWidth] = useState(0);
|
||||
const [visible, setVisible] = useState(false);
|
||||
const [status, setStatus] = useState<"loading" | "loaded" | "error">("loading");
|
||||
const [aspect, setAspect] = useState(16 / 9);
|
||||
const ioRef = useRef<IntersectionObserver | null>(null);
|
||||
const roRef = useRef<ResizeObserver | null>(null);
|
||||
|
||||
const setContainerRef = useCallback((el: HTMLDivElement | null) => {
|
||||
ioRef.current?.disconnect();
|
||||
roRef.current?.disconnect();
|
||||
if (!el) return;
|
||||
|
||||
const measured = el.parentElement?.clientWidth || el.clientWidth;
|
||||
setContainerWidth(measured);
|
||||
|
||||
ioRef.current = new IntersectionObserver(
|
||||
([entry]) => {
|
||||
if (entry.isIntersecting) {
|
||||
setVisible(true);
|
||||
ioRef.current?.disconnect();
|
||||
}
|
||||
const observerRef = useRef<ResizeObserver | null>(null);
|
||||
const request = useMemo(
|
||||
() => ({
|
||||
key: createThumbnailKey({ kind: "image", source: imageSrc, rich: Number(rich) }),
|
||||
projectId,
|
||||
sessionEpoch,
|
||||
kind: "image" as const,
|
||||
priority,
|
||||
rich,
|
||||
load: async (signal: AbortSignal) => {
|
||||
const aspect = await probeImageAspect(imageSrc, signal, true);
|
||||
return {
|
||||
value: { kind: "image" as const, url: imageSrc, aspect },
|
||||
weight:
|
||||
TIMELINE_VIEWPORT_BUDGETS.posterMaxPhysicalWidth *
|
||||
TIMELINE_VIEWPORT_BUDGETS.posterMaxPhysicalHeight *
|
||||
4,
|
||||
};
|
||||
},
|
||||
{ rootMargin: "200px" },
|
||||
);
|
||||
// fallow-ignore-next-line code-duplication
|
||||
ioRef.current.observe(el);
|
||||
}),
|
||||
[imageSrc, priority, projectId, rich, sessionEpoch],
|
||||
);
|
||||
const snapshot = useThumbnailLease(request);
|
||||
const value = snapshot.status === "ready" ? snapshot.value : null;
|
||||
const aspect = value?.kind === "image" ? value.aspect : 16 / 9;
|
||||
const { frameW, frameCount } = computeThumbnailStrip(containerWidth, aspect);
|
||||
|
||||
const target = el.parentElement || el;
|
||||
roRef.current = new ResizeObserver(([entry]) => {
|
||||
setContainerWidth(entry.contentRect.width);
|
||||
});
|
||||
roRef.current.observe(target);
|
||||
const setContainerRef = useCallback((element: HTMLDivElement | null) => {
|
||||
observerRef.current?.disconnect();
|
||||
if (!element) return;
|
||||
const target = element.parentElement ?? element;
|
||||
setContainerWidth(target.clientWidth);
|
||||
observerRef.current = new ResizeObserver(([entry]) =>
|
||||
setContainerWidth(entry.contentRect.width),
|
||||
);
|
||||
observerRef.current.observe(target);
|
||||
}, []);
|
||||
|
||||
useMountEffect(() => () => {
|
||||
ioRef.current?.disconnect();
|
||||
roRef.current?.disconnect();
|
||||
});
|
||||
|
||||
// Probe the image once visible — measures the natural aspect ratio so the
|
||||
// tile width matches, and flips to the error state (plain clip background)
|
||||
// if the src can't load. The browser cache makes the tile <img>s free.
|
||||
//
|
||||
// SVG handling: SVGs without intrinsic width/height report naturalWidth=0 on
|
||||
// load (treat as success with the 16:9 default aspect) and may fire onerror
|
||||
// in some environments even though the file is valid and can be displayed —
|
||||
// fall back to loaded-at-16:9 rather than hiding the strip entirely.
|
||||
// eslint-disable-next-line no-restricted-syntax
|
||||
useEffect(() => {
|
||||
if (!visible) return;
|
||||
let cancelled = false;
|
||||
setStatus("loading");
|
||||
|
||||
const isSvg = /\.svg($|\?)/i.test(imageSrc);
|
||||
|
||||
const probe = new Image();
|
||||
probe.onload = () => {
|
||||
if (cancelled) return;
|
||||
if (probe.naturalWidth > 0 && probe.naturalHeight > 0) {
|
||||
setAspect(probe.naturalWidth / probe.naturalHeight);
|
||||
}
|
||||
// naturalWidth===0 (e.g. SVG with no intrinsic dimensions) falls through
|
||||
// to "loaded" with the default 16:9 aspect already set in state.
|
||||
setStatus("loaded");
|
||||
};
|
||||
probe.onerror = () => {
|
||||
if (cancelled) return;
|
||||
// SVGs can fail the probe in certain browser/sandbox environments even
|
||||
// though the <img> tiles themselves render fine (different security
|
||||
// context). Show the strip at the 16:9 fallback rather than blanking.
|
||||
if (isSvg) {
|
||||
setStatus("loaded");
|
||||
} else {
|
||||
setStatus("error");
|
||||
}
|
||||
};
|
||||
probe.src = imageSrc;
|
||||
|
||||
return () => {
|
||||
cancelled = true;
|
||||
probe.onload = null;
|
||||
probe.onerror = null;
|
||||
probe.src = "";
|
||||
};
|
||||
}, [visible, imageSrc]);
|
||||
|
||||
const { frameW, frameCount } = computeThumbnailStrip(containerWidth, aspect);
|
||||
useMountEffect(() => () => observerRef.current?.disconnect());
|
||||
|
||||
return (
|
||||
<div ref={setContainerRef} className="absolute inset-0 overflow-hidden">
|
||||
{visible && status === "loaded" && (
|
||||
{value?.kind === "image" && (
|
||||
<div className="absolute inset-0 flex">
|
||||
{Array.from({ length: frameCount }).map((_, i) => (
|
||||
{Array.from({ length: frameCount }, (_, index) => (
|
||||
<div
|
||||
key={i}
|
||||
className="flex-shrink-0 h-full relative overflow-hidden bg-neutral-900"
|
||||
key={index}
|
||||
className="relative h-full flex-shrink-0 overflow-hidden bg-neutral-900"
|
||||
style={{ width: frameW }}
|
||||
>
|
||||
<img
|
||||
src={imageSrc}
|
||||
src={value.url}
|
||||
alt=""
|
||||
draggable={false}
|
||||
loading="lazy"
|
||||
className="absolute inset-0 w-full h-full object-cover"
|
||||
className="absolute inset-0 h-full w-full object-cover"
|
||||
/>
|
||||
</div>
|
||||
))}
|
||||
</div>
|
||||
)}
|
||||
|
||||
{visible && status === "loading" && (
|
||||
{snapshot.status === "loading" && (
|
||||
<div
|
||||
className="absolute inset-0 animate-pulse"
|
||||
style={{
|
||||
@@ -138,17 +96,16 @@ export const ImageThumbnail = memo(function ImageThumbnail({
|
||||
}}
|
||||
/>
|
||||
)}
|
||||
|
||||
{label && (
|
||||
<div
|
||||
className="absolute bottom-0 left-0 right-0 z-10 px-1.5 pb-0.5 pt-3"
|
||||
className="absolute inset-x-0 bottom-0 z-10 px-1.5 pb-0.5 pt-3"
|
||||
style={{
|
||||
background:
|
||||
"linear-gradient(to top, rgba(0,0,0,0.85) 0%, rgba(0,0,0,0.4) 60%, transparent 100%)",
|
||||
}}
|
||||
>
|
||||
<span
|
||||
className="text-[9px] font-semibold truncate block leading-tight"
|
||||
className="block truncate text-[9px] font-semibold leading-tight"
|
||||
style={{ color: labelColor, textShadow: "0 1px 2px rgba(0,0,0,0.9)" }}
|
||||
>
|
||||
{label}
|
||||
|
||||
@@ -64,7 +64,6 @@ export {
|
||||
getTimelineScrollTopForGeometryChange,
|
||||
getTimelineVisibleTimeRange,
|
||||
} from "./timelineViewportGeometry";
|
||||
|
||||
export const Timeline = memo(function Timeline({
|
||||
onSeek,
|
||||
onDrillDown,
|
||||
@@ -317,24 +316,25 @@ export const Timeline = memo(function Timeline({
|
||||
toggleSelectedKeyframe,
|
||||
});
|
||||
|
||||
const { clipIndex, renderTimeRange, pinnedClipIdentities } = useTimelineClipRenderWindow({
|
||||
tracks,
|
||||
viewport,
|
||||
pixelsPerSecond: pps,
|
||||
contentOrigin,
|
||||
duration: displayDuration,
|
||||
selectedElementId: selectedElementId ?? undefined,
|
||||
draggedElementId: draggedClip ? getTimelineElementIdentity(draggedClip.element) : undefined,
|
||||
resizingElementIds,
|
||||
focusedElementId: timelineFocus.pinnedElementId,
|
||||
focusedEaseElementId: focusedEaseSegment?.elementId,
|
||||
clipContextMenuElementId: clipContextMenu
|
||||
? getTimelineElementIdentity(clipContextMenu.element)
|
||||
: undefined,
|
||||
keyframeContextMenuElementId: kfContextMenu
|
||||
? getTimelineElementIdentity(kfContextMenu.element)
|
||||
: undefined,
|
||||
});
|
||||
const { clipIndex, renderTimeRange, visibleTimeRange, pinnedClipIdentities } =
|
||||
useTimelineClipRenderWindow({
|
||||
tracks,
|
||||
viewport,
|
||||
pixelsPerSecond: pps,
|
||||
contentOrigin,
|
||||
duration: displayDuration,
|
||||
selectedElementId: selectedElementId ?? undefined,
|
||||
draggedElementId: draggedClip ? getTimelineElementIdentity(draggedClip.element) : undefined,
|
||||
resizingElementIds,
|
||||
focusedElementId: timelineFocus.pinnedElementId,
|
||||
focusedEaseElementId: focusedEaseSegment?.elementId,
|
||||
clipContextMenuElementId: clipContextMenu
|
||||
? getTimelineElementIdentity(clipContextMenu.element)
|
||||
: undefined,
|
||||
keyframeContextMenuElementId: kfContextMenu
|
||||
? getTimelineElementIdentity(kfContextMenu.element)
|
||||
: undefined,
|
||||
});
|
||||
useTimelineActiveClips({
|
||||
scrollRef,
|
||||
currentTime,
|
||||
@@ -504,6 +504,7 @@ export const Timeline = memo(function Timeline({
|
||||
rowsVirtualized={timelineFocus.rowVirtualizationActive}
|
||||
clipIndex={clipIndex}
|
||||
renderTimeRange={renderTimeRange}
|
||||
visibleTimeRange={visibleTimeRange}
|
||||
pinnedClipIdentities={pinnedClipIdentities}
|
||||
trackOrder={trackOrder}
|
||||
tracks={tracks}
|
||||
|
||||
@@ -9,6 +9,7 @@ import { getTimelineDragOverlayPosition } from "./timelineClipDragPreview";
|
||||
import type { DraggedClipState } from "./timelineClipDragTypes";
|
||||
import type { TrackVisualStyle } from "./timelineIcons";
|
||||
import { isTimelineClipActive } from "./useTimelineActiveClips";
|
||||
import type { TimelineClipRenderContext } from "./TimelineTypes";
|
||||
|
||||
interface TimelineGestureOverlayProps {
|
||||
drag: DraggedClipState | null;
|
||||
@@ -22,6 +23,7 @@ interface TimelineGestureOverlayProps {
|
||||
renderClipContent?: (
|
||||
element: TimelineElement,
|
||||
style: { clip: string; label: string },
|
||||
context: TimelineClipRenderContext,
|
||||
) => ReactNode;
|
||||
renderClipOverlay?: (element: TimelineElement) => ReactNode;
|
||||
}
|
||||
@@ -87,6 +89,7 @@ export const TimelineGestureOverlay = memo(function TimelineGestureOverlay({
|
||||
getTrackStyle(element.tag),
|
||||
renderClipContent,
|
||||
renderClipOverlay,
|
||||
{ priority: "interaction", rich: true },
|
||||
)}
|
||||
</TimelineClip>
|
||||
</div>
|
||||
|
||||
@@ -76,10 +76,14 @@ function renderLanes(options: RenderLanesOptions = {}): {
|
||||
host: HTMLDivElement;
|
||||
root: Root;
|
||||
rerender: (next: RenderLanesOptions) => void;
|
||||
setSelectedElementId: ReturnType<typeof vi.fn>;
|
||||
onSelectElement: ReturnType<typeof vi.fn>;
|
||||
} {
|
||||
const host = document.createElement("div");
|
||||
document.body.append(host);
|
||||
const root = createRoot(host);
|
||||
const setSelectedElementId = vi.fn();
|
||||
const onSelectElement = vi.fn();
|
||||
const render = (next: RenderLanesOptions) => {
|
||||
const elements = next.elements ?? [element("clip-a", TRACK_A)];
|
||||
const gsapAnimations = next.animations ?? new Map<string, GsapAnimation[]>();
|
||||
@@ -118,6 +122,7 @@ function renderLanes(options: RenderLanesOptions = {}): {
|
||||
})}
|
||||
clipIndex={createTimelineClipIndex(tracks)}
|
||||
renderTimeRange={{ start: 0, end: Number.POSITIVE_INFINITY }}
|
||||
visibleTimeRange={{ start: 0, end: Number.POSITIVE_INFINITY }}
|
||||
pinnedClipIdentities={new Set()}
|
||||
trackOrder={displayTrackOrder}
|
||||
tracks={tracks}
|
||||
@@ -137,7 +142,7 @@ function renderLanes(options: RenderLanesOptions = {}): {
|
||||
setRangeSelection={vi.fn()}
|
||||
setResizingClip={vi.fn()}
|
||||
setDraggedClip={vi.fn()}
|
||||
setSelectedElementId={vi.fn()}
|
||||
setSelectedElementId={setSelectedElementId}
|
||||
shiftClickClipRef={createRef()}
|
||||
getPreviewElement={(el) => el}
|
||||
getTrackStyle={getTrackStyle}
|
||||
@@ -149,6 +154,7 @@ function renderLanes(options: RenderLanesOptions = {}): {
|
||||
onTogglePropertyGroupKeyframe={vi.fn()}
|
||||
onResizeElement={vi.fn()}
|
||||
onMoveElement={vi.fn()}
|
||||
onSelectElement={onSelectElement}
|
||||
onRazorSplit={vi.fn()}
|
||||
onRazorSplitAll={vi.fn()}
|
||||
/>,
|
||||
@@ -156,7 +162,7 @@ function renderLanes(options: RenderLanesOptions = {}): {
|
||||
});
|
||||
};
|
||||
render(options);
|
||||
return { host, root, rerender: render };
|
||||
return { host, root, rerender: render, setSelectedElementId, onSelectElement };
|
||||
}
|
||||
|
||||
function visibilityLabels(host: HTMLElement): (string | null)[] {
|
||||
@@ -347,3 +353,19 @@ describe("TimelineLanes disclosure target", () => {
|
||||
act(() => view.root.unmount());
|
||||
});
|
||||
});
|
||||
|
||||
describe("TimelineLanes selection", () => {
|
||||
it("keeps a selected clip selected when it is clicked again", () => {
|
||||
const selected = element("clip-a", TRACK_A);
|
||||
const view = renderLanes({
|
||||
elements: [selected],
|
||||
selectedElementIds: new Set([selected.id]),
|
||||
});
|
||||
|
||||
act(() => view.host.querySelector<HTMLButtonElement>('[data-el-id="clip-a"]')?.click());
|
||||
|
||||
expect(view.setSelectedElementId).toHaveBeenCalledWith(selected.id);
|
||||
expect(view.onSelectElement).toHaveBeenCalledWith(selected);
|
||||
act(() => view.root.unmount());
|
||||
});
|
||||
});
|
||||
|
||||
@@ -22,7 +22,7 @@ import type { TimelineEditCallbacks } from "./timelineCallbacks";
|
||||
import { trackStudioKeyframeLaneExpand } from "../../telemetry/events";
|
||||
import { SPLIT_BOUNDARY_EPSILON_S } from "../../utils/timelineElementSplit";
|
||||
import { isAudioTimelineElement, isMusicTrack } from "../../utils/timelineInspector";
|
||||
import { renderClipChildren } from "./timelineClipChildren";
|
||||
import { renderClipChildren, resolveClipRenderContext } from "./timelineClipChildren";
|
||||
import { TimelineTrackRow } from "./TimelineTrackRow";
|
||||
import { isTimelineClipActive } from "./useTimelineActiveClips";
|
||||
import { queryTimelineClipIndex } from "../lib/timelineClipIndex";
|
||||
@@ -57,6 +57,7 @@ export function TimelineLanes({
|
||||
rowsVirtualized,
|
||||
clipIndex,
|
||||
renderTimeRange,
|
||||
visibleTimeRange,
|
||||
pinnedClipIdentities,
|
||||
trackOrder,
|
||||
tracks,
|
||||
@@ -323,8 +324,7 @@ export function TimelineLanes({
|
||||
const isSelected =
|
||||
selectedElementId === elementKey || selectedElementIds.has(elementKey);
|
||||
const isComposition = !!el.compositionSrc;
|
||||
// The element identity is already unique per clip. Never fold in the map
|
||||
// index, or a splice/reorder remounts every clip at/after the change.
|
||||
// Element identity stays stable across clip splices and reorders.
|
||||
const clipKey = elementKey;
|
||||
const isDraggingClip =
|
||||
draggedClip?.started === true &&
|
||||
@@ -332,6 +332,11 @@ export function TimelineLanes({
|
||||
getTimelineElementIdentity(draggedElement) === elementKey;
|
||||
if (isDraggingClip) return null;
|
||||
const previewElement = getPreviewElement(el);
|
||||
const renderContext = resolveClipRenderContext(
|
||||
previewElement,
|
||||
visibleTimeRange,
|
||||
isSelected || hoveredClip === clipKey || pinnedClipIdentities.has(clipKey),
|
||||
);
|
||||
// Passenger of a live multi-drag: preserve the formation without changing
|
||||
// the passenger's timeline data until the owning drag commits.
|
||||
const isPassenger =
|
||||
@@ -475,15 +480,9 @@ export function TimelineLanes({
|
||||
}
|
||||
return;
|
||||
}
|
||||
// Plain click single-selects: drop any marquee multi-selection.
|
||||
// Only a click on the PRIMARY selection toggles it off — a click
|
||||
// on a marquee-selected clip narrows the selection to that clip.
|
||||
const hadMultiSelection = selectedElementIds.size > 0;
|
||||
usePlayerStore.getState().clearSelectedElementIds();
|
||||
const nextElement =
|
||||
selectedElementId === elementKey && !hadMultiSelection ? null : el;
|
||||
setSelectedElementId(nextElement ? elementKey : null);
|
||||
onSelectElement?.(nextElement);
|
||||
// Clip selection is idempotent; empty timeline space owns deselection.
|
||||
setSelectedElementId(elementKey);
|
||||
onSelectElement?.(el);
|
||||
}
|
||||
}
|
||||
onDoubleClick={(e) => {
|
||||
@@ -497,6 +496,7 @@ export function TimelineLanes({
|
||||
clipStyle,
|
||||
renderClipContent,
|
||||
renderClipOverlay,
|
||||
renderContext,
|
||||
)}
|
||||
</TimelineClip>
|
||||
);
|
||||
|
||||
@@ -4,6 +4,11 @@ import type { TimelineDropCallbacks } from "./timelineCallbacks";
|
||||
import type { TimelineTheme } from "./timelineTheme";
|
||||
import type { TimelineEditOverrides } from "./useResolvedTimelineEditCallbacks";
|
||||
|
||||
export interface TimelineClipRenderContext {
|
||||
priority: "overscan" | "visible" | "interaction";
|
||||
rich: boolean;
|
||||
}
|
||||
|
||||
export interface TimelineProps extends TimelineDropCallbacks, TimelineEditOverrides {
|
||||
/** Project-scoped reset boundary; soft source refreshes retain the same epoch. */
|
||||
sessionEpoch?: number;
|
||||
@@ -12,6 +17,7 @@ export interface TimelineProps extends TimelineDropCallbacks, TimelineEditOverri
|
||||
renderClipContent?: (
|
||||
element: TimelineElement,
|
||||
style: { clip: string; label: string },
|
||||
context: TimelineClipRenderContext,
|
||||
) => ReactNode;
|
||||
renderClipOverlay?: (element: TimelineElement) => ReactNode;
|
||||
onDeleteElement?: (element: TimelineElement) => Promise<void> | void;
|
||||
|
||||
@@ -2,65 +2,28 @@
|
||||
import React, { act } from "react";
|
||||
import { createRoot, type Root } from "react-dom/client";
|
||||
import { afterEach, beforeEach, describe, expect, it, vi } from "vitest";
|
||||
import { thumbnailScheduler } from "../lib/thumbnailScheduler";
|
||||
import { decodeVideoThumbnail } from "../lib/thumbnailVideoDecoder";
|
||||
import { VideoThumbnail } from "./VideoThumbnail";
|
||||
|
||||
vi.mock("../lib/thumbnailVideoDecoder", () => ({ decodeVideoThumbnail: vi.fn() }));
|
||||
|
||||
Object.defineProperty(globalThis, "IS_REACT_ACT_ENVIRONMENT", {
|
||||
configurable: true,
|
||||
value: true,
|
||||
});
|
||||
|
||||
// Fire "intersecting" immediately on observe so the extraction effect runs.
|
||||
class MockIntersectionObserver {
|
||||
private cb: IntersectionObserverCallback;
|
||||
constructor(cb: IntersectionObserverCallback) {
|
||||
this.cb = cb;
|
||||
}
|
||||
observe() {
|
||||
this.cb(
|
||||
[{ isIntersecting: true } as IntersectionObserverEntry],
|
||||
this as unknown as IntersectionObserver,
|
||||
);
|
||||
}
|
||||
disconnect() {}
|
||||
unobserve() {}
|
||||
takeRecords(): IntersectionObserverEntry[] {
|
||||
return [];
|
||||
}
|
||||
}
|
||||
|
||||
class MockResizeObserver {
|
||||
observe() {}
|
||||
disconnect() {}
|
||||
unobserve() {}
|
||||
}
|
||||
|
||||
const originalIO = globalThis.IntersectionObserver;
|
||||
const originalRO = globalThis.ResizeObserver;
|
||||
|
||||
let host: HTMLDivElement;
|
||||
let root: Root | null = null;
|
||||
let createdVideos: HTMLVideoElement[];
|
||||
|
||||
beforeEach(() => {
|
||||
globalThis.IntersectionObserver =
|
||||
MockIntersectionObserver as unknown as typeof IntersectionObserver;
|
||||
globalThis.ResizeObserver = MockResizeObserver as unknown as typeof ResizeObserver;
|
||||
|
||||
createdVideos = [];
|
||||
const origCreate = document.createElement.bind(document);
|
||||
vi.spyOn(document, "createElement").mockImplementation((tag: string) => {
|
||||
const el = origCreate(tag);
|
||||
if (tag === "video") createdVideos.push(el as HTMLVideoElement);
|
||||
return el;
|
||||
});
|
||||
|
||||
// happy-dom's <video>/<canvas> don't decode media; stub the seam the
|
||||
// extractor depends on so the effect can run deterministically.
|
||||
vi.spyOn(HTMLMediaElement.prototype, "load").mockImplementation(() => {});
|
||||
vi.spyOn(HTMLCanvasElement.prototype, "getContext").mockReturnValue({
|
||||
drawImage: () => {},
|
||||
} as unknown as CanvasRenderingContext2D);
|
||||
|
||||
host = document.createElement("div");
|
||||
document.body.append(host);
|
||||
});
|
||||
@@ -68,85 +31,68 @@ beforeEach(() => {
|
||||
afterEach(() => {
|
||||
act(() => root?.unmount());
|
||||
root = null;
|
||||
vi.restoreAllMocks();
|
||||
globalThis.IntersectionObserver = originalIO;
|
||||
globalThis.ResizeObserver = originalRO;
|
||||
thumbnailScheduler.invalidateProject("p");
|
||||
vi.clearAllMocks();
|
||||
document.body.innerHTML = "";
|
||||
});
|
||||
|
||||
function render(videoSrc: string) {
|
||||
async function render(rich = false) {
|
||||
root = createRoot(host);
|
||||
act(() => {
|
||||
root!.render(React.createElement(VideoThumbnail, { videoSrc, label: "", labelColor: "#fff" }));
|
||||
});
|
||||
}
|
||||
|
||||
function lastVideo(): HTMLVideoElement {
|
||||
const v = createdVideos.at(-1);
|
||||
expect(v).toBeDefined();
|
||||
return v!;
|
||||
}
|
||||
|
||||
describe("VideoThumbnail — tainted-canvas fallback", () => {
|
||||
it("stops the extractor and drops the shimmer when toDataURL throws a SecurityError", () => {
|
||||
vi.spyOn(HTMLCanvasElement.prototype, "toDataURL").mockImplementation(() => {
|
||||
throw new DOMException("Tainted canvases may not be exported.", "SecurityError");
|
||||
});
|
||||
|
||||
render("https://cdn.example.com/no-cors.mp4");
|
||||
|
||||
// The effect ran once visible → a hidden <video> was created.
|
||||
const video = lastVideo();
|
||||
|
||||
act(() => {
|
||||
video.dispatchEvent(new Event("loadedmetadata"));
|
||||
});
|
||||
act(() => {
|
||||
video.dispatchEvent(new Event("seeked"));
|
||||
});
|
||||
|
||||
// No frame captured, and crucially the shimmer is gone (not spinning
|
||||
// forever) — the clip falls back to its plain background.
|
||||
expect(host.querySelectorAll("img").length).toBe(0);
|
||||
expect(host.querySelector(".animate-pulse")).toBeNull();
|
||||
});
|
||||
|
||||
it("drops the shimmer when a no-CORS load fires the video error event (0 frames) (#2214)", () => {
|
||||
render("https://cdn.example.com/no-cors.mp4");
|
||||
// Before the load resolves, the shimmer placeholder is up.
|
||||
expect(host.querySelector(".animate-pulse")).not.toBeNull();
|
||||
|
||||
const video = lastVideo();
|
||||
// crossOrigin="anonymous" against a CORS-less server fails the load outright —
|
||||
// the error listener fires instead of loadedmetadata/seeked, so no frame is
|
||||
// ever captured. The shimmer must stop rather than spin forever.
|
||||
act(() => {
|
||||
video.dispatchEvent(new Event("error"));
|
||||
});
|
||||
|
||||
expect(host.querySelectorAll("img").length).toBe(0);
|
||||
expect(host.querySelector(".animate-pulse")).toBeNull();
|
||||
});
|
||||
|
||||
it("keeps streaming frames while the shimmer is up until a frame arrives", () => {
|
||||
vi.spyOn(HTMLCanvasElement.prototype, "toDataURL").mockReturnValue(
|
||||
"data:image/jpeg;base64,AAAA",
|
||||
await act(async () => {
|
||||
root!.render(
|
||||
<VideoThumbnail
|
||||
videoSrc="/api/projects/p/preview/assets/clip.mp4"
|
||||
label=""
|
||||
labelColor="#fff"
|
||||
projectId="p"
|
||||
sessionEpoch={1}
|
||||
priority="visible"
|
||||
rich={rich}
|
||||
/>,
|
||||
);
|
||||
await Promise.resolve();
|
||||
});
|
||||
}
|
||||
|
||||
render("/api/projects/p/preview/assets/clip.mp4");
|
||||
// Before any seek resolves, the shimmer placeholder is shown.
|
||||
expect(host.querySelector(".animate-pulse")).not.toBeNull();
|
||||
|
||||
const video = lastVideo();
|
||||
act(() => {
|
||||
video.dispatchEvent(new Event("loadedmetadata"));
|
||||
});
|
||||
act(() => {
|
||||
video.dispatchEvent(new Event("seeked"));
|
||||
describe("VideoThumbnail", () => {
|
||||
it("renders a scheduler-provided sparse poster", async () => {
|
||||
vi.mocked(decodeVideoThumbnail).mockResolvedValue({
|
||||
value: { kind: "image", url: "blob:poster", aspect: 16 / 9 },
|
||||
weight: 128,
|
||||
});
|
||||
|
||||
// A frame was captured, so tiles render and the shimmer clears.
|
||||
expect(host.querySelectorAll("img").length).toBeGreaterThanOrEqual(1);
|
||||
await render();
|
||||
|
||||
expect(decodeVideoThumbnail).toHaveBeenCalledWith(
|
||||
expect.objectContaining({ frameCount: 1 }),
|
||||
expect.any(AbortSignal),
|
||||
);
|
||||
expect(host.querySelector('img[src="blob:poster"]')).not.toBeNull();
|
||||
expect(host.querySelector(".animate-pulse")).toBeNull();
|
||||
});
|
||||
|
||||
it("requests a rich filmstrip only for interaction actors", async () => {
|
||||
vi.mocked(decodeVideoThumbnail).mockResolvedValue({
|
||||
value: { kind: "filmstrip", urls: ["blob:a", "blob:b"], aspect: 16 / 9 },
|
||||
weight: 256,
|
||||
});
|
||||
|
||||
await render(true);
|
||||
|
||||
expect(decodeVideoThumbnail).toHaveBeenCalledWith(
|
||||
expect.objectContaining({ frameCount: 6 }),
|
||||
expect.any(AbortSignal),
|
||||
);
|
||||
expect(host.querySelectorAll("img").length).toBeGreaterThan(0);
|
||||
});
|
||||
|
||||
it("clears the loading shimmer when the scheduled decode fails", async () => {
|
||||
vi.mocked(decodeVideoThumbnail).mockRejectedValue(new Error("decode failed"));
|
||||
|
||||
await render();
|
||||
await vi.waitFor(() => expect(thumbnailScheduler.getDiagnostics().active).toBe(0));
|
||||
|
||||
expect(host.querySelector(".animate-pulse")).toBeNull();
|
||||
expect(host.querySelector("img")).toBeNull();
|
||||
});
|
||||
});
|
||||
|
||||
@@ -1,5 +1,8 @@
|
||||
import { memo, useRef, useState, useCallback, useEffect } from "react";
|
||||
import { memo, useCallback, useMemo, useRef, useState } from "react";
|
||||
import { useMountEffect } from "../../hooks/useMountEffect";
|
||||
import { useThumbnailLease } from "../../hooks/useThumbnailLease";
|
||||
import { createThumbnailKey, type ThumbnailPriority } from "../lib/thumbnailScheduler";
|
||||
import { decodeVideoThumbnail } from "../lib/thumbnailVideoDecoder";
|
||||
import { computeThumbnailStrip, THUMBNAIL_CLIP_HEIGHT } from "./thumbnailUtils";
|
||||
|
||||
interface VideoThumbnailProps {
|
||||
@@ -7,189 +10,105 @@ interface VideoThumbnailProps {
|
||||
label: string;
|
||||
labelColor: string;
|
||||
duration?: number;
|
||||
sourceStart?: number;
|
||||
sourceRangeDuration?: number;
|
||||
projectId?: string;
|
||||
sessionEpoch?: number;
|
||||
priority?: ThumbnailPriority;
|
||||
rich?: boolean;
|
||||
}
|
||||
|
||||
const CLIP_HEIGHT = THUMBNAIL_CLIP_HEIGHT;
|
||||
const MAX_UNIQUE_FRAMES: number = 6;
|
||||
|
||||
/**
|
||||
* Renders a film-strip of video frames extracted client-side via a hidden
|
||||
* <video> + <canvas>. Each frame is a fixed-width tile; frames repeat to
|
||||
* fill the clip width — matching ClipThumbnail's visual pattern.
|
||||
*/
|
||||
/** Sparse, bounded video frames supplied by the shared thumbnail scheduler. */
|
||||
export const VideoThumbnail = memo(function VideoThumbnail({
|
||||
videoSrc,
|
||||
label,
|
||||
labelColor,
|
||||
duration = 5,
|
||||
sourceStart,
|
||||
sourceRangeDuration,
|
||||
projectId = videoSrc,
|
||||
sessionEpoch = 0,
|
||||
priority = "visible",
|
||||
rich = false,
|
||||
}: VideoThumbnailProps) {
|
||||
const [containerWidth, setContainerWidth] = useState(0);
|
||||
const [visible, setVisible] = useState(false);
|
||||
const [frames, setFrames] = useState<string[]>([]);
|
||||
const [failed, setFailed] = useState(false);
|
||||
const [aspect, setAspect] = useState(16 / 9);
|
||||
const ioRef = useRef<IntersectionObserver | null>(null);
|
||||
const roRef = useRef<ResizeObserver | null>(null);
|
||||
const extractingRef = useRef(false);
|
||||
const observerRef = useRef<ResizeObserver | null>(null);
|
||||
const request = useMemo(
|
||||
() => ({
|
||||
key: createThumbnailKey({
|
||||
kind: "video",
|
||||
source: videoSrc,
|
||||
start: sourceStart,
|
||||
duration: sourceRangeDuration ?? duration,
|
||||
frames: rich ? 6 : 1,
|
||||
}),
|
||||
projectId,
|
||||
sessionEpoch,
|
||||
kind: "video" as const,
|
||||
priority,
|
||||
rich,
|
||||
load: (signal: AbortSignal) =>
|
||||
decodeVideoThumbnail(
|
||||
{
|
||||
source: videoSrc,
|
||||
sourceStart,
|
||||
sourceRangeDuration: sourceRangeDuration ?? duration,
|
||||
frameCount: rich ? 6 : 1,
|
||||
fit: "cover",
|
||||
},
|
||||
signal,
|
||||
),
|
||||
}),
|
||||
[duration, priority, projectId, rich, sessionEpoch, sourceRangeDuration, sourceStart, videoSrc],
|
||||
);
|
||||
const snapshot = useThumbnailLease(request);
|
||||
const value = snapshot.status === "ready" ? snapshot.value : null;
|
||||
const urls =
|
||||
value?.kind === "filmstrip" ? value.urls : value?.kind === "image" ? [value.url] : [];
|
||||
const aspect = value?.kind === "image" || value?.kind === "filmstrip" ? value.aspect : 16 / 9;
|
||||
const { frameW, frameCount } = computeThumbnailStrip(
|
||||
containerWidth,
|
||||
aspect,
|
||||
THUMBNAIL_CLIP_HEIGHT,
|
||||
);
|
||||
|
||||
const setContainerRef = useCallback((el: HTMLDivElement | null) => {
|
||||
ioRef.current?.disconnect();
|
||||
roRef.current?.disconnect();
|
||||
if (!el) return;
|
||||
|
||||
const measured = el.parentElement?.clientWidth || el.clientWidth;
|
||||
setContainerWidth(measured);
|
||||
|
||||
ioRef.current = new IntersectionObserver(
|
||||
([entry]) => {
|
||||
if (entry.isIntersecting) {
|
||||
setVisible(true);
|
||||
ioRef.current?.disconnect();
|
||||
}
|
||||
},
|
||||
{ rootMargin: "200px" },
|
||||
const setContainerRef = useCallback((element: HTMLDivElement | null) => {
|
||||
observerRef.current?.disconnect();
|
||||
if (!element) return;
|
||||
const target = element.parentElement ?? element;
|
||||
setContainerWidth(target.clientWidth);
|
||||
observerRef.current = new ResizeObserver(([entry]) =>
|
||||
setContainerWidth(entry.contentRect.width),
|
||||
);
|
||||
// fallow-ignore-next-line code-duplication
|
||||
ioRef.current.observe(el);
|
||||
|
||||
const target = el.parentElement || el;
|
||||
roRef.current = new ResizeObserver(([entry]) => {
|
||||
setContainerWidth(entry.contentRect.width);
|
||||
});
|
||||
roRef.current.observe(target);
|
||||
observerRef.current.observe(target);
|
||||
}, []);
|
||||
|
||||
useMountEffect(() => () => {
|
||||
ioRef.current?.disconnect();
|
||||
roRef.current?.disconnect();
|
||||
});
|
||||
|
||||
// Extract frames progressively — each frame appears as soon as it's ready.
|
||||
// Note: useEffect with deps is acceptable — syncs with external video element API,
|
||||
// requires cleanup (cancel extraction, revoke URLs) when inputs change.
|
||||
// eslint-disable-next-line no-restricted-syntax
|
||||
useEffect(() => {
|
||||
if (!visible || extractingRef.current) return;
|
||||
extractingRef.current = true;
|
||||
|
||||
const video = document.createElement("video");
|
||||
video.crossOrigin = "anonymous";
|
||||
video.muted = true;
|
||||
video.preload = "auto";
|
||||
|
||||
const canvas = document.createElement("canvas");
|
||||
const ctx = canvas.getContext("2d");
|
||||
if (!ctx) {
|
||||
extractingRef.current = false;
|
||||
return;
|
||||
}
|
||||
|
||||
const timestamps: number[] = [];
|
||||
const minSeek = Math.min(0.4, duration * 0.05);
|
||||
for (let i = 0; i < MAX_UNIQUE_FRAMES; i++) {
|
||||
const raw =
|
||||
MAX_UNIQUE_FRAMES === 1 ? duration * 0.15 : (i / (MAX_UNIQUE_FRAMES - 1)) * duration;
|
||||
timestamps.push(Math.max(raw, minSeek));
|
||||
}
|
||||
|
||||
let idx = 0;
|
||||
let cancelled = false;
|
||||
|
||||
const extractNext = () => {
|
||||
if (cancelled || idx >= timestamps.length) {
|
||||
if (!cancelled) {
|
||||
video.src = "";
|
||||
video.load();
|
||||
}
|
||||
return;
|
||||
}
|
||||
video.currentTime = timestamps[idx];
|
||||
};
|
||||
|
||||
video.addEventListener("loadedmetadata", () => {
|
||||
if (video.videoWidth > 0 && video.videoHeight > 0) {
|
||||
setAspect(video.videoWidth / video.videoHeight);
|
||||
const h = CLIP_HEIGHT * 2;
|
||||
const w = Math.round(h * (video.videoWidth / video.videoHeight));
|
||||
canvas.width = w;
|
||||
canvas.height = h;
|
||||
}
|
||||
extractNext();
|
||||
});
|
||||
|
||||
video.addEventListener("seeked", () => {
|
||||
if (cancelled) return;
|
||||
let dataUrl: string;
|
||||
try {
|
||||
ctx.drawImage(video, 0, 0, canvas.width, canvas.height);
|
||||
dataUrl = canvas.toDataURL("image/jpeg", 0.6);
|
||||
} catch {
|
||||
// An external http(s) video served without CORS headers taints the
|
||||
// canvas, so toDataURL throws a SecurityError. Stop the extractor
|
||||
// cleanly and fall back to the no-thumbnail rendering (plain clip
|
||||
// background), matching ImageThumbnail's error path — otherwise the
|
||||
// shimmer placeholder would spin forever.
|
||||
cancelled = true;
|
||||
setFailed(true);
|
||||
video.src = "";
|
||||
video.load();
|
||||
return;
|
||||
}
|
||||
// Stream each frame immediately
|
||||
setFrames((prev) => [...prev, dataUrl]);
|
||||
idx++;
|
||||
extractNext();
|
||||
});
|
||||
|
||||
video.addEventListener("error", () => {
|
||||
// A no-CORS load fails outright (crossOrigin="anonymous" rejects a video
|
||||
// served without CORS headers), firing this instead of the taint path in
|
||||
// "seeked" — so 0 frames are ever extracted. Keep whatever frames we have,
|
||||
// but mark failed so the shimmer placeholder stops spinning forever and we
|
||||
// fall back to the plain clip background (#2214).
|
||||
setFailed(true);
|
||||
});
|
||||
|
||||
video.src = videoSrc;
|
||||
video.load();
|
||||
|
||||
return () => {
|
||||
cancelled = true;
|
||||
extractingRef.current = false;
|
||||
setFrames([]);
|
||||
setFailed(false);
|
||||
video.src = "";
|
||||
video.load();
|
||||
};
|
||||
}, [visible, videoSrc, duration]);
|
||||
|
||||
const { frameW, frameCount } = computeThumbnailStrip(containerWidth, aspect, CLIP_HEIGHT);
|
||||
useMountEffect(() => () => observerRef.current?.disconnect());
|
||||
|
||||
return (
|
||||
<div ref={setContainerRef} className="absolute inset-0 overflow-hidden">
|
||||
{visible && frames.length > 0 && (
|
||||
{urls.length > 0 && (
|
||||
<div className="absolute inset-0 flex">
|
||||
{Array.from({ length: frameCount }).map((_, i) => {
|
||||
const src = frames[i % frames.length];
|
||||
{Array.from({ length: frameCount }, (_, index) => {
|
||||
const src = urls[index % urls.length];
|
||||
return (
|
||||
<div
|
||||
key={i}
|
||||
className="flex-shrink-0 h-full relative overflow-hidden bg-neutral-900"
|
||||
key={index}
|
||||
className="relative h-full flex-shrink-0 overflow-hidden bg-neutral-900"
|
||||
style={{ width: frameW }}
|
||||
>
|
||||
<img
|
||||
src={src}
|
||||
alt=""
|
||||
draggable={false}
|
||||
className="absolute inset-0 w-full h-full object-cover"
|
||||
className="absolute inset-0 h-full w-full object-cover"
|
||||
/>
|
||||
</div>
|
||||
);
|
||||
})}
|
||||
</div>
|
||||
)}
|
||||
|
||||
{visible && frames.length === 0 && !failed && (
|
||||
{snapshot.status === "loading" && urls.length === 0 && (
|
||||
<div
|
||||
className="absolute inset-0 animate-pulse"
|
||||
style={{
|
||||
@@ -198,17 +117,16 @@ export const VideoThumbnail = memo(function VideoThumbnail({
|
||||
}}
|
||||
/>
|
||||
)}
|
||||
|
||||
{label && (
|
||||
<div
|
||||
className="absolute bottom-0 left-0 right-0 z-10 px-1.5 pb-0.5 pt-3"
|
||||
className="absolute inset-x-0 bottom-0 z-10 px-1.5 pb-0.5 pt-3"
|
||||
style={{
|
||||
background:
|
||||
"linear-gradient(to top, rgba(0,0,0,0.85) 0%, rgba(0,0,0,0.4) 60%, transparent 100%)",
|
||||
}}
|
||||
>
|
||||
<span
|
||||
className="text-[9px] font-semibold truncate block leading-tight"
|
||||
className="block truncate text-[9px] font-semibold leading-tight"
|
||||
style={{ color: labelColor, textShadow: "0 1px 2px rgba(0,0,0,0.9)" }}
|
||||
>
|
||||
{label}
|
||||
|
||||
@@ -41,6 +41,13 @@ describe("computeThumbnailStrip", () => {
|
||||
it("honors a custom clip height", () => {
|
||||
expect(computeThumbnailStrip(300, 2, 40).frameW).toBe(80);
|
||||
});
|
||||
|
||||
it("keeps narrow tiles above a caller-owned minimum", () => {
|
||||
expect(computeThumbnailStrip(300, 0.25, 40, 48)).toEqual({
|
||||
frameW: 48,
|
||||
frameCount: 7,
|
||||
});
|
||||
});
|
||||
});
|
||||
|
||||
describe("resolveMediaPreviewUrl", () => {
|
||||
|
||||
@@ -8,6 +8,52 @@ export interface ThumbnailStripLayout {
|
||||
frameCount: number;
|
||||
}
|
||||
|
||||
/**
|
||||
* Measure an image without mounting it in React's DOM. The scheduler owns the
|
||||
* abort signal, so an unmounted clip cannot leave Blink retaining a pending
|
||||
* image request and its former React tree.
|
||||
*/
|
||||
export function probeImageAspect(
|
||||
imageSrc: string,
|
||||
signal: AbortSignal,
|
||||
tolerateSvgError = false,
|
||||
): Promise<number> {
|
||||
return new Promise((resolve, reject) => {
|
||||
const image = new Image();
|
||||
const cleanup = () => {
|
||||
image.onload = null;
|
||||
image.onerror = null;
|
||||
signal.removeEventListener("abort", onAbort);
|
||||
};
|
||||
const onAbort = () => {
|
||||
cleanup();
|
||||
image.src = "";
|
||||
reject(new DOMException("Aborted", "AbortError"));
|
||||
};
|
||||
|
||||
if (signal.aborted) {
|
||||
onAbort();
|
||||
return;
|
||||
}
|
||||
|
||||
image.onload = () => {
|
||||
cleanup();
|
||||
resolve(
|
||||
image.naturalWidth > 0 && image.naturalHeight > 0
|
||||
? image.naturalWidth / image.naturalHeight
|
||||
: 16 / 9,
|
||||
);
|
||||
};
|
||||
image.onerror = () => {
|
||||
cleanup();
|
||||
if (tolerateSvgError && /\.svg($|\?)/i.test(imageSrc)) resolve(16 / 9);
|
||||
else reject(new Error("Image thumbnail failed to load"));
|
||||
};
|
||||
signal.addEventListener("abort", onAbort, { once: true });
|
||||
image.src = imageSrc;
|
||||
});
|
||||
}
|
||||
|
||||
/**
|
||||
* Compute the film-strip tile layout for a clip thumbnail: fixed-height tiles
|
||||
* sized by the media's aspect ratio, repeated to fill the clip width.
|
||||
@@ -17,9 +63,10 @@ export function computeThumbnailStrip(
|
||||
containerWidth: number,
|
||||
aspect: number,
|
||||
clipHeight: number = THUMBNAIL_CLIP_HEIGHT,
|
||||
minFrameWidth = 1,
|
||||
): ThumbnailStripLayout {
|
||||
const safeAspect = Number.isFinite(aspect) && aspect > 0 ? aspect : 16 / 9;
|
||||
const frameW = Math.max(1, Math.round(clipHeight * safeAspect));
|
||||
const frameW = Math.max(minFrameWidth, Math.round(clipHeight * safeAspect));
|
||||
const frameCount = containerWidth > 0 ? Math.max(1, Math.ceil(containerWidth / frameW)) : 1;
|
||||
return { frameW, frameCount };
|
||||
}
|
||||
|
||||
@@ -0,0 +1,31 @@
|
||||
import { describe, expect, it } from "vitest";
|
||||
import { resolveClipRenderContext } from "./timelineClipChildren";
|
||||
import type { TimelineElement } from "../store/playerStore";
|
||||
|
||||
const clip: TimelineElement = {
|
||||
id: "clip",
|
||||
tag: "video",
|
||||
start: 10,
|
||||
duration: 5,
|
||||
track: 0,
|
||||
};
|
||||
|
||||
describe("resolveClipRenderContext", () => {
|
||||
it("prioritizes interactive clips and enables rich thumbnails", () => {
|
||||
expect(resolveClipRenderContext(clip, { start: 0, end: 1 }, true)).toEqual({
|
||||
priority: "interaction",
|
||||
rich: true,
|
||||
});
|
||||
});
|
||||
|
||||
it("distinguishes visible clips from overscan clips", () => {
|
||||
expect(resolveClipRenderContext(clip, { start: 14, end: 16 }, false)).toEqual({
|
||||
priority: "visible",
|
||||
rich: false,
|
||||
});
|
||||
expect(resolveClipRenderContext(clip, { start: 15, end: 20 }, false)).toEqual({
|
||||
priority: "overscan",
|
||||
rich: false,
|
||||
});
|
||||
});
|
||||
});
|
||||
@@ -1,6 +1,20 @@
|
||||
import { type ReactNode } from "react";
|
||||
import { usePlayerStore, type TimelineElement } from "../store/playerStore";
|
||||
import type { TimelineTimeRange } from "../lib/timelineClipIndex";
|
||||
import type { TrackVisualStyle } from "./timelineIcons";
|
||||
import type { TimelineClipRenderContext } from "./TimelineTypes";
|
||||
|
||||
export function resolveClipRenderContext(
|
||||
element: TimelineElement,
|
||||
visibleTimeRange: TimelineTimeRange,
|
||||
interactive: boolean,
|
||||
): TimelineClipRenderContext {
|
||||
if (interactive) return { priority: "interaction", rich: true };
|
||||
const visible =
|
||||
element.start < visibleTimeRange.end &&
|
||||
element.start + element.duration > visibleTimeRange.start;
|
||||
return { priority: visible ? "visible" : "overscan", rich: false };
|
||||
}
|
||||
|
||||
function ClipLintDot({ element }: { element: TimelineElement }) {
|
||||
const lint = usePlayerStore((s) => s.lintFindingsByElement.get(element.key ?? element.id));
|
||||
@@ -18,9 +32,14 @@ export function renderClipChildren(
|
||||
element: TimelineElement,
|
||||
clipStyle: TrackVisualStyle,
|
||||
renderClipContent:
|
||||
| ((element: TimelineElement, style: { clip: string; label: string }) => ReactNode)
|
||||
| ((
|
||||
element: TimelineElement,
|
||||
style: { clip: string; label: string },
|
||||
context: TimelineClipRenderContext,
|
||||
) => ReactNode)
|
||||
| undefined,
|
||||
renderClipOverlay: ((element: TimelineElement) => ReactNode) | undefined,
|
||||
context: TimelineClipRenderContext = { priority: "visible", rich: false },
|
||||
): ReactNode {
|
||||
return (
|
||||
<>
|
||||
@@ -31,7 +50,7 @@ export function renderClipChildren(
|
||||
// diamonds hang outside its bounds), so the thumbnail layer must clip
|
||||
// itself to the clip's rounded corners or sharp corners poke out.
|
||||
<div className="absolute inset-0 overflow-hidden" style={{ borderRadius: "inherit" }}>
|
||||
{renderClipContent(element, clipStyle)}
|
||||
{renderClipContent(element, clipStyle, context)}
|
||||
</div>
|
||||
)}
|
||||
</>
|
||||
|
||||
@@ -10,6 +10,7 @@ import type { TimelineClipIndex, TimelineTimeRange } from "../lib/timelineClipIn
|
||||
import type { TimelineRowGeometry } from "./timelineLayout";
|
||||
import type { TimelineVirtualRow } from "./useTimelineVirtualRows";
|
||||
import type { TimelineLogicalRow } from "./timelineKeyboardNavigation";
|
||||
import type { TimelineClipRenderContext } from "./TimelineTypes";
|
||||
|
||||
/**
|
||||
* Props shared by the scroll container ({@link import("./TimelineCanvas")}) and
|
||||
@@ -33,6 +34,7 @@ export interface TimelineLaneBaseProps {
|
||||
rowsVirtualized: boolean;
|
||||
clipIndex: TimelineClipIndex;
|
||||
renderTimeRange: TimelineTimeRange;
|
||||
visibleTimeRange: TimelineTimeRange;
|
||||
pinnedClipIdentities: ReadonlySet<string>;
|
||||
trackOrder: number[];
|
||||
tracks: [number, TimelineElement[]][];
|
||||
@@ -48,6 +50,7 @@ export interface TimelineLaneBaseProps {
|
||||
renderClipContent?: (
|
||||
element: TimelineElement,
|
||||
style: { clip: string; label: string },
|
||||
context: TimelineClipRenderContext,
|
||||
) => ReactNode;
|
||||
renderClipOverlay?: (element: TimelineElement) => ReactNode;
|
||||
onDrillDown?: (element: TimelineElement) => void;
|
||||
|
||||
@@ -1,6 +1,9 @@
|
||||
import { useMemo } from "react";
|
||||
import { createTimelineClipIndex } from "../lib/timelineClipIndex";
|
||||
import { getTimelineRenderTimeRange } from "./timelineViewportGeometry";
|
||||
import {
|
||||
getTimelineRenderTimeRange,
|
||||
getTimelineVisibleTimeRange,
|
||||
} from "./timelineViewportGeometry";
|
||||
import type { TimelineScrollViewportSnapshot } from "./useTimelineScrollViewport";
|
||||
|
||||
interface UseTimelineClipRenderWindowInput {
|
||||
@@ -37,6 +40,10 @@ export function useTimelineClipRenderWindow({
|
||||
() => getTimelineRenderTimeRange(viewport, pixelsPerSecond, contentOrigin, duration),
|
||||
[contentOrigin, duration, pixelsPerSecond, viewport],
|
||||
);
|
||||
const visibleTimeRange = useMemo(
|
||||
() => getTimelineVisibleTimeRange(viewport, pixelsPerSecond, contentOrigin, duration),
|
||||
[contentOrigin, duration, pixelsPerSecond, viewport],
|
||||
);
|
||||
const pinnedClipIdentities = useMemo(
|
||||
() =>
|
||||
new Set(
|
||||
@@ -60,5 +67,5 @@ export function useTimelineClipRenderWindow({
|
||||
selectedElementId,
|
||||
],
|
||||
);
|
||||
return { clipIndex, renderTimeRange, pinnedClipIdentities };
|
||||
return { clipIndex, renderTimeRange, visibleTimeRange, pinnedClipIdentities };
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user