diff --git a/packages/studio/src/player/components/CompositionThumbnail.test.ts b/packages/studio/src/player/components/CompositionThumbnail.test.ts index 9a09b8b3b7..28bd9ad131 100644 --- a/packages/studio/src/player/components/CompositionThumbnail.test.ts +++ b/packages/studio/src/player/components/CompositionThumbnail.test.ts @@ -334,6 +334,20 @@ describe("CompositionThumbnail", () => { expect(slices().every((slice) => slice.style.aspectRatio === "")).toBe(true); }); + it("holds one lease per chunk for the whole clip, however many tiles show it", async () => { + const acquire = vi.spyOn(thumbnailScheduler, "acquire"); + try { + sizeHost(500, 40); + await renderThumbnail(eightSeconds); + await loadImage(0, 8 * 240, 135); + + expect(slices()).toHaveLength(8); + expect(acquire).toHaveBeenCalledTimes(1); + } finally { + acquire.mockRestore(); + } + }); + it("asks for nothing before the clip has a width, then only its strip", async () => { await renderThumbnail(eightSeconds); expect(globalThis.fetch).not.toHaveBeenCalled(); diff --git a/packages/studio/src/player/components/CompositionThumbnail.tsx b/packages/studio/src/player/components/CompositionThumbnail.tsx index 818d9693f6..2b84d7e4ff 100644 --- a/packages/studio/src/player/components/CompositionThumbnail.tsx +++ b/packages/studio/src/player/components/CompositionThumbnail.tsx @@ -1,14 +1,24 @@ -import { memo, useCallback, useEffect, useMemo, useState, type CSSProperties } from "react"; -import { useThumbnailLease } from "../../hooks/useThumbnailLease"; +import { + memo, + useCallback, + useEffect, + useLayoutEffect, + useMemo, + useReducer, + useRef, + useState, + type CSSProperties, +} from "react"; import { useThumbnailStripSize } from "../../hooks/useThumbnailStripSize"; import { createThumbnailKey, type ThumbnailPriority, type ThumbnailRequest, readyImage, + thumbnailScheduler, } from "../lib/thumbnailScheduler"; import { TIMELINE_VIEWPORT_BUDGETS } from "../lib/timelineViewportBudgets"; -import { ThumbnailTiles } from "./ThumbnailTiles"; +import { ThumbnailTiles, visibleTileRange } from "./ThumbnailTiles"; import { computeThumbnailStrip, probeImageAspect } from "./thumbnailUtils"; import { studioApiFetch } from "../../utils/studioApiFetch"; @@ -95,17 +105,13 @@ const STRIP_CHUNK_FRAMES = 8; const MIN_FRAME_STEP_SECONDS = 1 / 32; const LAST_FRAME_INSET_SECONDS = 0.001; +const gridStepFor = (tileSeconds: number) => + 2 ** Math.floor(Math.log2(Math.max(tileSeconds, MIN_FRAME_STEP_SECONDS))); + // Tiles map onto a power-of-two grid from 0, at least one cell per tile; chunks follow the grid, so zooms reuse frames. -export function planCompositionStrip( - sourceStart: number, - sourceRangeDuration: number, - tileSeconds: number, -) { - const step = 2 ** Math.floor(Math.log2(Math.max(tileSeconds, MIN_FRAME_STEP_SECONDS))); +function compositionStripGrid(sourceStart: number, sourceRangeDuration: number, step: number) { const firstCell = Math.floor(sourceStart / step); const lastCell = Math.ceil((sourceStart + sourceRangeDuration) / step) - 1; - const cellOf = (tile: number) => - Math.min(lastCell, Math.floor((sourceStart + (tile + 0.5) * tileSeconds) / step)); const chunkStart = (chunk: number) => Math.max(chunk * STRIP_CHUNK_FRAMES, firstCell); const chunkEnd = (chunk: number) => Math.min((chunk + 1) * STRIP_CHUNK_FRAMES, lastCell + 1); return { @@ -117,8 +123,11 @@ export function planCompositionStrip( const to = Math.min((cell + 1) * step, sourceStart + sourceRangeDuration); return Math.min((from + to) / 2, to - LAST_FRAME_INSET_SECONDS); }), - tile: (tile: number) => { - const cell = cellOf(tile); + tileAt: (tileSeconds: number) => (tile: number) => { + const cell = Math.min( + lastCell, + Math.floor((sourceStart + (tile + 0.5) * tileSeconds) / step), + ); const chunk = Math.floor(cell / STRIP_CHUNK_FRAMES); return { chunk, @@ -129,19 +138,13 @@ export function planCompositionStrip( }; } -function chunkUrl( - urlsByStepAndChunk: Map, - plan: ReturnType, - chunk: number, - options: Parameters[0], +export function planCompositionStrip( + sourceStart: number, + sourceRangeDuration: number, + tileSeconds: number, ) { - const key = `${plan.step}/${chunk}`; - let url = urlsByStepAndChunk.get(key); - if (url === undefined) { - url = buildCompositionThumbnailUrl({ ...options, times: plan.times(chunk) }); - urlsByStepAndChunk.set(key, url); - } - return url; + const grid = compositionStripGrid(sourceStart, sourceRangeDuration, gridStepFor(tileSeconds)); + return { times: grid.times, tile: grid.tileAt(tileSeconds) }; } /** The composition a preview URL renders: `/preview/comp/`, or the root for `/preview`. */ @@ -190,58 +193,64 @@ async function loadCompositionImage(url: string, signal: AbortSignal, frames: nu } } -interface TileImage { - url: string; - frame: number; - frames: number; -} - -interface CompositionTileProps extends TileImage { - posterUrl: string; - letterbox: boolean; - projectId: string; - sessionEpoch: number; - priority: ThumbnailPriority; - onAspect: (frameAspect: number) => void; -} - type StripImage = NonNullable>; -type TileCell = { request: ThumbnailRequest; frame: number; frames: number }; +type ShownCell = { request: ThumbnailRequest; frame: number; frames: number }; +type ShownTile = ShownCell & { strip: StripImage }; -/** The tile's strip once ready; until then what it showed last (kept leased), or the poster if it never had one. */ -function useShownStrip(cell: TileCell, posterCell: TileCell) { - const snapshot = useThumbnailLease(cell.request); - const next = readyImage(snapshot); - const [last, setLast] = useState(null); - const held = last ?? (!next && snapshot.status === "error" ? posterCell : null); - const fallback = readyImage(useThumbnailLease(held ? held.request : null)); - const view = viewOf(cell, next, held, fallback, posterCell); - if (view.remember && view.remember.request !== last?.request) setLast(view.remember); - return view; +const imageOf = (request: ThumbnailRequest) => readyImage(thumbnailScheduler.getSnapshot(request)); + +/** A tile's own frame once ready; until then what it showed last, or the poster if its strip failed. */ +function cellToShow(cell: ShownCell, last: ShownCell | undefined, posterCell: ShownCell) { + if (imageOf(cell.request)) return cell; + if (last && imageOf(last.request)) return last; + return thumbnailScheduler.getSnapshot(cell.request).status === "error" ? posterCell : null; } -const NOTHING_SHOWN = { shown: null, freshAspect: null, remember: null }; - -function viewOf( - cell: TileCell, - next: StripImage | null, - fallbackCell: TileCell | null, - fallback: StripImage | null, - posterCell: TileCell, +function showTiles( + [first, end]: readonly [number, number], + cellAt: (index: number) => ShownCell | null, + lastShown: ReadonlyMap, + posterCell: ShownCell, ) { - if (next) - return { - shown: { ...cell, strip: next }, - freshAspect: next.aspect / cell.frames, - remember: cell, + const tiles = new Map(); + const leased = new Set(); + let freshAspect: number | null = null; + for (let index = first; index < end; index++) { + const cell = cellAt(index); + if (!cell) continue; + leased.add(cell.request); + const shown = cellToShow(cell, lastShown.get(index), posterCell); + if (!shown) continue; + leased.add(shown.request); + const strip = imageOf(shown.request); + if (!strip) continue; + tiles.set(index, { ...shown, strip }); + if (shown === cell || shown === posterCell) freshAspect ??= strip.aspect / shown.frames; + } + return { tiles, leased, freshAspect }; +} + +/** Holds a lease on exactly the given requests, re-rendering when any of them changes. */ +function useThumbnailLeases(requests: ReadonlySet) { + const [, rerender] = useReducer((renders: number) => renders + 1, 0); + const leases = useRef(new Map()); + useLayoutEffect(() => { + const held = leases.current; + for (const request of requests) + if (!held.has(request)) held.set(request, thumbnailScheduler.acquire(request, rerender)); + for (const [request, lease] of held) { + if (requests.has(request)) continue; + lease.release(); + held.delete(request); + } + }, [requests]); + useEffect(() => { + const held = leases.current; + return () => { + for (const lease of held.values()) lease.release(); + held.clear(); }; - if (!fallback || !fallbackCell) return NOTHING_SHOWN; - const current = fallbackCell === posterCell; - return { - shown: { ...fallbackCell, strip: fallback }, - freshAspect: current ? fallback.aspect : null, - remember: current ? posterCell : null, - }; + }, []); } function StripSlice({ @@ -272,42 +281,6 @@ function StripSlice({ return letterbox ?
{slice}
: slice; } -const CompositionTile = memo(function CompositionTile({ - url, - posterUrl, - frame, - frames, - letterbox, - projectId, - sessionEpoch, - priority, - onAspect, -}: CompositionTileProps) { - const request = useMemo( - () => - compositionThumbnailRequest(url, projectId, { sessionEpoch, priority, rich: true }, frames), - [frames, priority, projectId, sessionEpoch, url], - ); - const cell = useMemo(() => ({ request, frame, frames }), [frame, frames, request]); - const posterCell = useMemo( - () => ({ - request: compositionThumbnailRequest(posterUrl, projectId, { - sessionEpoch, - priority, - rich: true, - }), - frame: 0, - frames: 1, - }), - [posterUrl, priority, projectId, sessionEpoch], - ); - const { shown, freshAspect } = useShownStrip(cell, posterCell); - useEffect(() => { - if (freshAspect) onAspect(freshAspect); - }, [freshAspect, onAspect]); - return shown ? : null; -}); - /** Server-rendered composition frames, deduplicated and budgeted by project/session. */ export const CompositionThumbnail = memo(function CompositionThumbnail({ previewUrl, @@ -350,26 +323,63 @@ export const CompositionThumbnail = memo(function CompositionThumbnail({ container.height, 48, ); - const plan = useMemo( + const measured = container.width > 0; + const tileSeconds = measured ? (sourceRangeDuration * frameW) / container.width : 0; + const step = gridStepFor(tileSeconds); + const grid = useMemo( () => - sourceRangeDuration > 0 && container.width > 0 - ? planCompositionStrip( - sourceStart, - sourceRangeDuration, - (sourceRangeDuration * frameW) / container.width, - ) + sourceRangeDuration > 0 && measured + ? compositionStripGrid(sourceStart, sourceRangeDuration, step) : null, - [container.width, frameW, sourceRangeDuration, sourceStart], + [measured, sourceRangeDuration, sourceStart, step], ); - const urlsByStepAndChunk = useMemo( - () => new Map(), - [urlOptions, sourceStart, sourceRangeDuration], + const posterCell = useMemo( + () => ({ + request: compositionThumbnailRequest(url, projectId, { sessionEpoch, priority, rich: true }), + frame: 0, + frames: 1, + }), + [priority, projectId, sessionEpoch, url], ); - const imageOf = (index: number): TileImage | null => { - if (!plan) return sourceRangeDuration > 0 ? null : { url, frame: 0, frames: 1 }; - const { chunk, frame, frames } = plan.tile(index); - return { url: chunkUrl(urlsByStepAndChunk, plan, chunk, urlOptions), frame, frames }; + const chunkRequest = useMemo(() => { + if (!grid) return null; + const requests = new Map(); + return (chunk: number, frames: number) => { + let request = requests.get(chunk); + if (!request) { + const chunkUrl = buildCompositionThumbnailUrl({ ...urlOptions, times: grid.times(chunk) }); + request = compositionThumbnailRequest( + chunkUrl, + projectId, + { sessionEpoch, priority, rich: true }, + frames, + ); + requests.set(chunk, request); + } + return request; + }; + }, [grid, priority, projectId, sessionEpoch, urlOptions]); + const tileOf = grid?.tileAt(tileSeconds); + const cellAt = (index: number): ShownCell | null => { + if (!tileOf || !chunkRequest) return sourceRangeDuration > 0 ? null : posterCell; + const { chunk, frame, frames } = tileOf(index); + return { request: chunkRequest(chunk, frames), frame, frames }; }; + const lastShown = useRef>(new Map()); + const { tiles, leased, freshAspect } = showTiles( + visibleTileRange(container, frameW, frameCount), + cellAt, + lastShown.current, + posterCell, + ); + useThumbnailLeases(leased); + useLayoutEffect(() => { + lastShown.current = tiles; + }); + useEffect(() => { + if (freshAspect) learnAspectOncePerRevision(freshAspect); + }, [freshAspect, learnAspectOncePerRevision]); + const letterbox = frameW > Math.round(container.height * frameAspect); return (
@@ -384,22 +394,19 @@ export const CompositionThumbnail = memo(function CompositionThumbnail({ }} > {(index) => { - const image = imageOf(index); + const tile = tiles.get(index); return (
- {image && ( - Math.round(container.height * frameAspect)} - projectId={projectId} - sessionEpoch={sessionEpoch} - priority={priority} - onAspect={learnAspectOncePerRevision} + {tile && ( + )}
diff --git a/packages/studio/src/player/components/ThumbnailTiles.tsx b/packages/studio/src/player/components/ThumbnailTiles.tsx index 2c7d146ba2..89b90c1b06 100644 --- a/packages/studio/src/player/components/ThumbnailTiles.tsx +++ b/packages/studio/src/player/components/ThumbnailTiles.tsx @@ -10,6 +10,15 @@ interface ThumbnailTilesProps { children: (index: number) => ReactNode; } +export function visibleTileRange( + strip: StripSize, + frameW: number, + frameCount: number, +): readonly [first: number, end: number] { + const end = Math.min(frameCount, Math.ceil(strip.inViewEnd / frameW)); + return [Math.min(end, Math.floor(strip.inViewStart / frameW)), end]; +} + /** A clip's tile row, mounting only the tiles in the strip's on-screen span, each at its own place. */ export function ThumbnailTiles({ strip, @@ -19,8 +28,7 @@ export function ThumbnailTiles({ style, children, }: ThumbnailTilesProps) { - const end = Math.min(frameCount, Math.ceil(strip.inViewEnd / frameW)); - const first = Math.min(end, Math.floor(strip.inViewStart / frameW)); + const [first, end] = visibleTileRange(strip, frameW, frameCount); return (