perf(studio-server): coordinate cancelable thumbnail generation (#2720)

* perf(studio): schedule adaptive timeline thumbnails

* perf(studio): bound thumbnail decoding resources

* perf(studio): virtualize timeline thumbnail media

* perf(studio): prioritize timeline thumbnail work

* perf(studio-server): coordinate cancelable thumbnail generation

---------

Co-authored-by: Codex <codex@local>
This commit is contained in:
Miguel Ángel
2026-08-05 20:41:35 -07:00
committed by GitHub
co-authored by Codex
parent 27d113e56c
commit 96861cbafc
59 changed files with 3537 additions and 879 deletions
@@ -5,13 +5,15 @@ import { createRoot } from "react-dom/client";
import { afterEach, describe, expect, it } from "vitest";
import { CompositionThumbnail, VideoThumbnail } from "../player";
import { AudioWaveform } from "../player/components/AudioWaveform";
import type { TimelineElement } from "../player/store/playerStore";
import type { TimelineClipRenderContext } from "../player/components/TimelineTypes";
import { usePlayerStore, type TimelineElement } from "../player/store/playerStore";
import { normalizeCompositionSrc } from "./useRenderClipContent";
import { useRenderClipContent } from "./useRenderClipContent";
(globalThis as unknown as { IS_REACT_ACT_ENVIRONMENT: boolean }).IS_REACT_ACT_ENVIRONMENT = true;
afterEach(() => {
usePlayerStore.setState({ thumbnailMode: "hidden" });
document.body.innerHTML = "";
});
@@ -67,6 +69,7 @@ describe("useRenderClipContent", () => {
function renderClipContent(
el: TimelineElement,
activePreviewUrl: string | null = "/api/projects/my-project/preview",
context?: TimelineClipRenderContext,
): ReactNode {
const host = document.createElement("div");
document.body.append(host);
@@ -80,7 +83,7 @@ describe("useRenderClipContent", () => {
activePreviewUrl,
effectiveTimelineDuration: 12,
});
content = render(el, { clip: "#222", label: "#fff" });
content = render(el, { clip: "#222", label: "#fff" }, context);
return null;
}
@@ -106,6 +109,8 @@ describe("useRenderClipContent", () => {
});
it("passes empty labels to thumbnail content so TimelineClip owns clip names", () => {
usePlayerStore.setState({ thumbnailMode: "adaptive" });
const cases: Array<{ content: ReactNode; type: unknown }> = [
{
content: renderClipContent({
@@ -166,4 +171,38 @@ describe("useRenderClipContent", () => {
}
}
});
it("forwards the viewport priority and interaction detail to media work", () => {
usePlayerStore.setState({ thumbnailMode: "adaptive", timelineSessionEpoch: 7 });
const content = renderClipContent(
{
id: "clip-video",
tag: "video",
start: 0,
duration: 4,
track: 0,
src: "assets/clip.mp4",
},
null,
{ priority: "interaction", rich: true },
);
expect(
isValidElement<{
projectId: string;
sessionEpoch: number;
priority: string;
rich: boolean;
}>(content),
).toBe(true);
if (isValidElement(content)) {
expect(content.props).toMatchObject({
projectId: "my-project",
sessionEpoch: 7,
priority: "interaction",
rich: true,
});
}
});
});
@@ -2,9 +2,12 @@ import { useCallback, type ReactNode } from "react";
import { createElement } from "react";
import { CompositionThumbnail, VideoThumbnail } from "../player";
import type { TimelineElement } from "../player";
import type { TimelineClipRenderContext } from "../player/components/TimelineTypes";
import { AudioWaveform } from "../player/components/AudioWaveform";
import { ImageThumbnail } from "../player/components/ImageThumbnail";
import { encodePreviewPath, resolveMediaPreviewUrl } from "../player/components/thumbnailUtils";
import { usePlayerStore } from "../player/store/playerStore";
import { effectiveThumbnailMode } from "../player/lib/thumbnailPolicy";
export function normalizeCompositionSrc(
compSrc: string,
@@ -51,7 +54,13 @@ function trimFractions(el: TimelineElement): { start?: number; end?: number } {
* Build the waveform element for an audio clip, windowing the rendered peaks to
* the trimmed source slice so the bars track the clip edges.
*/
function renderAudioClip(el: TimelineElement, pid: string, labelColor: string): ReactNode {
function renderAudioClip(
el: TimelineElement,
pid: string,
sessionEpoch: number,
labelColor: string,
context: TimelineClipRenderContext,
): ReactNode {
const srcRelative = resolvePreviewRelative(el.src, pid);
// Encode each path segment (spaces, parens, U+202F, unicode) so the URL matches
// what the assets panel loads — a raw segment 404s. resolvePreviewRelative
@@ -71,6 +80,9 @@ function renderAudioClip(el: TimelineElement, pid: string, labelColor: string):
labelColor,
trimStartFraction: start,
trimEndFraction: end,
projectId: pid,
sessionEpoch,
priority: context.priority,
});
}
@@ -87,13 +99,29 @@ export function useRenderClipContent({
activePreviewUrl,
effectiveTimelineDuration,
}: UseRenderClipContentOptions) {
// Self-sourced so the adaptive policy gates thumbnail generation without App plumbing.
const thumbnailMode = usePlayerStore((s) => s.thumbnailMode);
const effectiveMode = effectiveThumbnailMode(thumbnailMode);
const sessionEpoch = usePlayerStore((s) => s.timelineSessionEpoch);
return useCallback(
// Pre-existing clip-content dispatcher; reduced by extracting renderAudioClip.
// fallow-ignore-next-line complexity
(el: TimelineElement, style: { clip: string; label: string }): ReactNode => {
(
el: TimelineElement,
style: { clip: string; label: string },
context: TimelineClipRenderContext = { priority: "visible", rich: false },
): ReactNode => {
const pid = projectIdRef.current;
if (!pid) return null;
// Thumbnail generation disabled (perf) -> plain clip bars. Audio still shows
// its waveform (cheap, not a frame thumbnail). Toggle: timeline toolbar.
if (effectiveMode === "hidden") {
return el.tag === "audio"
? renderAudioClip(el, pid, sessionEpoch, style.label, context)
: null;
}
let compSrc = el.compositionSrc;
if (compSrc) {
compSrc = normalizeCompositionSrc(compSrc, pid, window.location.origin);
@@ -116,6 +144,10 @@ export function useRenderClipContent({
seekTime: 0,
duration: el.duration,
projectId: pid,
sessionEpoch,
priority: context.priority,
rich: context.rich,
});
}
@@ -123,7 +155,7 @@ export function useRenderClipContent({
// activePreviewUrl thumbnail branch; audio rows need waveform data, not a
// captured frame from the currently drilled composition preview.
if (el.tag === "audio") {
return renderAudioClip(el, pid, style.label);
return renderAudioClip(el, pid, sessionEpoch, style.label, context);
}
// When drilled into a composition, render all inner elements via
@@ -138,6 +170,10 @@ export function useRenderClipContent({
selectorIndex: el.selectorIndex,
seekTime: el.start,
duration: el.duration,
projectId: pid,
sessionEpoch,
priority: context.priority,
rich: context.rich,
});
}
@@ -157,6 +193,10 @@ export function useRenderClipContent({
imageSrc: mediaSrc,
label: "",
labelColor: style.label,
projectId: pid,
sessionEpoch,
priority: context.priority,
rich: context.rich,
});
}
return createElement(VideoThumbnail, {
@@ -164,6 +204,12 @@ export function useRenderClipContent({
label: "",
labelColor: style.label,
duration: el.duration,
sourceStart: el.playbackStart,
sourceRangeDuration: el.duration * (el.playbackRate ?? 1),
projectId: pid,
sessionEpoch,
priority: context.priority,
rich: context.rich,
});
}
@@ -177,11 +223,22 @@ export function useRenderClipContent({
selectorIndex: el.selectorIndex,
seekTime: el.start,
duration: el.duration,
projectId: pid,
sessionEpoch,
priority: context.priority,
rich: context.rich,
});
}
return null;
},
[projectIdRef, compIdToSrc, activePreviewUrl, effectiveTimelineDuration],
[
projectIdRef,
compIdToSrc,
activePreviewUrl,
effectiveTimelineDuration,
effectiveMode,
sessionEpoch,
],
);
}
@@ -0,0 +1,150 @@
// @vitest-environment happy-dom
import React, { act } from "react";
import { createRoot } from "react-dom/client";
import { afterEach, describe, expect, it, vi } from "vitest";
import { ThumbnailScheduler, type ThumbnailRequest } from "../player/lib/thumbnailScheduler";
import { useThumbnailLease } from "./useThumbnailLease";
(globalThis as { IS_REACT_ACT_ENVIRONMENT?: boolean }).IS_REACT_ACT_ENVIRONMENT = true;
afterEach(() => {
document.body.innerHTML = "";
});
describe("useThumbnailLease", () => {
it("subscribes once, publishes the result, and releases on unmount", async () => {
const scheduler = new ThumbnailScheduler();
const load = vi.fn(async () => ({
value: { kind: "image" as const, url: "blob:poster", aspect: 16 / 9 },
weight: 10,
}));
const request: ThumbnailRequest = {
key: "poster",
projectId: "demo",
sessionEpoch: 1,
kind: "image",
priority: "visible",
load,
};
let status = "missing";
function Probe() {
status = useThumbnailLease(request, scheduler).status;
return null;
}
const root = createRoot(document.createElement("div"));
await act(async () => {
root.render(React.createElement(Probe));
await Promise.resolve();
});
expect(load).toHaveBeenCalledTimes(1);
expect(status).toBe("ready");
expect(scheduler.getDiagnostics().leases).toBe(1);
act(() => root.unmount());
expect(scheduler.getDiagnostics().leases).toBe(0);
});
it("does not acquire work for a null request", () => {
const scheduler = new ThumbnailScheduler();
let status = "missing";
function Probe() {
status = useThumbnailLease(null, scheduler).status;
return null;
}
const root = createRoot(document.createElement("div"));
act(() => root.render(React.createElement(Probe)));
expect(status).toBe("idle");
expect(scheduler.getDiagnostics().leases).toBe(0);
act(() => root.unmount());
});
it("updates priority without restarting the active request", async () => {
const scheduler = new ThumbnailScheduler();
let resolve!: (value: {
value: { kind: "image"; url: string; aspect: number };
weight: number;
}) => void;
const pending = new Promise<{
value: { kind: "image"; url: string; aspect: number };
weight: number;
}>((accept) => {
resolve = accept;
});
const load = vi.fn(() => pending);
let priority: ThumbnailRequest["priority"] = "overscan";
function Probe() {
useThumbnailLease(
{
key: "same-content",
projectId: "demo",
sessionEpoch: 1,
kind: "image",
priority,
load,
},
scheduler,
);
return null;
}
const root = createRoot(document.createElement("div"));
act(() => root.render(React.createElement(Probe)));
priority = "interaction";
act(() => root.render(React.createElement(Probe)));
expect(load).toHaveBeenCalledTimes(1);
await act(async () => {
resolve({ value: { kind: "image", url: "blob:done", aspect: 1 }, weight: 1 });
await pending;
});
act(() => root.unmount());
});
it("resubscribes when the request work shape changes", async () => {
const scheduler = new ThumbnailScheduler();
const imageLoad = vi.fn(async () => ({
value: { kind: "image" as const, url: "blob:image", aspect: 1 },
weight: 1,
}));
const videoLoad = vi.fn(async () => ({
value: { kind: "image" as const, url: "blob:video", aspect: 1 },
weight: 1,
}));
let kind: ThumbnailRequest["kind"] = "image";
let rich = false;
function Probe() {
useThumbnailLease(
{
key: "same-content",
projectId: "demo",
sessionEpoch: 1,
kind,
rich,
priority: "visible",
load: kind === "image" ? imageLoad : videoLoad,
},
scheduler,
);
return null;
}
const root = createRoot(document.createElement("div"));
await act(async () => {
root.render(React.createElement(Probe));
await Promise.resolve();
});
kind = "video";
rich = true;
await act(async () => {
root.render(React.createElement(Probe));
await Promise.resolve();
});
expect(imageLoad).toHaveBeenCalledTimes(1);
expect(videoLoad).toHaveBeenCalledTimes(1);
expect(scheduler.getDiagnostics().leases).toBe(1);
act(() => root.unmount());
});
});
@@ -0,0 +1,44 @@
import { useCallback, useLayoutEffect, useRef, useSyncExternalStore } from "react";
import {
createThumbnailRequestIdentity,
thumbnailScheduler,
type ThumbnailRequest,
type ThumbnailScheduler,
type ThumbnailSnapshot,
} from "../player/lib/thumbnailScheduler";
const IDLE: ThumbnailSnapshot = Object.freeze({ status: "idle" });
export function useThumbnailLease(
request: ThumbnailRequest | null,
scheduler: ThumbnailScheduler = thumbnailScheduler,
): ThumbnailSnapshot {
const requestRef = useRef(request);
requestRef.current = request;
const leaseRef = useRef<ReturnType<ThumbnailScheduler["acquire"]> | null>(null);
const identity = request ? createThumbnailRequestIdentity(request) : null;
const priority = request?.priority;
const subscribe = useCallback(
(listener: () => void) => {
const current = requestRef.current;
if (!current || identity === null) return () => {};
const lease = scheduler.acquire(current, listener);
leaseRef.current = lease;
return () => {
if (leaseRef.current === lease) leaseRef.current = null;
lease.release();
};
},
[identity, scheduler],
);
const getSnapshot = useCallback(() => {
const current = requestRef.current;
return current && identity !== null ? scheduler.getSnapshot(current) : IDLE;
}, [identity, scheduler]);
useLayoutEffect(() => {
if (priority) leaseRef.current?.updatePriority(priority);
}, [priority]);
return useSyncExternalStore(subscribe, getSnapshot, () => IDLE);
}
@@ -24,6 +24,7 @@ interface HarnessProps {
options?: { revealPanel?: boolean; additive?: boolean; preserveGroup?: boolean },
) => void;
applyMarqueeSelection: (selections: DomEditSelection[], additive: boolean) => void;
onSelectionNotFound: () => void;
}
afterEach(() => {
@@ -67,8 +68,8 @@ function makeSyncFixture() {
const firstSelection = makeSelection("First", firstElement);
const secondSelection = makeSelection("Second", secondElement);
const timelineElements: TimelineElement[] = [
{ id: "clip-1", tag: "div", start: 0, duration: 1, track: 0 },
{ id: "clip-2", tag: "div", start: 1, duration: 1, track: 1 },
{ id: "clip-1", domId: "clip-1", tag: "div", start: 0, duration: 1, track: 0 },
{ id: "clip-2", domId: "clip-2", tag: "div", start: 1, duration: 1, track: 1 },
];
const selectionById = new Map([
["clip-1", firstSelection],
@@ -96,6 +97,7 @@ describe("useTimelineSelectionPreviewSync", () => {
buildDomSelectionForTimelineElement,
applyDomSelection,
applyMarqueeSelection,
onSelectionNotFound: vi.fn(),
});
expect(applyMarqueeSelection).toHaveBeenCalledWith([secondSelection, firstSelection], false);
@@ -108,6 +110,21 @@ describe("useTimelineSelectionPreviewSync", () => {
const applyDomSelection = vi.fn();
const applyMarqueeSelection = vi.fn();
const harness = renderHarness();
const buildDomSelectionForTimelineElement = vi.fn(async (element: TimelineElement) => {
return selectionById.get(element.id) ?? null;
});
await harness.rerender({
selectedElementId: "clip-1",
selectedElementIds: new Set(["clip-1"]),
timelineElements,
domEditSelection: firstSelection,
domEditGroupSelections: [firstSelection],
buildDomSelectionForTimelineElement,
applyDomSelection,
applyMarqueeSelection,
onSelectionNotFound: vi.fn(),
});
await harness.rerender({
selectedElementId: null,
@@ -115,15 +132,72 @@ describe("useTimelineSelectionPreviewSync", () => {
timelineElements,
domEditSelection: firstSelection,
domEditGroupSelections: [firstSelection],
buildDomSelectionForTimelineElement: vi.fn(async (element: TimelineElement) => {
return selectionById.get(element.id) ?? null;
}),
buildDomSelectionForTimelineElement,
applyDomSelection,
applyMarqueeSelection,
onSelectionNotFound: vi.fn(),
});
expect(applyDomSelection).toHaveBeenCalledWith(null, { revealPanel: false });
expect(applyMarqueeSelection).not.toHaveBeenCalled();
harness.cleanup();
});
it("warns once while retrying a timeline selection after preview refreshes", async () => {
const { secondSelection, timelineElements } = makeSyncFixture();
const applyDomSelection = vi.fn();
const applyMarqueeSelection = vi.fn();
const onSelectionNotFound = vi.fn();
let previewReady = false;
const buildDomSelectionForTimelineElement = vi.fn(async () =>
previewReady ? secondSelection : null,
);
const selectedElementIds = new Set(["clip-2"]);
const harness = renderHarness();
await harness.rerender({
selectedElementId: "clip-2",
selectedElementIds,
timelineElements,
domEditSelection: null,
domEditGroupSelections: [],
buildDomSelectionForTimelineElement,
applyDomSelection,
applyMarqueeSelection,
onSelectionNotFound,
});
expect(onSelectionNotFound).toHaveBeenCalledOnce();
expect(applyDomSelection).not.toHaveBeenCalled();
await harness.rerender({
selectedElementId: "clip-2",
selectedElementIds,
timelineElements: [...timelineElements],
domEditSelection: null,
domEditGroupSelections: [],
buildDomSelectionForTimelineElement,
applyDomSelection,
applyMarqueeSelection,
onSelectionNotFound,
});
expect(onSelectionNotFound).toHaveBeenCalledOnce();
previewReady = true;
await harness.rerender({
selectedElementId: "clip-2",
selectedElementIds,
timelineElements: [...timelineElements],
domEditSelection: null,
domEditGroupSelections: [],
buildDomSelectionForTimelineElement,
applyDomSelection,
applyMarqueeSelection,
onSelectionNotFound,
});
expect(applyDomSelection).toHaveBeenCalledWith(secondSelection);
harness.cleanup();
});
});
@@ -1,4 +1,4 @@
import { useEffect, useMemo } from "react";
import { useEffect, useMemo, useRef } from "react";
import type { TimelineElement } from "../player";
import type { DomEditSelection } from "../components/editor/domEditing";
import { resolveTimelineIdForSelection } from "../utils/studioHelpers";
@@ -18,6 +18,7 @@ interface UseTimelineSelectionPreviewSyncParams {
options?: { revealPanel?: boolean; additive?: boolean; preserveGroup?: boolean },
) => void;
applyMarqueeSelection: (selections: DomEditSelection[], additive: boolean) => void;
onSelectionNotFound: () => void;
}
function orderSelectedIds(ids: Set<string>, anchor: string | null): string[] {
@@ -56,34 +57,51 @@ export function useTimelineSelectionPreviewSync({
buildDomSelectionForTimelineElement,
applyDomSelection,
applyMarqueeSelection,
onSelectionNotFound,
}: UseTimelineSelectionPreviewSyncParams): void {
const selectedIds = useMemo(
() => orderSelectedIds(selectedElementIds, selectedElementId),
[selectedElementId, selectedElementIds],
);
const selectedKey = selectedIds.join("\0");
const domEditSelectionRef = useRef(domEditSelection);
const domEditGroupSelectionsRef = useRef(domEditGroupSelections);
const lastSyncedSelectedKeyRef = useRef("");
const missingSelectionKeyRef = useRef("");
domEditSelectionRef.current = domEditSelection;
domEditGroupSelectionsRef.current = domEditGroupSelections;
useEffect(() => {
const previousSelectedKey = lastSyncedSelectedKeyRef.current;
lastSyncedSelectedKeyRef.current = selectedKey;
const currentDomEditSelection = domEditSelectionRef.current;
const currentDomEditGroupSelections = domEditGroupSelectionsRef.current;
const currentSelections =
domEditGroupSelections.length > 1
? domEditGroupSelections
: domEditSelection
? [domEditSelection]
currentDomEditGroupSelections.length > 1
? currentDomEditGroupSelections
: currentDomEditSelection
? [currentDomEditSelection]
: [];
const currentIds = currentSelections
.map((selection) =>
resolveTimelineIdForSelection(selection, timelineElements, activeCompPath),
)
.filter((id): id is string => Boolean(id));
const currentAnchor = domEditSelection
? resolveTimelineIdForSelection(domEditSelection, timelineElements, activeCompPath)
const currentAnchor = currentDomEditSelection
? resolveTimelineIdForSelection(currentDomEditSelection, timelineElements, activeCompPath)
: null;
if (selectedIds.length === 0) {
if (currentSelections.length > 0) applyDomSelection(null, { revealPanel: false });
missingSelectionKeyRef.current = "";
if (previousSelectedKey.length > 0 && currentIds.length > 0) {
applyDomSelection(null, { revealPanel: false });
}
return;
}
if (selectionIdsMatch(currentIds, selectedIds, currentAnchor, selectedElementId)) {
missingSelectionKeyRef.current = "";
return;
}
if (selectionIdsMatch(currentIds, selectedIds, currentAnchor, selectedElementId)) return;
let cancelled = false;
const syncSelection = async () => {
@@ -101,11 +119,18 @@ export function useTimelineSelectionPreviewSync({
// shrunk set back and silently drop the members whose DOM node was not ready.
// Bail instead; a later effect run (on timelineElements/DOM change) applies the
// full set once every resolvable member has a live node.
if (selections.length < resolvableCount) return;
if (selections.length < resolvableCount) {
if (missingSelectionKeyRef.current !== selectedKey) {
missingSelectionKeyRef.current = selectedKey;
onSelectionNotFound();
}
return;
}
missingSelectionKeyRef.current = "";
if (selections.length === 0) {
applyDomSelection(null, { revealPanel: false });
} else if (selections.length === 1) {
applyDomSelection(selections[0], { revealPanel: false });
applyDomSelection(selections[0]);
} else {
applyMarqueeSelection(selections, false);
}
@@ -115,13 +140,15 @@ export function useTimelineSelectionPreviewSync({
return () => {
cancelled = true;
};
// DOM selection changes are read through refs. Depending on them directly
// would let the preview-to-timeline echo cancel an in-flight timeline click.
// eslint-disable-next-line react-hooks/exhaustive-deps
}, [
activeCompPath,
applyDomSelection,
applyMarqueeSelection,
buildDomSelectionForTimelineElement,
domEditGroupSelections,
domEditSelection,
onSelectionNotFound,
selectedElementId,
selectedIds,
selectedKey,