fix(studio): enable timeline resize for all elements, improve perf and UX

Enable trim-start and trim-end for all authored timeline elements (divs,
sections, compositions) — not just video/audio/img. The deterministic-window
gate was overly restrictive since all non-implicit elements have authored
data-start/data-duration that define their timeline window.

Replace iframe reload after resize/move with direct DOM attribute patching
via patchIframeDomTiming(). This eliminates playhead-jump-to-zero, visual
blinking, and race conditions from file-watcher echoes. File persistence
runs in a serialized background queue (persistTimelineEdit + enqueueEdit)
so rapid edits don't overwrite each other.

Add mediabunny-based media probe service (mediaProbe.ts) for fast metadata
extraction from file headers. Timeline elements missing sourceDuration are
enriched asynchronously without waiting for DOM loadedmetadata events.

Tune the runtime media preloader: lower lazy threshold from 6 to 3 clips,
add 3s lookbehind window for reverse scrub, adaptive promoted-clip cap.

Deduplicate getTimelineEditCapabilities — computed once in TimelineCanvas
and passed as a prop to TimelineClip instead of recomputing per clip.

Remove dead PlaybackAdapter re-export from useTimelinePlayer — all consumers
import directly from playbackTypes.
This commit is contained in:
Miguel Ángel
2026-05-19 20:40:39 -04:00
parent 4237165517
commit 89f9ca196d
13 changed files with 460 additions and 346 deletions
+2 -1
View File
@@ -38,7 +38,8 @@
"@codemirror/view": "6.40.0",
"@hyperframes/core": "workspace:*",
"@hyperframes/player": "workspace:*",
"@phosphor-icons/react": "^2.1.10"
"@phosphor-icons/react": "^2.1.10",
"mediabunny": "^1.45.3"
},
"devDependencies": {
"@hyperframes/producer": "workspace:*",
+5 -5
View File
@@ -117,12 +117,9 @@ export function StudioApp() {
});
const editHistory = usePersistentEditHistory({ projectId });
const domEditSaveTimestampRef = useRef(0);
const pendingTimelineEditPathRef = useRef<string | null>(null);
const reloadPreview = useCallback(() => {
try {
previewIframeRef.current?.contentWindow?.location.reload();
} catch {
setRefreshKey((k) => k + 1);
}
setRefreshKey((k) => k + 1);
}, []);
const fileManager = useFileManager({
@@ -155,6 +152,7 @@ export function StudioApp() {
activeCompPathRef,
domEditSaveTimestampRef,
reloadPreview: () => setRefreshKey((k) => k + 1),
pendingTimelineEditPathRef,
});
const timelineEditing = useTimelineEditing({
@@ -166,6 +164,8 @@ export function StudioApp() {
recordEdit: editHistory.recordEdit,
domEditSaveTimestampRef,
reloadPreview,
previewIframeRef,
pendingTimelineEditPathRef,
uploadProjectFiles: fileManager.uploadProjectFiles,
});
@@ -26,8 +26,11 @@ interface UseManifestPersistenceParams {
previewIframeRef: React.MutableRefObject<HTMLIFrameElement | null>;
activeCompPathRef: React.MutableRefObject<string | null>;
/** Shared timestamp ref — written by any studio save (code tab, timeline, DOM edits).
* Used to suppress SSE echoes so we don't double-reload after our own saves. */
* Used to suppress file-change echoes so we don't reload after our own saves. */
domEditSaveTimestampRef: React.MutableRefObject<number>;
/** Tracks in-flight timeline edits that patch the iframe DOM directly. File-change
* events for these paths are always suppressed since the preview is already up-to-date. */
pendingTimelineEditPathRef?: React.MutableRefObject<string | null>;
/** Called to reload the preview after undo/redo or external file changes. */
reloadPreview: () => void;
}
@@ -44,6 +47,7 @@ export function useManifestPersistence({
activeCompPathRef: _activeCompPathRef,
domEditSaveTimestampRef,
reloadPreview,
pendingTimelineEditPathRef,
}: UseManifestPersistenceParams) {
void _showToast;
void _recordEdit;
@@ -162,8 +166,12 @@ export function useManifestPersistence({
const handler = (payload?: unknown) => {
const changedPath = readStudioFileChangePath(payload);
if (!changedPath) return;
const recentDomEditSave = Date.now() - domEditSaveTimestampRef.current < 1200;
// External file change — reload unless it's an echo of our own save.
const recentDomEditSave = Date.now() - domEditSaveTimestampRef.current < 4000;
const pendingPath = pendingTimelineEditPathRef?.current;
if (pendingPath && changedPath.endsWith(pendingPath)) {
pendingTimelineEditPathRef!.current = null;
return;
}
if (!recentDomEditSave) {
reloadPreview();
}
+175 -101
View File
@@ -38,6 +38,8 @@ interface UseTimelineEditingOptions {
recordEdit: (input: RecordEditInput) => Promise<void>;
domEditSaveTimestampRef: React.MutableRefObject<number>;
reloadPreview: () => void;
previewIframeRef: React.RefObject<HTMLIFrameElement | null>;
pendingTimelineEditPathRef: React.MutableRefObject<string | null>;
uploadProjectFiles: (files: Iterable<File>, dir?: string) => Promise<string[]>;
}
@@ -53,6 +55,87 @@ function buildPatchTarget(element: { domId?: string; selector?: string; selector
return null;
}
function findIframeElement(
iframe: HTMLIFrameElement | null,
element: { domId?: string; selector?: string; selectorIndex?: number },
): Element | null {
const doc = iframe?.contentDocument;
if (!doc) return null;
if (element.domId) return doc.getElementById(element.domId);
if (!element.selector) return null;
return doc.querySelectorAll(element.selector)[element.selectorIndex ?? 0] ?? null;
}
const TIMING_ATTR_MAP: Record<string, string> = {
start: "data-start",
duration: "data-duration",
track: "data-track-index",
};
function patchIframeDomTiming(
iframe: HTMLIFrameElement | null,
element: TimelineElement,
updates: { start?: number; duration?: number; track?: number; playbackStart?: number },
): void {
try {
const el = findIframeElement(iframe, element);
if (!el) return;
for (const [key, attr] of Object.entries(TIMING_ATTR_MAP)) {
const val = updates[key as keyof typeof updates];
if (val != null) el.setAttribute(attr, formatTimelineAttributeNumber(val));
}
if (updates.playbackStart != null) {
const attr =
element.playbackStartAttr === "playback-start" ? "data-playback-start" : "data-media-start";
el.setAttribute(attr, formatTimelineAttributeNumber(updates.playbackStart));
}
} catch {
// Cross-origin or mid-navigation — safe to ignore, file is already saved.
}
}
type PatchTarget = NonNullable<ReturnType<typeof buildPatchTarget>>;
interface PersistTimelineEditInput {
projectId: string;
element: TimelineElement;
activeCompPath: string | null;
label: string;
buildPatches: (original: string, target: PatchTarget) => string;
writeProjectFile: (path: string, content: string) => Promise<void>;
recordEdit: (input: RecordEditInput) => Promise<void>;
domEditSaveTimestampRef: React.MutableRefObject<number>;
pendingTimelineEditPathRef: React.MutableRefObject<string | null>;
}
async function persistTimelineEdit(input: PersistTimelineEditInput): Promise<void> {
const targetPath = input.element.sourceFile || input.activeCompPath || "index.html";
const originalContent = await readFileContent(input.projectId, targetPath);
const patchTarget = buildPatchTarget(input.element);
if (!patchTarget) {
throw new Error(`Timeline element ${input.element.id} is missing a patchable target`);
}
const patchedContent = input.buildPatches(originalContent, patchTarget);
if (patchedContent === originalContent) {
throw new Error(`Unable to patch timeline element ${input.element.id} in ${targetPath}`);
}
input.pendingTimelineEditPathRef.current = targetPath;
input.domEditSaveTimestampRef.current = Date.now();
await saveProjectFilesWithHistory({
projectId: input.projectId,
label: input.label,
kind: "timeline",
files: { [targetPath]: patchedContent },
readFile: async () => originalContent,
writeFile: input.writeProjectFile,
recordEdit: input.recordEdit,
});
input.domEditSaveTimestampRef.current = Date.now();
}
async function readFileContent(projectId: string, targetPath: string): Promise<string> {
const response = await fetch(
`/api/projects/${projectId}/files/${encodeURIComponent(targetPath)}`,
@@ -78,127 +161,118 @@ export function useTimelineEditing({
recordEdit,
domEditSaveTimestampRef,
reloadPreview,
previewIframeRef,
pendingTimelineEditPathRef,
uploadProjectFiles,
}: UseTimelineEditingOptions) {
const projectIdRef = useRef(projectId);
projectIdRef.current = projectId;
const editQueueRef = useRef(Promise.resolve());
const lastBlockedTimelineToastAtRef = useRef(0);
const handleTimelineElementMove = useCallback(
async (element: TimelineElement, updates: Pick<TimelineElement, "start" | "track">) => {
const enqueueEdit = useCallback(
(
element: TimelineElement,
label: string,
buildPatches: PersistTimelineEditInput["buildPatches"],
) => {
const pid = projectIdRef.current;
if (!pid) throw new Error("No active project");
const targetPath = element.sourceFile || activeCompPath || "index.html";
const originalContent = await readFileContent(pid, targetPath);
const patchTarget = buildPatchTarget(element);
if (!patchTarget) {
throw new Error(`Timeline element ${element.id} is missing a patchable target`);
}
let patchedContent = applyPatchByTarget(originalContent, patchTarget, {
type: "attribute",
property: "start",
value: formatTimelineAttributeNumber(updates.start),
});
patchedContent = applyPatchByTarget(patchedContent, patchTarget, {
type: "attribute",
property: "track-index",
value: String(updates.track),
});
if (patchedContent === originalContent) {
throw new Error(`Unable to patch timeline element ${element.id} in ${targetPath}`);
}
domEditSaveTimestampRef.current = Date.now();
await saveProjectFilesWithHistory({
projectId: pid,
label: "Move timeline clip",
kind: "timeline",
files: { [targetPath]: patchedContent },
readFile: async () => originalContent,
writeFile: writeProjectFile,
recordEdit,
});
reloadPreview();
if (!pid) return;
editQueueRef.current = editQueueRef.current
.then(() =>
persistTimelineEdit({
projectId: pid,
element,
activeCompPath,
label,
buildPatches,
writeProjectFile,
recordEdit,
domEditSaveTimestampRef,
pendingTimelineEditPathRef,
}),
)
.catch((error) => {
console.error(`[Timeline] Failed to persist: ${label}`, error);
});
},
[activeCompPath, recordEdit, writeProjectFile, domEditSaveTimestampRef, reloadPreview],
[
activeCompPath,
recordEdit,
writeProjectFile,
domEditSaveTimestampRef,
pendingTimelineEditPathRef,
],
);
const handleTimelineElementMove = useCallback(
(element: TimelineElement, updates: Pick<TimelineElement, "start" | "track">) => {
patchIframeDomTiming(previewIframeRef.current, element, updates);
enqueueEdit(element, "Move timeline clip", (original, target) => {
let patched = applyPatchByTarget(original, target, {
type: "attribute",
property: "start",
value: formatTimelineAttributeNumber(updates.start),
});
return applyPatchByTarget(patched, target, {
type: "attribute",
property: "track-index",
value: String(updates.track),
});
});
},
[previewIframeRef, enqueueEdit],
);
const handleTimelineElementResize = useCallback(
async (
(
element: TimelineElement,
updates: Pick<TimelineElement, "start" | "duration" | "playbackStart">,
) => {
const pid = projectIdRef.current;
if (!pid) throw new Error("No active project");
patchIframeDomTiming(previewIframeRef.current, element, updates);
enqueueEdit(element, "Resize timeline clip", (original, target) => {
const playbackStartAttrName =
element.playbackStartAttr === "playback-start" ? "playback-start" : "media-start";
const currentPlaybackStartValue =
readAttributeByTarget(original, target, "playback-start") ??
readAttributeByTarget(original, target, "media-start");
const currentPlaybackStart =
currentPlaybackStartValue != null ? parseFloat(currentPlaybackStartValue) : undefined;
const trimDelta = updates.start - element.start;
const fallbackPlaybackStart =
updates.playbackStart == null &&
trimDelta !== 0 &&
Number.isFinite(currentPlaybackStart) &&
currentPlaybackStart != null
? Math.max(
0,
currentPlaybackStart + trimDelta * Math.max(element.playbackRate ?? 1, 0.1),
)
: undefined;
const nextPlaybackStart = updates.playbackStart ?? fallbackPlaybackStart;
const targetPath = element.sourceFile || activeCompPath || "index.html";
const originalContent = await readFileContent(pid, targetPath);
const patchTarget = buildPatchTarget(element);
if (!patchTarget) {
throw new Error(`Timeline element ${element.id} is missing a patchable target`);
}
const playbackStartAttrName =
element.playbackStartAttr === "playback-start" ? "playback-start" : "media-start";
const currentPlaybackStartValue =
readAttributeByTarget(originalContent, patchTarget, "playback-start") ??
readAttributeByTarget(originalContent, patchTarget, "media-start");
const currentPlaybackStart =
currentPlaybackStartValue != null ? parseFloat(currentPlaybackStartValue) : undefined;
const trimDelta = updates.start - element.start;
const fallbackPlaybackStart =
updates.playbackStart == null &&
trimDelta !== 0 &&
Number.isFinite(currentPlaybackStart) &&
currentPlaybackStart != null
? Math.max(0, currentPlaybackStart + trimDelta * Math.max(element.playbackRate ?? 1, 0.1))
: undefined;
const nextPlaybackStart = updates.playbackStart ?? fallbackPlaybackStart;
let patchedContent = originalContent;
patchedContent = applyPatchByTarget(patchedContent, patchTarget, {
type: "attribute",
property: "start",
value: formatTimelineAttributeNumber(updates.start),
});
patchedContent = applyPatchByTarget(patchedContent, patchTarget, {
type: "attribute",
property: "duration",
value: formatTimelineAttributeNumber(updates.duration),
});
if (nextPlaybackStart != null) {
patchedContent = applyPatchByTarget(patchedContent, patchTarget, {
let patched = applyPatchByTarget(original, target, {
type: "attribute",
property: playbackStartAttrName,
value: formatTimelineAttributeNumber(nextPlaybackStart),
property: "start",
value: formatTimelineAttributeNumber(updates.start),
});
}
if (patchedContent === originalContent) {
throw new Error(`Unable to patch timeline element ${element.id} in ${targetPath}`);
}
domEditSaveTimestampRef.current = Date.now();
await saveProjectFilesWithHistory({
projectId: pid,
label: "Resize timeline clip",
kind: "timeline",
files: { [targetPath]: patchedContent },
readFile: async () => originalContent,
writeFile: writeProjectFile,
recordEdit,
patched = applyPatchByTarget(patched, target, {
type: "attribute",
property: "duration",
value: formatTimelineAttributeNumber(updates.duration),
});
if (nextPlaybackStart != null) {
patched = applyPatchByTarget(patched, target, {
type: "attribute",
property: playbackStartAttrName,
value: formatTimelineAttributeNumber(nextPlaybackStart),
});
}
return patched;
});
reloadPreview();
},
[activeCompPath, recordEdit, writeProjectFile, domEditSaveTimestampRef, reloadPreview],
[previewIframeRef, enqueueEdit],
);
const handleTimelineElementDelete = useCallback(
@@ -252,6 +252,7 @@ export const TimelineCanvas = memo(function TimelineCanvas({
isHovered={hoveredClip === clipKey}
isDragging={false}
hasCustomContent={!!renderClipContent}
capabilities={capabilities}
theme={theme}
trackStyle={clipStyle}
isComposition={isComposition}
@@ -369,6 +370,7 @@ export const TimelineCanvas = memo(function TimelineCanvas({
isHovered={false}
isDragging={true}
hasCustomContent={!!renderClipContent}
capabilities={getTimelineEditCapabilities(activeDraggedElement)}
theme={theme}
trackStyle={getTrackStyle(activeDraggedElement.tag)}
isComposition={!!activeDraggedElement.compositionSrc}
@@ -3,7 +3,7 @@ import type { TimelineTrackStyle } from "./timelineTheme";
import { memo, type ReactNode } from "react";
import type { TimelineElement } from "../store/playerStore";
import { defaultTimelineTheme, getClipHandleOpacity, type TimelineTheme } from "./timelineTheme";
import { getTimelineEditCapabilities } from "./timelineEditing";
import type { TimelineEditCapabilities } from "./timelineEditing";
interface TimelineClipProps {
el: TimelineElement;
@@ -13,6 +13,7 @@ interface TimelineClipProps {
isHovered: boolean;
isDragging?: boolean;
hasCustomContent: boolean;
capabilities: TimelineEditCapabilities;
theme?: TimelineTheme;
trackStyle: TimelineTrackStyle;
isComposition: boolean;
@@ -33,6 +34,7 @@ export const TimelineClip = memo(function TimelineClip({
isHovered,
isDragging = false,
hasCustomContent,
capabilities,
theme = defaultTimelineTheme,
trackStyle,
isComposition,
@@ -47,6 +49,7 @@ export const TimelineClip = memo(function TimelineClip({
const leftPx = el.start * pps;
const widthPx = Math.max(el.duration * pps, 4);
const handleOpacity = getClipHandleOpacity({ isHovered, isSelected, isDragging });
const borderColor = isSelected
? theme.clipBorderActive
: isHovered
@@ -59,7 +62,6 @@ export const TimelineClip = memo(function TimelineClip({
: isHovered
? theme.clipShadowHover
: theme.clipShadow;
const capabilities = getTimelineEditCapabilities(el);
const displayLabel = el.label || el.id || el.tag;
const showHandles = handleOpacity > 0.01;
@@ -224,7 +224,7 @@ describe("getTimelineEditCapabilities", () => {
});
});
it("allows moving generic motion clips while keeping trims blocked", () => {
it("allows full editing of generic motion clips with authored timing", () => {
expect(
getTimelineEditCapabilities({
tag: "section",
@@ -233,8 +233,8 @@ describe("getTimelineEditCapabilities", () => {
}),
).toEqual({
canMove: true,
canTrimStart: false,
canTrimEnd: false,
canTrimStart: true,
canTrimEnd: true,
});
});
@@ -285,7 +285,7 @@ describe("getTimelineEditCapabilities", () => {
});
});
it("allows move and end trim for patchable composition hosts", () => {
it("allows full editing for patchable composition hosts", () => {
expect(
getTimelineEditCapabilities({
tag: "div",
@@ -295,7 +295,22 @@ describe("getTimelineEditCapabilities", () => {
}),
).toEqual({
canMove: true,
canTrimStart: false,
canTrimStart: true,
canTrimEnd: true,
});
});
it("allows full editing of explicitly authored generic elements", () => {
expect(
getTimelineEditCapabilities({
tag: "div",
duration: 4,
selector: "#hero-card",
timingSource: "authored",
}),
).toEqual({
canMove: true,
canTrimStart: true,
canTrimEnd: true,
});
});
@@ -576,6 +591,40 @@ describe("resolveTimelineResize", () => {
),
).toEqual({ start: 0.8, duration: 3.2, playbackStart: 0 });
});
it("trims generic element start without media offset", () => {
expect(
resolveTimelineResize(
{
start: 2,
duration: 4,
originClientX: 100,
pixelsPerSecond: 100,
minStart: 0,
maxEnd: 10,
},
"start",
200,
),
).toEqual({ start: 3, duration: 3, playbackStart: undefined });
});
it("extends generic element start leftward to time zero", () => {
expect(
resolveTimelineResize(
{
start: 1,
duration: 3,
originClientX: 100,
pixelsPerSecond: 100,
minStart: 0,
maxEnd: 10,
},
"start",
-200,
),
).toEqual({ start: 0, duration: 4, playbackStart: undefined });
});
});
describe("buildPromptCopyText", () => {
@@ -237,8 +237,8 @@ export function getTimelineEditCapabilities(input: {
const hasDeterministicWindow = isDeterministicTimelineWindow(input);
return {
canMove: canPatch && (hasDeterministicWindow || hasFiniteDuration),
canTrimEnd: canPatch && hasFiniteDuration && hasDeterministicWindow,
canTrimStart: canPatch && hasFiniteDuration && canOffsetTrimClipStart(input),
canTrimEnd: canPatch && hasFiniteDuration,
canTrimStart: canPatch && hasFiniteDuration,
};
}
@@ -7,7 +7,7 @@ import { useTimelineSyncCallbacks } from "./useTimelineSyncCallbacks";
// Re-export public API consumed by tests and external modules.
// All of these were previously defined in this file; they now live in focused
// sub-modules but are re-exported here so existing import sites don't change.
export type { PlaybackAdapter, ClipManifestClip } from "../lib/playbackTypes";
export type { ClipManifestClip } from "../lib/playbackTypes";
export { createStaticSeekPlaybackAdapter } from "../lib/playbackAdapter";
export {
getTimelineElementSelector,
@@ -42,6 +42,7 @@ import {
setPreviewPlaybackRate,
shouldMutePreviewAudio,
} from "../lib/timelineIframeHelpers";
import { probeMediaUrl, getCachedProbe } from "../lib/mediaProbe";
// ---------------------------------------------------------------------------
// Hook
@@ -106,6 +107,31 @@ export function useTimelinePlayer() {
if (!state.timelineReady) {
setTimelineReady(true);
}
// Asynchronously enrich media elements missing sourceDuration via mediabunny.
// The probe reads file headers only — no full decode — so this is cheap.
const needsProbe = mergedElements.filter(
(el) =>
el.src &&
el.sourceDuration == null &&
["video", "audio"].includes(el.tag.toLowerCase()) &&
!getCachedProbe(el.src),
);
if (needsProbe.length > 0) {
void Promise.allSettled(
needsProbe.map(async (el) => {
const result = await probeMediaUrl(el.src!);
if (!result) return;
const current = usePlayerStore.getState().elements;
const key = el.key ?? el.id;
const idx = current.findIndex((e) => (e.key ?? e.id) === key);
if (idx === -1 || current[idx].sourceDuration != null) return;
const patched = current.slice();
patched[idx] = { ...current[idx], sourceDuration: result.duration };
setElements(patched);
}),
);
}
},
[setElements, setTimelineReady, setDuration],
);
@@ -0,0 +1,68 @@
import { Input, UrlSource, ALL_FORMATS } from "mediabunny";
export interface MediaProbeResult {
duration: number;
width?: number;
height?: number;
hasVideo: boolean;
hasAudio: boolean;
}
const cache = new Map<string, MediaProbeResult>();
const inflight = new Map<string, Promise<MediaProbeResult | null>>();
function normalizeUrl(url: string): string {
try {
return new URL(url, window.location.href).href;
} catch {
return url;
}
}
async function probeOne(url: string): Promise<MediaProbeResult | null> {
const input = new Input({
source: new UrlSource(url),
formats: ALL_FORMATS,
});
try {
const duration = await input.getDurationFromMetadata();
if (duration == null || !Number.isFinite(duration) || duration <= 0) return null;
const videoTrack = await input.getPrimaryVideoTrack();
const audioTracks = await input.getAudioTracks();
const result: MediaProbeResult = {
duration,
width: videoTrack?.displayWidth,
height: videoTrack?.displayHeight,
hasVideo: videoTrack != null,
hasAudio: audioTracks.length > 0,
};
return result;
} catch {
return null;
} finally {
input.dispose();
}
}
export function getCachedProbe(url: string): MediaProbeResult | undefined {
return cache.get(normalizeUrl(url));
}
export async function probeMediaUrl(url: string): Promise<MediaProbeResult | null> {
const key = normalizeUrl(url);
const cached = cache.get(key);
if (cached) return cached;
let pending = inflight.get(key);
if (pending) return pending;
pending = probeOne(key).then((result) => {
inflight.delete(key);
if (result) cache.set(key, result);
return result;
});
inflight.set(key, pending);
return pending;
}