ext/hyperframes
2
0
Fork 0
mirror of https://github.com/heygen-com/hyperframes.git synced 2026-09-10 22:20:14 +00:00
Code Issues Packages Projects Releases Wiki Activity

fix: stabilize studio preview and runtime sync (#389)

Browse Source 
## Summary
Stabilize the Studio preview/runtime path so timeline data, preview rendering, and thumbnails stay in sync.

This PR includes:
- preview hot-refresh without remounting the iframe
- runtime duration/timeline fixes so Studio stops drifting from playback state
- thumbnail and selector-based preview fixes
- local Studio runtime serving and player-resolution fixes so dev/CI do not depend on prebuilt player artifacts
- tests around preview identity and thumbnail/runtime behavior

## Why This PR Exists
This is the foundation layer for timeline editing. Without it, the editor was prone to:
- iframe remount flashes after saves
- duration mismatches between preview and timeline
- stale or incorrect thumbnails
- CI/test failures when `@hyperframes/player` artifacts were not prebuilt

## Verification
- `bun run --filter @hyperframes/studio test`
- `bun run --filter @hyperframes/studio typecheck`
- `bun run --filter @hyperframes/core typecheck`
- `bunx oxlint packages/cli/src/server/studioServer.ts packages/core/src/runtime/timeline.ts packages/core/src/runtime/timeline.test.ts packages/core/src/studio-api/routes/thumbnail.ts packages/core/src/studio-api/routes/thumbnail.test.ts packages/core/src/studio-api/types.ts packages/studio/src/components/nle/NLELayout.tsx packages/studio/src/components/nle/NLEPreview.tsx packages/studio/src/components/nle/NLEPreview.test.ts packages/studio/src/player/components/CompositionThumbnail.tsx packages/studio/src/player/components/Player.tsx packages/studio/src/player/hooks/useTimelinePlayer.ts packages/studio/src/player/store/playerStore.ts packages/studio/vite.config.ts`
- `bunx oxfmt --check packages/cli/src/server/studioServer.ts packages/core/src/runtime/timeline.ts packages/core/src/runtime/timeline.test.ts packages/core/src/studio-api/routes/thumbnail.ts packages/core/src/studio-api/routes/thumbnail.test.ts packages/core/src/studio-api/types.ts packages/studio/src/components/nle/NLELayout.tsx packages/studio/src/components/nle/NLEPreview.tsx packages/studio/src/components/nle/NLEPreview.test.ts packages/studio/src/player/components/CompositionThumbnail.tsx packages/studio/src/player/components/Player.tsx packages/studio/src/player/hooks/useTimelinePlayer.ts packages/studio/src/player/store/playerStore.ts packages/studio/vite.config.ts`

## Stack
- base of stack
- followed by `feat: add studio timeline editing`
- followed by `fix: smooth scrubber end seeking`
This commit is contained in:
Miguel Ángel
2026-04-22 01:42:48 +02:00
committed by GitHub
parent 4ce1792601
commit 158204343d
19 changed files with 646 additions and 92 deletions
Download Patch File Download Diff File Expand all files Collapse all files
packages/studio/src/player/hooks/useTimelinePlayer.test.ts
+32
View File
@@ -0,0 +1,32 @@
import { describe, expect, it } from "vitest";
import { mergeTimelineElementsPreservingDowngrades } from "./useTimelinePlayer";
describe("mergeTimelineElementsPreservingDowngrades", () => {
it("preserves missing current elements when a shorter manifest arrives", () => {
expect(
mergeTimelineElementsPreservingDowngrades(
[
{ id: "hero", tag: "div", start: 0, duration: 4, track: 0 },
{ id: "cta", tag: "div", start: 4, duration: 2, track: 1 },
],
[{ id: "hero", tag: "div", start: 0, duration: 4, track: 0 }],
8,
8,
),
).toEqual([
{ id: "hero", tag: "div", start: 0, duration: 4, track: 0 },
{ id: "cta", tag: "div", start: 4, duration: 2, track: 1 },
]);
});
it("accepts longer-duration or same-size updates as authoritative", () => {
expect(
mergeTimelineElementsPreservingDowngrades(
[{ id: "hero", tag: "div", start: 0, duration: 4, track: 0 }],
[{ id: "hero", tag: "div", start: 0, duration: 4, track: 0 }],
4,
6,
),
).toEqual([{ id: "hero", tag: "div", start: 0, duration: 4, track: 0 }]);
});
});
packages/studio/src/player/hooks/useTimelinePlayer.ts
+181 -37
View File
@@ -61,6 +61,50 @@ function wrapTimeline(tl: TimelineLike): PlaybackAdapter {
};
}
function resolveMediaElement(el: Element): HTMLMediaElement | HTMLImageElement | null {
if (el instanceof HTMLMediaElement || el instanceof HTMLImageElement) return el;
const candidate = el.querySelector("video, audio, img");
return candidate instanceof HTMLMediaElement || candidate instanceof HTMLImageElement
? candidate
: null;
}
function applyMediaMetadataFromElement(entry: TimelineElement, el: Element): void {
const mediaStartAttr = el.getAttribute("data-playback-start")
? "playback-start"
: el.getAttribute("data-media-start")
? "media-start"
: undefined;
const mediaStartValue =
el.getAttribute("data-playback-start") ?? el.getAttribute("data-media-start");
if (mediaStartValue != null) {
const playbackStart = parseFloat(mediaStartValue);
if (Number.isFinite(playbackStart)) entry.playbackStart = playbackStart;
}
if (mediaStartAttr) entry.playbackStartAttr = mediaStartAttr;
const mediaEl = resolveMediaElement(el);
if (!mediaEl) return;
entry.tag = mediaEl.tagName.toLowerCase();
const src = mediaEl.getAttribute("src");
if (src) entry.src = src;
if (!(mediaEl instanceof HTMLMediaElement)) return;
const sourceDurationAttr =
el.getAttribute("data-source-duration") ?? mediaEl.getAttribute("data-source-duration");
const sourceDuration = sourceDurationAttr ? parseFloat(sourceDurationAttr) : mediaEl.duration;
if (Number.isFinite(sourceDuration) && sourceDuration > 0) {
entry.sourceDuration = sourceDuration;
}
const playbackRate = mediaEl.defaultPlaybackRate;
if (Number.isFinite(playbackRate) && playbackRate > 0) {
entry.playbackRate = playbackRate;
}
}
/**
* Parse [data-start] elements from a Document into TimelineElement[].
* Shared helper — used by onIframeLoad fallback, handleMessage, and enrichMissingCompositions.
@@ -78,37 +122,46 @@ function parseTimelineFromDOM(doc: Document, rootDuration: number): TimelineElem
if (startStr == null) return;
const start = parseFloat(startStr);
if (isNaN(start)) return;
if (Number.isFinite(rootDuration) && rootDuration > 0 && start >= rootDuration) return;
const tagLower = el.tagName.toLowerCase();
let dur = 0;
const durStr = el.getAttribute("data-duration");
if (durStr != null) dur = parseFloat(durStr);
if (isNaN(dur) || dur <= 0) dur = Math.max(0, rootDuration - start);
if (Number.isFinite(rootDuration) && rootDuration > 0) {
dur = Math.min(dur, Math.max(0, rootDuration - start));
}
if (!Number.isFinite(dur) || dur <= 0) return;
const trackStr = el.getAttribute("data-track-index");
const track = trackStr != null ? parseInt(trackStr, 10) : trackCounter++;
const compId = el.getAttribute("data-composition-id");
const entry: TimelineElement = {
id: el.id || el.className?.split(" ")[0] || tagLower,
id: el.id || compId || el.className?.split(" ")[0] || tagLower,
tag: tagLower,
start,
duration: dur,
track: isNaN(track) ? 0 : track,
selector: getTimelineElementSelector(el),
sourceFile: getTimelineElementSourceFile(el),
};
// Media elements
if (tagLower === "video" || tagLower === "audio" || tagLower === "img") {
const src = el.getAttribute("src");
const mediaEl = resolveMediaElement(el);
if (mediaEl) {
if (mediaEl.tagName === "IMG") {
entry.tag = "img";
}
const src = mediaEl.getAttribute("src");
if (src) entry.src = src;
const ms = el.getAttribute("data-media-start");
if (ms) entry.playbackStart = parseFloat(ms);
const vol = el.getAttribute("data-volume");
const vol = el.getAttribute("data-volume") ?? mediaEl.getAttribute("data-volume");
if (vol) entry.volume = parseFloat(vol);
applyMediaMetadataFromElement(entry, el);
}
// Sub-compositions
const compSrc =
el.getAttribute("data-composition-src") || el.getAttribute("data-composition-file");
const compId = el.getAttribute("data-composition-id");
if (compSrc) {
entry.compositionSrc = compSrc;
} else if (compId && compId !== rootComp?.getAttribute("data-composition-id")) {
@@ -126,6 +179,35 @@ function parseTimelineFromDOM(doc: Document, rootDuration: number): TimelineElem
return els;
}
function getTimelineElementSelector(el: Element): string | undefined {
if (el instanceof HTMLElement && el.id) return `#${el.id}`;
const compId = el.getAttribute("data-composition-id");
if (compId) return `[data-composition-id="${compId}"]`;
if (el instanceof HTMLElement) {
const firstClass = el.className.split(/\s+/).find(Boolean);
if (firstClass) return `.${firstClass}`;
}
return undefined;
}
function getTimelineElementSourceFile(el: Element): string | undefined {
const ownerRoot = el.parentElement?.closest("[data-composition-id]");
return (
ownerRoot?.getAttribute("data-composition-file") ??
ownerRoot?.getAttribute("data-composition-src") ??
undefined
);
}
function findTimelineDomNode(doc: Document, id: string): Element | null {
return (
doc.getElementById(id) ??
doc.querySelector(`[data-composition-id="${id}"]`) ??
doc.querySelector(`.${id}`) ??
null
);
}
function normalizePreviewViewport(doc: Document, win: Window): void {
if (doc.documentElement) {
doc.documentElement.style.overflow = "hidden";
@@ -212,6 +294,29 @@ export function resolveIframe(el: Element | null): HTMLIFrameElement | null {
return el.shadowRoot?.querySelector("iframe") ?? el.querySelector("iframe") ?? null;
}
export function mergeTimelineElementsPreservingDowngrades(
currentElements: TimelineElement[],
nextElements: TimelineElement[],
currentDuration: number,
nextDuration: number,
): TimelineElement[] {
const safeCurrentDuration = Number.isFinite(currentDuration) ? currentDuration : 0;
const safeNextDuration = Number.isFinite(nextDuration) ? nextDuration : 0;
if (
currentElements.length === 0 ||
nextElements.length >= currentElements.length ||
safeNextDuration > safeCurrentDuration
) {
return nextElements;
}
const nextIds = new Set(nextElements.map((element) => element.id));
const preserved = currentElements.filter((element) => !nextIds.has(element.id));
if (preserved.length === 0) return nextElements;
return [...nextElements, ...preserved];
}
export function useTimelinePlayer() {
const iframeRef = useRef<HTMLIFrameElement | null>(null);
const rafRef = useRef<number>(0);
@@ -224,6 +329,24 @@ export function useTimelinePlayer() {
const { setIsPlaying, setCurrentTime, setDuration, setTimelineReady, setElements } =
usePlayerStore.getState();
const syncTimelineElements = useCallback(
(elements: TimelineElement[], nextDuration?: number) => {
const state = usePlayerStore.getState();
const mergedElements = mergeTimelineElementsPreservingDowngrades(
state.elements,
elements,
state.duration,
nextDuration ?? state.duration,
);
setElements(mergedElements);
if (Number.isFinite(nextDuration) && (nextDuration ?? 0) > 0) {
setDuration(nextDuration ?? 0);
}
setTimelineReady(true);
},
[setElements, setTimelineReady, setDuration],
);
const getAdapter = useCallback((): PlaybackAdapter | null => {
try {
const iframe = iframeRef.current;
@@ -364,6 +487,7 @@ export function useTimelinePlayer() {
(clip) => !clip.parentCompositionId || !clipCompositionIds.has(clip.parentCompositionId),
);
const els: TimelineElement[] = filtered.map((clip) => {
let hostEl: Element | null = null;
const entry: TimelineElement = {
id: clip.id || clip.label || clip.tagName || "element",
tag: clip.tagName || clip.kind,
@@ -371,6 +495,19 @@ export function useTimelinePlayer() {
duration: clip.duration,
track: clip.track,
};
try {
const iframeDoc = iframeRef.current?.contentDocument;
if (iframeDoc && entry.id) {
hostEl = findTimelineDomNode(iframeDoc, entry.id);
}
} catch {
/* cross-origin */
}
if (hostEl) {
entry.selector = getTimelineElementSelector(hostEl);
entry.sourceFile = getTimelineElementSourceFile(hostEl);
applyMediaMetadataFromElement(entry, hostEl);
}
if (clip.assetUrl) entry.src = clip.assetUrl;
if (clip.kind === "composition" && clip.compositionId) {
// The bundler renames data-composition-src to data-composition-file
@@ -382,7 +519,7 @@ export function useTimelinePlayer() {
try {
const iframeDoc = iframeRef.current?.contentDocument;
hostEl =
iframeDoc?.querySelector(`[data-composition-id="${clip.compositionId}"]`) ?? null;
iframeDoc?.querySelector(`[data-composition-id="${clip.compositionId}"]`) ?? hostEl;
resolvedSrc =
hostEl?.getAttribute("data-composition-src") ??
hostEl?.getAttribute("data-composition-file") ??
@@ -401,32 +538,34 @@ export function useTimelinePlayer() {
entry.tag = "video";
}
}
if (hostEl) {
entry.selector = getTimelineElementSelector(hostEl);
entry.sourceFile = getTimelineElementSourceFile(hostEl);
}
}
return entry;
});
// Don't downgrade: if we already have more elements with a longer duration,
// skip updates that would show fewer clips (transient runtime state).
const currentElements = usePlayerStore.getState().elements;
const currentDuration = usePlayerStore.getState().duration;
const rawDuration = data.durationInFrames / 30;
// Clamp non-finite or absurdly large durations — the runtime can emit
// Infinity when it detects a loop-inflated GSAP timeline without an
// explicit data-duration on the root composition.
const newDuration = Number.isFinite(rawDuration) && rawDuration < 7200 ? rawDuration : 0;
if (currentElements.length > els.length && newDuration <= currentDuration) {
return; // skip transient downgrade
const effectiveDuration = newDuration > 0 ? newDuration : usePlayerStore.getState().duration;
const clampedEls =
effectiveDuration > 0
? els
.filter((element) => element.start < effectiveDuration)
.map((element) => ({
...element,
duration: Math.min(element.duration, effectiveDuration - element.start),
}))
.filter((element) => element.duration > 0)
: els;
if (clampedEls.length > 0) {
syncTimelineElements(clampedEls, newDuration > 0 ? newDuration : undefined);
}
setElements(els);
// Ensure duration covers the furthest clip end so fit-zoom shows everything
if (els.length > 0) {
const maxEnd = Math.max(...els.map((e) => e.start + e.duration));
const effectiveDur = Math.max(newDuration, maxEnd);
if (Number.isFinite(effectiveDur) && effectiveDur > currentDuration)
setDuration(effectiveDur);
}
if (els.length > 0) setTimelineReady(true);
},
[setElements, setTimelineReady, setDuration],
[syncTimelineElements],
);
/**
@@ -504,6 +643,12 @@ export function useTimelinePlayer() {
}
if (!Number.isFinite(dur) || dur <= 0) return;
if (!Number.isFinite(start)) start = 0;
const rootDuration = usePlayerStore.getState().duration;
if (Number.isFinite(rootDuration) && rootDuration > 0) {
if (start >= rootDuration) return;
dur = Math.min(dur, Math.max(0, rootDuration - start));
if (dur <= 0) return;
}
const trackStr = el.getAttribute("data-track-index");
const track = trackStr != null ? parseInt(trackStr, 10) : 0;
@@ -515,6 +660,8 @@ export function useTimelinePlayer() {
start,
duration: dur,
track: isNaN(track) ? 0 : track,
selector: getTimelineElementSelector(el),
sourceFile: getTimelineElementSourceFile(el),
};
if (compSrc) {
entry.compositionSrc = compSrc;
@@ -551,13 +698,12 @@ export function useTimelinePlayer() {
// Dedup: ensure no missing element duplicates an existing one
const finalIds = new Set(updatedEls.map((e) => e.id));
const dedupedMissing = missing.filter((m) => !finalIds.has(m.id));
setElements([...updatedEls, ...dedupedMissing]);
setTimelineReady(true);
syncTimelineElements([...updatedEls, ...dedupedMissing]);
}
} catch (err) {
console.warn("[useTimelinePlayer] enrichMissingCompositions failed", err);
}
}, [setElements, setTimelineReady]);
}, [syncTimelineElements]);
const onIframeLoad = useCallback(() => {
unmutePreviewMedia(iframeRef.current);
@@ -613,8 +759,7 @@ export function useTimelinePlayer() {
// Fallback: parse data-start elements directly from DOM (raw HTML without runtime)
const els = parseTimelineFromDOM(doc, adapter.getDuration());
if (els.length > 0) {
setElements(els);
setTimelineReady(true);
syncTimelineElements(els);
}
}
@@ -636,7 +781,7 @@ export function useTimelinePlayer() {
: undefined;
// Always show the root composition as a single clip — guarantees
// the timeline is never empty when a valid composition is loaded.
setElements([
syncTimelineElements([
{
id: rootId,
tag: (rootComp as HTMLElement).tagName?.toLowerCase() || "div",
@@ -646,7 +791,6 @@ export function useTimelinePlayer() {
compositionSrc,
},
]);
setTimelineReady(true);
}
}
// The runtime will also postMessage the full timeline after all compositions load.
@@ -672,6 +816,7 @@ export function useTimelinePlayer() {
setIsPlaying,
processTimelineMessage,
enrichMissingCompositions,
syncTimelineElements,
]);
/** Save the current playback time so the next onIframeLoad restores it. */
@@ -745,12 +890,12 @@ export function useTimelinePlayer() {
processTimelineMessageRef.current(data);
// Fill in composition hosts the manifest missed (element-reference starts)
enrichMissingCompositionsRef.current();
// Update duration only if the new value is longer (don't downgrade during generation)
if (data.durationInFrames > 0 && Number.isFinite(data.durationInFrames)) {
const fps = 30;
const dur = data.durationInFrames / fps;
const currentDur = usePlayerStore.getState().duration;
if (dur > currentDur) usePlayerStore.getState().setDuration(dur);
if (dur > 0 && dur < 7200) {
usePlayerStore.getState().setDuration(dur);
}
}
// If manifest produced 0 elements after filtering, try DOM fallback
if (usePlayerStore.getState().elements.length === 0) {
@@ -760,8 +905,7 @@ export function useTimelinePlayer() {
if (doc && adapter) {
const els = parseTimelineFromDOM(doc, adapter.getDuration());
if (els.length > 0) {
setElements(els);
setTimelineReady(true);
syncTimelineElements(els);
}
}
} catch (err) {