ext/hyperframes
2
0
Fork 0
mirror of https://github.com/heygen-com/hyperframes.git synced 2026-09-03 04:38:33 +00:00
Code Issues Packages Projects Releases Wiki Activity

fix(studio): enable timeline resize for all elements, improve perf and UX

Browse Source 
Enable trim-start and trim-end for all authored timeline elements (divs,
sections, compositions) — not just video/audio/img. The deterministic-window
gate was overly restrictive since all non-implicit elements have authored
data-start/data-duration that define their timeline window.

Replace iframe reload after resize/move with direct DOM attribute patching
via patchIframeDomTiming(). This eliminates playhead-jump-to-zero, visual
blinking, and race conditions from file-watcher echoes. File persistence
runs in a serialized background queue (persistTimelineEdit + enqueueEdit)
so rapid edits don't overwrite each other.

Add mediabunny-based media probe service (mediaProbe.ts) for fast metadata
extraction from file headers. Timeline elements missing sourceDuration are
enriched asynchronously without waiting for DOM loadedmetadata events.

Tune the runtime media preloader: lower lazy threshold from 6 to 3 clips,
add 3s lookbehind window for reverse scrub, adaptive promoted-clip cap.

Deduplicate getTimelineEditCapabilities — computed once in TimelineCanvas
and passed as a prop to TimelineClip instead of recomputing per clip.

Remove dead PlaybackAdapter re-export from useTimelinePlayer — all consumers
import directly from playbackTypes.
This commit is contained in:
Miguel Ángel
2026-05-19 20:40:39 -04:00
parent 4237165517
commit 89f9ca196d
13 changed files with 460 additions and 346 deletions
Download Patch File Download Diff File Expand all files Collapse all files
packages/core/src/runtime/mediaPreloader.test.ts
+75 -201
View File
@@ -43,6 +43,19 @@ function setupDOM(elements: HTMLMediaElement[]): void {
}) as typeof document.querySelectorAll;
}
function createTestFixture(
count: number,
options?: Parameters<typeof createMediaPreloadManager>[0],
) {
const elements = Array.from({ length: count }, (_, i) =>
mockMediaElement({ start: String(i * 5), duration: "5" }),
);
setupDOM(elements);
const manager = createMediaPreloadManager(options);
manager.refresh();
return { elements, manager };
}
describe("createMediaPreloadManager", () => {
let elements: HTMLMediaElement[];
@@ -50,7 +63,7 @@ describe("createMediaPreloadManager", () => {
elements = [];
});
it("is not lazy when fewer than 6 media elements", () => {
it("is not lazy when fewer than 3 media elements", () => {
elements = [
mockMediaElement({ start: "0", duration: "5" }),
mockMediaElement({ start: "5", duration: "5" }),
@@ -63,274 +76,135 @@ describe("createMediaPreloadManager", () => {
expect(manager.isLazy()).toBe(false);
});
it("activates lazy mode at exactly LAZY_THRESHOLD (6 elements)", () => {
elements = Array.from({ length: 6 }, (_, i) =>
mockMediaElement({ start: String(i * 5), duration: "5" }),
);
setupDOM(elements);
const manager = createMediaPreloadManager();
manager.refresh();
it("activates lazy mode at exactly LAZY_THRESHOLD (3 elements)", () => {
const { manager } = createTestFixture(3);
expect(manager.isLazy()).toBe(true);
});
it("is not lazy with 5 elements (below threshold)", () => {
elements = Array.from({ length: 5 }, (_, i) =>
mockMediaElement({ start: String(i * 5), duration: "5" }),
);
setupDOM(elements);
const manager = createMediaPreloadManager();
manager.refresh();
it("is not lazy with 2 elements (below threshold)", () => {
const { manager } = createTestFixture(2);
expect(manager.isLazy()).toBe(false);
});
it("activates lazy mode with 8 media elements", () => {
elements = Array.from({ length: 8 }, (_, i) =>
mockMediaElement({ start: String(i * 5), duration: "5" }),
);
setupDOM(elements);
const manager = createMediaPreloadManager();
manager.refresh();
const { manager } = createTestFixture(8);
expect(manager.isLazy()).toBe(true);
});
it("sync promotes clips in the lookahead window", () => {
elements = Array.from({ length: 8 }, (_, i) =>
mockMediaElement({ start: String(i * 5), duration: "5" }),
);
setupDOM(elements);
const manager = createMediaPreloadManager();
manager.refresh();
for (const el of elements) {
el.preload = "metadata";
}
manager.sync(0);
expect(elements[0].preload).toBe("auto");
expect(elements[1].preload).toBe("auto");
expect(elements[7].preload).toBe("metadata");
const f = createTestFixture(8);
for (const el of f.elements) el.preload = "metadata";
f.manager.sync(0);
expect(f.elements[0].preload).toBe("auto");
expect(f.elements[1].preload).toBe("auto");
expect(f.elements[7].preload).toBe("metadata");
});
it("preloadAroundTime promotes clips near seek target", () => {
elements = Array.from({ length: 10 }, (_, i) =>
mockMediaElement({ start: String(i * 5), duration: "5" }),
);
setupDOM(elements);
const manager = createMediaPreloadManager();
manager.refresh();
for (const el of elements) {
el.preload = "metadata";
}
manager.preloadAroundTime(30);
expect(elements[6].preload).toBe("auto");
expect(elements[7].preload).toBe("auto");
expect(elements[0].preload).toBe("metadata");
const f = createTestFixture(10);
for (const el of f.elements) el.preload = "metadata";
f.manager.preloadAroundTime(30);
expect(f.elements[6].preload).toBe("auto");
expect(f.elements[7].preload).toBe("auto");
expect(f.elements[0].preload).toBe("metadata");
});
it("sync is a no-op when not lazy", () => {
elements = [
mockMediaElement({ start: "0", duration: "5" }),
mockMediaElement({ start: "5", duration: "5" }),
];
setupDOM(elements);
const manager = createMediaPreloadManager();
manager.refresh();
manager.sync(0);
expect(manager.isLazy()).toBe(false);
const f = createTestFixture(2);
f.manager.sync(0);
expect(f.manager.isLazy()).toBe(false);
});
it("guarantees at least LOOKAHEAD_MIN_CLIPS are promoted", () => {
// Use 20s spacing so only 1 clip falls in the 10s lookahead window
elements = Array.from({ length: 8 }, (_, i) =>
mockMediaElement({ start: String(i * 20), duration: "5" }),
);
setupDOM(elements);
const manager = createMediaPreloadManager();
manager.refresh();
for (const el of elements) {
el.preload = "metadata";
}
for (const el of elements) el.preload = "metadata";
manager.sync(0);
const promotedCount = elements.filter((el) => el.preload === "auto").length;
expect(promotedCount).toBeGreaterThanOrEqual(2);
expect(elements.filter((el) => el.preload === "auto").length).toBeGreaterThanOrEqual(2);
});
it("evicts clips when scrubbing away from them", () => {
elements = Array.from({ length: 10 }, (_, i) =>
mockMediaElement({ start: String(i * 5), duration: "5" }),
);
setupDOM(elements);
const manager = createMediaPreloadManager();
manager.refresh();
for (const el of elements) {
el.preload = "metadata";
}
// Promote clips around t=0
manager.sync(0);
expect(elements[0].preload).toBe("auto");
expect(elements[1].preload).toBe("auto");
// Scrub to t=40 — clips 0,1 should be evicted
manager.sync(40);
expect(elements[0].preload).toBe("metadata");
expect(elements[0].src).toBe("");
expect(elements[8].preload).toBe("auto");
const f = createTestFixture(10);
for (const el of f.elements) el.preload = "metadata";
f.manager.sync(0);
expect(f.elements[0].preload).toBe("auto");
f.manager.sync(40);
expect(f.elements[0].preload).toBe("metadata");
expect(f.elements[0].src).toBe("");
expect(f.elements[8].preload).toBe("auto");
});
it("restores src when re-promoting a previously evicted clip", () => {
elements = Array.from({ length: 10 }, (_, i) =>
mockMediaElement({ start: String(i * 5), duration: "5" }),
);
setupDOM(elements);
const manager = createMediaPreloadManager();
manager.refresh();
for (const el of elements) {
el.preload = "metadata";
}
const originalSrc0 = elements[0].src;
// Promote at t=0, scrub away, scrub back
manager.sync(0);
manager.sync(40);
expect(elements[0].src).toBe("");
manager.sync(0);
expect(elements[0].src).toBe(originalSrc0);
expect(elements[0].preload).toBe("auto");
const f = createTestFixture(10);
for (const el of f.elements) el.preload = "metadata";
const originalSrc0 = f.elements[0].src;
f.manager.sync(0);
f.manager.sync(40);
expect(f.elements[0].src).toBe("");
f.manager.sync(0);
expect(f.elements[0].src).toBe(originalSrc0);
expect(f.elements[0].preload).toBe("auto");
});
it("does not exceed MAX_PROMOTED (5) clips", () => {
// 10 clips, each 5s long, spaced 5s apart
elements = Array.from({ length: 10 }, (_, i) =>
mockMediaElement({ start: String(i * 5), duration: "5" }),
);
setupDOM(elements);
const manager = createMediaPreloadManager();
manager.refresh();
for (const el of elements) {
el.preload = "metadata";
}
// Sync at t=0 — window covers clips 0,1,2 (0-15s lookahead)
manager.sync(0);
const promotedAfterFirst = elements.filter((el) => el.preload === "auto").length;
expect(promotedAfterFirst).toBeLessThanOrEqual(5);
// Sync at different position — should evict old ones
manager.sync(25);
const totalPromoted = elements.filter((el) => el.preload === "auto").length;
expect(totalPromoted).toBeLessThanOrEqual(5);
const f = createTestFixture(10);
for (const el of f.elements) el.preload = "metadata";
f.manager.sync(0);
expect(f.elements.filter((el) => el.preload === "auto").length).toBeLessThanOrEqual(5);
f.manager.sync(25);
expect(f.elements.filter((el) => el.preload === "auto").length).toBeLessThanOrEqual(5);
});
it("calls load() when evicting to release buffers", () => {
elements = Array.from({ length: 10 }, (_, i) =>
mockMediaElement({ start: String(i * 5), duration: "5" }),
const f = createTestFixture(10);
for (const el of f.elements) el.preload = "metadata";
f.manager.sync(0);
const loadCallsBefore = (f.elements[0].load as ReturnType<typeof vi.fn>).mock.calls.length;
f.manager.sync(40);
expect((f.elements[0].load as ReturnType<typeof vi.fn>).mock.calls.length).toBeGreaterThan(
loadCallsBefore,
);
setupDOM(elements);
const manager = createMediaPreloadManager();
manager.refresh();
for (const el of elements) {
el.preload = "metadata";
}
manager.sync(0);
const loadCallsBefore = (elements[0].load as ReturnType<typeof vi.fn>).mock.calls.length;
// Scrub away — eviction should call load() to release buffers
manager.sync(40);
const loadCallsAfter = (elements[0].load as ReturnType<typeof vi.fn>).mock.calls.length;
expect(loadCallsAfter).toBeGreaterThan(loadCallsBefore);
});
it("isLazy reports true with 6+ clips so caller can gate render-mode bypass", () => {
elements = Array.from({ length: 6 }, (_, i) =>
mockMediaElement({ start: String(i * 5), duration: "5" }),
);
setupDOM(elements);
const manager = createMediaPreloadManager();
manager.refresh();
const { manager } = createTestFixture(6);
expect(manager.isLazy()).toBe(true);
});
it("calls onActivation when lazy mode activates", () => {
elements = Array.from({ length: 8 }, (_, i) =>
mockMediaElement({ start: String(i * 5), duration: "5" }),
);
setupDOM(elements);
const onActivation = vi.fn();
const manager = createMediaPreloadManager({ onActivation });
manager.refresh();
createTestFixture(8, { onActivation });
expect(onActivation).toHaveBeenCalledOnce();
expect(onActivation).toHaveBeenCalledWith(8);
});
it("does not call onActivation below threshold", () => {
elements = [
mockMediaElement({ start: "0", duration: "5" }),
mockMediaElement({ start: "5", duration: "5" }),
];
setupDOM(elements);
const onActivation = vi.fn();
const manager = createMediaPreloadManager({ onActivation });
manager.refresh();
createTestFixture(2, { onActivation });
expect(onActivation).not.toHaveBeenCalled();
});
it("calls onActivation only once across multiple refreshes", () => {
elements = Array.from({ length: 8 }, (_, i) =>
mockMediaElement({ start: String(i * 5), duration: "5" }),
);
setupDOM(elements);
const onActivation = vi.fn();
const manager = createMediaPreloadManager({ onActivation });
const { manager } = createTestFixture(8, { onActivation });
manager.refresh();
manager.refresh();
manager.refresh();
expect(onActivation).toHaveBeenCalledOnce();
});
it("respects window.__HF_LAZY_PRELOAD_THRESHOLD override", () => {
elements = Array.from({ length: 4 }, (_, i) =>
elements = Array.from({ length: 2 }, (_, i) =>
mockMediaElement({ start: String(i * 5), duration: "5" }),
);
setupDOM(elements);
// 4 elements is below the default threshold (6) but at our custom one
(window as Record<string, unknown>).__HF_LAZY_PRELOAD_THRESHOLD = 4;
// 2 elements is below the default threshold (3) but at our custom one
(window as Record<string, unknown>).__HF_LAZY_PRELOAD_THRESHOLD = 2;
const manager = createMediaPreloadManager();
manager.refresh();
@@ -342,7 +216,7 @@ describe("createMediaPreloadManager", () => {
});
it("falls back to default threshold when __HF_LAZY_PRELOAD_THRESHOLD is not set", () => {
elements = Array.from({ length: 4 }, (_, i) =>
elements = Array.from({ length: 2 }, (_, i) =>
mockMediaElement({ start: String(i * 5), duration: "5" }),
);
setupDOM(elements);
packages/core/src/runtime/mediaPreloader.ts
+20 -17
View File
@@ -1,17 +1,18 @@
import { refreshRuntimeMediaCache, type RuntimeMediaClip } from "./media";
// Compositions with fewer than 6 timed clips rarely exceed browser memory
// limits during eager preload. The threshold avoids preload management
// overhead for typical compositions while catching the heavy-media case
// (e.g., 20 clips / 6GB reported in heygen-com/hyperframes#729).
const LAZY_THRESHOLD = 6;
// Start lazy preload management at 3 clips to keep memory pressure low from
// the start. The previous threshold of 6 let medium compositions (4–5 heavy
// videos) saturate browser memory before the preloader kicked in.
const LAZY_THRESHOLD = 3;
const LOOKAHEAD_SECONDS = 10;
const LOOKBEHIND_SECONDS = 3;
const LOOKAHEAD_MIN_CLIPS = 2;
// Cap on simultaneously promoted (buffered) clips. When the lookahead window
// contains more clips than this (e.g., many short clips), all window clips
// stay promoted — the cap is defense-in-depth, not a hard ceiling. The primary
// memory bound comes from window-based eviction in syncWindow().
const MAX_PROMOTED = 5;
// Adaptive cap: base of 4 for small sets, clamped to 6 for larger ones.
// The window-based eviction in syncWindow() is the primary memory bound;
// this cap is defense-in-depth for compositions with many short clips
// packed into the lookahead window.
const MAX_PROMOTED_BASE = 4;
const MAX_PROMOTED_CEIL = 6;
export interface MediaPreloadManager {
refresh(): void;
@@ -91,23 +92,23 @@ export function createMediaPreloadManager(options?: {
windowEls.add(clip.el);
}
// Evict clips no longer in window, oldest first
for (const clip of clips) {
if (promoted.has(clip.el) && !windowEls.has(clip.el)) {
evictClip(clip);
}
}
// If still over budget after removing out-of-window clips,
// evict the oldest promoted that isn't in the current window
while (promotionOrder.length > MAX_PROMOTED) {
const maxPromoted = Math.min(
MAX_PROMOTED_CEIL,
MAX_PROMOTED_BASE + Math.floor(clips.length / 10),
);
while (promotionOrder.length > maxPromoted) {
const oldest = promotionOrder[0];
if (windowEls.has(oldest)) break; // don't evict something currently needed
if (windowEls.has(oldest)) break;
const clip = clips.find((c) => c.el === oldest);
if (clip) {
evictClip(clip);
} else {
// Element no longer in clips list, just remove from tracking
promoted.delete(oldest);
promotionOrder.shift();
}
@@ -115,13 +116,15 @@ export function createMediaPreloadManager(options?: {
}
function getClipsInWindow(timeSeconds: number): Set<RuntimeMediaClip> {
const windowStart = timeSeconds - LOOKBEHIND_SECONDS;
const windowEnd = timeSeconds + LOOKAHEAD_SECONDS;
const inWindow = new Set<RuntimeMediaClip>();
for (const clip of clips) {
const active = timeSeconds >= clip.start && timeSeconds < clip.end;
const inLookahead = clip.start >= timeSeconds && clip.start <= windowEnd;
if (active || inLookahead) {
const inLookbehind = clip.end > windowStart && clip.end <= timeSeconds;
if (active || inLookahead || inLookbehind) {
inWindow.add(clip);
}
}