mirror of
https://github.com/heygen-com/hyperframes.git
synced 2026-11-17 23:47:33 +00:00
perf(runtime): scale media sync with the active window, not the document
Every seek rebuilt the media cache with a full-document
`querySelectorAll("video, audio")` and then evaluated relative time,
held-tail math and `isActive` for every clip in the composition. On the
gate fixture a single scrub drag issued 42 of those scans inside the
preview iframe.
The element list is now a registry rebuilt only when a mutation adds or
removes media. Pending mutation records are drained synchronously at read
time, so a sub-composition injected and seeked in the same task still
renders with its media rather than lagging a microtask behind.
The per-tick loop visits a transition set — clips in window now, plus the
ones that were in window last tick. Departures are the reason it is a
union and not a point query: the playhead crosses a clip's `end` between
frames, and the tick that has to pause the clip is the one where it is
already out of window. Nothing else pauses it (`hardSyncAllMedia` only
assigns `currentTime`), so a missed departure is audio running past its
clip edge for the rest of an exported video. An element first seen out of
window is checked once, which keeps an authored `autoplay` element from
starting unattended.
Scrub window of the runtime-cost gate: 51 full-document media queries
before, 12 after; the media-sync scan itself drops from 42 to 1. The
remaining 12 belong to the duration probes, `bindMediaMetadataListeners`
(U6) and three player-side call sites.
This commit is contained in:
@@ -1216,4 +1216,89 @@ describe("syncRuntimeMedia active window", () => {
|
||||
});
|
||||
expect(clip.el.volume).toBeCloseTo(0.4, 10);
|
||||
});
|
||||
|
||||
it("pauses an already-playing element the first time it is seen out of window", () => {
|
||||
// An authored `autoplay` element starts on its own. Before windowing every
|
||||
// tick visited it; now only its first sighting can catch it.
|
||||
const clip = createWindowClip({ start: 10, end: 15 });
|
||||
void clip.el.play();
|
||||
expect(clip.el.paused).toBe(false);
|
||||
syncRuntimeMedia({ clips: [clip], timeSeconds: 0, playing: false, playbackRate: 1 });
|
||||
expect(clip.el.paused).toBe(true);
|
||||
});
|
||||
|
||||
it("touches only the in-window and just-departed elements of a 100-clip composition", () => {
|
||||
const touches = new Map<number, number>();
|
||||
const clips = Array.from({ length: 100 }, (_, index) => {
|
||||
const clip = createWindowClip({ start: index, end: index + 3 });
|
||||
touches.set(index, 0);
|
||||
const bump = () => touches.set(index, (touches.get(index) ?? 0) + 1);
|
||||
const el = new Proxy(clip.el, {
|
||||
get(target, prop) {
|
||||
bump();
|
||||
const value = Reflect.get(target, prop, target);
|
||||
return typeof value === "function" ? value.bind(target) : value;
|
||||
},
|
||||
set(target, prop, value) {
|
||||
bump();
|
||||
return Reflect.set(target, prop, value, target);
|
||||
},
|
||||
});
|
||||
return { ...clip, el };
|
||||
});
|
||||
const touched = () =>
|
||||
[...touches.entries()].filter(([, count]) => count > 0).map(([index]) => index);
|
||||
const reset = () => {
|
||||
for (const index of touches.keys()) touches.set(index, 0);
|
||||
};
|
||||
|
||||
// The first tick legitimately sights every element once (autoplay guard).
|
||||
syncRuntimeMedia({ clips, timeSeconds: 50.5, playing: false, playbackRate: 1 });
|
||||
reset();
|
||||
syncRuntimeMedia({ clips, timeSeconds: 50.5, playing: false, playbackRate: 1 });
|
||||
expect(touched()).toEqual([48, 49, 50]);
|
||||
|
||||
reset();
|
||||
syncRuntimeMedia({ clips, timeSeconds: 51.5, playing: false, playbackRate: 1 });
|
||||
// 49-51 are in window; 48 is the departure that has to be paused.
|
||||
expect(touched()).toEqual([48, 49, 50, 51]);
|
||||
});
|
||||
});
|
||||
|
||||
describe("runtime media registry", () => {
|
||||
afterEach(() => {
|
||||
document.body.innerHTML = "";
|
||||
});
|
||||
|
||||
it("issues no full-document media query when nothing has been added or removed", () => {
|
||||
createVideo({ "data-start": "0", "data-duration": "5" });
|
||||
refreshRuntimeMediaCache();
|
||||
const querySelectorAll = vi.spyOn(document, "querySelectorAll");
|
||||
refreshRuntimeMediaCache();
|
||||
refreshRuntimeMediaCache();
|
||||
expect(querySelectorAll).not.toHaveBeenCalled();
|
||||
querySelectorAll.mockRestore();
|
||||
});
|
||||
|
||||
it("sees an element added by a composition mutation on the next call", () => {
|
||||
createVideo({ "data-start": "0", "data-duration": "5" });
|
||||
expect(refreshRuntimeMediaCache().mediaClips).toHaveLength(1);
|
||||
createAudio({ "data-start": "1", "data-duration": "2" });
|
||||
expect(refreshRuntimeMediaCache().mediaClips).toHaveLength(2);
|
||||
});
|
||||
|
||||
it("sees media inside a subtree injected in the same task", () => {
|
||||
refreshRuntimeMediaCache();
|
||||
const host = document.createElement("div");
|
||||
host.innerHTML = '<audio data-start="0" data-duration="2"></audio>';
|
||||
document.body.appendChild(host);
|
||||
expect(refreshRuntimeMediaCache().timedMediaEls).toHaveLength(1);
|
||||
});
|
||||
|
||||
it("does not leave a removed element in the registry", () => {
|
||||
const el = createVideo({ "data-start": "0", "data-duration": "5" });
|
||||
expect(refreshRuntimeMediaCache().timedMediaEls).toHaveLength(1);
|
||||
el.remove();
|
||||
expect(refreshRuntimeMediaCache().timedMediaEls).toHaveLength(0);
|
||||
});
|
||||
});
|
||||
|
||||
@@ -64,6 +64,57 @@ export type RuntimeMediaClip = {
|
||||
volumeKeyframes?: VolumeKeyframe[];
|
||||
};
|
||||
|
||||
type RuntimeMediaElement = HTMLVideoElement | HTMLAudioElement;
|
||||
|
||||
// Every media element in the document, in document order. Rebuilt only when a
|
||||
// mutation adds or removes one, not once per seek: the full-document
|
||||
// `querySelectorAll("video, audio")` this replaces walked the whole tree on
|
||||
// every scrub frame, which on a 273-clip project is the single largest
|
||||
// per-frame cost in the preview iframe.
|
||||
let mediaRegistry: RuntimeMediaElement[] | null = null;
|
||||
let mediaRegistryObserver: MutationObserver | null = null;
|
||||
|
||||
function nodesTouchMedia(nodes: NodeList): boolean {
|
||||
for (const node of nodes) {
|
||||
if (!(node instanceof Element)) continue;
|
||||
if (node.tagName === "VIDEO" || node.tagName === "AUDIO") return true;
|
||||
// Element-scoped, so it only walks the added/removed subtree — a
|
||||
// sub-composition being swapped in, not the document.
|
||||
if (node.querySelector("video, audio")) return true;
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
function invalidateMediaRegistryFor(records: MutationRecord[]): void {
|
||||
for (const record of records) {
|
||||
if (nodesTouchMedia(record.addedNodes) || nodesTouchMedia(record.removedNodes)) {
|
||||
mediaRegistry = null;
|
||||
return;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
function readMediaRegistry(): RuntimeMediaElement[] {
|
||||
if (!mediaRegistryObserver && typeof MutationObserver === "function") {
|
||||
mediaRegistryObserver = new MutationObserver(invalidateMediaRegistryFor);
|
||||
mediaRegistryObserver.observe(document, { childList: true, subtree: true });
|
||||
mediaRegistry = null;
|
||||
}
|
||||
if (mediaRegistryObserver) {
|
||||
// Drain synchronously rather than waiting for the observer's microtask:
|
||||
// the runtime injects a sub-composition and seeks in the same task, and a
|
||||
// registry that lagged by a microtask would render that seek without its
|
||||
// media. `takeRecords` is proportional to mutations since the last call —
|
||||
// zero during a scrub, where only inline styles change.
|
||||
invalidateMediaRegistryFor(mediaRegistryObserver.takeRecords());
|
||||
} else {
|
||||
// ponytail: no observer (non-DOM host), no cache — always re-query.
|
||||
mediaRegistry = null;
|
||||
}
|
||||
mediaRegistry ??= Array.from(document.querySelectorAll("video, audio")) as RuntimeMediaElement[];
|
||||
return mediaRegistry;
|
||||
}
|
||||
|
||||
export function refreshRuntimeMediaCache(params?: {
|
||||
resolveStartSeconds?: (element: Element) => number;
|
||||
resolveDurationSeconds?: (element: HTMLVideoElement | HTMLAudioElement) => number | null;
|
||||
@@ -74,9 +125,7 @@ export function refreshRuntimeMediaCache(params?: {
|
||||
videoClips: RuntimeMediaClip[];
|
||||
maxMediaEnd: number;
|
||||
} {
|
||||
const mediaEls = Array.from(document.querySelectorAll("video, audio")) as Array<
|
||||
HTMLVideoElement | HTMLAudioElement
|
||||
>;
|
||||
const mediaEls = readMediaRegistry();
|
||||
const timedMediaEls = params?.shouldIncludeElement
|
||||
? mediaEls.filter((el) => params.shouldIncludeElement?.(el))
|
||||
: mediaEls.filter((el) => el.hasAttribute("data-start"));
|
||||
@@ -183,6 +232,54 @@ export function evictMediaSyncState(el: HTMLMediaElement): void {
|
||||
lastRuntimeAppliedVolume.delete(el);
|
||||
}
|
||||
|
||||
// Elements inside the previous tick's time window, and every element this
|
||||
// module has ever been handed. Together they turn the per-tick visit list into
|
||||
// a transition set: clips in window now, plus the ones that just left it.
|
||||
// Departures cannot be recovered from a point query at the current time —
|
||||
// during playback the playhead crosses a clip's `end` between frames, and the
|
||||
// tick that has to call `pause()` on it is the one where it is already out of
|
||||
// window. Nothing else pauses it: `hardSyncAllMedia` only assigns
|
||||
// `currentTime`, so a missed departure means audio running past its clip edge
|
||||
// for the rest of an exported video.
|
||||
let lastWindowElements = new Set<HTMLMediaElement>();
|
||||
const knownMediaElements = new WeakSet<HTMLMediaElement>();
|
||||
|
||||
/**
|
||||
* Split `clips` into the ones whose window contains `timeSeconds` and pause
|
||||
* whatever left the window since the last call. Only in-window (and
|
||||
* newly-departed, and never-before-seen) elements are touched; the pass over
|
||||
* `clips` itself reads plain numbers, no DOM.
|
||||
*/
|
||||
function selectMediaTransitionSet(
|
||||
clips: RuntimeMediaClip[],
|
||||
timeSeconds: number,
|
||||
): RuntimeMediaClip[] {
|
||||
const inWindow: RuntimeMediaClip[] = [];
|
||||
const windowElements = new Set<HTMLMediaElement>();
|
||||
for (const clip of clips) {
|
||||
if (timeSeconds >= clip.start && timeSeconds < clip.end) {
|
||||
inWindow.push(clip);
|
||||
windowElements.add(clip.el);
|
||||
knownMediaElements.add(clip.el);
|
||||
continue;
|
||||
}
|
||||
if (knownMediaElements.has(clip.el)) continue;
|
||||
// First sight, already out of window: an authored `autoplay` element is
|
||||
// running right now and would never be visited again.
|
||||
knownMediaElements.add(clip.el);
|
||||
if (!clip.el.paused) clip.el.pause();
|
||||
}
|
||||
for (const el of lastWindowElements) {
|
||||
if (windowElements.has(el)) continue;
|
||||
// Clip left its active window — drop the offset baseline so the next
|
||||
// activation (e.g. re-entering a sub-composition) gets a hard resync.
|
||||
evictMediaSyncState(el);
|
||||
if (!el.paused) el.pause();
|
||||
}
|
||||
lastWindowElements = windowElements;
|
||||
return inWindow;
|
||||
}
|
||||
|
||||
/** Test-only seam: whether any per-source sync state is still tracked for `el`. */
|
||||
export function hasMediaSyncStateForTest(el: HTMLMediaElement): boolean {
|
||||
return (
|
||||
@@ -228,7 +325,7 @@ export function syncRuntimeMedia(params: {
|
||||
forceSync?: boolean;
|
||||
}): void {
|
||||
const forceMuteAll = !!(params.outputMuted || params.userMuted);
|
||||
for (const clip of params.clips) {
|
||||
for (const clip of selectMediaTransitionSet(params.clips, params.timeSeconds)) {
|
||||
const { el } = clip;
|
||||
if (!el.isConnected) continue;
|
||||
let relTime = (params.timeSeconds - clip.start) * clip.playbackRate + clip.mediaStart;
|
||||
@@ -439,8 +536,9 @@ export function syncRuntimeMedia(params: {
|
||||
}
|
||||
continue;
|
||||
}
|
||||
// Clip left its active window — drop the offset baseline so the next
|
||||
// activation (e.g. re-entering a sub-composition) gets a hard resync.
|
||||
// In window by time but not playable — audio that ended naturally, or a
|
||||
// negative `data-media-start` that puts relTime before zero. Same
|
||||
// treatment as a departure: drop the baseline so re-activation hard-syncs.
|
||||
evictMediaSyncState(el);
|
||||
if (!el.paused) el.pause();
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user