From 95fb938f2fb6be77abe9c5f26f111a9970af1469 Mon Sep 17 00:00:00 2001 From: Miguel Angel Simon Sierra Date: Sat, 1 Aug 2026 14:14:41 +0200 Subject: [PATCH] perf(runtime): scale media sync with the active window, not the document MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Every seek rebuilt the media cache with a full-document `querySelectorAll("video, audio")` and then evaluated relative time, held-tail math and `isActive` for every clip in the composition. On the gate fixture a single scrub drag issued 42 of those scans inside the preview iframe. The element list is now a registry rebuilt only when a mutation adds or removes media. Pending mutation records are drained synchronously at read time, so a sub-composition injected and seeked in the same task still renders with its media rather than lagging a microtask behind. The per-tick loop visits a transition set — clips in window now, plus the ones that were in window last tick. Departures are the reason it is a union and not a point query: the playhead crosses a clip's `end` between frames, and the tick that has to pause the clip is the one where it is already out of window. Nothing else pauses it (`hardSyncAllMedia` only assigns `currentTime`), so a missed departure is audio running past its clip edge for the rest of an exported video. An element first seen out of window is checked once, which keeps an authored `autoplay` element from starting unattended. Scrub window of the runtime-cost gate: 51 full-document media queries before, 12 after; the media-sync scan itself drops from 42 to 1. The remaining 12 belong to the duration probes, `bindMediaMetadataListeners` (U6) and three player-side call sites. --- packages/core/src/runtime/media.test.ts | 85 ++++++++++++++++++ packages/core/src/runtime/media.ts | 110 ++++++++++++++++++++++-- 2 files changed, 189 insertions(+), 6 deletions(-) diff --git a/packages/core/src/runtime/media.test.ts b/packages/core/src/runtime/media.test.ts index 9235d01c8a..0d0befbd44 100644 --- a/packages/core/src/runtime/media.test.ts +++ b/packages/core/src/runtime/media.test.ts @@ -1216,4 +1216,89 @@ describe("syncRuntimeMedia active window", () => { }); expect(clip.el.volume).toBeCloseTo(0.4, 10); }); + + it("pauses an already-playing element the first time it is seen out of window", () => { + // An authored `autoplay` element starts on its own. Before windowing every + // tick visited it; now only its first sighting can catch it. + const clip = createWindowClip({ start: 10, end: 15 }); + void clip.el.play(); + expect(clip.el.paused).toBe(false); + syncRuntimeMedia({ clips: [clip], timeSeconds: 0, playing: false, playbackRate: 1 }); + expect(clip.el.paused).toBe(true); + }); + + it("touches only the in-window and just-departed elements of a 100-clip composition", () => { + const touches = new Map(); + const clips = Array.from({ length: 100 }, (_, index) => { + const clip = createWindowClip({ start: index, end: index + 3 }); + touches.set(index, 0); + const bump = () => touches.set(index, (touches.get(index) ?? 0) + 1); + const el = new Proxy(clip.el, { + get(target, prop) { + bump(); + const value = Reflect.get(target, prop, target); + return typeof value === "function" ? value.bind(target) : value; + }, + set(target, prop, value) { + bump(); + return Reflect.set(target, prop, value, target); + }, + }); + return { ...clip, el }; + }); + const touched = () => + [...touches.entries()].filter(([, count]) => count > 0).map(([index]) => index); + const reset = () => { + for (const index of touches.keys()) touches.set(index, 0); + }; + + // The first tick legitimately sights every element once (autoplay guard). + syncRuntimeMedia({ clips, timeSeconds: 50.5, playing: false, playbackRate: 1 }); + reset(); + syncRuntimeMedia({ clips, timeSeconds: 50.5, playing: false, playbackRate: 1 }); + expect(touched()).toEqual([48, 49, 50]); + + reset(); + syncRuntimeMedia({ clips, timeSeconds: 51.5, playing: false, playbackRate: 1 }); + // 49-51 are in window; 48 is the departure that has to be paused. + expect(touched()).toEqual([48, 49, 50, 51]); + }); +}); + +describe("runtime media registry", () => { + afterEach(() => { + document.body.innerHTML = ""; + }); + + it("issues no full-document media query when nothing has been added or removed", () => { + createVideo({ "data-start": "0", "data-duration": "5" }); + refreshRuntimeMediaCache(); + const querySelectorAll = vi.spyOn(document, "querySelectorAll"); + refreshRuntimeMediaCache(); + refreshRuntimeMediaCache(); + expect(querySelectorAll).not.toHaveBeenCalled(); + querySelectorAll.mockRestore(); + }); + + it("sees an element added by a composition mutation on the next call", () => { + createVideo({ "data-start": "0", "data-duration": "5" }); + expect(refreshRuntimeMediaCache().mediaClips).toHaveLength(1); + createAudio({ "data-start": "1", "data-duration": "2" }); + expect(refreshRuntimeMediaCache().mediaClips).toHaveLength(2); + }); + + it("sees media inside a subtree injected in the same task", () => { + refreshRuntimeMediaCache(); + const host = document.createElement("div"); + host.innerHTML = ''; + document.body.appendChild(host); + expect(refreshRuntimeMediaCache().timedMediaEls).toHaveLength(1); + }); + + it("does not leave a removed element in the registry", () => { + const el = createVideo({ "data-start": "0", "data-duration": "5" }); + expect(refreshRuntimeMediaCache().timedMediaEls).toHaveLength(1); + el.remove(); + expect(refreshRuntimeMediaCache().timedMediaEls).toHaveLength(0); + }); }); diff --git a/packages/core/src/runtime/media.ts b/packages/core/src/runtime/media.ts index 21af57b1e0..6f3f603d2a 100644 --- a/packages/core/src/runtime/media.ts +++ b/packages/core/src/runtime/media.ts @@ -64,6 +64,57 @@ export type RuntimeMediaClip = { volumeKeyframes?: VolumeKeyframe[]; }; +type RuntimeMediaElement = HTMLVideoElement | HTMLAudioElement; + +// Every media element in the document, in document order. Rebuilt only when a +// mutation adds or removes one, not once per seek: the full-document +// `querySelectorAll("video, audio")` this replaces walked the whole tree on +// every scrub frame, which on a 273-clip project is the single largest +// per-frame cost in the preview iframe. +let mediaRegistry: RuntimeMediaElement[] | null = null; +let mediaRegistryObserver: MutationObserver | null = null; + +function nodesTouchMedia(nodes: NodeList): boolean { + for (const node of nodes) { + if (!(node instanceof Element)) continue; + if (node.tagName === "VIDEO" || node.tagName === "AUDIO") return true; + // Element-scoped, so it only walks the added/removed subtree — a + // sub-composition being swapped in, not the document. + if (node.querySelector("video, audio")) return true; + } + return false; +} + +function invalidateMediaRegistryFor(records: MutationRecord[]): void { + for (const record of records) { + if (nodesTouchMedia(record.addedNodes) || nodesTouchMedia(record.removedNodes)) { + mediaRegistry = null; + return; + } + } +} + +function readMediaRegistry(): RuntimeMediaElement[] { + if (!mediaRegistryObserver && typeof MutationObserver === "function") { + mediaRegistryObserver = new MutationObserver(invalidateMediaRegistryFor); + mediaRegistryObserver.observe(document, { childList: true, subtree: true }); + mediaRegistry = null; + } + if (mediaRegistryObserver) { + // Drain synchronously rather than waiting for the observer's microtask: + // the runtime injects a sub-composition and seeks in the same task, and a + // registry that lagged by a microtask would render that seek without its + // media. `takeRecords` is proportional to mutations since the last call — + // zero during a scrub, where only inline styles change. + invalidateMediaRegistryFor(mediaRegistryObserver.takeRecords()); + } else { + // ponytail: no observer (non-DOM host), no cache — always re-query. + mediaRegistry = null; + } + mediaRegistry ??= Array.from(document.querySelectorAll("video, audio")) as RuntimeMediaElement[]; + return mediaRegistry; +} + export function refreshRuntimeMediaCache(params?: { resolveStartSeconds?: (element: Element) => number; resolveDurationSeconds?: (element: HTMLVideoElement | HTMLAudioElement) => number | null; @@ -74,9 +125,7 @@ export function refreshRuntimeMediaCache(params?: { videoClips: RuntimeMediaClip[]; maxMediaEnd: number; } { - const mediaEls = Array.from(document.querySelectorAll("video, audio")) as Array< - HTMLVideoElement | HTMLAudioElement - >; + const mediaEls = readMediaRegistry(); const timedMediaEls = params?.shouldIncludeElement ? mediaEls.filter((el) => params.shouldIncludeElement?.(el)) : mediaEls.filter((el) => el.hasAttribute("data-start")); @@ -183,6 +232,54 @@ export function evictMediaSyncState(el: HTMLMediaElement): void { lastRuntimeAppliedVolume.delete(el); } +// Elements inside the previous tick's time window, and every element this +// module has ever been handed. Together they turn the per-tick visit list into +// a transition set: clips in window now, plus the ones that just left it. +// Departures cannot be recovered from a point query at the current time — +// during playback the playhead crosses a clip's `end` between frames, and the +// tick that has to call `pause()` on it is the one where it is already out of +// window. Nothing else pauses it: `hardSyncAllMedia` only assigns +// `currentTime`, so a missed departure means audio running past its clip edge +// for the rest of an exported video. +let lastWindowElements = new Set(); +const knownMediaElements = new WeakSet(); + +/** + * Split `clips` into the ones whose window contains `timeSeconds` and pause + * whatever left the window since the last call. Only in-window (and + * newly-departed, and never-before-seen) elements are touched; the pass over + * `clips` itself reads plain numbers, no DOM. + */ +function selectMediaTransitionSet( + clips: RuntimeMediaClip[], + timeSeconds: number, +): RuntimeMediaClip[] { + const inWindow: RuntimeMediaClip[] = []; + const windowElements = new Set(); + for (const clip of clips) { + if (timeSeconds >= clip.start && timeSeconds < clip.end) { + inWindow.push(clip); + windowElements.add(clip.el); + knownMediaElements.add(clip.el); + continue; + } + if (knownMediaElements.has(clip.el)) continue; + // First sight, already out of window: an authored `autoplay` element is + // running right now and would never be visited again. + knownMediaElements.add(clip.el); + if (!clip.el.paused) clip.el.pause(); + } + for (const el of lastWindowElements) { + if (windowElements.has(el)) continue; + // Clip left its active window — drop the offset baseline so the next + // activation (e.g. re-entering a sub-composition) gets a hard resync. + evictMediaSyncState(el); + if (!el.paused) el.pause(); + } + lastWindowElements = windowElements; + return inWindow; +} + /** Test-only seam: whether any per-source sync state is still tracked for `el`. */ export function hasMediaSyncStateForTest(el: HTMLMediaElement): boolean { return ( @@ -228,7 +325,7 @@ export function syncRuntimeMedia(params: { forceSync?: boolean; }): void { const forceMuteAll = !!(params.outputMuted || params.userMuted); - for (const clip of params.clips) { + for (const clip of selectMediaTransitionSet(params.clips, params.timeSeconds)) { const { el } = clip; if (!el.isConnected) continue; let relTime = (params.timeSeconds - clip.start) * clip.playbackRate + clip.mediaStart; @@ -439,8 +536,9 @@ export function syncRuntimeMedia(params: { } continue; } - // Clip left its active window — drop the offset baseline so the next - // activation (e.g. re-entering a sub-composition) gets a hard resync. + // In window by time but not playable — audio that ended naturally, or a + // negative `data-media-start` that puts relTime before zero. Same + // treatment as a departure: drop the baseline so re-activation hard-syncs. evictMediaSyncState(el); if (!el.paused) el.pause(); }