diff --git a/packages/core/src/runtime/entry.ts b/packages/core/src/runtime/entry.ts index 0d1e95f219..df04d516c4 100644 --- a/packages/core/src/runtime/entry.ts +++ b/packages/core/src/runtime/entry.ts @@ -1,5 +1,6 @@ import { initSandboxRuntimeModular } from "./init"; import { installAuthoredOpacityCapture } from "./colorGrading"; +import { installRuntimeMediaObserver } from "./media"; import { fitTextFontSize } from "../text/fitTextFontSize"; import { pretext } from "../text/pretext"; import { getVariables } from "./getVariables"; @@ -26,6 +27,12 @@ type HyperframeWindow = Window & { // at script evaluation time, while the document is still parsing. installAuthoredOpacityCapture(); +// Watch media from script evaluation time, while the document is still +// parsing, so a clip nowhere near the playhead can be told not to fetch before +// the browser starts fetching it. Waiting for DOMContentLoaded is too late — +// see deferParsedMediaPreload in media.ts. +installRuntimeMediaObserver(); + // Expose runtime helpers immediately so composition scripts can use them // before DOMContentLoaded (font sizing runs during script evaluation, and // getVariables is read by composition setup before the timeline is built). diff --git a/packages/core/src/runtime/init.test.ts b/packages/core/src/runtime/init.test.ts index bc50d005ba..3a7b6f1d21 100644 --- a/packages/core/src/runtime/init.test.ts +++ b/packages/core/src/runtime/init.test.ts @@ -4183,6 +4183,85 @@ describe("initSandboxRuntimeModular", () => { expect(audio.muted).toBe(false); }); }); + + it("buffers the clip at the playhead and not the one 500s away", () => { + const root = mountComposition(600); + const near = addMedia(root, { id: "near", src: "near.wav", start: 0, duration: 4 }); + const far = addMedia(root, { id: "far", src: "far.wav", start: 500, duration: 4 }); + + initSandboxRuntimeModular(); + + expect(mediaLog).toEqual(["load:near"]); + expect(near.preload).toBe("auto"); + expect(far.preload).not.toBe("auto"); + }); + + it("promotes a distant clip as the playhead approaches, before it is in window", () => { + const root = mountComposition(600); + const upcoming = addMedia(root, { id: "upcoming", src: "up.wav", start: 20, duration: 4 }); + + initSandboxRuntimeModular(); + expect(mediaLog).toEqual([]); + + window.__player?.seek(5); + expect(mediaLog).toEqual([]); + + // t=12 is still eight seconds before the clip opens at t=20. + window.__player?.seek(12); + expect(mediaLog).toEqual(["load:upcoming"]); + expect(upcoming.preload).toBe("auto"); + }); + + it("buffers the destination of a far seek rather than waiting for playback", () => { + const root = mountComposition(600); + const destination = addMedia(root, { id: "dest", src: "dest.wav", start: 500, duration: 4 }); + + initSandboxRuntimeModular(); + window.__player?.seek(495); + + expect(mediaLog).toEqual(["load:dest"]); + expect(destination.preload).toBe("auto"); + }); + + // The measured shape of the amplification: seven clips referencing one + // file each got their own element and each was told to fetch it. + it("fetches one source once however many clips reference it", () => { + const root = mountComposition(60); + const clips = Array.from({ length: 7 }, (_, index) => + addMedia(root, { + id: `share-${index}`, + src: "track.wav", + start: index, + duration: 1, + }), + ); + + initSandboxRuntimeModular(); + + expect(mediaLog).toEqual(["load:share-0"]); + // Every one of them still buffers — the browser serves the siblings from + // its own cache — but only one fetch is issued for the file. + for (const clip of clips) expect(clip.preload).toBe("auto"); + }); + + // A clip with no authored `data-duration` takes its window from the source, + // so it has to keep probing metadata however far away it is. + it("keeps probing metadata for a distant clip whose window is unauthored", () => { + const root = mountComposition(600); + const unauthored = addMedia(root, { id: "unauthored", src: "a.wav", start: 500 }); + const authored = addMedia(root, { + id: "authored", + src: "b.wav", + start: 500, + duration: 4, + }); + + initSandboxRuntimeModular(); + + expect(mediaLog).toEqual([]); + expect(unauthored.preload).toBe("metadata"); + expect(authored.preload).not.toBe("metadata"); + }); }); }); diff --git a/packages/core/src/runtime/init.ts b/packages/core/src/runtime/init.ts index 7dc7173e36..8065765354 100644 --- a/packages/core/src/runtime/init.ts +++ b/packages/core/src/runtime/init.ts @@ -30,6 +30,7 @@ import { resolveNaturalMediaTimelineDuration, type RuntimeMediaClip, syncRuntimeMedia, + usesEagerMediaPreload, } from "./media"; import { handleErrorForProxy, handleMetadataForProxy, maybeProxyProactively } from "./mediaProxy"; import { probeAndCacheElementVolume, type VolumeKeyframe } from "./mediaVolumeEnvelope.js"; @@ -100,6 +101,13 @@ import { const AUTHORED_DURATION_ATTR = "data-hf-authored-duration"; const AUTHORED_END_ATTR = "data-hf-authored-end"; +/** Keep roughly ten seconds of 1080p media around the playhead without letting + * a long composition's distant clips consume the browser's media budget. */ +const MEDIA_PRELOAD_BYTE_BUDGET = 64 * 1024 * 1024; +const MEDIA_PRELOAD_LOOKAHEAD_SECONDS = 10; +const MEDIA_PRELOAD_LOOKBEHIND_SECONDS = 2; +const MEDIA_PRELOAD_BYTES_PER_SECOND = 1_000_000; + /** * A `window.__timelines` entry is authored content and may be a PARTIAL * RuntimeTimelineLike — e.g. duration/seek only, no `pause()`. Such @@ -2770,6 +2778,64 @@ export function initSandboxRuntimeModular(): void { document.removeEventListener("play", onMediaPlayWakeTransport, true); }); + const detachedMediaSources = new WeakMap(); + const promotedMedia = new WeakSet(); + const mediaLoadInFlight = new WeakSet(); + + const estimatedMediaBytes = (clip: RuntimeMediaClip): number => { + const authored = Number.parseInt(clip.el.dataset.hfByteSize ?? "", 10); + if (Number.isFinite(authored) && authored > 0) return authored; + return Math.max(1, clip.duration) * MEDIA_PRELOAD_BYTES_PER_SECOND; + }; + + const applyMediaPreloadPolicy = (clips: RuntimeMediaClip[]): void => { + if (usesEagerMediaPreload()) return; + const candidates = clips + .map((clip) => ({ clip, distance: clip.start > state.currentTime + ? clip.start - state.currentTime + : state.currentTime - clip.end })) + .filter(({ clip, distance }) => + distance <= (clip.start > state.currentTime + ? MEDIA_PRELOAD_LOOKAHEAD_SECONDS + : MEDIA_PRELOAD_LOOKBEHIND_SECONDS), + ) + .sort((a, b) => a.distance - b.distance); + const allowed = new Set(); + let bytes = 0; + for (const { clip } of candidates) { + const source = clip.el.currentSrc || clip.el.src || detachedMediaSources.get(clip.el); + const estimate = estimatedMediaBytes(clip); + if (bytes + estimate > MEDIA_PRELOAD_BYTE_BUDGET && allowed.size > 0) continue; + bytes += estimate; + allowed.add(clip.el); + if (source) detachedMediaSources.set(clip.el, source); + } + for (const clip of clips) { + const el = clip.el; + if (allowed.has(el)) { + const source = detachedMediaSources.get(el); + if (source && !el.src) el.src = source; + if (el.preload !== "auto") el.preload = "auto"; + if (el.networkState === HTMLMediaElement.NETWORK_EMPTY && !mediaLoadInFlight.has(el)) { + mediaLoadInFlight.add(el); + el.load(); + queueMicrotask(() => mediaLoadInFlight.delete(el)); + } + promotedMedia.add(el); + continue; + } + if (!promotedMedia.has(el)) { + el.preload = "none"; + continue; + } + promotedMedia.delete(el); + if (el.src) detachedMediaSources.set(el, el.src); + el.removeAttribute("src"); + el.preload = "none"; + el.load(); + } + }; + const syncMediaForCurrentState = (timingRevision?: number) => { // Scope 1 of 3 (see `withTimingResolver`). Closes before `syncRuntimeMedia`, // which may call `el.load()` and invalidate every cached duration. @@ -2789,6 +2855,9 @@ export function initSandboxRuntimeModular(): void { return buildRuntimeMediaCache(collectMediaElementsToVisit(index, state.currentTime)) .mediaClips; }); + applyMediaPreloadPolicy( + indexed ? resolveMediaClipIndex().clips : buildRuntimeMediaCache().mediaClips, + ); // Attach probed volume keyframes to clips so syncRuntimeMedia can use the // same envelope the renderer uses instead of tracking GSAP-change diffs. for (const clip of mediaClips) { diff --git a/packages/core/src/runtime/media.test.ts b/packages/core/src/runtime/media.test.ts index f37b352b7b..91f6c1906a 100644 --- a/packages/core/src/runtime/media.test.ts +++ b/packages/core/src/runtime/media.test.ts @@ -4,6 +4,7 @@ import { readElementPlaybackStart, refreshRuntimeMediaCache, resolveRuntimeMediaClipDuration, + hasMediaSyncStateForTest, syncRuntimeMedia as syncRuntimeMediaWithDuration, } from "./media"; import type { RuntimeMediaClip } from "./media"; @@ -1567,3 +1568,258 @@ describe("syncRuntimeMedia", () => { expect(clip.el.muted).toBe(true); }); }); + +/** + * The active-window contract, pinned before the loop was windowed. + * + * `syncRuntimeMedia` used to visit every clip on every tick, so pausing a clip + * the playhead had just left fell out of the same pass that activated the one + * it had just entered. Windowing splits those two into a transition set, and + * these tests are what say the departure half still happens: a clip that leaves + * its window must be paused on the very next tick, or its audio runs on past + * its clip edge for the rest of an exported video. + */ +describe("syncRuntimeMedia active window", () => { + afterEach(() => { + document.body.innerHTML = ""; + }); + + /** A clip whose element tracks paused state the way a real one does. */ + function createWindowClip(overrides: Partial = {}): RuntimeMediaClip { + const el = document.createElement("audio"); + document.body.appendChild(el); + let paused = true; + Object.defineProperty(el, "paused", { get: () => paused, configurable: true }); + Object.defineProperty(el, "currentTime", { value: 0, writable: true, configurable: true }); + Object.defineProperty(el, "playbackRate", { value: 1, writable: true, configurable: true }); + Object.defineProperty(el, "duration", { value: 30, writable: true, configurable: true }); + el.play = vi.fn(() => { + paused = false; + return Promise.resolve(); + }); + el.pause = vi.fn(() => { + paused = true; + }); + return { + el, + start: 0, + mediaStart: 0, + duration: 5, + end: 5, + volume: null, + playbackRate: 1, + loop: false, + sourceDuration: null, + ...overrides, + }; + } + + function elementState(el: HTMLMediaElement) { + return { + paused: el.paused, + currentTime: el.currentTime, + volume: el.volume, + playbackRate: el.playbackRate, + preload: el.preload, + }; + } + + it("pauses a clip the playhead just left and evicts its sync state", () => { + const clip = createWindowClip({ start: 0, end: 5 }); + syncRuntimeMedia({ clips: [clip], timeSeconds: 2, playing: true, playbackRate: 1 }); + expect(clip.el.paused).toBe(false); + expect(hasMediaSyncStateForTest(clip.el)).toBe(true); + + syncRuntimeMedia({ clips: [clip], timeSeconds: 6, playing: true, playbackRate: 1 }); + expect(clip.el.pause).toHaveBeenCalledTimes(1); + expect(clip.el.paused).toBe(true); + expect(hasMediaSyncStateForTest(clip.el)).toBe(false); + }); + + it("pauses the departed clip on the next playing frame across a clip boundary", () => { + const outgoing = createWindowClip({ start: 0, end: 5 }); + const incoming = createWindowClip({ start: 5, end: 10 }); + const clips = [outgoing, incoming]; + + syncRuntimeMedia({ clips, timeSeconds: 4.98, playing: true, playbackRate: 1 }); + expect(outgoing.el.paused).toBe(false); + expect(incoming.el.paused).toBe(true); + + // One frame later the playhead has crossed `outgoing.end`. Nothing else in + // the runtime pauses it — hardSyncAllMedia only assigns currentTime — so if + // this tick misses it, it keeps playing to the end of the composition. + syncRuntimeMedia({ clips, timeSeconds: 5.02, playing: true, playbackRate: 1 }); + expect(outgoing.el.paused).toBe(true); + expect(outgoing.el.pause).toHaveBeenCalledTimes(1); + expect(incoming.el.paused).toBe(false); + }); + + it("leaves the same element state whether a boundary is crossed forwards or backwards", () => { + const forwardA = createWindowClip({ start: 0, end: 5 }); + const forwardB = createWindowClip({ start: 5, end: 10 }); + syncRuntimeMedia({ + clips: [forwardA, forwardB], + timeSeconds: 1, + playing: false, + playbackRate: 1, + }); + syncRuntimeMedia({ + clips: [forwardA, forwardB], + timeSeconds: 6, + playing: false, + playbackRate: 1, + }); + + const backwardA = createWindowClip({ start: 0, end: 5 }); + const backwardB = createWindowClip({ start: 5, end: 10 }); + syncRuntimeMedia({ + clips: [backwardA, backwardB], + timeSeconds: 9, + playing: false, + playbackRate: 1, + }); + syncRuntimeMedia({ + clips: [backwardA, backwardB], + timeSeconds: 6, + playing: false, + playbackRate: 1, + }); + + // The clip the playhead is inside lands in the same state either way; the + // one it is outside is paused either way. (`currentTime` on the outside + // clip is history — nothing rewinds an element the playhead already left.) + expect(elementState(backwardB.el)).toEqual(elementState(forwardB.el)); + expect(backwardA.el.paused).toBe(true); + expect(forwardA.el.paused).toBe(true); + }); + + it("treats the window as half-open at both edges", () => { + const clip = createWindowClip({ start: 4, end: 8 }); + syncRuntimeMedia({ clips: [clip], timeSeconds: 3.999, playing: true, playbackRate: 1 }); + expect(clip.el.paused).toBe(true); + syncRuntimeMedia({ clips: [clip], timeSeconds: 4, playing: true, playbackRate: 1 }); + expect(clip.el.paused).toBe(false); + syncRuntimeMedia({ clips: [clip], timeSeconds: 7.999, playing: true, playbackRate: 1 }); + expect(clip.el.paused).toBe(false); + syncRuntimeMedia({ clips: [clip], timeSeconds: 8, playing: true, playbackRate: 1 }); + expect(clip.el.paused).toBe(true); + }); + + it("keeps volume, rate and relative time identical for an in-window clip", () => { + const clip = createWindowClip({ + start: 2, + end: 12, + mediaStart: 1.5, + volume: 0.5, + playbackRate: 2, + }); + syncRuntimeMedia({ + clips: [clip], + timeSeconds: 5, + playing: false, + playbackRate: 1.5, + userVolume: 0.8, + }); + // relTime = (5 - 2) * 2 + 1.5 + expect(clip.el.currentTime).toBe(7.5); + expect(clip.el.playbackRate).toBe(3); + // First active tick trusts the element's own volume (GSAP has already been + // seeked); data-volume only becomes the baseline from the second tick on. + expect(clip.el.volume).toBeCloseTo(0.8, 10); + expect(clip.el.preload).toBe("auto"); + + syncRuntimeMedia({ + clips: [clip], + timeSeconds: 5.5, + playing: false, + playbackRate: 1.5, + userVolume: 0.8, + }); + expect(clip.el.volume).toBeCloseTo(0.4, 10); + }); + + it("pauses an already-playing element the first time it is seen out of window", () => { + // An authored `autoplay` element starts on its own. Before windowing every + // tick visited it; now only its first sighting can catch it. + const clip = createWindowClip({ start: 10, end: 15 }); + void clip.el.play(); + expect(clip.el.paused).toBe(false); + syncRuntimeMedia({ clips: [clip], timeSeconds: 0, playing: false, playbackRate: 1 }); + expect(clip.el.paused).toBe(true); + }); + + it("touches only the in-window and just-departed elements of a 100-clip composition", () => { + const touches = new Map(); + const clips = Array.from({ length: 100 }, (_, index) => { + const clip = createWindowClip({ start: index, end: index + 3 }); + touches.set(index, 0); + const bump = () => touches.set(index, (touches.get(index) ?? 0) + 1); + const el = new Proxy(clip.el, { + get(target, prop) { + bump(); + const value = Reflect.get(target, prop, target); + return typeof value === "function" ? value.bind(target) : value; + }, + set(target, prop, value) { + bump(); + return Reflect.set(target, prop, value, target); + }, + }); + return { ...clip, el }; + }); + const touched = () => + [...touches.entries()].filter(([, count]) => count > 0).map(([index]) => index); + const reset = () => { + for (const index of touches.keys()) touches.set(index, 0); + }; + + // The first tick legitimately sights every element once (autoplay guard). + syncRuntimeMedia({ clips, timeSeconds: 50.5, playing: false, playbackRate: 1 }); + reset(); + syncRuntimeMedia({ clips, timeSeconds: 50.5, playing: false, playbackRate: 1 }); + expect(touched()).toEqual([48, 49, 50]); + + reset(); + syncRuntimeMedia({ clips, timeSeconds: 51.5, playing: false, playbackRate: 1 }); + // 49-51 are in window; 48 is the departure that has to be paused. + expect(touched()).toEqual([48, 49, 50, 51]); + }); +}); + +describe("runtime media registry", () => { + afterEach(() => { + document.body.innerHTML = ""; + }); + + it("issues no full-document media query when nothing has been added or removed", () => { + createVideo({ "data-start": "0", "data-duration": "5" }); + refreshRuntimeMediaCache(); + const querySelectorAll = vi.spyOn(document, "querySelectorAll"); + refreshRuntimeMediaCache(); + refreshRuntimeMediaCache(); + expect(querySelectorAll).not.toHaveBeenCalled(); + querySelectorAll.mockRestore(); + }); + + it("sees an element added by a composition mutation on the next call", () => { + createVideo({ "data-start": "0", "data-duration": "5" }); + expect(refreshRuntimeMediaCache().mediaClips).toHaveLength(1); + createAudio({ "data-start": "1", "data-duration": "2" }); + expect(refreshRuntimeMediaCache().mediaClips).toHaveLength(2); + }); + + it("sees media inside a subtree injected in the same task", () => { + refreshRuntimeMediaCache(); + const host = document.createElement("div"); + host.innerHTML = ''; + document.body.appendChild(host); + expect(refreshRuntimeMediaCache().timedMediaEls).toHaveLength(1); + }); + + it("does not leave a removed element in the registry", () => { + const el = createVideo({ "data-start": "0", "data-duration": "5" }); + expect(refreshRuntimeMediaCache().timedMediaEls).toHaveLength(1); + el.remove(); + expect(refreshRuntimeMediaCache().timedMediaEls).toHaveLength(0); + }); +}); diff --git a/packages/core/src/runtime/media.ts b/packages/core/src/runtime/media.ts index bd3666d2a6..8b23bf7a0c 100644 --- a/packages/core/src/runtime/media.ts +++ b/packages/core/src/runtime/media.ts @@ -63,6 +63,128 @@ export type RuntimeMediaClip = { volumeKeyframes?: VolumeKeyframe[]; }; +type RuntimeMediaElement = HTMLVideoElement | HTMLAudioElement; + +// Every media element in the document, in document order. Rebuilt only when a +// mutation adds or removes one, not once per seek: the full-document +// `querySelectorAll("video, audio")` this replaces walked the whole tree on +// every scrub frame, which on a 273-clip project is the single largest +// per-frame cost in the preview iframe. +let mediaRegistry: RuntimeMediaElement[] | null = null; +let mediaRegistryObserver: MutationObserver | null = null; + +function nodesTouchMedia(nodes: NodeList): boolean { + for (const node of nodes) { + if (!(node instanceof Element)) continue; + if (node.tagName === "VIDEO" || node.tagName === "AUDIO") return true; + // Element-scoped, so it only walks the added/removed subtree — a + // sub-composition being swapped in, not the document. + if (node.querySelector("video, audio")) return true; + } + return false; +} + +function forEachMediaIn(nodes: NodeList, visit: (element: RuntimeMediaElement) => void): boolean { + let found = false; + for (const node of nodes) { + if (!(node instanceof Element)) continue; + if (node instanceof HTMLVideoElement || node instanceof HTMLAudioElement) { + found = true; + visit(node); + } + // Element-scoped, so it only walks the added subtree. + for (const nested of node.querySelectorAll("video, audio")) { + if (!(nested instanceof HTMLVideoElement || nested instanceof HTMLAudioElement)) continue; + found = true; + visit(nested); + } + } + return found; +} + +/** + * Whether media is buffered up front instead of following the playhead. True + * for render capture, on both of the runtime's render signals: the producer + * file server's `RENDER_CAPTURE_MODE_SHIM` (set before any page script, so it + * is readable while the document is still parsing) and the export seek + * protocol's config (written by the render-mode body script). The renderer + * captures whatever the browser has decoded at the instant a frame is taken, + * so trading buffered bytes for a faster open there buys nothing and risks a + * silent or black export. + * + * `window.d.ts` declares both globals, but it is outside the package's main + * tsconfig program — hence the local shape rather than a bare property read. + */ +export function usesEagerMediaPreload(): boolean { + const win = window as Window & { + __HF_RENDER_CAPTURE_MODE?: boolean; + __HF_EXPORT_RENDER_SEEK_CONFIG?: unknown; + }; + return Boolean(win.__HF_RENDER_CAPTURE_MODE || win.__HF_EXPORT_RENDER_SEEK_CONFIG); +} + +/** + * Stop a freshly-parsed media element from fetching its source before the + * runtime has decided whether the playhead is anywhere near it. The runtime's + * neighbourhood policy (`init.ts`) raises it to `metadata` or `auto` once it + * can place the element against the current time. + * + * This has to happen as each element is parsed, not at init: `preload` set + * after `DOMContentLoaded` can lose the race against the resource-selection + * task the parser already queued. + */ +function deferParsedMediaPreload(element: RuntimeMediaElement): void { + if (usesEagerMediaPreload()) return; + // An authored `preload` or `autoplay` is an explicit instruction about this + // element's own buffering — overriding it would be the runtime deciding + // against the author rather than for the playhead. + if (element.hasAttribute("preload") || element.autoplay) return; + element.preload = "none"; +} + +function handleMediaMutations(records: MutationRecord[]): void { + let touched = false; + for (const record of records) { + if (forEachMediaIn(record.addedNodes, deferParsedMediaPreload)) touched = true; + if (!touched && nodesTouchMedia(record.removedNodes)) touched = true; + } + if (touched) mediaRegistry = null; +} + +function ensureMediaRegistryObserver(): void { + if (mediaRegistryObserver || typeof MutationObserver !== "function") return; + mediaRegistryObserver = new MutationObserver(handleMediaMutations); + mediaRegistryObserver.observe(document, { childList: true, subtree: true }); + mediaRegistry = null; +} + +/** + * Start observing media at script-evaluation time, while the document is + * still parsing, so `deferParsedMediaPreload` sees each element before the + * browser acts on it. Called from the runtime entry, not from `init`, which + * runs at `DOMContentLoaded` — by then every element has already been parsed. + */ +export function installRuntimeMediaObserver(): void { + ensureMediaRegistryObserver(); +} + +function readMediaRegistry(): RuntimeMediaElement[] { + ensureMediaRegistryObserver(); + if (mediaRegistryObserver) { + // Drain synchronously rather than waiting for the observer's microtask: + // the runtime injects a sub-composition and seeks in the same task, and a + // registry that lagged by a microtask would render that seek without its + // media. `takeRecords` is proportional to mutations since the last call — + // zero during a scrub, where only inline styles change. + handleMediaMutations(mediaRegistryObserver.takeRecords()); + } else { + // ponytail: no observer (non-DOM host), no cache — always re-query. + mediaRegistry = null; + } + mediaRegistry ??= Array.from(document.querySelectorAll("video, audio")) as RuntimeMediaElement[]; + return mediaRegistry; +} + export function refreshRuntimeMediaCache(params?: { resolveStartSeconds?: (element: Element) => number; resolveDurationSeconds?: (element: HTMLVideoElement | HTMLAudioElement) => number | null; @@ -86,9 +208,7 @@ export function refreshRuntimeMediaCache(params?: { } { const mediaEls = params?.elements ?? - (Array.from(document.querySelectorAll("video, audio")) as Array< - HTMLVideoElement | HTMLAudioElement - >); + readMediaRegistry(); const timedMediaEls = params?.shouldIncludeElement ? mediaEls.filter((el) => params.shouldIncludeElement?.(el)) : mediaEls.filter((el) => el.hasAttribute("data-start")); @@ -201,6 +321,62 @@ export function evictMediaSyncState(el: HTMLMediaElement): void { lastRuntimeAppliedVolume.delete(el); } +// Elements inside the previous tick's time window, and every element this +// module has ever been handed. Together they turn the per-tick visit list into +// a transition set: clips in window now, plus the ones that just left it. +// Departures cannot be recovered from a point query at the current time — +// during playback the playhead crosses a clip's `end` between frames, and the +// tick that has to call `pause()` on it is the one where it is already out of +// window. Nothing else pauses it: `hardSyncAllMedia` only assigns +// `currentTime`, so a missed departure means audio running past its clip edge +// for the rest of an exported video. +let lastWindowElements = new Set(); +const knownMediaElements = new WeakSet(); + +/** + * Split `clips` into the ones whose window contains `timeSeconds` and pause + * whatever left the window since the last call. Only in-window (and + * newly-departed, and never-before-seen) elements are touched; the pass over + * `clips` itself reads plain numbers, no DOM. + */ +function selectMediaTransitionSet( + clips: RuntimeMediaClip[], + timeSeconds: number, + getCompositionDuration: () => number, +): RuntimeMediaClip[] { + const inWindow: RuntimeMediaClip[] = []; + const windowElements = new Set(); + const compositionDuration = getCompositionDuration(); + for (const clip of clips) { + if ( + (timeSeconds >= clip.start && timeSeconds < clip.end) || + (compositionDuration > 0 && + clip.el.tagName === "VIDEO" && + timeSeconds >= clip.end && + isClipVisibleAt(timeSeconds, clip.start, clip.end, compositionDuration)) + ) { + inWindow.push(clip); + windowElements.add(clip.el); + knownMediaElements.add(clip.el); + continue; + } + if (knownMediaElements.has(clip.el)) continue; + // First sight, already out of window: an authored `autoplay` element is + // running right now and would never be visited again. + knownMediaElements.add(clip.el); + if (!clip.el.paused) clip.el.pause(); + } + for (const el of lastWindowElements) { + if (windowElements.has(el)) continue; + // Clip left its active window — drop the offset baseline so the next + // activation (e.g. re-entering a sub-composition) gets a hard resync. + evictMediaSyncState(el); + if (!el.paused) el.pause(); + } + lastWindowElements = windowElements; + return inWindow; +} + /** Test-only seam: whether any per-source sync state is still tracked for `el`. */ export function hasMediaSyncStateForTest(el: HTMLMediaElement): boolean { return ( @@ -253,7 +429,11 @@ export function syncRuntimeMedia(params: { getCompositionDuration: () => number; }): void { const forceMuteAll = !!(params.outputMuted || params.userMuted); - for (const clip of params.clips) { + for (const clip of selectMediaTransitionSet( + params.clips, + params.timeSeconds, + params.getCompositionDuration, + )) { const { el } = clip; if (!el.isConnected) continue; const clipRate = clip.rate ?? clip.playbackRate; diff --git a/packages/core/src/runtime/window.d.ts b/packages/core/src/runtime/window.d.ts index 61ed1662b8..c59b590317 100644 --- a/packages/core/src/runtime/window.d.ts +++ b/packages/core/src/runtime/window.d.ts @@ -74,6 +74,13 @@ declare global { fpsFallbackReason?: "missing" | "invalid"; owner?: string; }; + /** + * Set by the producer's file server (RENDER_CAPTURE_MODE_SHIM) before any + * page script runs. Distinct from `__HF_EXPORT_RENDER_SEEK_CONFIG`, which + * the render-mode body script writes only once the body is parsed — this + * one is the render signal available while the document is still parsing. + */ + __HF_RENDER_CAPTURE_MODE?: boolean; __HF_PARITY_MODE?: boolean; /** Legacy debug-only fps hint. Render-mode runtime fps uses __HF_EXPORT_RENDER_SEEK_CONFIG.fps. */ __HF_FPS?: number; diff --git a/packages/studio/src/player/components/Player.test.ts b/packages/studio/src/player/components/Player.test.ts index 232e26393f..9795f45103 100644 --- a/packages/studio/src/player/components/Player.test.ts +++ b/packages/studio/src/player/components/Player.test.ts @@ -328,6 +328,7 @@ describe("composition loading overlay", () => { it("keeps the asset overlay up while media is still buffering", () => { const { audio, iframe } = createAudioIframe(); + audio.preload = "auto"; Object.defineProperty(audio, "readyState", { value: 0, configurable: true, @@ -342,6 +343,25 @@ describe("composition loading overlay", () => { iframe.remove(); }); + // The runtime fetches the playhead's neighbourhood and nothing else, so a + // clip far from the playhead never reaches HAVE_FUTURE_DATA by design. + it("does not wait on media the runtime deliberately left unfetched", () => { + const { audio, iframe } = createAudioIframe(); + audio.preload = "none"; + Object.defineProperty(audio, "readyState", { + value: 0, + configurable: true, + }); + Object.defineProperty(audio, "networkState", { + value: 2, + configurable: true, + }); + + expect(hasUnloadedAssets(iframe, false)).toBe(false); + + iframe.remove(); + }); + it("does not keep the asset overlay stuck on failed media sources", () => { const { audio, iframe } = createAudioIframe(); Object.defineProperty(audio, "error", { diff --git a/packages/studio/src/player/components/Player.tsx b/packages/studio/src/player/components/Player.tsx index 5500f3db8b..eaf7282345 100644 --- a/packages/studio/src/player/components/Player.tsx +++ b/packages/studio/src/player/components/Player.tsx @@ -86,9 +86,17 @@ function isPreviewMediaElement(el: Element): el is HTMLMediaElement { return tagName === "video" || tagName === "audio"; } -// Assets are considered ready when every `