From 35f33d60dbf8056e8a0ce513001cd6db9f2840aa Mon Sep 17 00:00:00 2001 From: Miguel Angel Simon Sierra Date: Mon, 9 Nov 2026 04:39:33 -0500 Subject: [PATCH] perf(core): Safari keeps a later clip's src off until it is near --- packages/core/src/runtime/init.ts | 6 +- .../core/src/runtime/mediaLoadFailure.test.ts | 10 --- .../core/src/runtime/preloadMedia.test.ts | 79 ++++++++++--------- packages/core/src/runtime/preloadMedia.ts | 38 +++++---- packages/core/src/runtime/proxySrc.ts | 22 ++++-- .../src/runtime/webAudioTransport.test.ts | 16 ---- .../core/src/runtime/webAudioTransport.ts | 5 +- 7 files changed, 83 insertions(+), 93 deletions(-) diff --git a/packages/core/src/runtime/init.ts b/packages/core/src/runtime/init.ts index eba57f3b5e..bd0f275ee7 100644 --- a/packages/core/src/runtime/init.ts +++ b/packages/core/src/runtime/init.ts @@ -23,6 +23,7 @@ import { preloadMedia, prepareUpcomingMedia, releaseMedia, + restoreHeldBackSrc, lengthIsAuthored, stopMediaDownload, } from "./preloadMedia"; @@ -176,7 +177,7 @@ import { forgetPositionEdit, installPositionEditsSeekReapply, } from "./positionEdits"; -import { stillStandsIn, unheldPoster, unproxiedSrc } from "./proxySrc"; +import { isSrcHeldBack, stillStandsIn, unheldPoster, unproxiedSrc } from "./proxySrc"; import { applyVariableBindings, unproxiedMediaSrc } from "./applyVariableBindings"; import { createColorGradingRuntime, type RuntimeColorGradingApi } from "./colorGrading"; import { COLOR_GRADING_AUTHORED_OPACITY_ATTR } from "../colorGrading"; @@ -3230,6 +3231,7 @@ export function initSandboxRuntimeModular(): void { time: number, forLoop: boolean, ) => { + if (visible && isSrcHeldBack(el)) preloadMedia(el); // A hold for data is still playing: fetching other clips now would take bandwidth from the starved one. const playing = clock.isPlaying() || bufferingFor.length > 0; if (!(visible && playing)) armedFrom.delete(el); @@ -6446,6 +6448,8 @@ export function initSandboxRuntimeModular(): void { // Decided BEFORE the transport is asked, because the two verdicts want // two different fallback chains — and only one of them is the chain // that existed before (#3458). + // The route reads the src, which a clip not yet near does not have; preload none keeps it from loading. + restoreHeldBackSrc(rawEl); const route = classifyWebAudioMediaRoute(rawEl); reportWebAudioMediaRoute(rawEl, route); // The cross-origin verdict's BEST outcome is decode, since a CDN that diff --git a/packages/core/src/runtime/mediaLoadFailure.test.ts b/packages/core/src/runtime/mediaLoadFailure.test.ts index 959b87886f..2a391b1326 100644 --- a/packages/core/src/runtime/mediaLoadFailure.test.ts +++ b/packages/core/src/runtime/mediaLoadFailure.test.ts @@ -2,7 +2,6 @@ import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; import { postRuntimeMessage } from "./bridge"; import { watchAudioLoadErrors } from "./mediaLoadFailure"; import { RUNTIME_MEDIA_LOAD_FAILED } from "./protocol"; -import { recordOpenedAtInPoint } from "./proxySrc"; import { WebAudioTransport } from "./webAudioTransport"; vi.mock("./bridge", () => ({ postRuntimeMessage: vi.fn() })); @@ -38,15 +37,6 @@ afterEach(() => { }); describe("a preview media load that fails is reported, not swallowed", () => { - it("names the src the author wrote, not the in-point fragment the runtime added", async () => { - vi.stubGlobal("fetch", vi.fn(async () => ({ ok: false, status: 404 }))); - const el = audio("assets/take.wav", "take"); - recordOpenedAtInPoint(el, "assets/take.wav", "assets/take.wav#t=30"); - el.setAttribute("src", "assets/take.wav#t=30"); - await transport(async () => ({}) as AudioBuffer).decodeAudioElement(el); - expect(failures()[0]).toMatchObject({ asset: "assets/take.wav" }); - }); - it("a fetch the server refuses says its status, once however often playback retries", async () => { vi.stubGlobal( "fetch", diff --git a/packages/core/src/runtime/preloadMedia.test.ts b/packages/core/src/runtime/preloadMedia.test.ts index 8ae74cef5f..1f7af3083f 100644 --- a/packages/core/src/runtime/preloadMedia.test.ts +++ b/packages/core/src/runtime/preloadMedia.test.ts @@ -7,6 +7,7 @@ import { preloadMedia, prepareUpcomingMedia, releaseMedia, + restoreHeldBackSrc, } from "./preloadMedia"; function media(tagName = "VIDEO", networkState = 2, readyState = 0, id = "", renderId = "") { @@ -264,11 +265,13 @@ describe("preparing upcoming video playback", () => { }); }); -describe("a clip opening at its in-point on Safari", () => { +describe("a clip not yet near, in the player on Safari", () => { const SAFARI = "Mozilla/5.0 (iPhone; CPU iPhone OS 18_0 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/18.0 Mobile/15E148 Safari/604.1"; const CHROME = "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 Chrome/120.0 Safari/537.36"; let meta: HTMLMetaElement; + const stopWatching = () => + (window as Window & { __hfMediaDeferral?: MutationObserver }).__hfMediaDeferral?.disconnect(); const inPlayer = (on: boolean) => { const frame = document.createElement("iframe"); if (on) frame.setAttribute("data-hf-player-frame", ""); @@ -279,74 +282,72 @@ describe("a clip opening at its in-point on Safari", () => { meta.name = "hyperframes-studio-preview"; document.head.appendChild(meta); vi.spyOn(navigator, "userAgent", "get").mockReturnValue(SAFARI); + vi.spyOn(HTMLMediaElement.prototype, "load").mockImplementation(() => {}); inPlayer(true); deferMediaUntilDue(); }); afterEach(() => { - (window as Window & { __hfMediaDeferral?: MutationObserver }).__hfMediaDeferral?.disconnect(); + stopWatching(); vi.restoreAllMocks(); Reflect.deleteProperty(window, "frameElement"); meta.remove(); document.body.replaceChildren(); }); + const LATER = ''; async function added(html: string): Promise { document.body.insertAdjacentHTML("beforeend", html); await Promise.resolve(); return document.body.lastElementChild as HTMLVideoElement; } - it("gets a start fragment and keeps its authored src for everything that reads it", async () => { - const v = await added( - '', - ); - expect(v.getAttribute("src")).toBe("/m/film.mp4#t=30"); + it("has no src, so it does not load, and still reports the src the author wrote", async () => { + const v = await added(LATER); + expect(v.hasAttribute("src")).toBe(false); + expect(v.preload).toBe("none"); expect(unproxiedSrc(v)).toBe("/m/film.mp4"); }); - it("stops reporting the authored src once something else sets the src", async () => { - const v = await added( - '', - ); - v.setAttribute("src", "/m/other.mp4"); - expect(unproxiedSrc(v)).toBe("/m/other.mp4"); + it("gets its src back, and loads, when it is preloaded", async () => { + const v = await added(LATER); + preloadMedia(v); + expect(v.getAttribute("src")).toBe("/m/film.mp4"); + expect(v.preload).toBe("auto"); + expect(unproxiedSrc(v)).toBe("/m/film.mp4"); + }); + + it("gets its src back without loading when the audio route needs to read it", async () => { + const v = await added(LATER); + expect(restoreHeldBackSrc(v)).toBe(true); + expect(v.getAttribute("src")).toBe("/m/film.mp4"); + expect(v.preload).toBe("none"); + expect(restoreHeldBackSrc(v)).toBe(false); }); it("does so for a clip already parsed when the runtime arrives", () => { - (window as Window & { __hfMediaDeferral?: MutationObserver }).__hfMediaDeferral?.disconnect(); - document.body.insertAdjacentHTML( - "beforeend", - '', - ); + stopWatching(); + document.body.insertAdjacentHTML("beforeend", LATER); deferMediaUntilDue(); - expect(document.body.lastElementChild!.getAttribute("src")).toBe("/m/film.mp4#t=30"); + expect(document.body.lastElementChild!.hasAttribute("src")).toBe(false); }); it.each([ - ["starts at the head of the file", "/m/film.mp4", 'data-media-start="0"'], - ["loops", "/m/film.mp4", 'data-media-start="30" loop'], - ["already carries a fragment", "/m/film.mp4#t=1", 'data-media-start="30"'], - ])("is left alone when it %s", async (_name, src, attrs) => { - const v = await added( - ``, - ); - expect(v.getAttribute("src")).toBe(src); - expect(unproxiedSrc(v)).toBe(src); - }); - - it("is left alone in Studio's preview, which copies and saves what it reads from the page", async () => { - inPlayer(false); - const v = await added( - '', - ); + ["is the first take", ''], + ["has no authored length", ''], + ])("keeps its src when it %s", async (_name, html) => { + const v = await added(html); expect(v.getAttribute("src")).toBe("/m/film.mp4"); }); - it("is left alone off Safari", async () => { + it("keeps its src in Studio's preview, which copies and saves what it reads from the page", async () => { + inPlayer(false); + const v = await added(LATER); + expect(v.getAttribute("src")).toBe("/m/film.mp4"); + }); + + it("keeps its src off Safari", async () => { vi.spyOn(navigator, "userAgent", "get").mockReturnValue(CHROME); - const v = await added( - '', - ); + const v = await added(LATER); expect(v.getAttribute("src")).toBe("/m/film.mp4"); }); }); diff --git a/packages/core/src/runtime/preloadMedia.ts b/packages/core/src/runtime/preloadMedia.ts index e92b4e4f67..ca5f165965 100644 --- a/packages/core/src/runtime/preloadMedia.ts +++ b/packages/core/src/runtime/preloadMedia.ts @@ -1,7 +1,8 @@ import { parseStrictFiniteTimingNumber, readMediaStart } from "./playbackRate"; -import { recordOpenedAtInPoint } from "./proxySrc"; +import { holdBackSrc, takeHeldBackSrc } from "./proxySrc"; import { createRuntimeStartTimeResolver } from "./startResolver"; import { readMediaRenderId } from "./renderFrameSibling"; +import { isMediaElement } from "./domRealm"; import { defersMedia } from "./timedClipHide"; import { inPlayerFrame } from "../studioPreviewMark"; import { swallow } from "./diagnostics"; @@ -24,6 +25,7 @@ export function isRenderFrameVideo( export function preloadMedia(media: PreloadableMedia): void { if (isRenderFrameVideo(media)) return; + if (isMediaElement(media)) restoreHeldBackSrc(media); if (media.preload !== "auto") media.preload = "auto"; // load() resets an in-flight video fetch, discarding its selected resource and buffered data. const videoAlreadyLoading = media.tagName === "VIDEO" && media.networkState === 2; @@ -99,19 +101,24 @@ export function lengthIsAuthored(media: Element): boolean { return parseStrictFiniteTimingNumber(media.getAttribute("data-duration")) != null; } -/** WebKit reads about 40 MB from the head of the file for every fresh element however late its in-point - * seek lands; a start fragment makes it open at the in-point, so many clips of one long file stop pulling - * many times its size, and each is cheap enough to fetch ahead. */ -export function openAtInPoint(el: HTMLMediaElement): void { +/** WebKit reads about 35 MB from the head of the file for every element that has a src, however late the clip + * is due, so a film of many cuts from one file moves many times its size and starves the clip on screen. + * The player keeps a clip's src off until it is near. */ +function holdBackUntilNear(el: HTMLMediaElement): void { const src = el.getAttribute("src"); - const start = readMediaStart(el); - // Only the player: Studio copies and saves what it reads from the page, and must never see the fragment. - if (!inPlayerFrame(document) || !src || src.includes("#") || el.loop) return; - if (el.querySelector("source")) return; - if (!(start > 0)) return; - const written = `${src}#t=${start}`; - recordOpenedAtInPoint(el, src, written); - el.setAttribute("src", written); + if (!inPlayerFrame(document) || !src || el.querySelector("source")) return; + holdBackSrc(el, src); + el.removeAttribute("src"); + // A load the parser started before this ran is only ended by a load of the now empty element. + el.load(); +} + +/** Puts a held-back src on its element; true when there was one. The element loads only when preloaded. */ +export function restoreHeldBackSrc(el: HTMLMediaElement): boolean { + const src = takeHeldBackSrc(el); + if (src === null) return false; + el.setAttribute("src", src); + return true; } type DeferralWindow = Window & { __hfMediaDeferral?: MutationObserver }; @@ -127,8 +134,9 @@ export function deferMediaUntilDue(): void { const start = parseStrictFiniteTimingNumber(el.getAttribute("data-start")); if (start == null || !lengthIsAuthored(el)) return; const media = el as HTMLMediaElement; - openAtInPoint(media); - if (startInFilm(el) > 0) media.preload = "none"; + if (startInFilm(el) <= 0) return; + media.preload = "none"; + holdBackUntilNear(media); }; win.__hfMediaDeferral = new MutationObserver((records) => { const startInFilm = createRuntimeStartTimeResolver({}).resolveMediaStartForElement; diff --git a/packages/core/src/runtime/proxySrc.ts b/packages/core/src/runtime/proxySrc.ts index 450ef3a4c1..dfdb270941 100644 --- a/packages/core/src/runtime/proxySrc.ts +++ b/packages/core/src/runtime/proxySrc.ts @@ -1,21 +1,27 @@ /** Media elements swapped to their proxy, each with the src attribute it had before the swap. */ export const swappedElements = new WeakMap(); -/** Media elements the runtime opened at their in-point: the src the author wrote and the one written in its place. */ -const openedAtInPoint = new WeakMap(); +/** Media elements whose src is held back until the clip is near, each with the src the author wrote. */ +const heldBackSrc = new WeakMap(); -export function recordOpenedAtInPoint(el: Element, authored: string, written: string): void { - openedAtInPoint.set(el, { authored, written }); +export function holdBackSrc(el: Element, src: string): void { + heldBackSrc.set(el, src); } -/** True while the src is still the one the runtime wrote: a script or binding that sets another ends it. */ -export function isOpenedAtInPoint(el: Element): boolean { - return openedAtInPoint.get(el)?.written === el.getAttribute("src"); +export function isSrcHeldBack(el: Element): boolean { + return heldBackSrc.has(el); +} + +/** The held-back src, forgotten: the caller puts it back on the element. */ +export function takeHeldBackSrc(el: Element): string | null { + const src = heldBackSrc.get(el) ?? null; + heldBackSrc.delete(el); + return src; } export function unproxiedSrc(el: Element): string | null { if (swappedElements.has(el)) return swappedElements.get(el) ?? null; - return isOpenedAtInPoint(el) ? openedAtInPoint.get(el)!.authored : el.getAttribute("src"); + return heldBackSrc.get(el) ?? el.getAttribute("src"); } /** Videos showing a still of their last frame as poster across a copy swap: the poster they had before, and when. */ diff --git a/packages/core/src/runtime/webAudioTransport.test.ts b/packages/core/src/runtime/webAudioTransport.test.ts index da96a07dd2..1e97fbeffa 100644 --- a/packages/core/src/runtime/webAudioTransport.test.ts +++ b/packages/core/src/runtime/webAudioTransport.test.ts @@ -1868,22 +1868,6 @@ describe("WebAudioTransport", () => { vi.unstubAllGlobals(); }); - it("decodes a file once for clips that open at different in-points of it", async () => { - const transport = transportWithDecode(async () => ({}) as AudioBuffer); - const fetchMock = vi.fn(async () => ({ - ok: true, - arrayBuffer: async () => new ArrayBuffer(8), - })); - vi.stubGlobal("fetch", fetchMock); - await Promise.all([ - transport.decodeAudioElement(el("take.wav#t=10")), - transport.decodeAudioElement(el("take.wav#t=90")), - ]); - expect(fetchMock).toHaveBeenCalledTimes(1); - expect(fetchMock.mock.calls[0]![0]).toBe(key("take.wav")); - vi.unstubAllGlobals(); - }); - it("DOES blacklist genuinely undecodable bytes — not retried", async () => { const transport = transportWithDecode(async () => { throw new Error("unsupported codec"); diff --git a/packages/core/src/runtime/webAudioTransport.ts b/packages/core/src/runtime/webAudioTransport.ts index f404d79b62..6f9f721ef8 100644 --- a/packages/core/src/runtime/webAudioTransport.ts +++ b/packages/core/src/runtime/webAudioTransport.ts @@ -210,14 +210,11 @@ function isBufferSource( return source.sourceKind === "buffer"; } -// The fragment narrowing a clip's span is not part of the file: every clip of it shares one decode. const decodeKey = (el: HTMLMediaElement): string | null => { const src = el.currentSrc || el.getAttribute("src"); if (!src) return null; try { - const url = new URL(assetUrl(src)); - url.hash = ""; - return url.href; + return assetUrl(src); } catch { return src; }