From fecde9992fba422e56df4dedc84940bda47ce75e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Miguel=20=C3=81ngel?= Date: Tue, 10 Nov 2026 12:38:47 +0000 Subject: [PATCH] perf(core): Safari stops a long video's clips from loading the whole file at once (#7436) * perf(core): Safari clips open at their in-point and are fetched ahead * perf(core): Safari opens a long file's clips at their in-point and leaves short files as they were * perf(core): Safari holds a long file's later clips with no source until each one is due * perf(core): Safari plays a long file's sound from its element instead of fetching the whole file * perf(core): Safari holds a later clip of a long file even when WebKit began reading it at parse * style(core): fit the long-file comments to the comment budget and the formatter * refactor(core): one proxy trigger path, and the producer budget at its measured size * perf(core): Safari opens a later clip WebKit read at parse at its in-point rather than holding it * build: the producer size budget at its measured 3162274 bytes * refactor(core): name the condition that holds a long file's later clip --- .../src/runtime/init.longSourceHold.test.ts | 83 ++++++++++ packages/core/src/runtime/init.test.ts | 21 ++- packages/core/src/runtime/init.ts | 13 +- .../core/src/runtime/mediaLoadFailure.test.ts | 13 ++ packages/core/src/runtime/mediaLoadFailure.ts | 3 +- packages/core/src/runtime/mediaProxy.test.ts | 12 ++ packages/core/src/runtime/mediaProxy.ts | 31 ++-- packages/core/src/runtime/picker.ts | 3 +- .../core/src/runtime/preloadMedia.test.ts | 152 +++++++++++++++++- packages/core/src/runtime/preloadMedia.ts | 113 ++++++++++++- packages/core/src/runtime/proxySrc.ts | 39 ++++- packages/core/src/runtime/timeline.ts | 5 +- .../src/runtime/webAudioTransport.test.ts | 16 ++ .../core/src/runtime/webAudioTransport.ts | 4 +- scripts/package-size-budgets.json | 2 +- 15 files changed, 471 insertions(+), 39 deletions(-) create mode 100644 packages/core/src/runtime/init.longSourceHold.test.ts diff --git a/packages/core/src/runtime/init.longSourceHold.test.ts b/packages/core/src/runtime/init.longSourceHold.test.ts new file mode 100644 index 0000000000..42ff6712e2 --- /dev/null +++ b/packages/core/src/runtime/init.longSourceHold.test.ts @@ -0,0 +1,83 @@ +import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; +import { initSandboxRuntimeModular } from "./init"; +import { deferMediaUntilDue } from "./preloadMedia"; +import { unproxiedSrc } from "./proxySrc"; +import { createMockTimeline, resetRuntimeFixtureDom } from "./runtimeSeekFixture.test-helpers"; + +const IPHONE = + "Mozilla/5.0 (iPhone; CPU iPhone OS 26_5 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/26.5 Mobile/15E148 Safari/604.1"; +let seq = 0; + +describe("later clips of a long file in the player on WebKit", () => { + let file = ""; + const clip = (id: string) => document.getElementById(id) as HTMLVideoElement; + const src = (id: string) => clip(id).getAttribute("src"); + + beforeEach(async () => { + resetRuntimeFixtureDom(); + vi.spyOn(navigator, "userAgent", "get").mockReturnValue(IPHONE); + const frame = document.createElement("iframe"); + frame.setAttribute("data-hf-player-frame", ""); + Object.defineProperty(window, "frameElement", { value: frame, configurable: true }); + vi.stubGlobal( + "fetch", + vi.fn( + async () => + new Response(null, { status: 206, headers: { "content-range": "bytes 0-0/68000000" } }), + ), + ); + // Each test names its own file: a file's size is asked once per page. + file = `/m/film-${seq++}.mp4`; + deferMediaUntilDue(); + document.body.innerHTML = + `
` + + [ + ["v0", 0, 30], + ["v1", 20, 50], + ["v2", 40, 70], + ] + .map( + ([id, start, inPoint]) => + ``, + ) + .join("") + + `
`; + for (const video of document.querySelectorAll("video")) video.load = vi.fn(); + await vi.waitFor(() => expect(src("v2")).toBeNull()); + window.__timelines = { main: createMockTimeline(60) }; + initSandboxRuntimeModular(); + }); + + afterEach(() => { + window.__hfRuntimeTeardown?.(); + (window as Window & { __hfMediaDeferral?: MutationObserver }).__hfMediaDeferral?.disconnect(); + vi.unstubAllGlobals(); + vi.restoreAllMocks(); + Reflect.deleteProperty(window, "frameElement"); + resetRuntimeFixtureDom(); + }); + + it("hold no src until due, past the next take, and still name the file they play", () => { + expect([src("v0"), src("v1"), src("v2")]).toEqual([`${file}#t=30`, `${file}#t=50`, null]); + expect(unproxiedSrc(clip("v2"))).toBe(file); + }); + + it("get their src at their in-point when a paused seek lands on one", () => { + window.__player?.seek(41); + expect(src("v2")).toBe(`${file}#t=70`); + }); + + it("keep a src a binding sets while one is held, and play that one when due", () => { + clip("v2").setAttribute("src", "/m/other.mp4"); + window.__player?.seek(41); + expect(src("v2")).toBe("/m/other.mp4"); + expect(unproxiedSrc(clip("v2"))).toBe("/m/other.mp4"); + }); + + it("let go of the file once passed, and get it back on a seek back", () => { + window.__player?.seek(41); + expect(src("v1")).toBeNull(); + window.__player?.seek(21); + expect(src("v1")).toBe(`${file}#t=50`); + }); +}); diff --git a/packages/core/src/runtime/init.test.ts b/packages/core/src/runtime/init.test.ts index 143c9359fe..3cbae1edd4 100644 --- a/packages/core/src/runtime/init.test.ts +++ b/packages/core/src/runtime/init.test.ts @@ -13,6 +13,7 @@ import { runtimeProtocolMetadata } from "./protocol"; import { collectRuntimeTimelinePayload } from "./timeline"; import { TYPEGPU_PRESENT_HEARTBEAT_MS } from "./adapters/typegpu"; import { WebAudioTransport } from "./webAudioTransport"; +import { bytesOfSource } from "./preloadMedia"; import type { RuntimeTimelineChildLike, RuntimeTimelineLike } from "./types"; import { registerRuntimeDataHandler, @@ -6047,9 +6048,9 @@ describe("initSandboxRuntimeModular", () => { const SAFARI = "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/18.0 Safari/605.1.15"; - const startRouted = async (userAgent?: string, duration = 10) => { + const startRouted = async (userAgent?: string, duration = 10, src = "/assets/track.mp3") => { if (userAgent) vi.spyOn(navigator, "userAgent", "get").mockReturnValue(userAgent); - const audio = mountAudio("/assets/track.mp3"); + const audio = mountAudio(src); Object.defineProperty(audio, "duration", { value: duration }); const capture = vi .spyOn(WebAudioTransport.prototype, "scheduleMediaElementPlayback") @@ -6088,6 +6089,22 @@ describe("initSandboxRuntimeModular", () => { expect(release).not.toHaveBeenCalled(); }); + it("leaves a clip of a long file on its element: decoding it would fetch the whole file first", async () => { + vi.stubGlobal( + "fetch", + vi.fn( + async () => + new Response(null, { status: 206, headers: { "content-range": "bytes 0-0/68000000" } }), + ), + ); + await bytesOfSource("/assets/long-film.mp4"); + vi.unstubAllGlobals(); + const { schedule, release } = await startRouted(SAFARI, 321, "/assets/long-film.mp4"); + await Promise.resolve(); + expect(schedule).not.toHaveBeenCalled(); + expect(release).not.toHaveBeenCalled(); + }); + it("gives a clip that has not loaded its source's verdict", async () => { vi.spyOn(WebAudioTransport.prototype, "fitsDecodeBudget").mockReturnValue(true); const { audio, release } = await startRouted(SAFARI, Number.NaN); diff --git a/packages/core/src/runtime/init.ts b/packages/core/src/runtime/init.ts index 843b64ebf5..b551dc4fee 100644 --- a/packages/core/src/runtime/init.ts +++ b/packages/core/src/runtime/init.ts @@ -20,6 +20,8 @@ import { clipsFirstRenderedIn } from "./firstRenderReveal"; import { refreshSvgSelectorAliases } from "../compiler/svgSelectorAliases"; import { RUNTIME_FILLER } from "./protocol"; import { + giveBackWhenDue, + isLongSource, preloadMedia, prepareUpcomingMedia, releaseMedia, @@ -523,6 +525,7 @@ function decodesFirst(el: HTMLMediaElement, webAudio: WebAudioTransport): boolea isWebKit() && isAudioElement(el) && !el.loop && + !isLongSource(el) && webAudio.fitsDecodeBudget(el) && typeof readElementRateSpec(el) === "number" && readElementPlaybackRate(el) === 1 @@ -1314,7 +1317,7 @@ export function initSandboxRuntimeModular(): void { ? { from: "media", tag: media.tagName.toLowerCase(), - src: media.getAttribute("src") ?? media.currentSrc ?? "", + src: unproxiedSrc(media) ?? media.currentSrc ?? "", trimmed: media.hasAttribute("data-duration") || media.hasAttribute("data-end"), } : { from }; @@ -2530,10 +2533,7 @@ export function initSandboxRuntimeModular(): void { } const tagName = node.tagName.toLowerCase(); const assetUrl = - node.getAttribute("src") ?? - node.getAttribute("href") ?? - node.getAttribute("poster") ?? - null; + unproxiedSrc(node) ?? node.getAttribute("href") ?? node.getAttribute("poster") ?? null; const diagnosticCode = tagName === "link" ? "runtime_stylesheet_load_failed" : "runtime_asset_load_failed"; postRuntimeDiagnosticOnce( @@ -3053,7 +3053,7 @@ export function initSandboxRuntimeModular(): void { const NEXT_TAKE_SPAN_SECONDS = 0.5; let nextTakes: { t: number; bound: number; clips: Set } | null = null; const fileOf = (el: HTMLMediaElement) => - el.getAttribute("src") ?? el.querySelector("source")?.getAttribute("src") ?? ""; + unproxiedSrc(el) ?? el.querySelector("source")?.getAttribute("src") ?? ""; const nextTakeClips = (t: number): Set => { if (nextTakes?.t === t && nextTakes.bound === metadataBoundMedia.size) return nextTakes.clips; const byFile = new Map(); @@ -3253,6 +3253,7 @@ export function initSandboxRuntimeModular(): void { resolveAbsoluteMediaStartSeconds(el) + (resolveDurationForElement(el) ?? Number.NaN) <= time; const spent = passed && (decided !== true || (el.networkState === 2 && el.readyState < 4)); const near = visible || armed || (upcoming && !unslotted && !spent); + if (near) giveBackWhenDue(el); if ((visible || !near) && endPrefetch(el)) queueMicrotask(fillPrefetchSlots); const approaching = near && !visible && resolveAbsoluteMediaStartSeconds(el) > clock.now(); const wasApproaching = approachingMedia.has(el); diff --git a/packages/core/src/runtime/mediaLoadFailure.test.ts b/packages/core/src/runtime/mediaLoadFailure.test.ts index 2a391b1326..cd84e90457 100644 --- a/packages/core/src/runtime/mediaLoadFailure.test.ts +++ b/packages/core/src/runtime/mediaLoadFailure.test.ts @@ -2,6 +2,7 @@ import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; import { postRuntimeMessage } from "./bridge"; import { watchAudioLoadErrors } from "./mediaLoadFailure"; import { RUNTIME_MEDIA_LOAD_FAILED } from "./protocol"; +import { recordOpenedAtInPoint } from "./proxySrc"; import { WebAudioTransport } from "./webAudioTransport"; vi.mock("./bridge", () => ({ postRuntimeMessage: vi.fn() })); @@ -37,6 +38,18 @@ afterEach(() => { }); describe("a preview media load that fails is reported, not swallowed", () => { + it("names the src the author wrote, not the in-point fragment the runtime added", async () => { + vi.stubGlobal( + "fetch", + vi.fn(async () => ({ ok: false, status: 404 })), + ); + const el = audio("assets/take.wav", "take"); + recordOpenedAtInPoint(el, "assets/take.wav", "assets/take.wav#t=30"); + el.setAttribute("src", "assets/take.wav#t=30"); + await transport(async () => ({}) as AudioBuffer).decodeAudioElement(el); + expect(failures()[0]).toMatchObject({ asset: "assets/take.wav" }); + }); + it("a fetch the server refuses says its status, once however often playback retries", async () => { vi.stubGlobal( "fetch", diff --git a/packages/core/src/runtime/mediaLoadFailure.ts b/packages/core/src/runtime/mediaLoadFailure.ts index b15e005658..5b140a6f42 100644 --- a/packages/core/src/runtime/mediaLoadFailure.ts +++ b/packages/core/src/runtime/mediaLoadFailure.ts @@ -1,4 +1,5 @@ import { postRuntimeMessage } from "./bridge"; +import { unproxiedSrc } from "./proxySrc"; import { isAudioElement, isSourceElement } from "./domRealm"; import { isRenderMode } from "./mediaProxy"; import { RUNTIME_MEDIA_LOAD_FAILED, type MediaLoadStage } from "./protocol"; @@ -30,7 +31,7 @@ export function reportMediaLoadFailure( if (stages.has(stage)) return; stages.add(stage); reported.set(el, stages); - const asset = el.getAttribute("src") ?? ""; + const asset = unproxiedSrc(el) ?? ""; const details: Record = { asset, elementId: el.id || null, diff --git a/packages/core/src/runtime/mediaProxy.test.ts b/packages/core/src/runtime/mediaProxy.test.ts index fc293b8f61..18423c63c1 100644 --- a/packages/core/src/runtime/mediaProxy.test.ts +++ b/packages/core/src/runtime/mediaProxy.test.ts @@ -18,6 +18,7 @@ import { swapToProxy, type MediaCodecMapEntry, } from "./mediaProxy"; +import { recordOpenedAtInPoint, unproxiedSrc } from "./proxySrc"; vi.mock("./bridge", () => ({ postRuntimeMessage: vi.fn(), @@ -497,6 +498,17 @@ describe("handleErrorForProxy (tertiary trigger)", () => { }); describe("swapToProxy", () => { + it("swaps a clip held with no src after giving it back its file", async () => { + const el = createVideo("/video.mp4"); + recordOpenedAtInPoint(el, "/video.mp4", null); + el.removeAttribute("src"); + swapToProxy(el, HEVC_ENTRY, "proactive"); + await settle(); + expect(isProxied(el)).toBe(true); + expect(new URL(el.src).pathname).toBe("/video.mp4"); + expect(unproxiedSrc(el)).toBe("/video.mp4"); + }); + it("does not poison swap state when the source URL is malformed", async () => { const el = createVideo("/video.mp4"); Object.defineProperty(el, "currentSrc", { diff --git a/packages/core/src/runtime/mediaProxy.ts b/packages/core/src/runtime/mediaProxy.ts index d315f074d8..25c7481beb 100644 --- a/packages/core/src/runtime/mediaProxy.ts +++ b/packages/core/src/runtime/mediaProxy.ts @@ -10,7 +10,8 @@ import { import { findInjectedRenderFrame } from "./renderFrameSibling"; import type { RuntimeJson } from "./types"; import { isVideoElement } from "./domRealm"; -import { heldPosters, swappedElements } from "./proxySrc"; +import { heldPosters, swappedElements, unproxiedSrc } from "./proxySrc"; +import { giveBackWhenDue } from "./preloadMedia"; import { restartPlaybackHealth } from "./playbackHealth"; import { failedCopyElements, noteCopy, setCopyReportingLevel } from "./previewCopies"; import { waitForServedProxy } from "./proxyWait"; @@ -523,7 +524,7 @@ async function makeCopy(el: HTMLMediaElement, job: CopyJob, scale: number | null return; } noteCopy(el, "ready", null, light); - swappedElements.set(el, originalAttr); + swappedElements.set(el, unproxiedSrc(el)); if (lightFacts) lightElements.add(el); loadProxy(el, proxiedSrc, originalSrc, variant, box, lightFacts !== undefined); // The shown size may have grown while this copy was being made. @@ -551,6 +552,7 @@ export function swapToProxy( lightFacts?: PreviewSourceFacts, ): void { if (swappedElements.has(el)) return; + giveBackWhenDue(el); if (proxyRequested.has(el) && proxyRequested.get(el) === el.getAttribute("src")) return; const originalSrc = currentSrcValue(el); const variant = copyVariant(entry); @@ -761,24 +763,7 @@ export function handleMetadataForProxy(el: HTMLMediaElement): void { if (isRenderMode(el)) return; if (!isVideoElement(el)) return; if (el.videoWidth !== 0) return; - const src = currentSrcValue(el); - if (swappedElements.has(el)) { - emitUnavailableDiagnostic(el, "proxy_playback_failed", src); - return; - } - const map = window.__HF_MEDIA_CODEC_MAP__; - if (!map) return; - const key = deriveCodecMapKey(el); - if (key === null) { - emitUnavailableDiagnostic(el, "cross_origin", src); - return; - } - const entry = lookupCodecMapEntry(key, map); - if (entry && !entry.browserHostile) { - emitUnavailableDiagnostic(el, "browser_safe_codec", src); - return; - } - swapToProxy(el, entry, "reactive"); + swapOrDiagnose(el, "reactive"); } /** @@ -792,6 +777,10 @@ export function handleMetadataForProxy(el: HTMLMediaElement): void { export function handleErrorForProxy(el: HTMLMediaElement): void { if (isRenderMode(el)) return; if (!isVideoElement(el)) return; + swapOrDiagnose(el, "tertiary"); +} + +function swapOrDiagnose(el: HTMLMediaElement, trigger: ProxyTrigger): void { const src = currentSrcValue(el); if (swappedElements.has(el)) { emitUnavailableDiagnostic(el, "proxy_playback_failed", src); @@ -809,5 +798,5 @@ export function handleErrorForProxy(el: HTMLMediaElement): void { emitUnavailableDiagnostic(el, "browser_safe_codec", src); return; } - swapToProxy(el, entry, "tertiary"); + swapToProxy(el, entry, trigger); } diff --git a/packages/core/src/runtime/picker.ts b/packages/core/src/runtime/picker.ts index 5e658425c0..b05564d578 100644 --- a/packages/core/src/runtime/picker.ts +++ b/packages/core/src/runtime/picker.ts @@ -3,6 +3,7 @@ import { PREVIEW_VIDEO_FRAME_ATTR } from "../studioPreviewMark"; import { COLOR_GRADING_SOURCE_HIDDEN_ATTR } from "../colorGrading"; import { swallow } from "./diagnostics"; import { isElementNode } from "./domRealm"; +import { unproxiedSrc } from "./proxySrc"; import { createDrawnProbe } from "./pickerDrawn"; type PickerModuleDeps = { @@ -311,7 +312,7 @@ export function createPickerModule(deps: PickerModuleDeps): PickerModule { label: buildElementLabel(el), boundingBox: { x: rect.left, y: rect.top, width: rect.width, height: rect.height }, textContent: el.textContent ? el.textContent.trim().slice(0, 200) : null, - src: el.getAttribute("src") || el.getAttribute("data-composition-src") || null, + src: unproxiedSrc(el) || el.getAttribute("data-composition-src") || null, dataAttributes, }; } diff --git a/packages/core/src/runtime/preloadMedia.test.ts b/packages/core/src/runtime/preloadMedia.test.ts index 08e08a1095..7209701d64 100644 --- a/packages/core/src/runtime/preloadMedia.test.ts +++ b/packages/core/src/runtime/preloadMedia.test.ts @@ -1,6 +1,9 @@ -import { afterEach, describe, expect, it, vi } from "vitest"; +import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; import { mediaElementTime } from "./mediaClock"; +import { unproxiedSrc } from "./proxySrc"; import { + deferMediaUntilDue, + giveBackWhenDue, isRenderFrameVideo, preloadMedia, prepareUpcomingMedia, @@ -261,3 +264,150 @@ describe("preparing upcoming video playback", () => { element.remove(); }); }); + +describe("a clip of a long file opening at its in-point on Safari", () => { + const SAFARI = + "Mozilla/5.0 (iPhone; CPU iPhone OS 18_0 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/18.0 Mobile/15E148 Safari/604.1"; + const CHROME = "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 Chrome/120.0 Safari/537.36"; + let meta: HTMLMetaElement; + let seq = 0; + const stopWatching = () => + (window as Window & { __hfMediaDeferral?: MutationObserver }).__hfMediaDeferral?.disconnect(); + const inPlayer = (on: boolean) => { + const frame = document.createElement("iframe"); + if (on) frame.setAttribute("data-hf-player-frame", ""); + Object.defineProperty(window, "frameElement", { value: frame, configurable: true }); + }; + const sizeOf = (bytes: number | "unreadable") => + vi.stubGlobal( + "fetch", + vi.fn(async () => { + if (bytes === "unreadable") throw new TypeError("blocked"); + return new Response(null, { + status: 206, + headers: { "content-range": `bytes 0-0/${bytes}` }, + }); + }), + ); + beforeEach(() => { + meta = document.createElement("meta"); + meta.name = "hyperframes-studio-preview"; + document.head.appendChild(meta); + vi.spyOn(navigator, "userAgent", "get").mockReturnValue(SAFARI); + inPlayer(true); + sizeOf(68_000_000); + deferMediaUntilDue(); + }); + afterEach(() => { + stopWatching(); + vi.unstubAllGlobals(); + vi.restoreAllMocks(); + Reflect.deleteProperty(window, "frameElement"); + meta.remove(); + document.body.replaceChildren(); + }); + + // Each test names its own file: the size of a file is asked once per page. + const clip = (attrs = 'data-media-start="30"', start = "0") => + ``; + const later = () => clip('data-media-start="30"', "4"); + async function added(html: string): Promise { + document.body.insertAdjacentHTML("beforeend", html); + await Promise.resolve(); + return document.body.lastElementChild as HTMLVideoElement; + } + const opened = (v: HTMLVideoElement) => v.getAttribute("src")!.includes("#t="); + + it("gets a start fragment and keeps its authored src for everything that reads it", async () => { + const v = await added(clip()); + const authored = v.getAttribute("src")!; + await vi.waitFor(() => expect(v.getAttribute("src")).toBe(`${authored}#t=30`)); + expect(unproxiedSrc(v)).toBe(authored); + }); + + it("holds a later clip with no src, still naming its file, until it is due", async () => { + const v = await added(later()); + const authored = v.getAttribute("src")!; + await vi.waitFor(() => expect(v.getAttribute("src")).toBeNull()); + expect(unproxiedSrc(v)).toBe(authored); + giveBackWhenDue(v); + expect(v.getAttribute("src")).toBe(`${authored}#t=30`); + }); + + it("opens a later clip WebKit already began reading at its in-point, and gives its preload back", async () => { + vi.spyOn(HTMLMediaElement.prototype, "networkState", "get").mockReturnValue(2); + const v = await added(later()); + await vi.waitFor(() => expect(opened(v)).toBe(true)); + expect(v.preload).not.toBe("none"); + }); + + it("holds a later clip again when it is released after playing", async () => { + const v = await added(later()); + await vi.waitFor(() => expect(v.getAttribute("src")).toBeNull()); + giveBackWhenDue(v); + releaseMedia(v, true); + expect(v.getAttribute("src")).toBeNull(); + }); + + it("does not hold a later clip a binding has given another src", async () => { + const v = await added(later()); + v.setAttribute("src", "/m/bound.mp4"); + await new Promise((r) => setTimeout(r, 10)); + expect(v.getAttribute("src")).toBe("/m/bound.mp4"); + }); + + it("does so for a clip already parsed when the runtime arrives", async () => { + stopWatching(); + document.body.insertAdjacentHTML("beforeend", clip()); + deferMediaUntilDue(); + const v = document.body.lastElementChild as HTMLVideoElement; + await vi.waitFor(() => expect(opened(v)).toBe(true)); + }); + + it("stops reporting the authored src once something else sets the src", async () => { + const v = await added(clip()); + await vi.waitFor(() => expect(opened(v)).toBe(true)); + v.setAttribute("src", "/m/other.mp4"); + expect(unproxiedSrc(v)).toBe("/m/other.mp4"); + }); + + it("is left alone when its file is small, so a short film loads exactly as before", async () => { + sizeOf(5_600_000); + const v = await added(later()); + await vi.waitFor(() => expect(fetch).toHaveBeenCalled()); + await new Promise((r) => setTimeout(r, 10)); + expect(opened(v)).toBe(false); + expect(v.preload).toBe("none"); + }); + + it("is left alone when the size of its file cannot be read", async () => { + sizeOf("unreadable"); + const v = await added(clip()); + await vi.waitFor(() => expect(fetch).toHaveBeenCalled()); + await new Promise((r) => setTimeout(r, 10)); + expect(opened(v)).toBe(false); + }); + + it.each([ + ["starts at the head of the file", 'data-media-start="0"'], + ["loops", 'data-media-start="30" loop'], + ])("is left alone when it %s", async (_name, attrs) => { + const v = await added(clip(attrs)); + await new Promise((r) => setTimeout(r, 10)); + expect(opened(v)).toBe(false); + }); + + it("is left alone in Studio's preview, which copies and saves what it reads from the page", async () => { + inPlayer(false); + const v = await added(clip()); + await new Promise((r) => setTimeout(r, 10)); + expect(opened(v)).toBe(false); + }); + + it("is left alone off Safari", async () => { + vi.spyOn(navigator, "userAgent", "get").mockReturnValue(CHROME); + const v = await added(clip()); + await new Promise((r) => setTimeout(r, 10)); + expect(opened(v)).toBe(false); + }); +}); diff --git a/packages/core/src/runtime/preloadMedia.ts b/packages/core/src/runtime/preloadMedia.ts index 91eaaf3093..2e9933d4b6 100644 --- a/packages/core/src/runtime/preloadMedia.ts +++ b/packages/core/src/runtime/preloadMedia.ts @@ -1,7 +1,15 @@ import { parseStrictFiniteTimingNumber, readMediaStart } from "./playbackRate"; +import { + authoredSrcOf, + holdsWhenIdle, + isHeldUntilDue, + recordOpenedAtInPoint, + unproxiedSrc, +} from "./proxySrc"; import { createRuntimeStartTimeResolver } from "./startResolver"; import { readMediaRenderId } from "./renderFrameSibling"; import { defersMedia } from "./timedClipHide"; +import { inPlayerFrame } from "../studioPreviewMark"; import { swallow } from "./diagnostics"; import { isWebKit, mediaElementTime } from "./mediaClock"; @@ -88,6 +96,7 @@ function wholeFileBuffered(media: HTMLMediaElement): boolean { export function releaseMedia(media: HTMLMediaElement, whenBuffered = false): void { if (media.querySelector("source") || (!whenBuffered && wholeFileBuffered(media))) return; + if (holdsWhenIdle(media)) return holdUntilDue(media); const src = media.getAttribute("src"); stopMediaDownload(media); if (src !== null) media.setAttribute("src", src); @@ -97,6 +106,98 @@ export function lengthIsAuthored(media: Element): boolean { return parseStrictFiniteTimingNumber(media.getAttribute("data-duration")) != null; } +const LONG_SOURCE_BYTES = 32 * 1024 * 1024; +const sourceBytes = new Map>(); + +const knownBytes = new Map(); + +export function isLongSource(el: Element): boolean { + const src = unproxiedSrc(el); + if (!src) return false; + try { + return (knownBytes.get(new URL(src, document.baseURI).href) ?? 0) >= LONG_SOURCE_BYTES; + } catch { + return false; + } +} + +export function bytesOfSource(src: string): Promise { + let url: string; + try { + url = new URL(src, document.baseURI).href; + } catch { + return Promise.resolve(null); + } + const known = sourceBytes.get(url); + if (known) return known; + const asked = (async () => { + const abort = new AbortController(); + try { + const response = await fetch(url, { headers: { Range: "bytes=0-0" }, signal: abort.signal }); + const total = + /\/(\d+)$/.exec(response.headers.get("content-range") ?? "")?.[1] ?? + (response.status === 200 ? response.headers.get("content-length") : null); + if (total === null) return null; + knownBytes.set(url, Number(total)); + return Number(total); + } catch { + return null; + } finally { + abort.abort(); + } + })(); + sourceBytes.set(url, asked); + return asked; +} + +function openAtInPoint(el: HTMLMediaElement): void { + const src = el.getAttribute("src"); + const start = readMediaStart(el); + // Only the player: Studio copies and saves what it reads from the page, and must never see the fragment. + if (!inPlayerFrame(document) || !src || src.includes("#") || el.loop) return; + if (el.querySelector("source")) return; + if (!(start > 0)) return; + const written = `${src}#t=${start}`; + recordOpenedAtInPoint(el, src, written); + el.setAttribute("src", written); +} + +// WebKit opens the file for any clip that has a src, whatever its preload: a later clip of a long file has none. +function holdUntilDue(el: HTMLMediaElement): void { + const authored = unproxiedSrc(el); + if (authored === null) return; + recordOpenedAtInPoint(el, authored, null, true); + stopMediaDownload(el); +} + +export function giveBackWhenDue(el: HTMLMediaElement): void { + if (!isHeldUntilDue(el)) return; + const authored = authoredSrcOf(el)!; + const start = readMediaStart(el); + const written = start > 0 ? `${authored}#t=${start}` : authored; + recordOpenedAtInPoint(el, authored, written); + el.setAttribute("src", written); +} + +// One WebKit began reading at parse opens at its in-point instead: its metadata then costs little. +const holdable = (el: HTMLMediaElement, src: string) => + el.preload === "none" && + el.networkState !== 2 && + !src.includes("#") && + !el.loop && + !el.querySelector("source"); + +function openLongSourceAtInPoint(el: HTMLMediaElement, later: boolean, opened: () => void): void { + const src = el.getAttribute("src"); + if (!isWebKit() || !inPlayerFrame(document) || !src) return; + void bytesOfSource(src).then((bytes) => { + if (bytes === null || bytes < LONG_SOURCE_BYTES || el.getAttribute("src") !== src) return; + if (later && holdable(el, src)) return holdUntilDue(el); + openAtInPoint(el); + opened(); + }); +} + type DeferralWindow = Window & { __hfMediaDeferral?: MutationObserver }; /** Preview only: later clips parse at preload none, as Chromium ignores a none set mid-fetch. @@ -108,8 +209,12 @@ export function deferMediaUntilDue(): void { if (!defersMedia()) return; const defer = (el: Element, startInFilm: (el: Element) => number) => { const start = parseStrictFiniteTimingNumber(el.getAttribute("data-start")); - if (start != null && lengthIsAuthored(el) && startInFilm(el) > 0) - (el as HTMLMediaElement).preload = "none"; + if (start == null || !lengthIsAuthored(el)) return; + const media = el as HTMLMediaElement; + const authoredPreload = media.preload; + const later = startInFilm(el) > 0; + if (later) media.preload = "none"; + openLongSourceAtInPoint(media, later, () => (media.preload = authoredPreload)); }; win.__hfMediaDeferral = new MutationObserver((records) => { const startInFilm = createRuntimeStartTimeResolver({}).resolveMediaStartForElement; @@ -122,4 +227,8 @@ export function deferMediaUntilDue(): void { } }); win.__hfMediaDeferral.observe(document.documentElement, { childList: true, subtree: true }); + if (isWebKit()) { + const startInFilm = createRuntimeStartTimeResolver({}).resolveMediaStartForElement; + for (const media of document.querySelectorAll("video, audio")) defer(media, startInFilm); + } } diff --git a/packages/core/src/runtime/proxySrc.ts b/packages/core/src/runtime/proxySrc.ts index 82f62315c9..eae5759c8e 100644 --- a/packages/core/src/runtime/proxySrc.ts +++ b/packages/core/src/runtime/proxySrc.ts @@ -1,8 +1,45 @@ /** Media elements swapped to their proxy, each with the src attribute it had before the swap. */ export const swappedElements = new WeakMap(); +/** Media elements the runtime opened at their in-point: the src the author wrote, the one written in its + * place (null while the clip is held with no src until it is due), and whether it is held again when idle. */ +const openedAtInPoint = new WeakMap< + Element, + { authored: string; written: string | null; holdsWhenIdle: boolean } +>(); + +export function recordOpenedAtInPoint( + el: Element, + authored: string, + written: string | null, + holdsWhenIdle = openedAtInPoint.get(el)?.holdsWhenIdle ?? false, +): void { + openedAtInPoint.set(el, { authored, written, holdsWhenIdle }); +} + +/** True for a clip the runtime holds whenever it is not due, while its src is still the runtime's. */ +export function holdsWhenIdle(el: Element): boolean { + return isOpenedAtInPoint(el) && openedAtInPoint.get(el)!.holdsWhenIdle; +} + +/** True while the src is still the one the runtime wrote: a script or binding that sets another ends it. */ +function isOpenedAtInPoint(el: Element): boolean { + const record = openedAtInPoint.get(el); + return record !== undefined && record.written === el.getAttribute("src"); +} + +export function isHeldUntilDue(el: Element): boolean { + return isOpenedAtInPoint(el) && openedAtInPoint.get(el)!.written === null; +} + +/** The src the author wrote for a clip the runtime opened or holds. */ +export function authoredSrcOf(el: Element): string | null { + return openedAtInPoint.get(el)?.authored ?? null; +} + export function unproxiedSrc(el: Element): string | null { - return swappedElements.has(el) ? (swappedElements.get(el) ?? null) : el.getAttribute("src"); + if (swappedElements.has(el)) return swappedElements.get(el) ?? null; + return isOpenedAtInPoint(el) ? openedAtInPoint.get(el)!.authored : el.getAttribute("src"); } /** Videos showing a still of their last frame as poster across a copy swap: the poster they had before, and when. */ diff --git a/packages/core/src/runtime/timeline.ts b/packages/core/src/runtime/timeline.ts index aac2d0bd46..f37af327c1 100644 --- a/packages/core/src/runtime/timeline.ts +++ b/packages/core/src/runtime/timeline.ts @@ -4,6 +4,7 @@ import type { RuntimeTimelineScene, RuntimeTimelineLike, } from "./types"; +import { unproxiedSrc } from "./proxySrc"; import { stableClipId } from "./clipTree"; import { findRootCompositionElement, parseCompositionDimension } from "./compositionDimension"; import { @@ -166,13 +167,13 @@ function toAbsoluteAssetUrl(rawValue: string | null | undefined): string | null } function resolveNodeAssetUrl(node: Element): string | null { - const src = node.getAttribute("src") ?? node.getAttribute("data-src"); + const src = unproxiedSrc(node) ?? node.getAttribute("data-src"); if (src) return toAbsoluteAssetUrl(src); const compositionSrc = node.getAttribute("data-composition-src"); if (compositionSrc) return toAbsoluteAssetUrl(compositionSrc); const mediaDescendant = node.querySelector("img[src], video[src], audio[src], source[src]"); if (!mediaDescendant) return null; - return toAbsoluteAssetUrl(mediaDescendant.getAttribute("src")); + return toAbsoluteAssetUrl(unproxiedSrc(mediaDescendant)); } function getFirstClassToken(node: Element): string | null { diff --git a/packages/core/src/runtime/webAudioTransport.test.ts b/packages/core/src/runtime/webAudioTransport.test.ts index 1e97fbeffa..da96a07dd2 100644 --- a/packages/core/src/runtime/webAudioTransport.test.ts +++ b/packages/core/src/runtime/webAudioTransport.test.ts @@ -1868,6 +1868,22 @@ describe("WebAudioTransport", () => { vi.unstubAllGlobals(); }); + it("decodes a file once for clips that open at different in-points of it", async () => { + const transport = transportWithDecode(async () => ({}) as AudioBuffer); + const fetchMock = vi.fn(async () => ({ + ok: true, + arrayBuffer: async () => new ArrayBuffer(8), + })); + vi.stubGlobal("fetch", fetchMock); + await Promise.all([ + transport.decodeAudioElement(el("take.wav#t=10")), + transport.decodeAudioElement(el("take.wav#t=90")), + ]); + expect(fetchMock).toHaveBeenCalledTimes(1); + expect(fetchMock.mock.calls[0]![0]).toBe(key("take.wav")); + vi.unstubAllGlobals(); + }); + it("DOES blacklist genuinely undecodable bytes — not retried", async () => { const transport = transportWithDecode(async () => { throw new Error("unsupported codec"); diff --git a/packages/core/src/runtime/webAudioTransport.ts b/packages/core/src/runtime/webAudioTransport.ts index 6f9f721ef8..c381e58543 100644 --- a/packages/core/src/runtime/webAudioTransport.ts +++ b/packages/core/src/runtime/webAudioTransport.ts @@ -214,7 +214,9 @@ const decodeKey = (el: HTMLMediaElement): string | null => { const src = el.currentSrc || el.getAttribute("src"); if (!src) return null; try { - return assetUrl(src); + const url = new URL(assetUrl(src)); + url.hash = ""; + return url.href; } catch { return src; } diff --git a/scripts/package-size-budgets.json b/scripts/package-size-budgets.json index ea7584e509..76310afae7 100644 --- a/scripts/package-size-budgets.json +++ b/scripts/package-size-budgets.json @@ -37,7 +37,7 @@ }, "@hyperframes/producer": { "packed": 870000, - "unpacked": 3130000 + "unpacked": 3162274 }, "@hyperframes/sdk": { "packed": 90662,