perf(core): Safari stops a long video's clips from loading the whole file at once (#7436)

* perf(core): Safari clips open at their in-point and are fetched ahead

* perf(core): Safari opens a long file's clips at their in-point and leaves short files as they were

* perf(core): Safari holds a long file's later clips with no source until each one is due

* perf(core): Safari plays a long file's sound from its element instead of fetching the whole file

* perf(core): Safari holds a later clip of a long file even when WebKit began reading it at parse

* style(core): fit the long-file comments to the comment budget and the formatter

* refactor(core): one proxy trigger path, and the producer budget at its measured size

* perf(core): Safari opens a later clip WebKit read at parse at its in-point rather than holding it

* build: the producer size budget at its measured 3162274 bytes

* refactor(core): name the condition that holds a long file's later clip
This commit is contained in:
Miguel Ángel
2026-11-10 12:38:47 +00:00
committed by GitHub
parent ec30de0627
commit fecde9992f
15 changed files with 471 additions and 39 deletions
@@ -0,0 +1,83 @@
import { afterEach, beforeEach, describe, expect, it, vi } from "vitest";
import { initSandboxRuntimeModular } from "./init";
import { deferMediaUntilDue } from "./preloadMedia";
import { unproxiedSrc } from "./proxySrc";
import { createMockTimeline, resetRuntimeFixtureDom } from "./runtimeSeekFixture.test-helpers";
const IPHONE =
"Mozilla/5.0 (iPhone; CPU iPhone OS 26_5 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/26.5 Mobile/15E148 Safari/604.1";
let seq = 0;
describe("later clips of a long file in the player on WebKit", () => {
let file = "";
const clip = (id: string) => document.getElementById(id) as HTMLVideoElement;
const src = (id: string) => clip(id).getAttribute("src");
beforeEach(async () => {
resetRuntimeFixtureDom();
vi.spyOn(navigator, "userAgent", "get").mockReturnValue(IPHONE);
const frame = document.createElement("iframe");
frame.setAttribute("data-hf-player-frame", "");
Object.defineProperty(window, "frameElement", { value: frame, configurable: true });
vi.stubGlobal(
"fetch",
vi.fn(
async () =>
new Response(null, { status: 206, headers: { "content-range": "bytes 0-0/68000000" } }),
),
);
// Each test names its own file: a file's size is asked once per page.
file = `/m/film-${seq++}.mp4`;
deferMediaUntilDue();
document.body.innerHTML =
`<div data-composition-id="main" data-root="true" data-start="0" data-duration="60" data-width="1920" data-height="1080">` +
[
["v0", 0, 30],
["v1", 20, 50],
["v2", 40, 70],
]
.map(
([id, start, inPoint]) =>
`<video id="${id}" src="${file}" data-start="${start}" data-duration="2" data-media-start="${inPoint}" data-track-index="0"></video>`,
)
.join("") +
`</div>`;
for (const video of document.querySelectorAll("video")) video.load = vi.fn();
await vi.waitFor(() => expect(src("v2")).toBeNull());
window.__timelines = { main: createMockTimeline(60) };
initSandboxRuntimeModular();
});
afterEach(() => {
window.__hfRuntimeTeardown?.();
(window as Window & { __hfMediaDeferral?: MutationObserver }).__hfMediaDeferral?.disconnect();
vi.unstubAllGlobals();
vi.restoreAllMocks();
Reflect.deleteProperty(window, "frameElement");
resetRuntimeFixtureDom();
});
it("hold no src until due, past the next take, and still name the file they play", () => {
expect([src("v0"), src("v1"), src("v2")]).toEqual([`${file}#t=30`, `${file}#t=50`, null]);
expect(unproxiedSrc(clip("v2"))).toBe(file);
});
it("get their src at their in-point when a paused seek lands on one", () => {
window.__player?.seek(41);
expect(src("v2")).toBe(`${file}#t=70`);
});
it("keep a src a binding sets while one is held, and play that one when due", () => {
clip("v2").setAttribute("src", "/m/other.mp4");
window.__player?.seek(41);
expect(src("v2")).toBe("/m/other.mp4");
expect(unproxiedSrc(clip("v2"))).toBe("/m/other.mp4");
});
it("let go of the file once passed, and get it back on a seek back", () => {
window.__player?.seek(41);
expect(src("v1")).toBeNull();
window.__player?.seek(21);
expect(src("v1")).toBe(`${file}#t=50`);
});
});
+19 -2
View File
@@ -13,6 +13,7 @@ import { runtimeProtocolMetadata } from "./protocol";
import { collectRuntimeTimelinePayload } from "./timeline";
import { TYPEGPU_PRESENT_HEARTBEAT_MS } from "./adapters/typegpu";
import { WebAudioTransport } from "./webAudioTransport";
import { bytesOfSource } from "./preloadMedia";
import type { RuntimeTimelineChildLike, RuntimeTimelineLike } from "./types";
import {
registerRuntimeDataHandler,
@@ -6047,9 +6048,9 @@ describe("initSandboxRuntimeModular", () => {
const SAFARI =
"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/18.0 Safari/605.1.15";
const startRouted = async (userAgent?: string, duration = 10) => {
const startRouted = async (userAgent?: string, duration = 10, src = "/assets/track.mp3") => {
if (userAgent) vi.spyOn(navigator, "userAgent", "get").mockReturnValue(userAgent);
const audio = mountAudio("/assets/track.mp3");
const audio = mountAudio(src);
Object.defineProperty(audio, "duration", { value: duration });
const capture = vi
.spyOn(WebAudioTransport.prototype, "scheduleMediaElementPlayback")
@@ -6088,6 +6089,22 @@ describe("initSandboxRuntimeModular", () => {
expect(release).not.toHaveBeenCalled();
});
it("leaves a clip of a long file on its element: decoding it would fetch the whole file first", async () => {
vi.stubGlobal(
"fetch",
vi.fn(
async () =>
new Response(null, { status: 206, headers: { "content-range": "bytes 0-0/68000000" } }),
),
);
await bytesOfSource("/assets/long-film.mp4");
vi.unstubAllGlobals();
const { schedule, release } = await startRouted(SAFARI, 321, "/assets/long-film.mp4");
await Promise.resolve();
expect(schedule).not.toHaveBeenCalled();
expect(release).not.toHaveBeenCalled();
});
it("gives a clip that has not loaded its source's verdict", async () => {
vi.spyOn(WebAudioTransport.prototype, "fitsDecodeBudget").mockReturnValue(true);
const { audio, release } = await startRouted(SAFARI, Number.NaN);
+7 -6
View File
@@ -20,6 +20,8 @@ import { clipsFirstRenderedIn } from "./firstRenderReveal";
import { refreshSvgSelectorAliases } from "../compiler/svgSelectorAliases";
import { RUNTIME_FILLER } from "./protocol";
import {
giveBackWhenDue,
isLongSource,
preloadMedia,
prepareUpcomingMedia,
releaseMedia,
@@ -523,6 +525,7 @@ function decodesFirst(el: HTMLMediaElement, webAudio: WebAudioTransport): boolea
isWebKit() &&
isAudioElement(el) &&
!el.loop &&
!isLongSource(el) &&
webAudio.fitsDecodeBudget(el) &&
typeof readElementRateSpec(el) === "number" &&
readElementPlaybackRate(el) === 1
@@ -1314,7 +1317,7 @@ export function initSandboxRuntimeModular(): void {
? {
from: "media",
tag: media.tagName.toLowerCase(),
src: media.getAttribute("src") ?? media.currentSrc ?? "",
src: unproxiedSrc(media) ?? media.currentSrc ?? "",
trimmed: media.hasAttribute("data-duration") || media.hasAttribute("data-end"),
}
: { from };
@@ -2530,10 +2533,7 @@ export function initSandboxRuntimeModular(): void {
}
const tagName = node.tagName.toLowerCase();
const assetUrl =
node.getAttribute("src") ??
node.getAttribute("href") ??
node.getAttribute("poster") ??
null;
unproxiedSrc(node) ?? node.getAttribute("href") ?? node.getAttribute("poster") ?? null;
const diagnosticCode =
tagName === "link" ? "runtime_stylesheet_load_failed" : "runtime_asset_load_failed";
postRuntimeDiagnosticOnce(
@@ -3053,7 +3053,7 @@ export function initSandboxRuntimeModular(): void {
const NEXT_TAKE_SPAN_SECONDS = 0.5;
let nextTakes: { t: number; bound: number; clips: Set<HTMLMediaElement> } | null = null;
const fileOf = (el: HTMLMediaElement) =>
el.getAttribute("src") ?? el.querySelector("source")?.getAttribute("src") ?? "";
unproxiedSrc(el) ?? el.querySelector("source")?.getAttribute("src") ?? "";
const nextTakeClips = (t: number): Set<HTMLMediaElement> => {
if (nextTakes?.t === t && nextTakes.bound === metadataBoundMedia.size) return nextTakes.clips;
const byFile = new Map<string, { el: HTMLMediaElement; start: number }[]>();
@@ -3253,6 +3253,7 @@ export function initSandboxRuntimeModular(): void {
resolveAbsoluteMediaStartSeconds(el) + (resolveDurationForElement(el) ?? Number.NaN) <= time;
const spent = passed && (decided !== true || (el.networkState === 2 && el.readyState < 4));
const near = visible || armed || (upcoming && !unslotted && !spent);
if (near) giveBackWhenDue(el);
if ((visible || !near) && endPrefetch(el)) queueMicrotask(fillPrefetchSlots);
const approaching = near && !visible && resolveAbsoluteMediaStartSeconds(el) > clock.now();
const wasApproaching = approachingMedia.has(el);
@@ -2,6 +2,7 @@ import { afterEach, beforeEach, describe, expect, it, vi } from "vitest";
import { postRuntimeMessage } from "./bridge";
import { watchAudioLoadErrors } from "./mediaLoadFailure";
import { RUNTIME_MEDIA_LOAD_FAILED } from "./protocol";
import { recordOpenedAtInPoint } from "./proxySrc";
import { WebAudioTransport } from "./webAudioTransport";
vi.mock("./bridge", () => ({ postRuntimeMessage: vi.fn() }));
@@ -37,6 +38,18 @@ afterEach(() => {
});
describe("a preview media load that fails is reported, not swallowed", () => {
it("names the src the author wrote, not the in-point fragment the runtime added", async () => {
vi.stubGlobal(
"fetch",
vi.fn(async () => ({ ok: false, status: 404 })),
);
const el = audio("assets/take.wav", "take");
recordOpenedAtInPoint(el, "assets/take.wav", "assets/take.wav#t=30");
el.setAttribute("src", "assets/take.wav#t=30");
await transport(async () => ({}) as AudioBuffer).decodeAudioElement(el);
expect(failures()[0]).toMatchObject({ asset: "assets/take.wav" });
});
it("a fetch the server refuses says its status, once however often playback retries", async () => {
vi.stubGlobal(
"fetch",
@@ -1,4 +1,5 @@
import { postRuntimeMessage } from "./bridge";
import { unproxiedSrc } from "./proxySrc";
import { isAudioElement, isSourceElement } from "./domRealm";
import { isRenderMode } from "./mediaProxy";
import { RUNTIME_MEDIA_LOAD_FAILED, type MediaLoadStage } from "./protocol";
@@ -30,7 +31,7 @@ export function reportMediaLoadFailure(
if (stages.has(stage)) return;
stages.add(stage);
reported.set(el, stages);
const asset = el.getAttribute("src") ?? "";
const asset = unproxiedSrc(el) ?? "";
const details: Record<string, RuntimeJson> = {
asset,
elementId: el.id || null,
@@ -18,6 +18,7 @@ import {
swapToProxy,
type MediaCodecMapEntry,
} from "./mediaProxy";
import { recordOpenedAtInPoint, unproxiedSrc } from "./proxySrc";
vi.mock("./bridge", () => ({
postRuntimeMessage: vi.fn(),
@@ -497,6 +498,17 @@ describe("handleErrorForProxy (tertiary trigger)", () => {
});
describe("swapToProxy", () => {
it("swaps a clip held with no src after giving it back its file", async () => {
const el = createVideo("/video.mp4");
recordOpenedAtInPoint(el, "/video.mp4", null);
el.removeAttribute("src");
swapToProxy(el, HEVC_ENTRY, "proactive");
await settle();
expect(isProxied(el)).toBe(true);
expect(new URL(el.src).pathname).toBe("/video.mp4");
expect(unproxiedSrc(el)).toBe("/video.mp4");
});
it("does not poison swap state when the source URL is malformed", async () => {
const el = createVideo("/video.mp4");
Object.defineProperty(el, "currentSrc", {
+10 -21
View File
@@ -10,7 +10,8 @@ import {
import { findInjectedRenderFrame } from "./renderFrameSibling";
import type { RuntimeJson } from "./types";
import { isVideoElement } from "./domRealm";
import { heldPosters, swappedElements } from "./proxySrc";
import { heldPosters, swappedElements, unproxiedSrc } from "./proxySrc";
import { giveBackWhenDue } from "./preloadMedia";
import { restartPlaybackHealth } from "./playbackHealth";
import { failedCopyElements, noteCopy, setCopyReportingLevel } from "./previewCopies";
import { waitForServedProxy } from "./proxyWait";
@@ -523,7 +524,7 @@ async function makeCopy(el: HTMLMediaElement, job: CopyJob, scale: number | null
return;
}
noteCopy(el, "ready", null, light);
swappedElements.set(el, originalAttr);
swappedElements.set(el, unproxiedSrc(el));
if (lightFacts) lightElements.add(el);
loadProxy(el, proxiedSrc, originalSrc, variant, box, lightFacts !== undefined);
// The shown size may have grown while this copy was being made.
@@ -551,6 +552,7 @@ export function swapToProxy(
lightFacts?: PreviewSourceFacts,
): void {
if (swappedElements.has(el)) return;
giveBackWhenDue(el);
if (proxyRequested.has(el) && proxyRequested.get(el) === el.getAttribute("src")) return;
const originalSrc = currentSrcValue(el);
const variant = copyVariant(entry);
@@ -761,24 +763,7 @@ export function handleMetadataForProxy(el: HTMLMediaElement): void {
if (isRenderMode(el)) return;
if (!isVideoElement(el)) return;
if (el.videoWidth !== 0) return;
const src = currentSrcValue(el);
if (swappedElements.has(el)) {
emitUnavailableDiagnostic(el, "proxy_playback_failed", src);
return;
}
const map = window.__HF_MEDIA_CODEC_MAP__;
if (!map) return;
const key = deriveCodecMapKey(el);
if (key === null) {
emitUnavailableDiagnostic(el, "cross_origin", src);
return;
}
const entry = lookupCodecMapEntry(key, map);
if (entry && !entry.browserHostile) {
emitUnavailableDiagnostic(el, "browser_safe_codec", src);
return;
}
swapToProxy(el, entry, "reactive");
swapOrDiagnose(el, "reactive");
}
/**
@@ -792,6 +777,10 @@ export function handleMetadataForProxy(el: HTMLMediaElement): void {
export function handleErrorForProxy(el: HTMLMediaElement): void {
if (isRenderMode(el)) return;
if (!isVideoElement(el)) return;
swapOrDiagnose(el, "tertiary");
}
function swapOrDiagnose(el: HTMLMediaElement, trigger: ProxyTrigger): void {
const src = currentSrcValue(el);
if (swappedElements.has(el)) {
emitUnavailableDiagnostic(el, "proxy_playback_failed", src);
@@ -809,5 +798,5 @@ export function handleErrorForProxy(el: HTMLMediaElement): void {
emitUnavailableDiagnostic(el, "browser_safe_codec", src);
return;
}
swapToProxy(el, entry, "tertiary");
swapToProxy(el, entry, trigger);
}
+2 -1
View File
@@ -3,6 +3,7 @@ import { PREVIEW_VIDEO_FRAME_ATTR } from "../studioPreviewMark";
import { COLOR_GRADING_SOURCE_HIDDEN_ATTR } from "../colorGrading";
import { swallow } from "./diagnostics";
import { isElementNode } from "./domRealm";
import { unproxiedSrc } from "./proxySrc";
import { createDrawnProbe } from "./pickerDrawn";
type PickerModuleDeps = {
@@ -311,7 +312,7 @@ export function createPickerModule(deps: PickerModuleDeps): PickerModule {
label: buildElementLabel(el),
boundingBox: { x: rect.left, y: rect.top, width: rect.width, height: rect.height },
textContent: el.textContent ? el.textContent.trim().slice(0, 200) : null,
src: el.getAttribute("src") || el.getAttribute("data-composition-src") || null,
src: unproxiedSrc(el) || el.getAttribute("data-composition-src") || null,
dataAttributes,
};
}
+151 -1
View File
@@ -1,6 +1,9 @@
import { afterEach, describe, expect, it, vi } from "vitest";
import { afterEach, beforeEach, describe, expect, it, vi } from "vitest";
import { mediaElementTime } from "./mediaClock";
import { unproxiedSrc } from "./proxySrc";
import {
deferMediaUntilDue,
giveBackWhenDue,
isRenderFrameVideo,
preloadMedia,
prepareUpcomingMedia,
@@ -261,3 +264,150 @@ describe("preparing upcoming video playback", () => {
element.remove();
});
});
describe("a clip of a long file opening at its in-point on Safari", () => {
const SAFARI =
"Mozilla/5.0 (iPhone; CPU iPhone OS 18_0 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/18.0 Mobile/15E148 Safari/604.1";
const CHROME = "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 Chrome/120.0 Safari/537.36";
let meta: HTMLMetaElement;
let seq = 0;
const stopWatching = () =>
(window as Window & { __hfMediaDeferral?: MutationObserver }).__hfMediaDeferral?.disconnect();
const inPlayer = (on: boolean) => {
const frame = document.createElement("iframe");
if (on) frame.setAttribute("data-hf-player-frame", "");
Object.defineProperty(window, "frameElement", { value: frame, configurable: true });
};
const sizeOf = (bytes: number | "unreadable") =>
vi.stubGlobal(
"fetch",
vi.fn(async () => {
if (bytes === "unreadable") throw new TypeError("blocked");
return new Response(null, {
status: 206,
headers: { "content-range": `bytes 0-0/${bytes}` },
});
}),
);
beforeEach(() => {
meta = document.createElement("meta");
meta.name = "hyperframes-studio-preview";
document.head.appendChild(meta);
vi.spyOn(navigator, "userAgent", "get").mockReturnValue(SAFARI);
inPlayer(true);
sizeOf(68_000_000);
deferMediaUntilDue();
});
afterEach(() => {
stopWatching();
vi.unstubAllGlobals();
vi.restoreAllMocks();
Reflect.deleteProperty(window, "frameElement");
meta.remove();
document.body.replaceChildren();
});
// Each test names its own file: the size of a file is asked once per page.
const clip = (attrs = 'data-media-start="30"', start = "0") =>
`<video src="/m/film-${seq++}.mp4" data-start="${start}" data-duration="2" ${attrs}></video>`;
const later = () => clip('data-media-start="30"', "4");
async function added(html: string): Promise<HTMLVideoElement> {
document.body.insertAdjacentHTML("beforeend", html);
await Promise.resolve();
return document.body.lastElementChild as HTMLVideoElement;
}
const opened = (v: HTMLVideoElement) => v.getAttribute("src")!.includes("#t=");
it("gets a start fragment and keeps its authored src for everything that reads it", async () => {
const v = await added(clip());
const authored = v.getAttribute("src")!;
await vi.waitFor(() => expect(v.getAttribute("src")).toBe(`${authored}#t=30`));
expect(unproxiedSrc(v)).toBe(authored);
});
it("holds a later clip with no src, still naming its file, until it is due", async () => {
const v = await added(later());
const authored = v.getAttribute("src")!;
await vi.waitFor(() => expect(v.getAttribute("src")).toBeNull());
expect(unproxiedSrc(v)).toBe(authored);
giveBackWhenDue(v);
expect(v.getAttribute("src")).toBe(`${authored}#t=30`);
});
it("opens a later clip WebKit already began reading at its in-point, and gives its preload back", async () => {
vi.spyOn(HTMLMediaElement.prototype, "networkState", "get").mockReturnValue(2);
const v = await added(later());
await vi.waitFor(() => expect(opened(v)).toBe(true));
expect(v.preload).not.toBe("none");
});
it("holds a later clip again when it is released after playing", async () => {
const v = await added(later());
await vi.waitFor(() => expect(v.getAttribute("src")).toBeNull());
giveBackWhenDue(v);
releaseMedia(v, true);
expect(v.getAttribute("src")).toBeNull();
});
it("does not hold a later clip a binding has given another src", async () => {
const v = await added(later());
v.setAttribute("src", "/m/bound.mp4");
await new Promise((r) => setTimeout(r, 10));
expect(v.getAttribute("src")).toBe("/m/bound.mp4");
});
it("does so for a clip already parsed when the runtime arrives", async () => {
stopWatching();
document.body.insertAdjacentHTML("beforeend", clip());
deferMediaUntilDue();
const v = document.body.lastElementChild as HTMLVideoElement;
await vi.waitFor(() => expect(opened(v)).toBe(true));
});
it("stops reporting the authored src once something else sets the src", async () => {
const v = await added(clip());
await vi.waitFor(() => expect(opened(v)).toBe(true));
v.setAttribute("src", "/m/other.mp4");
expect(unproxiedSrc(v)).toBe("/m/other.mp4");
});
it("is left alone when its file is small, so a short film loads exactly as before", async () => {
sizeOf(5_600_000);
const v = await added(later());
await vi.waitFor(() => expect(fetch).toHaveBeenCalled());
await new Promise((r) => setTimeout(r, 10));
expect(opened(v)).toBe(false);
expect(v.preload).toBe("none");
});
it("is left alone when the size of its file cannot be read", async () => {
sizeOf("unreadable");
const v = await added(clip());
await vi.waitFor(() => expect(fetch).toHaveBeenCalled());
await new Promise((r) => setTimeout(r, 10));
expect(opened(v)).toBe(false);
});
it.each([
["starts at the head of the file", 'data-media-start="0"'],
["loops", 'data-media-start="30" loop'],
])("is left alone when it %s", async (_name, attrs) => {
const v = await added(clip(attrs));
await new Promise((r) => setTimeout(r, 10));
expect(opened(v)).toBe(false);
});
it("is left alone in Studio's preview, which copies and saves what it reads from the page", async () => {
inPlayer(false);
const v = await added(clip());
await new Promise((r) => setTimeout(r, 10));
expect(opened(v)).toBe(false);
});
it("is left alone off Safari", async () => {
vi.spyOn(navigator, "userAgent", "get").mockReturnValue(CHROME);
const v = await added(clip());
await new Promise((r) => setTimeout(r, 10));
expect(opened(v)).toBe(false);
});
});
+111 -2
View File
@@ -1,7 +1,15 @@
import { parseStrictFiniteTimingNumber, readMediaStart } from "./playbackRate";
import {
authoredSrcOf,
holdsWhenIdle,
isHeldUntilDue,
recordOpenedAtInPoint,
unproxiedSrc,
} from "./proxySrc";
import { createRuntimeStartTimeResolver } from "./startResolver";
import { readMediaRenderId } from "./renderFrameSibling";
import { defersMedia } from "./timedClipHide";
import { inPlayerFrame } from "../studioPreviewMark";
import { swallow } from "./diagnostics";
import { isWebKit, mediaElementTime } from "./mediaClock";
@@ -88,6 +96,7 @@ function wholeFileBuffered(media: HTMLMediaElement): boolean {
export function releaseMedia(media: HTMLMediaElement, whenBuffered = false): void {
if (media.querySelector("source") || (!whenBuffered && wholeFileBuffered(media))) return;
if (holdsWhenIdle(media)) return holdUntilDue(media);
const src = media.getAttribute("src");
stopMediaDownload(media);
if (src !== null) media.setAttribute("src", src);
@@ -97,6 +106,98 @@ export function lengthIsAuthored(media: Element): boolean {
return parseStrictFiniteTimingNumber(media.getAttribute("data-duration")) != null;
}
const LONG_SOURCE_BYTES = 32 * 1024 * 1024;
const sourceBytes = new Map<string, Promise<number | null>>();
const knownBytes = new Map<string, number>();
export function isLongSource(el: Element): boolean {
const src = unproxiedSrc(el);
if (!src) return false;
try {
return (knownBytes.get(new URL(src, document.baseURI).href) ?? 0) >= LONG_SOURCE_BYTES;
} catch {
return false;
}
}
export function bytesOfSource(src: string): Promise<number | null> {
let url: string;
try {
url = new URL(src, document.baseURI).href;
} catch {
return Promise.resolve(null);
}
const known = sourceBytes.get(url);
if (known) return known;
const asked = (async () => {
const abort = new AbortController();
try {
const response = await fetch(url, { headers: { Range: "bytes=0-0" }, signal: abort.signal });
const total =
/\/(\d+)$/.exec(response.headers.get("content-range") ?? "")?.[1] ??
(response.status === 200 ? response.headers.get("content-length") : null);
if (total === null) return null;
knownBytes.set(url, Number(total));
return Number(total);
} catch {
return null;
} finally {
abort.abort();
}
})();
sourceBytes.set(url, asked);
return asked;
}
function openAtInPoint(el: HTMLMediaElement): void {
const src = el.getAttribute("src");
const start = readMediaStart(el);
// Only the player: Studio copies and saves what it reads from the page, and must never see the fragment.
if (!inPlayerFrame(document) || !src || src.includes("#") || el.loop) return;
if (el.querySelector("source")) return;
if (!(start > 0)) return;
const written = `${src}#t=${start}`;
recordOpenedAtInPoint(el, src, written);
el.setAttribute("src", written);
}
// WebKit opens the file for any clip that has a src, whatever its preload: a later clip of a long file has none.
function holdUntilDue(el: HTMLMediaElement): void {
const authored = unproxiedSrc(el);
if (authored === null) return;
recordOpenedAtInPoint(el, authored, null, true);
stopMediaDownload(el);
}
export function giveBackWhenDue(el: HTMLMediaElement): void {
if (!isHeldUntilDue(el)) return;
const authored = authoredSrcOf(el)!;
const start = readMediaStart(el);
const written = start > 0 ? `${authored}#t=${start}` : authored;
recordOpenedAtInPoint(el, authored, written);
el.setAttribute("src", written);
}
// One WebKit began reading at parse opens at its in-point instead: its metadata then costs little.
const holdable = (el: HTMLMediaElement, src: string) =>
el.preload === "none" &&
el.networkState !== 2 &&
!src.includes("#") &&
!el.loop &&
!el.querySelector("source");
function openLongSourceAtInPoint(el: HTMLMediaElement, later: boolean, opened: () => void): void {
const src = el.getAttribute("src");
if (!isWebKit() || !inPlayerFrame(document) || !src) return;
void bytesOfSource(src).then((bytes) => {
if (bytes === null || bytes < LONG_SOURCE_BYTES || el.getAttribute("src") !== src) return;
if (later && holdable(el, src)) return holdUntilDue(el);
openAtInPoint(el);
opened();
});
}
type DeferralWindow = Window & { __hfMediaDeferral?: MutationObserver };
/** Preview only: later clips parse at preload none, as Chromium ignores a none set mid-fetch.
@@ -108,8 +209,12 @@ export function deferMediaUntilDue(): void {
if (!defersMedia()) return;
const defer = (el: Element, startInFilm: (el: Element) => number) => {
const start = parseStrictFiniteTimingNumber(el.getAttribute("data-start"));
if (start != null && lengthIsAuthored(el) && startInFilm(el) > 0)
(el as HTMLMediaElement).preload = "none";
if (start == null || !lengthIsAuthored(el)) return;
const media = el as HTMLMediaElement;
const authoredPreload = media.preload;
const later = startInFilm(el) > 0;
if (later) media.preload = "none";
openLongSourceAtInPoint(media, later, () => (media.preload = authoredPreload));
};
win.__hfMediaDeferral = new MutationObserver((records) => {
const startInFilm = createRuntimeStartTimeResolver({}).resolveMediaStartForElement;
@@ -122,4 +227,8 @@ export function deferMediaUntilDue(): void {
}
});
win.__hfMediaDeferral.observe(document.documentElement, { childList: true, subtree: true });
if (isWebKit()) {
const startInFilm = createRuntimeStartTimeResolver({}).resolveMediaStartForElement;
for (const media of document.querySelectorAll("video, audio")) defer(media, startInFilm);
}
}
+38 -1
View File
@@ -1,8 +1,45 @@
/** Media elements swapped to their proxy, each with the src attribute it had before the swap. */
export const swappedElements = new WeakMap<Element, string | null>();
/** Media elements the runtime opened at their in-point: the src the author wrote, the one written in its
* place (null while the clip is held with no src until it is due), and whether it is held again when idle. */
const openedAtInPoint = new WeakMap<
Element,
{ authored: string; written: string | null; holdsWhenIdle: boolean }
>();
export function recordOpenedAtInPoint(
el: Element,
authored: string,
written: string | null,
holdsWhenIdle = openedAtInPoint.get(el)?.holdsWhenIdle ?? false,
): void {
openedAtInPoint.set(el, { authored, written, holdsWhenIdle });
}
/** True for a clip the runtime holds whenever it is not due, while its src is still the runtime's. */
export function holdsWhenIdle(el: Element): boolean {
return isOpenedAtInPoint(el) && openedAtInPoint.get(el)!.holdsWhenIdle;
}
/** True while the src is still the one the runtime wrote: a script or binding that sets another ends it. */
function isOpenedAtInPoint(el: Element): boolean {
const record = openedAtInPoint.get(el);
return record !== undefined && record.written === el.getAttribute("src");
}
export function isHeldUntilDue(el: Element): boolean {
return isOpenedAtInPoint(el) && openedAtInPoint.get(el)!.written === null;
}
/** The src the author wrote for a clip the runtime opened or holds. */
export function authoredSrcOf(el: Element): string | null {
return openedAtInPoint.get(el)?.authored ?? null;
}
export function unproxiedSrc(el: Element): string | null {
return swappedElements.has(el) ? (swappedElements.get(el) ?? null) : el.getAttribute("src");
if (swappedElements.has(el)) return swappedElements.get(el) ?? null;
return isOpenedAtInPoint(el) ? openedAtInPoint.get(el)!.authored : el.getAttribute("src");
}
/** Videos showing a still of their last frame as poster across a copy swap: the poster they had before, and when. */
+3 -2
View File
@@ -4,6 +4,7 @@ import type {
RuntimeTimelineScene,
RuntimeTimelineLike,
} from "./types";
import { unproxiedSrc } from "./proxySrc";
import { stableClipId } from "./clipTree";
import { findRootCompositionElement, parseCompositionDimension } from "./compositionDimension";
import {
@@ -166,13 +167,13 @@ function toAbsoluteAssetUrl(rawValue: string | null | undefined): string | null
}
function resolveNodeAssetUrl(node: Element): string | null {
const src = node.getAttribute("src") ?? node.getAttribute("data-src");
const src = unproxiedSrc(node) ?? node.getAttribute("data-src");
if (src) return toAbsoluteAssetUrl(src);
const compositionSrc = node.getAttribute("data-composition-src");
if (compositionSrc) return toAbsoluteAssetUrl(compositionSrc);
const mediaDescendant = node.querySelector("img[src], video[src], audio[src], source[src]");
if (!mediaDescendant) return null;
return toAbsoluteAssetUrl(mediaDescendant.getAttribute("src"));
return toAbsoluteAssetUrl(unproxiedSrc(mediaDescendant));
}
function getFirstClassToken(node: Element): string | null {
@@ -1868,6 +1868,22 @@ describe("WebAudioTransport", () => {
vi.unstubAllGlobals();
});
it("decodes a file once for clips that open at different in-points of it", async () => {
const transport = transportWithDecode(async () => ({}) as AudioBuffer);
const fetchMock = vi.fn(async () => ({
ok: true,
arrayBuffer: async () => new ArrayBuffer(8),
}));
vi.stubGlobal("fetch", fetchMock);
await Promise.all([
transport.decodeAudioElement(el("take.wav#t=10")),
transport.decodeAudioElement(el("take.wav#t=90")),
]);
expect(fetchMock).toHaveBeenCalledTimes(1);
expect(fetchMock.mock.calls[0]![0]).toBe(key("take.wav"));
vi.unstubAllGlobals();
});
it("DOES blacklist genuinely undecodable bytes — not retried", async () => {
const transport = transportWithDecode(async () => {
throw new Error("unsupported codec");
@@ -214,7 +214,9 @@ const decodeKey = (el: HTMLMediaElement): string | null => {
const src = el.currentSrc || el.getAttribute("src");
if (!src) return null;
try {
return assetUrl(src);
const url = new URL(assetUrl(src));
url.hash = "";
return url.href;
} catch {
return src;
}