diff --git a/packages/core/src/runtime/init.longSourceHold.test.ts b/packages/core/src/runtime/init.longSourceHold.test.ts
new file mode 100644
index 0000000000..42ff6712e2
--- /dev/null
+++ b/packages/core/src/runtime/init.longSourceHold.test.ts
@@ -0,0 +1,83 @@
+import { afterEach, beforeEach, describe, expect, it, vi } from "vitest";
+import { initSandboxRuntimeModular } from "./init";
+import { deferMediaUntilDue } from "./preloadMedia";
+import { unproxiedSrc } from "./proxySrc";
+import { createMockTimeline, resetRuntimeFixtureDom } from "./runtimeSeekFixture.test-helpers";
+
+const IPHONE =
+ "Mozilla/5.0 (iPhone; CPU iPhone OS 26_5 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/26.5 Mobile/15E148 Safari/604.1";
+let seq = 0;
+
+describe("later clips of a long file in the player on WebKit", () => {
+ let file = "";
+ const clip = (id: string) => document.getElementById(id) as HTMLVideoElement;
+ const src = (id: string) => clip(id).getAttribute("src");
+
+ beforeEach(async () => {
+ resetRuntimeFixtureDom();
+ vi.spyOn(navigator, "userAgent", "get").mockReturnValue(IPHONE);
+ const frame = document.createElement("iframe");
+ frame.setAttribute("data-hf-player-frame", "");
+ Object.defineProperty(window, "frameElement", { value: frame, configurable: true });
+ vi.stubGlobal(
+ "fetch",
+ vi.fn(
+ async () =>
+ new Response(null, { status: 206, headers: { "content-range": "bytes 0-0/68000000" } }),
+ ),
+ );
+ // Each test names its own file: a file's size is asked once per page.
+ file = `/m/film-${seq++}.mp4`;
+ deferMediaUntilDue();
+ document.body.innerHTML =
+ `
` +
+ [
+ ["v0", 0, 30],
+ ["v1", 20, 50],
+ ["v2", 40, 70],
+ ]
+ .map(
+ ([id, start, inPoint]) =>
+ ``,
+ )
+ .join("") +
+ `
`;
+ for (const video of document.querySelectorAll("video")) video.load = vi.fn();
+ await vi.waitFor(() => expect(src("v2")).toBeNull());
+ window.__timelines = { main: createMockTimeline(60) };
+ initSandboxRuntimeModular();
+ });
+
+ afterEach(() => {
+ window.__hfRuntimeTeardown?.();
+ (window as Window & { __hfMediaDeferral?: MutationObserver }).__hfMediaDeferral?.disconnect();
+ vi.unstubAllGlobals();
+ vi.restoreAllMocks();
+ Reflect.deleteProperty(window, "frameElement");
+ resetRuntimeFixtureDom();
+ });
+
+ it("hold no src until due, past the next take, and still name the file they play", () => {
+ expect([src("v0"), src("v1"), src("v2")]).toEqual([`${file}#t=30`, `${file}#t=50`, null]);
+ expect(unproxiedSrc(clip("v2"))).toBe(file);
+ });
+
+ it("get their src at their in-point when a paused seek lands on one", () => {
+ window.__player?.seek(41);
+ expect(src("v2")).toBe(`${file}#t=70`);
+ });
+
+ it("keep a src a binding sets while one is held, and play that one when due", () => {
+ clip("v2").setAttribute("src", "/m/other.mp4");
+ window.__player?.seek(41);
+ expect(src("v2")).toBe("/m/other.mp4");
+ expect(unproxiedSrc(clip("v2"))).toBe("/m/other.mp4");
+ });
+
+ it("let go of the file once passed, and get it back on a seek back", () => {
+ window.__player?.seek(41);
+ expect(src("v1")).toBeNull();
+ window.__player?.seek(21);
+ expect(src("v1")).toBe(`${file}#t=50`);
+ });
+});
diff --git a/packages/core/src/runtime/init.test.ts b/packages/core/src/runtime/init.test.ts
index 143c9359fe..3cbae1edd4 100644
--- a/packages/core/src/runtime/init.test.ts
+++ b/packages/core/src/runtime/init.test.ts
@@ -13,6 +13,7 @@ import { runtimeProtocolMetadata } from "./protocol";
import { collectRuntimeTimelinePayload } from "./timeline";
import { TYPEGPU_PRESENT_HEARTBEAT_MS } from "./adapters/typegpu";
import { WebAudioTransport } from "./webAudioTransport";
+import { bytesOfSource } from "./preloadMedia";
import type { RuntimeTimelineChildLike, RuntimeTimelineLike } from "./types";
import {
registerRuntimeDataHandler,
@@ -6047,9 +6048,9 @@ describe("initSandboxRuntimeModular", () => {
const SAFARI =
"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/18.0 Safari/605.1.15";
- const startRouted = async (userAgent?: string, duration = 10) => {
+ const startRouted = async (userAgent?: string, duration = 10, src = "/assets/track.mp3") => {
if (userAgent) vi.spyOn(navigator, "userAgent", "get").mockReturnValue(userAgent);
- const audio = mountAudio("/assets/track.mp3");
+ const audio = mountAudio(src);
Object.defineProperty(audio, "duration", { value: duration });
const capture = vi
.spyOn(WebAudioTransport.prototype, "scheduleMediaElementPlayback")
@@ -6088,6 +6089,22 @@ describe("initSandboxRuntimeModular", () => {
expect(release).not.toHaveBeenCalled();
});
+ it("leaves a clip of a long file on its element: decoding it would fetch the whole file first", async () => {
+ vi.stubGlobal(
+ "fetch",
+ vi.fn(
+ async () =>
+ new Response(null, { status: 206, headers: { "content-range": "bytes 0-0/68000000" } }),
+ ),
+ );
+ await bytesOfSource("/assets/long-film.mp4");
+ vi.unstubAllGlobals();
+ const { schedule, release } = await startRouted(SAFARI, 321, "/assets/long-film.mp4");
+ await Promise.resolve();
+ expect(schedule).not.toHaveBeenCalled();
+ expect(release).not.toHaveBeenCalled();
+ });
+
it("gives a clip that has not loaded its source's verdict", async () => {
vi.spyOn(WebAudioTransport.prototype, "fitsDecodeBudget").mockReturnValue(true);
const { audio, release } = await startRouted(SAFARI, Number.NaN);
diff --git a/packages/core/src/runtime/init.ts b/packages/core/src/runtime/init.ts
index 843b64ebf5..b551dc4fee 100644
--- a/packages/core/src/runtime/init.ts
+++ b/packages/core/src/runtime/init.ts
@@ -20,6 +20,8 @@ import { clipsFirstRenderedIn } from "./firstRenderReveal";
import { refreshSvgSelectorAliases } from "../compiler/svgSelectorAliases";
import { RUNTIME_FILLER } from "./protocol";
import {
+ giveBackWhenDue,
+ isLongSource,
preloadMedia,
prepareUpcomingMedia,
releaseMedia,
@@ -523,6 +525,7 @@ function decodesFirst(el: HTMLMediaElement, webAudio: WebAudioTransport): boolea
isWebKit() &&
isAudioElement(el) &&
!el.loop &&
+ !isLongSource(el) &&
webAudio.fitsDecodeBudget(el) &&
typeof readElementRateSpec(el) === "number" &&
readElementPlaybackRate(el) === 1
@@ -1314,7 +1317,7 @@ export function initSandboxRuntimeModular(): void {
? {
from: "media",
tag: media.tagName.toLowerCase(),
- src: media.getAttribute("src") ?? media.currentSrc ?? "",
+ src: unproxiedSrc(media) ?? media.currentSrc ?? "",
trimmed: media.hasAttribute("data-duration") || media.hasAttribute("data-end"),
}
: { from };
@@ -2530,10 +2533,7 @@ export function initSandboxRuntimeModular(): void {
}
const tagName = node.tagName.toLowerCase();
const assetUrl =
- node.getAttribute("src") ??
- node.getAttribute("href") ??
- node.getAttribute("poster") ??
- null;
+ unproxiedSrc(node) ?? node.getAttribute("href") ?? node.getAttribute("poster") ?? null;
const diagnosticCode =
tagName === "link" ? "runtime_stylesheet_load_failed" : "runtime_asset_load_failed";
postRuntimeDiagnosticOnce(
@@ -3053,7 +3053,7 @@ export function initSandboxRuntimeModular(): void {
const NEXT_TAKE_SPAN_SECONDS = 0.5;
let nextTakes: { t: number; bound: number; clips: Set } | null = null;
const fileOf = (el: HTMLMediaElement) =>
- el.getAttribute("src") ?? el.querySelector("source")?.getAttribute("src") ?? "";
+ unproxiedSrc(el) ?? el.querySelector("source")?.getAttribute("src") ?? "";
const nextTakeClips = (t: number): Set => {
if (nextTakes?.t === t && nextTakes.bound === metadataBoundMedia.size) return nextTakes.clips;
const byFile = new Map();
@@ -3253,6 +3253,7 @@ export function initSandboxRuntimeModular(): void {
resolveAbsoluteMediaStartSeconds(el) + (resolveDurationForElement(el) ?? Number.NaN) <= time;
const spent = passed && (decided !== true || (el.networkState === 2 && el.readyState < 4));
const near = visible || armed || (upcoming && !unslotted && !spent);
+ if (near) giveBackWhenDue(el);
if ((visible || !near) && endPrefetch(el)) queueMicrotask(fillPrefetchSlots);
const approaching = near && !visible && resolveAbsoluteMediaStartSeconds(el) > clock.now();
const wasApproaching = approachingMedia.has(el);
diff --git a/packages/core/src/runtime/mediaLoadFailure.test.ts b/packages/core/src/runtime/mediaLoadFailure.test.ts
index 2a391b1326..cd84e90457 100644
--- a/packages/core/src/runtime/mediaLoadFailure.test.ts
+++ b/packages/core/src/runtime/mediaLoadFailure.test.ts
@@ -2,6 +2,7 @@ import { afterEach, beforeEach, describe, expect, it, vi } from "vitest";
import { postRuntimeMessage } from "./bridge";
import { watchAudioLoadErrors } from "./mediaLoadFailure";
import { RUNTIME_MEDIA_LOAD_FAILED } from "./protocol";
+import { recordOpenedAtInPoint } from "./proxySrc";
import { WebAudioTransport } from "./webAudioTransport";
vi.mock("./bridge", () => ({ postRuntimeMessage: vi.fn() }));
@@ -37,6 +38,18 @@ afterEach(() => {
});
describe("a preview media load that fails is reported, not swallowed", () => {
+ it("names the src the author wrote, not the in-point fragment the runtime added", async () => {
+ vi.stubGlobal(
+ "fetch",
+ vi.fn(async () => ({ ok: false, status: 404 })),
+ );
+ const el = audio("assets/take.wav", "take");
+ recordOpenedAtInPoint(el, "assets/take.wav", "assets/take.wav#t=30");
+ el.setAttribute("src", "assets/take.wav#t=30");
+ await transport(async () => ({}) as AudioBuffer).decodeAudioElement(el);
+ expect(failures()[0]).toMatchObject({ asset: "assets/take.wav" });
+ });
+
it("a fetch the server refuses says its status, once however often playback retries", async () => {
vi.stubGlobal(
"fetch",
diff --git a/packages/core/src/runtime/mediaLoadFailure.ts b/packages/core/src/runtime/mediaLoadFailure.ts
index b15e005658..5b140a6f42 100644
--- a/packages/core/src/runtime/mediaLoadFailure.ts
+++ b/packages/core/src/runtime/mediaLoadFailure.ts
@@ -1,4 +1,5 @@
import { postRuntimeMessage } from "./bridge";
+import { unproxiedSrc } from "./proxySrc";
import { isAudioElement, isSourceElement } from "./domRealm";
import { isRenderMode } from "./mediaProxy";
import { RUNTIME_MEDIA_LOAD_FAILED, type MediaLoadStage } from "./protocol";
@@ -30,7 +31,7 @@ export function reportMediaLoadFailure(
if (stages.has(stage)) return;
stages.add(stage);
reported.set(el, stages);
- const asset = el.getAttribute("src") ?? "";
+ const asset = unproxiedSrc(el) ?? "";
const details: Record = {
asset,
elementId: el.id || null,
diff --git a/packages/core/src/runtime/mediaProxy.test.ts b/packages/core/src/runtime/mediaProxy.test.ts
index fc293b8f61..18423c63c1 100644
--- a/packages/core/src/runtime/mediaProxy.test.ts
+++ b/packages/core/src/runtime/mediaProxy.test.ts
@@ -18,6 +18,7 @@ import {
swapToProxy,
type MediaCodecMapEntry,
} from "./mediaProxy";
+import { recordOpenedAtInPoint, unproxiedSrc } from "./proxySrc";
vi.mock("./bridge", () => ({
postRuntimeMessage: vi.fn(),
@@ -497,6 +498,17 @@ describe("handleErrorForProxy (tertiary trigger)", () => {
});
describe("swapToProxy", () => {
+ it("swaps a clip held with no src after giving it back its file", async () => {
+ const el = createVideo("/video.mp4");
+ recordOpenedAtInPoint(el, "/video.mp4", null);
+ el.removeAttribute("src");
+ swapToProxy(el, HEVC_ENTRY, "proactive");
+ await settle();
+ expect(isProxied(el)).toBe(true);
+ expect(new URL(el.src).pathname).toBe("/video.mp4");
+ expect(unproxiedSrc(el)).toBe("/video.mp4");
+ });
+
it("does not poison swap state when the source URL is malformed", async () => {
const el = createVideo("/video.mp4");
Object.defineProperty(el, "currentSrc", {
diff --git a/packages/core/src/runtime/mediaProxy.ts b/packages/core/src/runtime/mediaProxy.ts
index d315f074d8..25c7481beb 100644
--- a/packages/core/src/runtime/mediaProxy.ts
+++ b/packages/core/src/runtime/mediaProxy.ts
@@ -10,7 +10,8 @@ import {
import { findInjectedRenderFrame } from "./renderFrameSibling";
import type { RuntimeJson } from "./types";
import { isVideoElement } from "./domRealm";
-import { heldPosters, swappedElements } from "./proxySrc";
+import { heldPosters, swappedElements, unproxiedSrc } from "./proxySrc";
+import { giveBackWhenDue } from "./preloadMedia";
import { restartPlaybackHealth } from "./playbackHealth";
import { failedCopyElements, noteCopy, setCopyReportingLevel } from "./previewCopies";
import { waitForServedProxy } from "./proxyWait";
@@ -523,7 +524,7 @@ async function makeCopy(el: HTMLMediaElement, job: CopyJob, scale: number | null
return;
}
noteCopy(el, "ready", null, light);
- swappedElements.set(el, originalAttr);
+ swappedElements.set(el, unproxiedSrc(el));
if (lightFacts) lightElements.add(el);
loadProxy(el, proxiedSrc, originalSrc, variant, box, lightFacts !== undefined);
// The shown size may have grown while this copy was being made.
@@ -551,6 +552,7 @@ export function swapToProxy(
lightFacts?: PreviewSourceFacts,
): void {
if (swappedElements.has(el)) return;
+ giveBackWhenDue(el);
if (proxyRequested.has(el) && proxyRequested.get(el) === el.getAttribute("src")) return;
const originalSrc = currentSrcValue(el);
const variant = copyVariant(entry);
@@ -761,24 +763,7 @@ export function handleMetadataForProxy(el: HTMLMediaElement): void {
if (isRenderMode(el)) return;
if (!isVideoElement(el)) return;
if (el.videoWidth !== 0) return;
- const src = currentSrcValue(el);
- if (swappedElements.has(el)) {
- emitUnavailableDiagnostic(el, "proxy_playback_failed", src);
- return;
- }
- const map = window.__HF_MEDIA_CODEC_MAP__;
- if (!map) return;
- const key = deriveCodecMapKey(el);
- if (key === null) {
- emitUnavailableDiagnostic(el, "cross_origin", src);
- return;
- }
- const entry = lookupCodecMapEntry(key, map);
- if (entry && !entry.browserHostile) {
- emitUnavailableDiagnostic(el, "browser_safe_codec", src);
- return;
- }
- swapToProxy(el, entry, "reactive");
+ swapOrDiagnose(el, "reactive");
}
/**
@@ -792,6 +777,10 @@ export function handleMetadataForProxy(el: HTMLMediaElement): void {
export function handleErrorForProxy(el: HTMLMediaElement): void {
if (isRenderMode(el)) return;
if (!isVideoElement(el)) return;
+ swapOrDiagnose(el, "tertiary");
+}
+
+function swapOrDiagnose(el: HTMLMediaElement, trigger: ProxyTrigger): void {
const src = currentSrcValue(el);
if (swappedElements.has(el)) {
emitUnavailableDiagnostic(el, "proxy_playback_failed", src);
@@ -809,5 +798,5 @@ export function handleErrorForProxy(el: HTMLMediaElement): void {
emitUnavailableDiagnostic(el, "browser_safe_codec", src);
return;
}
- swapToProxy(el, entry, "tertiary");
+ swapToProxy(el, entry, trigger);
}
diff --git a/packages/core/src/runtime/picker.ts b/packages/core/src/runtime/picker.ts
index 5e658425c0..b05564d578 100644
--- a/packages/core/src/runtime/picker.ts
+++ b/packages/core/src/runtime/picker.ts
@@ -3,6 +3,7 @@ import { PREVIEW_VIDEO_FRAME_ATTR } from "../studioPreviewMark";
import { COLOR_GRADING_SOURCE_HIDDEN_ATTR } from "../colorGrading";
import { swallow } from "./diagnostics";
import { isElementNode } from "./domRealm";
+import { unproxiedSrc } from "./proxySrc";
import { createDrawnProbe } from "./pickerDrawn";
type PickerModuleDeps = {
@@ -311,7 +312,7 @@ export function createPickerModule(deps: PickerModuleDeps): PickerModule {
label: buildElementLabel(el),
boundingBox: { x: rect.left, y: rect.top, width: rect.width, height: rect.height },
textContent: el.textContent ? el.textContent.trim().slice(0, 200) : null,
- src: el.getAttribute("src") || el.getAttribute("data-composition-src") || null,
+ src: unproxiedSrc(el) || el.getAttribute("data-composition-src") || null,
dataAttributes,
};
}
diff --git a/packages/core/src/runtime/preloadMedia.test.ts b/packages/core/src/runtime/preloadMedia.test.ts
index 08e08a1095..7209701d64 100644
--- a/packages/core/src/runtime/preloadMedia.test.ts
+++ b/packages/core/src/runtime/preloadMedia.test.ts
@@ -1,6 +1,9 @@
-import { afterEach, describe, expect, it, vi } from "vitest";
+import { afterEach, beforeEach, describe, expect, it, vi } from "vitest";
import { mediaElementTime } from "./mediaClock";
+import { unproxiedSrc } from "./proxySrc";
import {
+ deferMediaUntilDue,
+ giveBackWhenDue,
isRenderFrameVideo,
preloadMedia,
prepareUpcomingMedia,
@@ -261,3 +264,150 @@ describe("preparing upcoming video playback", () => {
element.remove();
});
});
+
+describe("a clip of a long file opening at its in-point on Safari", () => {
+ const SAFARI =
+ "Mozilla/5.0 (iPhone; CPU iPhone OS 18_0 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/18.0 Mobile/15E148 Safari/604.1";
+ const CHROME = "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 Chrome/120.0 Safari/537.36";
+ let meta: HTMLMetaElement;
+ let seq = 0;
+ const stopWatching = () =>
+ (window as Window & { __hfMediaDeferral?: MutationObserver }).__hfMediaDeferral?.disconnect();
+ const inPlayer = (on: boolean) => {
+ const frame = document.createElement("iframe");
+ if (on) frame.setAttribute("data-hf-player-frame", "");
+ Object.defineProperty(window, "frameElement", { value: frame, configurable: true });
+ };
+ const sizeOf = (bytes: number | "unreadable") =>
+ vi.stubGlobal(
+ "fetch",
+ vi.fn(async () => {
+ if (bytes === "unreadable") throw new TypeError("blocked");
+ return new Response(null, {
+ status: 206,
+ headers: { "content-range": `bytes 0-0/${bytes}` },
+ });
+ }),
+ );
+ beforeEach(() => {
+ meta = document.createElement("meta");
+ meta.name = "hyperframes-studio-preview";
+ document.head.appendChild(meta);
+ vi.spyOn(navigator, "userAgent", "get").mockReturnValue(SAFARI);
+ inPlayer(true);
+ sizeOf(68_000_000);
+ deferMediaUntilDue();
+ });
+ afterEach(() => {
+ stopWatching();
+ vi.unstubAllGlobals();
+ vi.restoreAllMocks();
+ Reflect.deleteProperty(window, "frameElement");
+ meta.remove();
+ document.body.replaceChildren();
+ });
+
+ // Each test names its own file: the size of a file is asked once per page.
+ const clip = (attrs = 'data-media-start="30"', start = "0") =>
+ ``;
+ const later = () => clip('data-media-start="30"', "4");
+ async function added(html: string): Promise {
+ document.body.insertAdjacentHTML("beforeend", html);
+ await Promise.resolve();
+ return document.body.lastElementChild as HTMLVideoElement;
+ }
+ const opened = (v: HTMLVideoElement) => v.getAttribute("src")!.includes("#t=");
+
+ it("gets a start fragment and keeps its authored src for everything that reads it", async () => {
+ const v = await added(clip());
+ const authored = v.getAttribute("src")!;
+ await vi.waitFor(() => expect(v.getAttribute("src")).toBe(`${authored}#t=30`));
+ expect(unproxiedSrc(v)).toBe(authored);
+ });
+
+ it("holds a later clip with no src, still naming its file, until it is due", async () => {
+ const v = await added(later());
+ const authored = v.getAttribute("src")!;
+ await vi.waitFor(() => expect(v.getAttribute("src")).toBeNull());
+ expect(unproxiedSrc(v)).toBe(authored);
+ giveBackWhenDue(v);
+ expect(v.getAttribute("src")).toBe(`${authored}#t=30`);
+ });
+
+ it("opens a later clip WebKit already began reading at its in-point, and gives its preload back", async () => {
+ vi.spyOn(HTMLMediaElement.prototype, "networkState", "get").mockReturnValue(2);
+ const v = await added(later());
+ await vi.waitFor(() => expect(opened(v)).toBe(true));
+ expect(v.preload).not.toBe("none");
+ });
+
+ it("holds a later clip again when it is released after playing", async () => {
+ const v = await added(later());
+ await vi.waitFor(() => expect(v.getAttribute("src")).toBeNull());
+ giveBackWhenDue(v);
+ releaseMedia(v, true);
+ expect(v.getAttribute("src")).toBeNull();
+ });
+
+ it("does not hold a later clip a binding has given another src", async () => {
+ const v = await added(later());
+ v.setAttribute("src", "/m/bound.mp4");
+ await new Promise((r) => setTimeout(r, 10));
+ expect(v.getAttribute("src")).toBe("/m/bound.mp4");
+ });
+
+ it("does so for a clip already parsed when the runtime arrives", async () => {
+ stopWatching();
+ document.body.insertAdjacentHTML("beforeend", clip());
+ deferMediaUntilDue();
+ const v = document.body.lastElementChild as HTMLVideoElement;
+ await vi.waitFor(() => expect(opened(v)).toBe(true));
+ });
+
+ it("stops reporting the authored src once something else sets the src", async () => {
+ const v = await added(clip());
+ await vi.waitFor(() => expect(opened(v)).toBe(true));
+ v.setAttribute("src", "/m/other.mp4");
+ expect(unproxiedSrc(v)).toBe("/m/other.mp4");
+ });
+
+ it("is left alone when its file is small, so a short film loads exactly as before", async () => {
+ sizeOf(5_600_000);
+ const v = await added(later());
+ await vi.waitFor(() => expect(fetch).toHaveBeenCalled());
+ await new Promise((r) => setTimeout(r, 10));
+ expect(opened(v)).toBe(false);
+ expect(v.preload).toBe("none");
+ });
+
+ it("is left alone when the size of its file cannot be read", async () => {
+ sizeOf("unreadable");
+ const v = await added(clip());
+ await vi.waitFor(() => expect(fetch).toHaveBeenCalled());
+ await new Promise((r) => setTimeout(r, 10));
+ expect(opened(v)).toBe(false);
+ });
+
+ it.each([
+ ["starts at the head of the file", 'data-media-start="0"'],
+ ["loops", 'data-media-start="30" loop'],
+ ])("is left alone when it %s", async (_name, attrs) => {
+ const v = await added(clip(attrs));
+ await new Promise((r) => setTimeout(r, 10));
+ expect(opened(v)).toBe(false);
+ });
+
+ it("is left alone in Studio's preview, which copies and saves what it reads from the page", async () => {
+ inPlayer(false);
+ const v = await added(clip());
+ await new Promise((r) => setTimeout(r, 10));
+ expect(opened(v)).toBe(false);
+ });
+
+ it("is left alone off Safari", async () => {
+ vi.spyOn(navigator, "userAgent", "get").mockReturnValue(CHROME);
+ const v = await added(clip());
+ await new Promise((r) => setTimeout(r, 10));
+ expect(opened(v)).toBe(false);
+ });
+});
diff --git a/packages/core/src/runtime/preloadMedia.ts b/packages/core/src/runtime/preloadMedia.ts
index 91eaaf3093..2e9933d4b6 100644
--- a/packages/core/src/runtime/preloadMedia.ts
+++ b/packages/core/src/runtime/preloadMedia.ts
@@ -1,7 +1,15 @@
import { parseStrictFiniteTimingNumber, readMediaStart } from "./playbackRate";
+import {
+ authoredSrcOf,
+ holdsWhenIdle,
+ isHeldUntilDue,
+ recordOpenedAtInPoint,
+ unproxiedSrc,
+} from "./proxySrc";
import { createRuntimeStartTimeResolver } from "./startResolver";
import { readMediaRenderId } from "./renderFrameSibling";
import { defersMedia } from "./timedClipHide";
+import { inPlayerFrame } from "../studioPreviewMark";
import { swallow } from "./diagnostics";
import { isWebKit, mediaElementTime } from "./mediaClock";
@@ -88,6 +96,7 @@ function wholeFileBuffered(media: HTMLMediaElement): boolean {
export function releaseMedia(media: HTMLMediaElement, whenBuffered = false): void {
if (media.querySelector("source") || (!whenBuffered && wholeFileBuffered(media))) return;
+ if (holdsWhenIdle(media)) return holdUntilDue(media);
const src = media.getAttribute("src");
stopMediaDownload(media);
if (src !== null) media.setAttribute("src", src);
@@ -97,6 +106,98 @@ export function lengthIsAuthored(media: Element): boolean {
return parseStrictFiniteTimingNumber(media.getAttribute("data-duration")) != null;
}
+const LONG_SOURCE_BYTES = 32 * 1024 * 1024;
+const sourceBytes = new Map>();
+
+const knownBytes = new Map();
+
+export function isLongSource(el: Element): boolean {
+ const src = unproxiedSrc(el);
+ if (!src) return false;
+ try {
+ return (knownBytes.get(new URL(src, document.baseURI).href) ?? 0) >= LONG_SOURCE_BYTES;
+ } catch {
+ return false;
+ }
+}
+
+export function bytesOfSource(src: string): Promise {
+ let url: string;
+ try {
+ url = new URL(src, document.baseURI).href;
+ } catch {
+ return Promise.resolve(null);
+ }
+ const known = sourceBytes.get(url);
+ if (known) return known;
+ const asked = (async () => {
+ const abort = new AbortController();
+ try {
+ const response = await fetch(url, { headers: { Range: "bytes=0-0" }, signal: abort.signal });
+ const total =
+ /\/(\d+)$/.exec(response.headers.get("content-range") ?? "")?.[1] ??
+ (response.status === 200 ? response.headers.get("content-length") : null);
+ if (total === null) return null;
+ knownBytes.set(url, Number(total));
+ return Number(total);
+ } catch {
+ return null;
+ } finally {
+ abort.abort();
+ }
+ })();
+ sourceBytes.set(url, asked);
+ return asked;
+}
+
+function openAtInPoint(el: HTMLMediaElement): void {
+ const src = el.getAttribute("src");
+ const start = readMediaStart(el);
+ // Only the player: Studio copies and saves what it reads from the page, and must never see the fragment.
+ if (!inPlayerFrame(document) || !src || src.includes("#") || el.loop) return;
+ if (el.querySelector("source")) return;
+ if (!(start > 0)) return;
+ const written = `${src}#t=${start}`;
+ recordOpenedAtInPoint(el, src, written);
+ el.setAttribute("src", written);
+}
+
+// WebKit opens the file for any clip that has a src, whatever its preload: a later clip of a long file has none.
+function holdUntilDue(el: HTMLMediaElement): void {
+ const authored = unproxiedSrc(el);
+ if (authored === null) return;
+ recordOpenedAtInPoint(el, authored, null, true);
+ stopMediaDownload(el);
+}
+
+export function giveBackWhenDue(el: HTMLMediaElement): void {
+ if (!isHeldUntilDue(el)) return;
+ const authored = authoredSrcOf(el)!;
+ const start = readMediaStart(el);
+ const written = start > 0 ? `${authored}#t=${start}` : authored;
+ recordOpenedAtInPoint(el, authored, written);
+ el.setAttribute("src", written);
+}
+
+// One WebKit began reading at parse opens at its in-point instead: its metadata then costs little.
+const holdable = (el: HTMLMediaElement, src: string) =>
+ el.preload === "none" &&
+ el.networkState !== 2 &&
+ !src.includes("#") &&
+ !el.loop &&
+ !el.querySelector("source");
+
+function openLongSourceAtInPoint(el: HTMLMediaElement, later: boolean, opened: () => void): void {
+ const src = el.getAttribute("src");
+ if (!isWebKit() || !inPlayerFrame(document) || !src) return;
+ void bytesOfSource(src).then((bytes) => {
+ if (bytes === null || bytes < LONG_SOURCE_BYTES || el.getAttribute("src") !== src) return;
+ if (later && holdable(el, src)) return holdUntilDue(el);
+ openAtInPoint(el);
+ opened();
+ });
+}
+
type DeferralWindow = Window & { __hfMediaDeferral?: MutationObserver };
/** Preview only: later clips parse at preload none, as Chromium ignores a none set mid-fetch.
@@ -108,8 +209,12 @@ export function deferMediaUntilDue(): void {
if (!defersMedia()) return;
const defer = (el: Element, startInFilm: (el: Element) => number) => {
const start = parseStrictFiniteTimingNumber(el.getAttribute("data-start"));
- if (start != null && lengthIsAuthored(el) && startInFilm(el) > 0)
- (el as HTMLMediaElement).preload = "none";
+ if (start == null || !lengthIsAuthored(el)) return;
+ const media = el as HTMLMediaElement;
+ const authoredPreload = media.preload;
+ const later = startInFilm(el) > 0;
+ if (later) media.preload = "none";
+ openLongSourceAtInPoint(media, later, () => (media.preload = authoredPreload));
};
win.__hfMediaDeferral = new MutationObserver((records) => {
const startInFilm = createRuntimeStartTimeResolver({}).resolveMediaStartForElement;
@@ -122,4 +227,8 @@ export function deferMediaUntilDue(): void {
}
});
win.__hfMediaDeferral.observe(document.documentElement, { childList: true, subtree: true });
+ if (isWebKit()) {
+ const startInFilm = createRuntimeStartTimeResolver({}).resolveMediaStartForElement;
+ for (const media of document.querySelectorAll("video, audio")) defer(media, startInFilm);
+ }
}
diff --git a/packages/core/src/runtime/proxySrc.ts b/packages/core/src/runtime/proxySrc.ts
index 82f62315c9..eae5759c8e 100644
--- a/packages/core/src/runtime/proxySrc.ts
+++ b/packages/core/src/runtime/proxySrc.ts
@@ -1,8 +1,45 @@
/** Media elements swapped to their proxy, each with the src attribute it had before the swap. */
export const swappedElements = new WeakMap();
+/** Media elements the runtime opened at their in-point: the src the author wrote, the one written in its
+ * place (null while the clip is held with no src until it is due), and whether it is held again when idle. */
+const openedAtInPoint = new WeakMap<
+ Element,
+ { authored: string; written: string | null; holdsWhenIdle: boolean }
+>();
+
+export function recordOpenedAtInPoint(
+ el: Element,
+ authored: string,
+ written: string | null,
+ holdsWhenIdle = openedAtInPoint.get(el)?.holdsWhenIdle ?? false,
+): void {
+ openedAtInPoint.set(el, { authored, written, holdsWhenIdle });
+}
+
+/** True for a clip the runtime holds whenever it is not due, while its src is still the runtime's. */
+export function holdsWhenIdle(el: Element): boolean {
+ return isOpenedAtInPoint(el) && openedAtInPoint.get(el)!.holdsWhenIdle;
+}
+
+/** True while the src is still the one the runtime wrote: a script or binding that sets another ends it. */
+function isOpenedAtInPoint(el: Element): boolean {
+ const record = openedAtInPoint.get(el);
+ return record !== undefined && record.written === el.getAttribute("src");
+}
+
+export function isHeldUntilDue(el: Element): boolean {
+ return isOpenedAtInPoint(el) && openedAtInPoint.get(el)!.written === null;
+}
+
+/** The src the author wrote for a clip the runtime opened or holds. */
+export function authoredSrcOf(el: Element): string | null {
+ return openedAtInPoint.get(el)?.authored ?? null;
+}
+
export function unproxiedSrc(el: Element): string | null {
- return swappedElements.has(el) ? (swappedElements.get(el) ?? null) : el.getAttribute("src");
+ if (swappedElements.has(el)) return swappedElements.get(el) ?? null;
+ return isOpenedAtInPoint(el) ? openedAtInPoint.get(el)!.authored : el.getAttribute("src");
}
/** Videos showing a still of their last frame as poster across a copy swap: the poster they had before, and when. */
diff --git a/packages/core/src/runtime/timeline.ts b/packages/core/src/runtime/timeline.ts
index aac2d0bd46..f37af327c1 100644
--- a/packages/core/src/runtime/timeline.ts
+++ b/packages/core/src/runtime/timeline.ts
@@ -4,6 +4,7 @@ import type {
RuntimeTimelineScene,
RuntimeTimelineLike,
} from "./types";
+import { unproxiedSrc } from "./proxySrc";
import { stableClipId } from "./clipTree";
import { findRootCompositionElement, parseCompositionDimension } from "./compositionDimension";
import {
@@ -166,13 +167,13 @@ function toAbsoluteAssetUrl(rawValue: string | null | undefined): string | null
}
function resolveNodeAssetUrl(node: Element): string | null {
- const src = node.getAttribute("src") ?? node.getAttribute("data-src");
+ const src = unproxiedSrc(node) ?? node.getAttribute("data-src");
if (src) return toAbsoluteAssetUrl(src);
const compositionSrc = node.getAttribute("data-composition-src");
if (compositionSrc) return toAbsoluteAssetUrl(compositionSrc);
const mediaDescendant = node.querySelector("img[src], video[src], audio[src], source[src]");
if (!mediaDescendant) return null;
- return toAbsoluteAssetUrl(mediaDescendant.getAttribute("src"));
+ return toAbsoluteAssetUrl(unproxiedSrc(mediaDescendant));
}
function getFirstClassToken(node: Element): string | null {
diff --git a/packages/core/src/runtime/webAudioTransport.test.ts b/packages/core/src/runtime/webAudioTransport.test.ts
index 1e97fbeffa..da96a07dd2 100644
--- a/packages/core/src/runtime/webAudioTransport.test.ts
+++ b/packages/core/src/runtime/webAudioTransport.test.ts
@@ -1868,6 +1868,22 @@ describe("WebAudioTransport", () => {
vi.unstubAllGlobals();
});
+ it("decodes a file once for clips that open at different in-points of it", async () => {
+ const transport = transportWithDecode(async () => ({}) as AudioBuffer);
+ const fetchMock = vi.fn(async () => ({
+ ok: true,
+ arrayBuffer: async () => new ArrayBuffer(8),
+ }));
+ vi.stubGlobal("fetch", fetchMock);
+ await Promise.all([
+ transport.decodeAudioElement(el("take.wav#t=10")),
+ transport.decodeAudioElement(el("take.wav#t=90")),
+ ]);
+ expect(fetchMock).toHaveBeenCalledTimes(1);
+ expect(fetchMock.mock.calls[0]![0]).toBe(key("take.wav"));
+ vi.unstubAllGlobals();
+ });
+
it("DOES blacklist genuinely undecodable bytes — not retried", async () => {
const transport = transportWithDecode(async () => {
throw new Error("unsupported codec");
diff --git a/packages/core/src/runtime/webAudioTransport.ts b/packages/core/src/runtime/webAudioTransport.ts
index 6f9f721ef8..c381e58543 100644
--- a/packages/core/src/runtime/webAudioTransport.ts
+++ b/packages/core/src/runtime/webAudioTransport.ts
@@ -214,7 +214,9 @@ const decodeKey = (el: HTMLMediaElement): string | null => {
const src = el.currentSrc || el.getAttribute("src");
if (!src) return null;
try {
- return assetUrl(src);
+ const url = new URL(assetUrl(src));
+ url.hash = "";
+ return url.href;
} catch {
return src;
}
diff --git a/scripts/package-size-budgets.json b/scripts/package-size-budgets.json
index ea7584e509..76310afae7 100644
--- a/scripts/package-size-budgets.json
+++ b/scripts/package-size-budgets.json
@@ -37,7 +37,7 @@
},
"@hyperframes/producer": {
"packed": 870000,
- "unpacked": 3130000
+ "unpacked": 3162274
},
"@hyperframes/sdk": {
"packed": 90662,