From 036e41fa09e6670f155fa3f2c361b58b8ed77e02 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Miguel=20=C3=81ngel?= Date: Mon, 2 Nov 2026 13:15:47 +0000 Subject: [PATCH] fix(core): iPhone Safari keeps one connection per file for the take on screen and the next (#7012) * fix(core): iPhone Safari keeps one connection per file for the take on screen and the next * fix(core): drop the doc comments the comment ratchet counts in studioPreviewMark --- packages/core/src/compositionReadiness.ts | 6 +- packages/core/src/runtime/entry.test.ts | 106 ++++++++++++++++++++- packages/core/src/runtime/init.ts | 30 +++++- packages/core/src/runtime/timedClipHide.ts | 3 +- packages/core/src/studioPreviewMark.ts | 13 +++ packages/player/src/iframe-dom.ts | 3 + 6 files changed, 155 insertions(+), 6 deletions(-) diff --git a/packages/core/src/compositionReadiness.ts b/packages/core/src/compositionReadiness.ts index 7361ecad92..efa4ebc957 100644 --- a/packages/core/src/compositionReadiness.ts +++ b/packages/core/src/compositionReadiness.ts @@ -1,7 +1,7 @@ import { createRuntimeStartTimeResolver } from "./runtime/startResolver.js"; import { isRuntimeElementVisibleAt } from "./runtime/timeline.js"; import type { RuntimeTimelineLike } from "./runtime/types.js"; -import { STUDIO_PREVIEW_MARK_META } from "./studioPreviewMark.js"; +import { defersMediaIn } from "./studioPreviewMark.js"; /** A composition is "ready" once every declared input settles, not just once * its duration is known. Each input returns null (nothing to wait on) or a @@ -104,8 +104,8 @@ export function scanPendingCompositionAssets( timelineRegistry: runtimeWindow?.__timelines, includeAuthoredTimingAttrs: true, }); - // Studio's preview buffers a clip off the first frame only once it comes due, as with a lazy image. - const defersMedia = doc.querySelector(`meta[name="${STUDIO_PREVIEW_MARK_META}"]`) !== null; + // Studio's preview and the player buffer a clip off the first frame only once it comes due, as with a lazy image. + const defersMedia = defersMediaIn(doc); const pendingMedia = Array.from(doc.querySelectorAll("video, audio")) .filter(isRealmHtmlMediaElement) .filter((el) => diff --git a/packages/core/src/runtime/entry.test.ts b/packages/core/src/runtime/entry.test.ts index e2e58a158b..36d4f33417 100644 --- a/packages/core/src/runtime/entry.test.ts +++ b/packages/core/src/runtime/entry.test.ts @@ -2,7 +2,11 @@ import { afterEach, describe, expect, it, vi } from "vitest"; import { mediaElementTime } from "./mediaClock"; import { HF_COLOR_GRADING_ATTR, serializeHfColorGrading } from "../colorGrading"; -import { STUDIO_PREVIEW_LAZY_ATTR, STUDIO_PREVIEW_MARK_META } from "../studioPreviewMark"; +import { + PLAYER_FRAME_ATTR, + STUDIO_PREVIEW_LAZY_ATTR, + STUDIO_PREVIEW_MARK_META, +} from "../studioPreviewMark"; import { revealClip } from "./clipReveal"; import type { RuntimeTimelineLike } from "./types"; @@ -91,6 +95,12 @@ const servePreview = () => document.head.appendChild( Object.assign(document.createElement("meta"), { name: STUDIO_PREVIEW_MARK_META }), ); +const asSafari = () => + vi + .spyOn(navigator, "userAgent", "get") + .mockReturnValue( + "Mozilla/5.0 AppleWebKit/605.1.15 (KHTML, like Gecko) Version/17.4 Safari/605.1.15", + ); const neverDecodes = (clip: HTMLElement) => { withImage(clip).querySelector("img")!.decode = () => new Promise(() => {}); return clip; @@ -429,6 +439,64 @@ describe("runtime entry", () => { expect(now.getAttribute("src")).toBe("now.mp4"); }); + // Three takes of one file, each a picture and a sound, as an edit cut from a single source is. + const takesOfOneFile = () => { + const root = mountRoot(); + const take = (start: string) => { + const pair = [timed(root, "video", start), timed(root, "audio", start)]; + for (const el of pair) el.setAttribute("src", "/source.mp4"); + return pair; + }; + return [take("0"), take("3"), take("9")]; + }; + const inPlayerFrame = (userAgent: "safari" | "chrome") => { + const frame = document.createElement("iframe"); + frame.setAttribute(PLAYER_FRAME_ATTR, ""); + vi.spyOn(window, "frameElement", "get").mockReturnValue(frame); + if (userAgent === "safari") asSafari(); + }; + + it("in the player on Safari a file keeps the take on screen and the next one loading, and frees the rest", async () => { + spyLoad(); + inPlayerFrame("safari"); + const [first, second, third] = takesOfOneFile(); + + await evaluateRuntime(); + expect([first, second, third].map((take) => armed(...take))).toEqual([ + [true, true], + [true, true], + [false, false], + ]); + + window.__player?.seek(3.5); + expect([first, second, third].map((take) => armed(...take))).toEqual([ + [false, false], + [true, true], + [true, true], + ]); + }); + + it("in the player on Chromium every take of a file loads at parse", async () => { + spyLoad(); + inPlayerFrame("chrome"); + const takes = takesOfOneFile(); + + await evaluateRuntime(); + expect(takes.map((take) => armed(...take))).toEqual([ + [true, true], + [true, true], + [true, true], + ]); + }); + + it("in Studio's preview a clip 5 s ahead stays unloaded, as the look-ahead is 2 s", async () => { + servePreview(); + const [now, soon] = videos(mountRoot(), "0", "5"); + + await evaluateRuntime(); + expect(armed(now, soon)).toEqual([true, false]); + }); + it("loads a clip with no authored length as a render does, so no reload drops its duration", async () => { spyLoad(); servePreview(); @@ -940,6 +1008,42 @@ describe("runtime entry", () => { expect([first.preload, later.preload, untrimmed.preload]).toEqual(["", "none", ""]); }); + it("gives each parsed clip preload none in the player's frame, where no clip fetches ahead of the first take", async () => { + const frame = document.createElement("iframe"); + frame.setAttribute(PLAYER_FRAME_ATTR, ""); + vi.spyOn(window, "frameElement", "get").mockReturnValue(frame); + asSafari(); + const root = mountRoot(); + timed(root, "div", "0"); + await evaluateRuntime(); + const [first, later] = videos(root, "0", "5"); + await Promise.resolve(); + expect([first.preload, later.preload]).toEqual(["", "none"]); + }); + + it("leaves Chromium loading every clip at parse in the player's frame", async () => { + const frame = document.createElement("iframe"); + frame.setAttribute(PLAYER_FRAME_ATTR, ""); + vi.spyOn(window, "frameElement", "get").mockReturnValue(frame); + const root = mountRoot(); + timed(root, "div", "0"); + await evaluateRuntime(); + const [first, later] = videos(root, "0", "5"); + await Promise.resolve(); + expect([first.preload, later.preload]).toEqual(["", ""]); + }); + + it("leaves a frame the player did not mark loading every clip at parse", async () => { + vi.spyOn(window, "frameElement", "get").mockReturnValue(document.createElement("iframe")); + asSafari(); + const root = mountRoot(); + timed(root, "div", "0"); + await evaluateRuntime(); + const [first, later] = videos(root, "0", "5"); + await Promise.resolve(); + expect([first.preload, later.preload]).toEqual(["", ""]); + }); + it("defers a clip at 0 inside a later scene, by its start in the film", async () => { servePreview(); const root = mountRoot(); diff --git a/packages/core/src/runtime/init.ts b/packages/core/src/runtime/init.ts index 4f3666fee8..719fda0a60 100644 --- a/packages/core/src/runtime/init.ts +++ b/packages/core/src/runtime/init.ts @@ -42,6 +42,7 @@ import { skipsHiddenImages, } from "./timedClipHide"; import { + inPlayerFrame, PREVIEW_ONLY_ATTRS, RUNTIME_STAMPED_ATTR, STUDIO_PREVIEW_FAR_SCENE_ATTR as FAR_SCENE_ATTR, @@ -2996,6 +2997,29 @@ export function initSandboxRuntimeModular(): void { let moveSceneWindow = (_time: number): boolean => false; const LOOKAHEAD_SECONDS = 2; const RELEASE_LOOKAHEAD_SECONDS = 4; + // The player keeps a connection per file for the clips on screen and the next take only (see inPlayerFrame). + const playerFrame = inPlayerFrame(document); + const NEXT_TAKE_SPAN_SECONDS = 0.5; + let nextTakes: { t: number; bound: number; clips: Set } | null = null; + const fileOf = (el: HTMLMediaElement) => + el.getAttribute("src") ?? el.querySelector("source")?.getAttribute("src") ?? ""; + const nextTakeClips = (t: number): Set => { + if (nextTakes?.t === t && nextTakes.bound === metadataBoundMedia.size) return nextTakes.clips; + const byFile = new Map(); + for (const el of metadataBoundMedia) { + const file = fileOf(el); + const start = resolveAbsoluteMediaStartSeconds(el); + if (!el.isConnected || !file || !(start > t)) continue; + byFile.set(file, [...(byFile.get(file) ?? []), { el, start }]); + } + const clips = new Set(); + for (const later of byFile.values()) { + const next = Math.min(...later.map((clip) => clip.start)); + for (const clip of later) if (clip.start <= next + NEXT_TAKE_SPAN_SECONDS) clips.add(clip.el); + } + nextTakes = { t, bound: metadataBoundMedia.size, clips }; + return clips; + }; // Unskipped while within the window on either side of the playhead, so a step or shuttle back // across a cut finds the clip it left still loaded; the edge checks catch a clip shorter than it. const dueSoon = ( @@ -3248,6 +3272,8 @@ export function initSandboxRuntimeModular(): void { ): number[] | null => { const own = windowOf(node); if (!own || !Number.isFinite(own.start)) return null; + // Which clip is next on its file flips at every other clip's start. + if (playerFrame && isMediaElement(node)) return null; const edges: number[] = []; for (const ahead of [0, LOOKAHEAD_SECONDS, RELEASE_LOOKAHEAD_SECONDS]) edges.push(own.start - ahead, own.end - ahead, own.end + ahead); @@ -3392,7 +3418,9 @@ export function initSandboxRuntimeModular(): void { preloadNearPlayhead( rawNode, isVisibleNow, - upcoming || mediaStaysDue(rawNode, visibleAt, currentTime), + playerFrame + ? nextTakeClips(currentTime).has(rawNode) + : upcoming || mediaStaysDue(rawNode, visibleAt, currentTime), currentTime, dueAtLoopStart(rawNode, visibleAt), ); diff --git a/packages/core/src/runtime/timedClipHide.ts b/packages/core/src/runtime/timedClipHide.ts index a905d55efa..814e233fa0 100644 --- a/packages/core/src/runtime/timedClipHide.ts +++ b/packages/core/src/runtime/timedClipHide.ts @@ -1,5 +1,6 @@ import { CLIP_HIDDEN_ATTR, olderShaderBuildOnPage, revealClip } from "./clipReveal"; import { + defersMediaIn, RUNTIME_STAMPED_ATTR, STUDIO_PREVIEW_CAPTURE_MODE, STUDIO_PREVIEW_MARK_META, @@ -60,7 +61,7 @@ function appendStyle(parent: Element, attr: string, css: string): void { } export function defersMedia(): boolean { - return document.querySelector(STUDIO_PREVIEW_MARK) !== null; + return defersMediaIn(document); } /** True in Studio's preview, where hidden clips' images are neither fetched nor decoded until due. */ diff --git a/packages/core/src/studioPreviewMark.ts b/packages/core/src/studioPreviewMark.ts index 2899d0edfc..81acab6575 100644 --- a/packages/core/src/studioPreviewMark.ts +++ b/packages/core/src/studioPreviewMark.ts @@ -1,5 +1,18 @@ +import { isWebKit } from "./runtime/mediaClock.js"; + export const STUDIO_PREVIEW_MARK_META = "hyperframes-studio-preview"; export const STUDIO_PREVIEW_CAPTURE_MODE = "capture"; +export const PLAYER_FRAME_ATTR = "data-hf-player-frame"; + +export function inPlayerFrame(doc: Document): boolean { + return isWebKit() && doc.defaultView?.frameElement?.hasAttribute(PLAYER_FRAME_ATTR) === true; +} + +export function defersMediaIn(doc: Document): boolean { + return ( + doc.querySelector(`meta[name="${STUDIO_PREVIEW_MARK_META}"]`) !== null || inPlayerFrame(doc) + ); +} export const STUDIO_PREVIEW_UPCOMING_ATTR = "data-hf-upcoming"; export const STUDIO_PREVIEW_LAZY_ATTR = "data-hf-preview-lazy"; export const STUDIO_PREVIEW_ERRORS = "__hfPreviewErrors"; diff --git a/packages/player/src/iframe-dom.ts b/packages/player/src/iframe-dom.ts index 537d861867..f9665eebc8 100644 --- a/packages/player/src/iframe-dom.ts +++ b/packages/player/src/iframe-dom.ts @@ -4,6 +4,8 @@ * Keeps constructor boilerplate out of the web component class body. */ +import { PLAYER_FRAME_ATTR } from "@hyperframes/core/studio-preview-mark"; + // Cached Constructable Stylesheet shared across all player instances. let _sharedSheet: CSSStyleSheet | null = null; @@ -49,6 +51,7 @@ export function createCompositionIframe(): { iframe.allow = "autoplay; fullscreen"; iframe.referrerPolicy = "no-referrer"; iframe.title = "HyperFrames Composition"; + iframe.setAttribute(PLAYER_FRAME_ATTR, ""); container.appendChild(iframe); return { container, iframe };