fix(core): iPhone Safari keeps one connection per file for the take on screen and the next (#7012)

* fix(core): iPhone Safari keeps one connection per file for the take on screen and the next

* fix(core): drop the doc comments the comment ratchet counts in studioPreviewMark

packages/core/src/compositionReadiness.ts

@@ -1,7 +1,7 @@
 import { createRuntimeStartTimeResolver } from "./runtime/startResolver.js";
 import { isRuntimeElementVisibleAt } from "./runtime/timeline.js";
 import type { RuntimeTimelineLike } from "./runtime/types.js";
-import { STUDIO_PREVIEW_MARK_META } from "./studioPreviewMark.js";
+import { defersMediaIn } from "./studioPreviewMark.js";
 
 /** A composition is "ready" once every declared input settles, not just once
  * its duration is known. Each input returns null (nothing to wait on) or a
@@ -104,8 +104,8 @@ export function scanPendingCompositionAssets(
     timelineRegistry: runtimeWindow?.__timelines,
     includeAuthoredTimingAttrs: true,
   });
-  // Studio's preview buffers a clip off the first frame only once it comes due, as with a lazy image.
-  const defersMedia = doc.querySelector(`meta[name="${STUDIO_PREVIEW_MARK_META}"]`) !== null;
+  // Studio's preview and the player buffer a clip off the first frame only once it comes due, as with a lazy image.
+  const defersMedia = defersMediaIn(doc);
   const pendingMedia = Array.from(doc.querySelectorAll("video, audio"))
     .filter(isRealmHtmlMediaElement)
     .filter((el) =>

packages/core/src/runtime/entry.test.ts

@@ -2,7 +2,11 @@
 import { afterEach, describe, expect, it, vi } from "vitest";
 import { mediaElementTime } from "./mediaClock";
 import { HF_COLOR_GRADING_ATTR, serializeHfColorGrading } from "../colorGrading";
-import { STUDIO_PREVIEW_LAZY_ATTR, STUDIO_PREVIEW_MARK_META } from "../studioPreviewMark";
+import {
+  PLAYER_FRAME_ATTR,
+  STUDIO_PREVIEW_LAZY_ATTR,
+  STUDIO_PREVIEW_MARK_META,
+} from "../studioPreviewMark";
 import { revealClip } from "./clipReveal";
 import type { RuntimeTimelineLike } from "./types";
 
@@ -91,6 +95,12 @@ const servePreview = () =>
   document.head.appendChild(
     Object.assign(document.createElement("meta"), { name: STUDIO_PREVIEW_MARK_META }),
   );
+const asSafari = () =>
+  vi
+    .spyOn(navigator, "userAgent", "get")
+    .mockReturnValue(
+      "Mozilla/5.0 AppleWebKit/605.1.15 (KHTML, like Gecko) Version/17.4 Safari/605.1.15",
+    );
 const neverDecodes = (clip: HTMLElement) => {
   withImage(clip).querySelector("img")!.decode = () => new Promise<void>(() => {});
   return clip;
@@ -429,6 +439,64 @@ describe("runtime entry", () => {
       expect(now.getAttribute("src")).toBe("now.mp4");
     });
 
+    // Three takes of one file, each a picture and a sound, as an edit cut from a single source is.
+    const takesOfOneFile = () => {
+      const root = mountRoot();
+      const take = (start: string) => {
+        const pair = [timed(root, "video", start), timed(root, "audio", start)];
+        for (const el of pair) el.setAttribute("src", "/source.mp4");
+        return pair;
+      };
+      return [take("0"), take("3"), take("9")];
+    };
+    const inPlayerFrame = (userAgent: "safari" | "chrome") => {
+      const frame = document.createElement("iframe");
+      frame.setAttribute(PLAYER_FRAME_ATTR, "");
+      vi.spyOn(window, "frameElement", "get").mockReturnValue(frame);
+      if (userAgent === "safari") asSafari();
+    };
+
+    it("in the player on Safari a file keeps the take on screen and the next one loading, and frees the rest", async () => {
+      spyLoad();
+      inPlayerFrame("safari");
+      const [first, second, third] = takesOfOneFile();
+
+      await evaluateRuntime();
+      expect([first, second, third].map((take) => armed(...take))).toEqual([
+        [true, true],
+        [true, true],
+        [false, false],
+      ]);
+
+      window.__player?.seek(3.5);
+      expect([first, second, third].map((take) => armed(...take))).toEqual([
+        [false, false],
+        [true, true],
+        [true, true],
+      ]);
+    });
+
+    it("in the player on Chromium every take of a file loads at parse", async () => {
+      spyLoad();
+      inPlayerFrame("chrome");
+      const takes = takesOfOneFile();
+
+      await evaluateRuntime();
+      expect(takes.map((take) => armed(...take))).toEqual([
+        [true, true],
+        [true, true],
+        [true, true],
+      ]);
+    });
+
+    it("in Studio's preview a clip 5 s ahead stays unloaded, as the look-ahead is 2 s", async () => {
+      servePreview();
+      const [now, soon] = videos(mountRoot(), "0", "5");
+
+      await evaluateRuntime();
+      expect(armed(now, soon)).toEqual([true, false]);
+    });
+
     it("loads a clip with no authored length as a render does, so no reload drops its duration", async () => {
       spyLoad();
       servePreview();
@@ -940,6 +1008,42 @@ describe("runtime entry", () => {
       expect([first.preload, later.preload, untrimmed.preload]).toEqual(["", "none", ""]);
     });
 
+    it("gives each parsed clip preload none in the player's frame, where no clip fetches ahead of the first take", async () => {
+      const frame = document.createElement("iframe");
+      frame.setAttribute(PLAYER_FRAME_ATTR, "");
+      vi.spyOn(window, "frameElement", "get").mockReturnValue(frame);
+      asSafari();
+      const root = mountRoot();
+      timed(root, "div", "0");
+      await evaluateRuntime();
+      const [first, later] = videos(root, "0", "5");
+      await Promise.resolve();
+      expect([first.preload, later.preload]).toEqual(["", "none"]);
+    });
+
+    it("leaves Chromium loading every clip at parse in the player's frame", async () => {
+      const frame = document.createElement("iframe");
+      frame.setAttribute(PLAYER_FRAME_ATTR, "");
+      vi.spyOn(window, "frameElement", "get").mockReturnValue(frame);
+      const root = mountRoot();
+      timed(root, "div", "0");
+      await evaluateRuntime();
+      const [first, later] = videos(root, "0", "5");
+      await Promise.resolve();
+      expect([first.preload, later.preload]).toEqual(["", ""]);
+    });
+
+    it("leaves a frame the player did not mark loading every clip at parse", async () => {
+      vi.spyOn(window, "frameElement", "get").mockReturnValue(document.createElement("iframe"));
+      asSafari();
+      const root = mountRoot();
+      timed(root, "div", "0");
+      await evaluateRuntime();
+      const [first, later] = videos(root, "0", "5");
+      await Promise.resolve();
+      expect([first.preload, later.preload]).toEqual(["", ""]);
+    });
+
     it("defers a clip at 0 inside a later scene, by its start in the film", async () => {
       servePreview();
       const root = mountRoot();

packages/core/src/runtime/init.ts

@@ -42,6 +42,7 @@ import {
   skipsHiddenImages,
 } from "./timedClipHide";
 import {
+  inPlayerFrame,
   PREVIEW_ONLY_ATTRS,
   RUNTIME_STAMPED_ATTR,
   STUDIO_PREVIEW_FAR_SCENE_ATTR as FAR_SCENE_ATTR,
@@ -2996,6 +2997,29 @@ export function initSandboxRuntimeModular(): void {
   let moveSceneWindow = (_time: number): boolean => false;
   const LOOKAHEAD_SECONDS = 2;
   const RELEASE_LOOKAHEAD_SECONDS = 4;
+  // The player keeps a connection per file for the clips on screen and the next take only (see inPlayerFrame).
+  const playerFrame = inPlayerFrame(document);
+  const NEXT_TAKE_SPAN_SECONDS = 0.5;
+  let nextTakes: { t: number; bound: number; clips: Set<HTMLMediaElement> } | null = null;
+  const fileOf = (el: HTMLMediaElement) =>
+    el.getAttribute("src") ?? el.querySelector("source")?.getAttribute("src") ?? "";
+  const nextTakeClips = (t: number): Set<HTMLMediaElement> => {
+    if (nextTakes?.t === t && nextTakes.bound === metadataBoundMedia.size) return nextTakes.clips;
+    const byFile = new Map<string, { el: HTMLMediaElement; start: number }[]>();
+    for (const el of metadataBoundMedia) {
+      const file = fileOf(el);
+      const start = resolveAbsoluteMediaStartSeconds(el);
+      if (!el.isConnected || !file || !(start > t)) continue;
+      byFile.set(file, [...(byFile.get(file) ?? []), { el, start }]);
+    }
+    const clips = new Set<HTMLMediaElement>();
+    for (const later of byFile.values()) {
+      const next = Math.min(...later.map((clip) => clip.start));
+      for (const clip of later) if (clip.start <= next + NEXT_TAKE_SPAN_SECONDS) clips.add(clip.el);
+    }
+    nextTakes = { t, bound: metadataBoundMedia.size, clips };
+    return clips;
+  };
   // Unskipped while within the window on either side of the playhead, so a step or shuttle back
   // across a cut finds the clip it left still loaded; the edge checks catch a clip shorter than it.
   const dueSoon = (
@@ -3248,6 +3272,8 @@ export function initSandboxRuntimeModular(): void {
   ): number[] | null => {
     const own = windowOf(node);
     if (!own || !Number.isFinite(own.start)) return null;
+    // Which clip is next on its file flips at every other clip's start.
+    if (playerFrame && isMediaElement(node)) return null;
     const edges: number[] = [];
     for (const ahead of [0, LOOKAHEAD_SECONDS, RELEASE_LOOKAHEAD_SECONDS])
       edges.push(own.start - ahead, own.end - ahead, own.end + ahead);
@@ -3392,7 +3418,9 @@ export function initSandboxRuntimeModular(): void {
         preloadNearPlayhead(
           rawNode,
           isVisibleNow,
-          upcoming || mediaStaysDue(rawNode, visibleAt, currentTime),
+          playerFrame
+            ? nextTakeClips(currentTime).has(rawNode)
+            : upcoming || mediaStaysDue(rawNode, visibleAt, currentTime),
           currentTime,
           dueAtLoopStart(rawNode, visibleAt),
         );

packages/core/src/runtime/timedClipHide.ts

@@ -1,5 +1,6 @@
 import { CLIP_HIDDEN_ATTR, olderShaderBuildOnPage, revealClip } from "./clipReveal";
 import {
+  defersMediaIn,
   RUNTIME_STAMPED_ATTR,
   STUDIO_PREVIEW_CAPTURE_MODE,
   STUDIO_PREVIEW_MARK_META,
@@ -60,7 +61,7 @@ function appendStyle(parent: Element, attr: string, css: string): void {
 }
 
 export function defersMedia(): boolean {
-  return document.querySelector(STUDIO_PREVIEW_MARK) !== null;
+  return defersMediaIn(document);
 }
 
 /** True in Studio's preview, where hidden clips' images are neither fetched nor decoded until due. */

packages/core/src/studioPreviewMark.ts

@@ -1,5 +1,18 @@
+import { isWebKit } from "./runtime/mediaClock.js";
+
 export const STUDIO_PREVIEW_MARK_META = "hyperframes-studio-preview";
 export const STUDIO_PREVIEW_CAPTURE_MODE = "capture";
+export const PLAYER_FRAME_ATTR = "data-hf-player-frame";
+
+export function inPlayerFrame(doc: Document): boolean {
+  return isWebKit() && doc.defaultView?.frameElement?.hasAttribute(PLAYER_FRAME_ATTR) === true;
+}
+
+export function defersMediaIn(doc: Document): boolean {
+  return (
+    doc.querySelector(`meta[name="${STUDIO_PREVIEW_MARK_META}"]`) !== null || inPlayerFrame(doc)
+  );
+}
 export const STUDIO_PREVIEW_UPCOMING_ATTR = "data-hf-upcoming";
 export const STUDIO_PREVIEW_LAZY_ATTR = "data-hf-preview-lazy";
 export const STUDIO_PREVIEW_ERRORS = "__hfPreviewErrors";

packages/player/src/iframe-dom.ts

@@ -4,6 +4,8 @@
  * Keeps constructor boilerplate out of the web component class body.
  */
 
+import { PLAYER_FRAME_ATTR } from "@hyperframes/core/studio-preview-mark";
+
 // Cached Constructable Stylesheet shared across all player instances.
 let _sharedSheet: CSSStyleSheet | null = null;
 
@@ -49,6 +51,7 @@ export function createCompositionIframe(): {
   iframe.allow = "autoplay; fullscreen";
   iframe.referrerPolicy = "no-referrer";
   iframe.title = "HyperFrames Composition";
+  iframe.setAttribute(PLAYER_FRAME_ATTR, "");
 
   container.appendChild(iframe);
   return { container, iframe };