From a7282dcd0eb888ab496dcbd41a1e35ac2a3bb1c7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Miguel=20=C3=81ngel?= Date: Fri, 18 Sep 2026 17:27:16 -0400 Subject: [PATCH] fix(player): wait for a cross-origin composition's assets before playing (#4103) * fix(player): wait for a cross-origin composition's assets before playing The player's readiness gate scans the iframe document, which is null when the composition is embedded from another origin, so it released playback at once and the first frames could play with images still loading. The runtime now runs the same readiness scan inside the iframe and posts assets-ready; its timeline message reports whether assets have settled so an older runtime, which never posts it, is not waited on. Also declares the animation-frame abort handler before a host whose requestAnimationFrame calls back synchronously can reach it. * fix(player): keep a ready cross-origin player ready across a late iframe load The iframe load event reset readiness for any document the player cannot read. The runtime posts its timeline before load and a paused runtime never posts it again, so a load arriving after the assets settled left the player not ready. A ready cross-origin player already holds this document's handshake, because every navigation clears readiness first. * fix(core): stop RuntimeReadinessWindow widening Window.__hf so the runtime typecheck passes * fix(core): a torn-down runtime no longer posts assets-ready --- .../core/src/compositionReadiness.test.ts | 10 +++ packages/core/src/compositionReadiness.ts | 14 ++-- packages/core/src/runtime/README.md | 4 +- packages/core/src/runtime/init.test.ts | 36 +++++++++ packages/core/src/runtime/init.ts | 13 +++- .../core/src/runtime/transportPark.test.ts | 3 + packages/core/src/runtime/types.ts | 14 ++++ .../player/src/hyperframes-player.test.ts | 78 +++++++++++++++++++ packages/player/src/hyperframes-player.ts | 48 ++++++++---- .../src/runtime-message-handler.test.ts | 32 +++++++- .../player/src/runtime-message-handler.ts | 14 +++- 11 files changed, 242 insertions(+), 24 deletions(-) diff --git a/packages/core/src/compositionReadiness.test.ts b/packages/core/src/compositionReadiness.test.ts index 539c520f4c..157a3ceb36 100644 --- a/packages/core/src/compositionReadiness.test.ts +++ b/packages/core/src/compositionReadiness.test.ts @@ -170,6 +170,16 @@ describe("paintAndIdleReadinessInput", () => { expect(paintAndIdleReadinessInput(docWith(""), new AbortController().signal)).toBeNull(); }); + it("settles on a host whose requestAnimationFrame calls back synchronously", async () => { + let ts = 0; + const win = { requestAnimationFrame: (cb: (t: number) => void) => (cb((ts += 16)), 1) }; + const doc = { defaultView: win } as unknown as Document; + + await expect( + paintAndIdleReadinessInput(doc, new AbortController().signal), + ).resolves.toBeUndefined(); + }); + it("waits for first paint, then two consecutive quiet frame gaps", async () => { const { fireFrame, isResolved } = startPaintAndIdleTracking(); diff --git a/packages/core/src/compositionReadiness.ts b/packages/core/src/compositionReadiness.ts index 54f44062e1..cf10e4684f 100644 --- a/packages/core/src/compositionReadiness.ts +++ b/packages/core/src/compositionReadiness.ts @@ -96,7 +96,8 @@ const RENDER_READY_POLL_MS = 50; // file lives under src/runtime — excluded from this package's own build // program (see tsconfig.json) — so it isn't visible here. Same flag, declared // locally instead of depending on a global merge from outside this file's scope. -interface RuntimeReadinessWindow extends Window { +// `__hf` is omitted from Window so the runtime program's strict shape can't conflict with `unknown`. +interface RuntimeReadinessWindow extends Omit { __renderReady?: boolean; // __renderReady is only ever set by init.ts, which always sets __hf // first (`window.__hf = window.__hf || {}`) — a doc with no __hf can @@ -160,15 +161,18 @@ function nextAnimationFrame(win: Window, signal: AbortSignal): Promise { resolve(-1); return; } - const id = win.requestAnimationFrame((ts) => { - signal.removeEventListener("abort", onAbort); - resolve(ts); - }); + let id = 0; const onAbort = () => { win.cancelAnimationFrame?.(id); resolve(-1); }; signal.addEventListener("abort", onAbort, { once: true }); + // A host whose rAF calls back synchronously reaches onAbort in the + // callback, so it must already be declared above. + id = win.requestAnimationFrame((ts) => { + signal.removeEventListener("abort", onAbort); + resolve(ts); + }); }); } diff --git a/packages/core/src/runtime/README.md b/packages/core/src/runtime/README.md index 67d852f094..691df2a05d 100644 --- a/packages/core/src/runtime/README.md +++ b/packages/core/src/runtime/README.md @@ -31,7 +31,9 @@ postMessage: - actions: `play`, `pause`, `seek`, `set-muted`, `set-playback-rate`, `enable-pick-mode`, `disable-pick-mode` - runtime -> parent events: - `source: "hf-preview"` - - `type: "state"` and `type: "timeline"` + - `type: "state"` and `type: "timeline"`; `timeline` carries `assetsReady` (whether the + composition's media, images and fonts have settled) and the runtime then posts + `type: "assets-ready"` once, so a parent that cannot read the iframe can gate playback - `type: "ready"` — emitted once when `installRuntimeControlBridge` registers the control-message listener. The parent uses it to replay current playback state (`set-muted`, `set-volume`, `set-playback-rate`) so any control diff --git a/packages/core/src/runtime/init.test.ts b/packages/core/src/runtime/init.test.ts index 415fa6ee6b..2e533380ce 100644 --- a/packages/core/src/runtime/init.test.ts +++ b/packages/core/src/runtime/init.test.ts @@ -3094,6 +3094,42 @@ describe("initSandboxRuntimeModular", () => { expect(seekTimes[seekTimes.length - 1]).toBe(0); }); + it("posts assets-ready once, after the timeline, and only once a pending image settles", async () => { + const root = document.createElement("div"); + root.setAttribute("data-composition-id", "root"); + root.setAttribute("data-root", "true"); + root.setAttribute("data-duration", "5"); + root.setAttribute("data-width", "1920"); + root.setAttribute("data-height", "1080"); + const img = document.createElement("img"); + const decodes: Array<() => void> = []; + Object.defineProperty(img, "complete", { value: false, configurable: true }); + img.decode = () => new Promise((resolve) => decodes.push(resolve)); + root.appendChild(img); + document.body.appendChild(root); + window.__timelines = { root: createMockTimeline(5) }; + const outbound: Array> = []; + vi.spyOn(window.parent, "postMessage").mockImplementation((message: unknown) => { + if (typeof message === "object" && message !== null) { + outbound.push(message as Record); + } + }); + + initSandboxRuntimeModular(); + const types = () => outbound.map((m) => m.type); + expect(outbound.find((m) => m.type === "timeline")?.assetsReady).toBe(false); + await new Promise((r) => setTimeout(r, 200)); + expect(types()).not.toContain("assets-ready"); + + decodes.forEach((resolve) => resolve()); + await vi.waitFor(() => expect(types()).toContain("assets-ready")); + window.__player!.renderSeek(1); + await new Promise((r) => setTimeout(r, 100)); + expect(types().filter((t) => t === "assets-ready")).toHaveLength(1); + expect(decodes).toHaveLength(1); + expect(types().indexOf("assets-ready")).toBeGreaterThan(types().indexOf("timeline")); + }); + it("accepts replayed transport controls when the bridge announces ready without duplicate listeners", () => { const root = document.createElement("div"); root.setAttribute("data-composition-id", "root"); diff --git a/packages/core/src/runtime/init.ts b/packages/core/src/runtime/init.ts index ad6453efad..215e523d57 100644 --- a/packages/core/src/runtime/init.ts +++ b/packages/core/src/runtime/init.ts @@ -77,6 +77,7 @@ import { import { installStudioCustomEase } from "./customEase"; import { parseStrictFiniteTimingNumber, resolveMediaElementDurationSeconds } from "./playbackRate"; import { MEDIA_START_BASIS_ATTR } from "../mediaTiming"; +import { settleCompositionReadiness } from "../compositionReadiness"; import { clearRuntimeData, setRuntimeData, @@ -2758,6 +2759,8 @@ export function initSandboxRuntimeModular(): void { // transport tick. A plain count misses same-count swaps (one sub-comp unloads // as another loads), so the signature keys on id+tag in document order. let clipTreeSignature = ""; + let assetsReadyStarted = false; + let assetsSettled = false; let liveRootDurationOverrideSeconds = 0; const computeClipTreeSignature = (): string => { let sig = ""; @@ -2807,7 +2810,15 @@ export function initSandboxRuntimeModular(): void { clipTreeSignature = currentSignature; } - postRuntimeMessage(payload); + postRuntimeMessage({ ...payload, assetsReady: assetsSettled }); + if (!assetsReadyStarted) { + assetsReadyStarted = true; + settleCompositionReadiness(document, ({ timedOut }) => { + if (state.tornDown) return; + assetsSettled = true; + postRuntimeMessage({ source: "hf-preview", type: "assets-ready", timedOut }); + }); + } scheduleRootStageLayoutDiagnostics(); }; diff --git a/packages/core/src/runtime/transportPark.test.ts b/packages/core/src/runtime/transportPark.test.ts index b20bf99923..d70a3127c8 100644 --- a/packages/core/src/runtime/transportPark.test.ts +++ b/packages/core/src/runtime/transportPark.test.ts @@ -3,6 +3,9 @@ import { initSandboxRuntimeModular } from "./init"; import { STUDIO_MANUAL_EDIT_GESTURE_ATTR } from "../editing/draftMarkers"; import type { RuntimeTimelineLike } from "./types"; +// The readiness gate's 8s timeout is a one-shot timer these transport tests must not count. +vi.mock("../compositionReadiness", () => ({ settleCompositionReadiness: vi.fn() })); + /** * The transport parks itself when the editor is paused and settled. Everything * it used to discover by looking again on the next frame has to arrive by some diff --git a/packages/core/src/runtime/types.ts b/packages/core/src/runtime/types.ts index 221b5a660c..fbd02201d4 100644 --- a/packages/core/src/runtime/types.ts +++ b/packages/core/src/runtime/types.ts @@ -96,6 +96,10 @@ export type RuntimeTimelineMessage = RuntimeProtocolV1 & { scenes: RuntimeTimelineScene[]; compositionWidth: number; compositionHeight: number; + /** Present when this runtime will post `assets-ready`; the value is whether + * the composition's assets have settled yet. Absent on older runtimes, whose + * parents must not wait for a message that never comes. */ + assetsReady?: boolean; }; export type RuntimeDiagnosticMessage = { @@ -171,6 +175,15 @@ export type RuntimeReadyMessage = { type: "ready"; }; +/** Posted once per runtime instance, after the first timeline message, when + * the composition's media, images and fonts have settled (or timed out). It + * lets a parent that cannot read the iframe (opaque origin) gate playback. */ +export type RuntimeAssetsReadyMessage = { + source: "hf-preview"; + type: "assets-ready"; + timedOut: boolean; +}; + export type RuntimeDataErrorMessage = { source: "hf-preview"; type: "runtime-data-error"; @@ -236,6 +249,7 @@ export type RuntimeOutboundMessage = | RuntimeStageSizeMessage | RuntimeMediaAutoplayBlockedMessage | RuntimeReadyMessage + | RuntimeAssetsReadyMessage | RuntimeDataErrorMessage | RuntimeDataAppliedMessage | RuntimeAnalyticsMessage diff --git a/packages/player/src/hyperframes-player.test.ts b/packages/player/src/hyperframes-player.test.ts index a7ce1d643d..e267f95095 100644 --- a/packages/player/src/hyperframes-player.test.ts +++ b/packages/player/src/hyperframes-player.test.ts @@ -2823,6 +2823,84 @@ describe("HyperframesPlayer asset-ready gate", () => { return { doc, video }; } + const post = (player: PlayerInternal, data: Record) => + (player as unknown as { _onMessage(e: MessageEvent): void })._onMessage({ + source: player.iframe.contentWindow, + data: { source: "hf-preview", ...data }, + } as unknown as MessageEvent); + + it("holds an opaque-origin composition until its runtime posts assets-ready", async () => { + const player = await createConnectedPlayer(); + player._ready = false; + Object.defineProperty(player.iframe, "contentDocument", { get: () => null }); + post(player, { type: "timeline", durationInFrames: 60, assetsReady: false }); + expect(player.assetsReady).toBe(false); + + post(player, { type: "assets-ready", timedOut: false }); + expect(player.assetsReady).toBe(true); + + player.remove(); + }); + + it("keeps a queued play across the iframe load event while the runtime still reports assets pending", async () => { + const player = await createConnectedPlayer(); + player._ready = false; + Object.defineProperty(player.iframe, "contentDocument", { get: () => null }); + post(player, { type: "timeline", durationInFrames: 60, assetsReady: false }); + player.play(); + expect(player._pendingPlay).toBe(true); + + player._onIframeLoad(); + post(player, { type: "assets-ready", timedOut: false }); + + expect(player.assetsReady).toBe(true); + expect(player._ready).toBe(true); + expect(player._pendingPlay).toBe(false); + expect(player._paused).toBe(false); + + player.remove(); + }); + + it("stays ready when a late iframe load follows a settled opaque-origin wait", async () => { + const player = await createConnectedPlayer(); + player._ready = false; + Object.defineProperty(player.iframe, "contentDocument", { get: () => null }); + post(player, { type: "timeline", durationInFrames: 60, assetsReady: false }); + post(player, { type: "assets-ready", timedOut: false }); + expect(player._ready).toBe(true); + + player._onIframeLoad(); + + expect(player._ready).toBe(true); + expect(player.assetsReady).toBe(true); + player.play(); + expect(player._paused).toBe(false); + + player.remove(); + }); + + it("does not wait on an opaque-origin runtime that already settled its assets", async () => { + const player = await createConnectedPlayer(); + player._ready = false; + Object.defineProperty(player.iframe, "contentDocument", { get: () => null }); + post(player, { type: "timeline", durationInFrames: 60, assetsReady: true }); + + expect(player.assetsReady).toBe(true); + + player.remove(); + }); + + it("does not wait on an opaque-origin runtime that never announced the capability", async () => { + const player = await createConnectedPlayer(); + player._ready = false; + Object.defineProperty(player.iframe, "contentDocument", { get: () => null }); + post(player, { type: "timeline", durationInFrames: 60 }); + + expect(player.assetsReady).toBe(true); + + player.remove(); + }); + it("settles immediately for a cross-origin composition (doc === null)", async () => { const player = await createConnectedPlayer(); diff --git a/packages/player/src/hyperframes-player.ts b/packages/player/src/hyperframes-player.ts index 567524b1e4..ae296c6eb3 100644 --- a/packages/player/src/hyperframes-player.ts +++ b/packages/player/src/hyperframes-player.ts @@ -130,6 +130,7 @@ class HyperframesPlayer extends HTMLElement { private _scenes: { id: string; start: number; duration: number }[] = []; private _runtimeFps = 30; private _runtimeBridgeReady = false; + private _runtimeAssetsReadyGeneration = -1; private _runtimeData = new Map(); private _runtimeDataRequestId = 0; private _pendingRuntimeData = new Map(); @@ -955,11 +956,17 @@ class HyperframesPlayer extends HTMLElement { this._replayBridgeState(); this._replayRuntimeData(); }, + onRuntimeAssetsReady: () => { + if (this._runtimeAssetsReadyGeneration === this._assetsGeneration) { + this._settleAssetsReady(this._assetsGeneration); + } + }, onRuntimeDataApplied: (channel, requestId) => this._resolveRuntimeDataDelivery(channel, requestId), onRuntimeDataError: (channel, requestId, message) => this._rejectRuntimeDataDelivery(channel, requestId, message), - onRuntimeTimelineReady: (duration) => this._onRuntimeTimelineReady(duration), + onRuntimeTimelineReady: (duration, assetsReady) => + this._onRuntimeTimelineReady(duration, assetsReady), setRuntimeFps: (fps) => { this._runtimeFps = fps; }, @@ -978,7 +985,7 @@ class HyperframesPlayer extends HTMLElement { }); } - private _onRuntimeTimelineReady(duration: number) { + private _onRuntimeTimelineReady(duration: number, assetsReady: boolean | undefined) { if (this._ready) return; this.probe.stop(); this._duration = duration; @@ -996,7 +1003,7 @@ class HyperframesPlayer extends HTMLElement { this._replayBridgeState(); this._setIframeMediaMuted(this.muted); - this._waitForAssetsReady(doc); + this._waitForAssetsReady(doc, assetsReady); if (this.hasAttribute("autoplay") || this._pendingPlay) this.play(); } @@ -1022,7 +1029,7 @@ class HyperframesPlayer extends HTMLElement { * bounded by ASSETS_READY_TIMEOUT_MS. The overlay is debounced by * ASSETS_LOADING_SHOW_DELAY_MS rather than shown the instant a wait * starts, since one is now pending on nearly every Play. */ - private _waitForAssetsReady(doc: Document | null): void { + private _waitForAssetsReady(doc: Document | null, runtimeAssetsReady?: boolean): void { this._clearAssetsLoadingShowTimer(); this._assetsReady = false; // Invalidates any earlier wait still in flight (a composition swap, or @@ -1030,7 +1037,15 @@ class HyperframesPlayer extends HTMLElement { // rather than resolving a since-superseded generation. const generation = ++this._assetsGeneration; if (!doc) { - this._settleAssetsReady(generation); + // An opaque-origin iframe cannot be scanned from here; a runtime that + // reports `assetsReady: false` runs the same scan itself and posts the result. + if (runtimeAssetsReady === false) { + this._runtimeAssetsReadyGeneration = generation; + this._startAssetsLoadingOverlayTimer(generation); + setTimeout(() => this._settleAssetsReady(generation), ASSETS_READY_TIMEOUT_MS); + } else { + this._settleAssetsReady(generation); + } return; } settleCompositionReadiness( @@ -1042,14 +1057,16 @@ class HyperframesPlayer extends HTMLElement { }, { timeoutMs: ASSETS_READY_TIMEOUT_MS }, ); - if (!this._assetsReady) { - this._assetsLoadingShowTimer = setTimeout(() => { - this._assetsLoadingShowTimer = null; - if (generation !== this._assetsGeneration || this._assetsReady) return; - this.setAttribute(ASSETS_LOADING_ATTR, ""); - this.shaderLoader.showAssetsLoading(); - }, ASSETS_LOADING_SHOW_DELAY_MS); - } + if (!this._assetsReady) this._startAssetsLoadingOverlayTimer(generation); + } + + private _startAssetsLoadingOverlayTimer(generation: number): void { + this._assetsLoadingShowTimer = setTimeout(() => { + this._assetsLoadingShowTimer = null; + if (generation !== this._assetsGeneration || this._assetsReady) return; + this.setAttribute(ASSETS_LOADING_ATTR, ""); + this.shaderLoader.showAssetsLoading(); + }, ASSETS_LOADING_SHOW_DELAY_MS); } /** Timeout diagnostic. Re-scans since some assets may have resolved by @@ -1128,6 +1145,11 @@ class HyperframesPlayer extends HTMLElement { } private _onIframeLoad() { + // The runtime posts its timeline at DOMContentLoaded, before `load`, and every + // host-initiated navigation clears `_ready` first. So a ready opaque-origin player already + // holds this document's handshake; a paused runtime would never post it again. + if (this._ready && this._getSameOriginIframeDocument() === null) return; + this._ready = false; // The runtime installs its bridge at DOMContentLoaded, posts `ready`, and only then does the // iframe's load event fire. Do not erase that authoritative handshake here: doing so strands diff --git a/packages/player/src/runtime-message-handler.test.ts b/packages/player/src/runtime-message-handler.test.ts index 5bd2a5f7a5..6e19ee2b65 100644 --- a/packages/player/src/runtime-message-handler.test.ts +++ b/packages/player/src/runtime-message-handler.test.ts @@ -171,7 +171,7 @@ describe("handleRuntimeMessage timeline ready", () => { handleRuntimeMessage(timelineEvent(120, frameWindow), frameWindow, callbacks); - expect(callbacks.onRuntimeTimelineReady).toHaveBeenCalledWith(4); + expect(callbacks.onRuntimeTimelineReady).toHaveBeenCalledWith(4, undefined); }); it.each([ @@ -191,6 +191,7 @@ describe("handleRuntimeMessage timeline ready", () => { expect(callbacks.setRuntimeFps).toHaveBeenCalledWith(expect.closeTo(fps, 6)); expect(callbacks.onRuntimeTimelineReady).toHaveBeenCalledWith( expect.closeTo(expectedSeconds, 6), + undefined, ); }); @@ -209,7 +210,7 @@ describe("handleRuntimeMessage timeline ready", () => { callbacks, ); - expect(callbacks.onRuntimeTimelineReady).toHaveBeenCalledWith(4.25); + expect(callbacks.onRuntimeTimelineReady).toHaveBeenCalledWith(4.25, undefined); expect(callbacks.setCompositionSize).toHaveBeenCalledWith(1080, 1920); }); @@ -241,3 +242,30 @@ describe("handleRuntimeMessage timeline ready", () => { expect(callbacks.onRuntimeTimelineReady).not.toHaveBeenCalled(); }); }); + +describe("handleRuntimeMessage assets-ready", () => { + const frame = {}; + const send = (callbacks: MessageHandlerCallbacks, data: Record) => + handleRuntimeMessage( + { source: frame, data: { source: "hf-preview", ...data } } as unknown as MessageEvent, + frame as Window, + callbacks, + ); + + it("reports whether the runtime has settled its assets on its timeline", () => { + const callbacks = makeCallbacks(); + send(callbacks, { type: "timeline", durationInFrames: 60, assetsReady: false }); + send(callbacks, { type: "timeline", durationInFrames: 60, assetsReady: true }); + send(callbacks, { type: "timeline", durationInFrames: 60 }); + expect(callbacks.onRuntimeTimelineReady).toHaveBeenNthCalledWith(1, 2, false); + expect(callbacks.onRuntimeTimelineReady).toHaveBeenNthCalledWith(2, 2, true); + expect(callbacks.onRuntimeTimelineReady).toHaveBeenNthCalledWith(3, 2, undefined); + }); + + it("forwards the runtime's assets-ready result", () => { + const callbacks = makeCallbacks(); + callbacks.onRuntimeAssetsReady = vi.fn(); + send(callbacks, { type: "assets-ready", timedOut: true }); + expect(callbacks.onRuntimeAssetsReady).toHaveBeenCalledWith(true); + }); +}); diff --git a/packages/player/src/runtime-message-handler.ts b/packages/player/src/runtime-message-handler.ts index 8ccb3c027d..d3507ba54e 100644 --- a/packages/player/src/runtime-message-handler.ts +++ b/packages/player/src/runtime-message-handler.ts @@ -40,12 +40,14 @@ export interface MessageHandlerCallbacks extends PlaybackStateCallbacks { * uses it to replay current bridge state (mute, volume, playback rate) so * control messages sent before the iframe's listener registered aren't lost. */ onRuntimeReady: () => void; + /** Invoked when the runtime reports its composition assets settled. */ + onRuntimeAssetsReady?: (timedOut: boolean) => void; onRuntimeDataApplied?: (channel: unknown, requestId: unknown) => void; onRuntimeDataError?: (channel: unknown, requestId: unknown, message: unknown) => void; /** Invoked when the runtime posts a finite positive timeline duration. The * player uses this as the cross-origin readiness signal because the * same-origin composition probe cannot inspect CDN iframes. */ - onRuntimeTimelineReady: (duration: number) => void; + onRuntimeTimelineReady: (duration: number, assetsReady: boolean | undefined) => void; setRuntimeFps?: (fps: number) => void; /** Called with the scene list whenever a "timeline" message is received. */ setScenes: (scenes: SceneRecord[]) => void; @@ -94,6 +96,11 @@ export function handleRuntimeMessage( return; } + if (data["type"] === "assets-ready") { + callbacks.onRuntimeAssetsReady?.(data["timedOut"] === true); + return; + } + if (data["type"] === "runtime-data-error") { callbacks.onRuntimeDataError?.(data["channel"], data["requestId"], data["message"]); return; @@ -142,7 +149,10 @@ export function handleRuntimeMessage( const pb = callbacks.getPlaybackState(); callbacks.setPlaybackState({ ...pb, duration }); callbacks.updateControlsTime(pb.currentTime, duration); - callbacks.onRuntimeTimelineReady(duration); + callbacks.onRuntimeTimelineReady( + duration, + typeof data["assetsReady"] === "boolean" ? data["assetsReady"] : undefined, + ); } if ( Number.isFinite(data["compositionWidth"]) &&