diff --git a/packages/core/src/runtime/editingAgent.test.ts b/packages/core/src/runtime/editingAgent.test.ts index d03cb7e10d..b3119162fe 100644 --- a/packages/core/src/runtime/editingAgent.test.ts +++ b/packages/core/src/runtime/editingAgent.test.ts @@ -48,8 +48,7 @@ describe("installEditingAgent", () => { return (handoff.mock.calls[0][2] as MessagePort[])[0]; }; - it("hands one port to the parent, once", () => { - post(parent, CONNECT); + it("hands a port to the parent on its connect", () => { post(parent, CONNECT); expect(handoff).toHaveBeenCalledOnce(); const [message, targetOrigin, transfer] = handoff.mock.calls[0]; @@ -58,6 +57,27 @@ describe("installEditingAgent", () => { expect(transfer).toHaveLength(1); }); + it("hands a fresh port to each later connect from the parent and keeps the earlier one working", async () => { + const old = connectedPort(); + post(parent, CONNECT); + expect(handoff).toHaveBeenCalledTimes(2); + const fresh = (handoff.mock.calls[1][2] as MessagePort[])[0]; + expect(fresh).not.toBe(old); + expect(await ask(old, { v: 1, id: 1, kind: "hello" })).toMatchObject({ id: 1, ok: true }); + expect(await ask(fresh, { v: 1, id: 2, kind: "hello" })).toMatchObject({ id: 2, ok: true }); + }); + + it("closes every port it handed over when disposed", async () => { + const first = connectedPort(); + post(parent, CONNECT); + const second = (handoff.mock.calls[1][2] as MessagePort[])[0]; + const closed = [first, second].map( + (port) => new Promise((resolve) => port.addEventListener("close", () => resolve())), + ); + agent.dispose(); + await Promise.all(closed); + }); + it("ignores a connect from the page itself or another frame, without using up the handoff", () => { post(window, CONNECT); post(frameWindow(), CONNECT); @@ -69,6 +89,14 @@ describe("installEditingAgent", () => { expect(handoff).toHaveBeenCalledOnce(); }); + it("keeps the parent's port when another frame asks after it", async () => { + const port = connectedPort(); + post(frameWindow(), CONNECT); + post(window, CONNECT); + expect(handoff).toHaveBeenCalledOnce(); + expect(await ask(port, { v: 1, id: 1, kind: "hello" })).toMatchObject({ id: 1, ok: true }); + }); + it("never hands off on a top-level page, where the page is its own parent", () => { const selfPost = vi.spyOn(window, "postMessage").mockImplementation(() => {}); const topLevel = installEditingAgent(window); diff --git a/packages/core/src/runtime/editingAgent.ts b/packages/core/src/runtime/editingAgent.ts index 2220afb224..c0e3a8c4af 100644 --- a/packages/core/src/runtime/editingAgent.ts +++ b/packages/core/src/runtime/editingAgent.ts @@ -1,10 +1,10 @@ /** * The runtime's end of the editing channel (wire types in `editingPort.ts`). * - * The port goes to `window.parent` once, when the parent asks for it, so only - * the parent holds a channel. A connect posted by anything else, including the - * page itself, is ignored and does not use up the handoff. Requests on the port - * are validated before they run; a bad one gets a typed error reply, never a throw. + * Each connect from `window.parent` gets a fresh port, so only the parent holds a channel and a + * remounted client is served too; a connect from anything else is ignored. Earlier ports stay + * open: a client may take any one of them, and a disposed client closes its own. + * Requests on the port are validated; a bad one gets a typed error reply, never a throw. */ import { swallow } from "./diagnostics"; import { @@ -99,7 +99,7 @@ function failure(id: number, err: unknown): EditingReply { } export function installEditingAgent(parent: Window = window.parent): { dispose: () => void } { - let port: MessagePort | null = null; + const ports = new Set(); const onMessage = (event: MessageEvent) => { // A top-level page is its own parent, so a self-post would pass the source check. if (parent === window || event.source !== parent) return; @@ -107,9 +107,9 @@ export function installEditingAgent(parent: Window = window.parent): { dispose: if (!isPlainRecord(data) || data.source !== "hf-parent" || data.type !== EDITING_CONNECT) return; if (data.v !== EDITING_PROTOCOL_VERSION) return; - window.removeEventListener("message", onMessage); const channel = new MessageChannel(); - port = channel.port1; + const port = channel.port1; + ports.add(port); serve(port); try { // "*" as every runtime post does (bridge.ts): the frame is not told its parent's origin. @@ -121,13 +121,15 @@ export function installEditingAgent(parent: Window = window.parent): { dispose: } catch (err) { swallow("editingAgent.handoff", err); port.close(); + ports.delete(port); } }; window.addEventListener("message", onMessage); return { dispose() { window.removeEventListener("message", onMessage); - port?.close(); + for (const port of ports) port.close(); + ports.clear(); }, }; } diff --git a/packages/core/src/runtime/editingPort.test.ts b/packages/core/src/runtime/editingPort.test.ts index 8d9f205636..5f30e0a763 100644 --- a/packages/core/src/runtime/editingPort.test.ts +++ b/packages/core/src/runtime/editingPort.test.ts @@ -67,6 +67,69 @@ describe("createEditingPortClient", () => { } }); + it("serves a new client for the same frame after the first is disposed", async () => { + const { iframe, frame } = previewFrame(); + const { host, deliver } = fakeHost(); + const parent = document.body.appendChild(document.createElement("iframe")).contentWindow!; + parent.postMessage = ((data: unknown, _origin: string, transfer: MessagePort[]) => { + ports.push(...transfer); + deliver({ source: frame, data, ports: transfer }); + }) as Window["postMessage"]; + frame.postMessage = ((data: unknown) => + window.dispatchEvent( + new MessageEvent("message", { source: parent, data }), + )) as Window["postMessage"]; + const agent = installEditingAgent(parent); + try { + const first = createEditingPortClient({ iframe, host }); + await first.ready; + first.dispose(); + const second = createEditingPortClient({ iframe, host, timeoutMs: 1_000 }); + clients.push(second); + await second.ready; + expect(await second.request("wait-for-seek", {})).toEqual({ waited: false }); + } finally { + agent.dispose(); + } + }); + + // The runtime posts `ready` before it installs its agent, so a client can send its connect + // twice and receive two ports. Each step below is one queued delivery, run in the order given. + it.each([ + ["ready before either port", ["ready", "agent", "agent", "port", "port"]], + ["ready between the connects", ["agent", "ready", "port", "agent", "port"]], + ["first port before ready", ["agent", "port", "ready", "agent", "port"]], + ])("keeps a working port when %s", async (_name, steps) => { + const { iframe, frame } = previewFrame(); + const { host, deliver } = fakeHost(); + const parent = document.body.appendChild(document.createElement("iframe")).contentWindow!; + const toAgent: unknown[] = []; + const toHost: Partial[] = []; + parent.postMessage = ((data: unknown, _origin: string, transfer: MessagePort[]) => { + ports.push(...transfer); + toHost.push({ source: frame, data, ports: transfer }); + }) as Window["postMessage"]; + frame.postMessage = ((data: unknown) => toAgent.push(data)) as Window["postMessage"]; + const agent = installEditingAgent(parent); + try { + const client = createEditingPortClient({ iframe, host, timeoutMs: 300 }); + clients.push(client); + for (const step of steps) { + if (step === "ready") + deliver({ source: frame, data: { source: "hf-preview", type: "ready" } }); + else if (step === "port") deliver(toHost.shift() ?? {}); + else if (toAgent.length > 0) + window.dispatchEvent( + new MessageEvent("message", { source: parent, data: toAgent.shift() }), + ); + } + await client.ready; + await expect(client.request("hello", {})).resolves.toMatchObject({ version: 1 }); + } finally { + agent.dispose(); + } + }); + it("asks for the port at once and on ready, and takes it only from its own frame", async () => { const { iframe, frame, connects } = previewFrame(); const { host, deliver } = fakeHost(); diff --git a/packages/studio/src/player/components/Player.tsx b/packages/studio/src/player/components/Player.tsx index 78370ae000..e6809982d0 100644 --- a/packages/studio/src/player/components/Player.tsx +++ b/packages/studio/src/player/components/Player.tsx @@ -8,6 +8,7 @@ import { HyperframesLoader } from "../../components/ui"; import { usePlayerStore } from "../store/playerStore"; import { announcePreviewDocumentLoaded } from "../sceneSwap"; import type { IframeWindow } from "../lib/playbackTypes"; +import { watchPreviewFrame } from "../lib/bridgePlayback"; // Importing "@hyperframes/player" registers a class extending HTMLElement at // module load, which throws under SSR, hence the dynamic import behind a // `typeof window` guard. Kicking it here rather than in the mount effect puts @@ -353,6 +354,7 @@ export const Player = forwardRef( loadIdRef.current += 1; player.setAttribute("src", src); container.appendChild(player); + watchPreviewFrame(iframe); enableInteractiveIframe(player); diff --git a/packages/studio/src/player/hooks/playbackAdapterResolution.ts b/packages/studio/src/player/hooks/playbackAdapterResolution.ts index a816b74181..c338bce496 100644 --- a/packages/studio/src/player/hooks/playbackAdapterResolution.ts +++ b/packages/studio/src/player/hooks/playbackAdapterResolution.ts @@ -11,6 +11,9 @@ import { type StaticSeekCacheEntry, } from "../lib/playbackAdapter"; import { readTimelineDurationFromDocument } from "../lib/timelineDOM"; +import { attachPausedVideoProvider } from "../lib/pausedVideoProvider"; +import { previewFrameOpaque } from "../lib/previewChannel"; +import { bridgePlaybackAdapter } from "../lib/bridgePlayback"; /** * Timeline-based adapters (`__timeline`, then `__timelines`): `match` is the first that covers @@ -96,3 +99,15 @@ export function resolvePlaybackAdapter( return bestAdapter; } + +/** A frame Studio cannot script (opaque origin) is driven over the control bridge. */ +export function resolveFramePlaybackAdapter( + iframe: HTMLIFrameElement, + refs: StaticSeekRefs, +): PlaybackAdapter | null { + if (previewFrameOpaque(iframe)) return bridgePlaybackAdapter(iframe); + const win = iframe.contentWindow as IframeWindow | null; + if (!win) return null; + attachPausedVideoProvider(iframe, win); + return resolvePlaybackAdapter(iframe, win, refs); +} diff --git a/packages/studio/src/player/hooks/previewMessageRouter.manifest.test.ts b/packages/studio/src/player/hooks/previewMessageRouter.manifest.test.ts new file mode 100644 index 0000000000..e386a0497a --- /dev/null +++ b/packages/studio/src/player/hooks/previewMessageRouter.manifest.test.ts @@ -0,0 +1,67 @@ +// @vitest-environment happy-dom + +import { afterEach, describe, expect, it, vi } from "vitest"; +import { usePlayerStore } from "../store/playerStore"; +import { createPreviewMessageHandler } from "./previewMessageRouter"; +import { detachedFakeIframe, serveEditingRequests } from "./timelinePlayerTestHarness"; + +const closers: Array<() => void> = []; +afterEach(() => { + closers.splice(0).forEach((close) => close()); + usePlayerStore.getState().reset(); +}); + +const PORT_MANIFEST = { clips: [{ id: "a" }], durationInFrames: 60, scenes: [] }; +const GLOBAL_MANIFEST = { clips: [{ id: "x" }], durationInFrames: 30, scenes: [] }; + +/** A frame whose globals and editing port carry different manifests, so a test sees which was read. */ +function routeStateTicks(opaque: boolean) { + const frame: Record = { __clipManifest: GLOBAL_MANIFEST }; + const iframe = detachedFakeIframe(frame); + if (opaque) { + Object.defineProperty(iframe, "contentDocument", { value: null }); + Object.defineProperty(frame, "document", { + get: () => { + throw new DOMException("blocked", "SecurityError"); + }, + }); + } + const asked = vi.fn(() => ({ clipManifest: PORT_MANIFEST })); + closers.push(serveEditingRequests(frame, asked)); + const processed = vi.fn(); + const handle = createPreviewMessageHandler({ + iframeRef: { current: iframe }, + processTimelineMessageRef: { current: processed }, + enrichMissingCompositionsRef: { current: vi.fn() }, + lastTimelineMessageRef: { current: Date.now() }, + getAdapter: () => null, + syncTimelineElements: vi.fn(), + }); + const tick = () => + handle( + new MessageEvent("message", { + source: frame as unknown as Window, + data: { source: "hf-preview", type: "state", isPlaying: false }, + }), + ); + return { tick, asked, processed }; +} + +describe("a state tick while the timeline is empty", () => { + it("reads a scriptable frame's manifest straight off it, at once, with no port", async () => { + const { tick, asked, processed } = routeStateTicks(false); + tick(); + expect(processed).toHaveBeenCalledWith(GLOBAL_MANIFEST); + await new Promise((resolve) => setTimeout(resolve, 20)); + expect(asked).not.toHaveBeenCalled(); + }); + + it("asks an opaque frame's runtime over the editing port, one request at a time", async () => { + const { tick, asked, processed } = routeStateTicks(true); + tick(); + tick(); + await vi.waitFor(() => expect(processed).toHaveBeenCalledWith(PORT_MANIFEST)); + expect(asked).toHaveBeenCalledOnce(); + expect(processed).toHaveBeenCalledOnce(); + }); +}); diff --git a/packages/studio/src/player/hooks/previewMessageRouter.ts b/packages/studio/src/player/hooks/previewMessageRouter.ts index 88cb80a40f..a6686f7171 100644 --- a/packages/studio/src/player/hooks/previewMessageRouter.ts +++ b/packages/studio/src/player/hooks/previewMessageRouter.ts @@ -23,6 +23,7 @@ import { parsePreviewFacts, } from "../store/previewQualitySlice"; import { parseTimelineFromDOM } from "../lib/timelineDOM"; +import { previewClipManifest, previewFrameOpaque } from "../lib/previewChannel"; /** What `processTimelineMessage` accepts — the clip-manifest postMessage. */ export interface ClipManifestMessage { @@ -83,6 +84,39 @@ function enrichDue(lastTimelineMessage: number, enriched: { at: number }): boole return true; } +/** Asks the runtime for its clip manifest, one request at a time, while the store has no elements. */ +function recoverClipManifest( + deps: PreviewMessageRouterDeps, + iframe: HTMLIFrameElement, + recovery: { asking: boolean }, +): void { + if (recovery.asking) return; + recovery.asking = true; + previewClipManifest(iframe) + .then((manifest) => { + const stillWanted = + usePlayerStore.getState().elements.length === 0 && + deps.iframeRef.current === iframe && + !hasTimelinePerformanceFixtureLease(); + if (manifest && manifest.clips.length > 0 && stillWanted) + deps.processTimelineMessageRef.current(manifest); + }) + .catch(() => {}) + .finally(() => (recovery.asking = false)); +} + +function recoverEmptyTimeline( + deps: PreviewMessageRouterDeps, + iframe: HTMLIFrameElement | null, + recovery: { asking: boolean }, +): void { + if (iframe && previewFrameOpaque(iframe)) return recoverClipManifest(deps, iframe, recovery); + const manifest = (iframe?.contentWindow as IframeWindow | null)?.__clipManifest; + if (manifest && manifest.clips.length > 0) { + deps.processTimelineMessageRef.current(manifest); + } +} + /** * A `state` tick doubles as a recovery hook: if the store still has no * elements, read the manifest straight off the iframe, and if no `timeline` @@ -92,15 +126,11 @@ function handleStateMessage( deps: PreviewMessageRouterDeps, iframe: HTMLIFrameElement | null, playingHoldsClipListPosts: boolean, - enriched: { at: number }, + enriched: { at: number; asking: boolean }, ): void { try { - if (usePlayerStore.getState().elements.length === 0) { - const manifest = (iframe?.contentWindow as IframeWindow | null)?.__clipManifest; - if (manifest && manifest.clips.length > 0) { - deps.processTimelineMessageRef.current(manifest); - } - } + if (usePlayerStore.getState().elements.length === 0) + recoverEmptyTimeline(deps, iframe, enriched); if (!playingHoldsClipListPosts && enrichDue(deps.lastTimelineMessageRef.current, enriched)) { deps.enrichMissingCompositionsRef.current(); } @@ -156,7 +186,7 @@ function applyStoreReport(data: PreviewMessage): boolean { export function createPreviewMessageHandler( deps: PreviewMessageRouterDeps, ): (e: MessageEvent) => void { - const enriched = { at: 0 }; + const enriched = { at: 0, asking: false }; return (e: MessageEvent) => { const iframe = deps.iframeRef.current; const data = acceptedPreviewMessage(e, iframe); diff --git a/packages/studio/src/player/hooks/timelinePlayerTestHarness.ts b/packages/studio/src/player/hooks/timelinePlayerTestHarness.ts index 94843e2a5b..4541464e1a 100644 --- a/packages/studio/src/player/hooks/timelinePlayerTestHarness.ts +++ b/packages/studio/src/player/hooks/timelinePlayerTestHarness.ts @@ -60,13 +60,18 @@ export function renderTimelinePlayerHarness(options?: UseTimelinePlayerOptions) }; } -export function makeFakeIframe(iframeWindow: Record): HTMLIFrameElement { +export function detachedFakeIframe(iframeWindow: Record): HTMLIFrameElement { const iframe = document.createElement("iframe"); Object.defineProperty(iframe, "contentWindow", { value: iframeWindow, configurable: true }); Object.defineProperty(iframe, "contentDocument", { value: document.implementation.createHTMLDocument("preview"), configurable: true, }); + return iframe; +} + +export function makeFakeIframe(iframeWindow: Record): HTMLIFrameElement { + const iframe = detachedFakeIframe(iframeWindow); document.body.appendChild(iframe); return iframe; } @@ -148,3 +153,23 @@ export async function paintShadow(getApi: () => TimelinePlayerApi, shadow: HTMLI }); return slot.gen; } + +export function serveEditingRequests( + frame: Record, + answer: (kind: string, params: unknown) => unknown, +): () => void { + const ports: MessagePort[] = []; + frame.postMessage = (message: { type?: string } | null) => { + if (message?.type !== "editing-connect") return; + const { port1, port2 } = new MessageChannel(); + ports.push(port1); + port1.onmessage = (event: MessageEvent<{ id: number; kind: string; params: unknown }>) => { + const { id, kind, params } = event.data; + port1.postMessage({ v: 1, id, ok: true, result: answer(kind, params) }); + }; + const handoff = { source: "hf-preview", type: "editing-port", v: 1 }; + const source = frame as unknown as Window; + window.dispatchEvent(new MessageEvent("message", { source, data: handoff, ports: [port2] })); + }; + return () => ports.forEach((port) => port.close()); +} diff --git a/packages/studio/src/player/hooks/useShadowPreviewReload.ts b/packages/studio/src/player/hooks/useShadowPreviewReload.ts index a8520a8675..f3f90bae83 100644 --- a/packages/studio/src/player/hooks/useShadowPreviewReload.ts +++ b/packages/studio/src/player/hooks/useShadowPreviewReload.ts @@ -20,6 +20,7 @@ import { } from "./useTimelineSyncCallbacks"; import type { IframeWindow, PlaybackAdapter } from "../lib/playbackTypes"; import { setPreviewReloading } from "../previewReloading"; +import { previewFrameOpaque, previewSeekPainted } from "../lib/previewChannel"; import { studioFileRevision } from "../../utils/studioFileVersion"; import { announcePreviewPromoted } from "../sceneSwap"; import { afterStudioPendingEdits, isStudioEditSaving } from "../../utils/studioPendingEdits"; @@ -34,7 +35,9 @@ export const SHADOW_READY_TIMEOUT_MS = 15_000; export const SHADOW_READY_BUDGETS = 3; function restoreSeekPainted(iframe: HTMLIFrameElement | null): Promise | undefined { - const win = iframe?.contentWindow as IframeWindow | null | undefined; + if (!iframe) return undefined; + if (previewFrameOpaque(iframe)) return previewSeekPainted(iframe, SHADOW_READY_TIMEOUT_MS); + const win = iframe.contentWindow as IframeWindow | null; return win?.__hfWaitForSeekCompletion?.().catch(() => {}); } diff --git a/packages/studio/src/player/hooks/useTimelinePlayer.ts b/packages/studio/src/player/hooks/useTimelinePlayer.ts index 14fc807220..aea248d14c 100644 --- a/packages/studio/src/player/hooks/useTimelinePlayer.ts +++ b/packages/studio/src/player/hooks/useTimelinePlayer.ts @@ -4,7 +4,7 @@ import { useMountEffect } from "../../hooks/useMountEffect"; import { usePlaybackKeyboard } from "./usePlaybackKeyboard"; import { useTimelineSyncCallbacks } from "./useTimelineSyncCallbacks"; import { useShadowPreviewReload } from "./useShadowPreviewReload"; -import { resolvePlaybackAdapter } from "./playbackAdapterResolution"; +import { resolveFramePlaybackAdapter } from "./playbackAdapterResolution"; import { subscribePreviewFrame, transportAdapter } from "../store/previewFrameStore"; import { clampToDuration } from "../lib/time"; import { useTimelinePlayerLoop } from "./useTimelinePlayerLoop"; @@ -28,7 +28,7 @@ export { } from "../lib/playbackShortcuts"; import type { PlaybackAdapter, IframeWindow } from "../lib/playbackTypes"; -import { attachPausedVideoProvider } from "../lib/pausedVideoProvider"; +import { setFrameLoopStart } from "../lib/previewChannel"; import { releaseStaticSeekCache, type StaticSeekCacheEntry } from "../lib/playbackAdapter"; import { mergeTimelineElementsPreservingDowngrades } from "../lib/timelineDOM"; import { findTimelineElementInIframe } from "../../hooks/timelineEditingHelpers"; @@ -149,10 +149,8 @@ export function useTimelinePlayer({ try { // undefined means "no override"; an explicit null override means "no adapter". const iframe = overrideIframe !== undefined ? overrideIframe : iframeRef.current; - const win = iframe?.contentWindow as IframeWindow | null; - if (!iframe || !win) return null; - attachPausedVideoProvider(iframe, win); - const adapter = resolvePlaybackAdapter(iframe, win, { + if (!iframe) return null; + const adapter = resolveFramePlaybackAdapter(iframe, { cache: staticSeekAdapterRef, warned: staticSeekWarnedRef, }); @@ -165,9 +163,7 @@ export function useTimelinePlayer({ ); const setLoopStart = useCallback((seconds: number | null) => { - try { - (iframeRef.current?.contentWindow as IframeWindow | null)?.__hf?.setLoopStart?.(seconds); - } catch {} + if (iframeRef.current) setFrameLoopStart(iframeRef.current, seconds); }, []); const { startRAFLoop, stopRAFLoop, stopReverseLoop } = useTimelinePlayerLoop({ rafRef, diff --git a/packages/studio/src/player/lib/bridgePlayback.ts b/packages/studio/src/player/lib/bridgePlayback.ts new file mode 100644 index 0000000000..ff9b1eb2a2 --- /dev/null +++ b/packages/studio/src/player/lib/bridgePlayback.ts @@ -0,0 +1,133 @@ +// Playback of a frame Studio cannot script: control-bridge commands, with time and duration mirrored +// from the runtime's `state` and `timeline` posts. Player registers each frame before it loads, +// because a runtime posts its first `timeline` while it boots. +import type { PlaybackAdapter } from "./playbackTypes"; +import { inspectStudioRuntimeMessage, postRuntimeControlMessage } from "./runtimeProtocol"; + +interface PlaybackMirror { + iframe: HTMLIFrameElement; + time: number; + duration: number; + playing: boolean; + /** A seek the runtime has not confirmed yet, and until when to wait for it. */ + seekTo: number | null; + seekUntil: number; +} + +const SEEK_CONFIRM_MS = 500; +// A post without fps metadata is matched within half a frame of the lowest frame rate a film may use. +const LOWEST_FPS = 1; + +/** Keyed by the frame's window, which a frame keeps across its loads. */ +const mirrors = new WeakMap(); + +function mirrorOf(iframe: HTMLIFrameElement): PlaybackMirror | null { + const frame = iframe.contentWindow; + if (!frame) return null; + let mirror = mirrors.get(frame); + if (!mirror) { + mirror = { iframe, time: 0, duration: 0, playing: false, seekTo: null, seekUntil: 0 }; + mirrors.set(frame, mirror); + } + mirror.iframe = iframe; + return mirror; +} + +/** Starts mirroring a preview frame Studio built; call it before the frame loads. */ +export function watchPreviewFrame(iframe: HTMLIFrameElement): void { + mirrorOf(iframe); +} + +/** The origins a post from this frame can carry, given its sandbox and what it loads. */ +function frameOrigins(iframe: HTMLIFrameElement): string[] { + if (iframe.hasAttribute("sandbox") && !iframe.sandbox.contains("allow-same-origin")) { + return ["null"]; + } + const src = iframe.getAttribute("src"); + if (iframe.hasAttribute("srcdoc") || !src) return [window.location.origin]; + try { + const url = new URL(src, window.location.href); + if (url.protocol === "about:") return [window.location.origin]; + return [url.protocol === "data:" ? "null" : url.origin]; + } catch { + return []; + } +} + +const finiteAtLeastZero = (value: unknown): value is number => + typeof value === "number" && Number.isFinite(value) && value >= 0; + +/** The runtime lands a seek on the frame at or before it, so only that frame confirms it. */ +function confirmsSeek(currentTime: number, seekTo: number, fps: number | null): boolean { + if (fps === null) return Math.abs(currentTime - seekTo) <= 0.5 / LOWEST_FPS; + return Math.abs(currentTime - Math.floor(seekTo * fps + 1e-9) / fps) < 0.5 / fps; +} + +/** The runtime's playback post and its frame rate, or null for anything else. */ +function runtimePost( + data: unknown, +): { message: Record; fps: number | null } | null { + if (typeof data !== "object" || data === null) return null; + const message = data as Record; + if (message.source !== "hf-preview") return null; + const inspection = inspectStudioRuntimeMessage(message); + if (inspection.status === "unsupported") return null; + return { message, fps: inspection.status === "supported" ? inspection.fps : null }; +} + +function mirrorState( + mirror: PlaybackMirror, + currentTime: number, + playing: boolean, + fps: number | null, +) { + // A state posted before the runtime handled our seek would put the playhead back. + const { seekTo } = mirror; + const waiting = seekTo !== null && performance.now() < mirror.seekUntil; + if (waiting && !confirmsSeek(currentTime, seekTo, fps)) return; + mirror.seekTo = null; + mirror.time = currentTime; + mirror.playing = playing; +} + +function mirrorRuntimePlayback(event: MessageEvent): void { + // Only a frame Studio built, posting from an origin that frame can have. + const mirror = event.source ? mirrors.get(event.source) : undefined; + if (!mirror || !frameOrigins(mirror.iframe).includes(event.origin)) return; + const post = runtimePost(event.data); + if (!post) return; + const { message, fps } = post; + if (message.type === "timeline" && finiteAtLeastZero(message.durationSeconds)) { + mirror.duration = message.durationSeconds; + } else if (message.type === "state" && finiteAtLeastZero(message.currentTime)) { + mirrorState(mirror, message.currentTime, message.isPlaying === true, fps); + } +} + +if (typeof window !== "undefined") window.addEventListener("message", mirrorRuntimePlayback); + +export function bridgePlaybackAdapter(iframe: HTMLIFrameElement): PlaybackAdapter | null { + const frame = iframe.contentWindow; + const mirror = mirrorOf(iframe); + if (!frame || !mirror) return null; + const send = (action: string, payload?: Record) => + postRuntimeControlMessage(frame, action, payload); + return { + play: () => { + send("play"); + mirror.playing = true; + }, + pause: () => { + send("pause"); + mirror.playing = false; + }, + // The bridge's seek takes no keepPlaying or settled option yet. + seek: (time) => { + send("seek", { timeSeconds: time }); + Object.assign(mirror, { time, seekTo: time, seekUntil: performance.now() + SEEK_CONFIRM_MS }); + }, + getTime: () => mirror.time, + getDuration: () => mirror.duration, + isPlaying: () => mirror.playing, + }; +} diff --git a/packages/studio/src/player/lib/previewChannel.test.ts b/packages/studio/src/player/lib/previewChannel.test.ts new file mode 100644 index 0000000000..f93227b2f0 --- /dev/null +++ b/packages/studio/src/player/lib/previewChannel.test.ts @@ -0,0 +1,260 @@ +// @vitest-environment happy-dom + +import { afterEach, describe, expect, it, vi } from "vitest"; +import { + AGENT_PROBE_MS, + previewClipManifest, + previewEditingClient, + previewFrameOpaque, + previewSeekPainted, + setFrameLoopStart, +} from "./previewChannel"; +import { bridgePlaybackAdapter, watchPreviewFrame } from "./bridgePlayback"; +import { runtimeProtocolMetadata } from "@hyperframes/core/runtime/protocol"; +import { resolveFramePlaybackAdapter } from "../hooks/playbackAdapterResolution"; +import { detachedFakeIframe, serveEditingRequests } from "../hooks/timelinePlayerTestHarness"; + +const closers: Array<() => void> = []; +afterEach(() => closers.splice(0).forEach((close) => close())); + +/** A frame on another origin: any read of its window throws, as a browser's does. */ +function opaqueFrame() { + const posted: unknown[] = []; + const postMessage = (message: unknown) => posted.push(message); + const frame = new Proxy({} as Record, { + get: (_target, key) => { + if (key === "postMessage") return postMessage; + throw new DOMException(`blocked read of ${String(key)}`, "SecurityError"); + }, + }); + const iframe = document.createElement("iframe"); + iframe.setAttribute("sandbox", "allow-scripts"); + Object.defineProperty(iframe, "contentWindow", { value: frame }); + Object.defineProperty(iframe, "contentDocument", { value: null }); + return { iframe, frame, posted }; +} + +/** Posts from `frame` as its runtime does, from an opaque origin unless told otherwise. */ +function fromFrame(frame: object, data: Record, origin = "null") { + const source = frame as Window; + window.dispatchEvent( + new MessageEvent("message", { source, origin, data: { source: "hf-preview", ...data } }), + ); +} + +/** A fake frame, same-origin unless `opaque`, whose runtime answers editing requests with `answer`. */ +function servedFrame(answer: (kind: string, params: unknown) => unknown, opaque = false) { + const frame: Record = {}; + const iframe = detachedFakeIframe(frame); + if (opaque) { + Object.defineProperty(iframe, "contentDocument", { value: null }); + Object.defineProperty(frame, "document", { + get: () => { + throw new DOMException("blocked", "SecurityError"); + }, + }); + } + const handed = vi.fn(answer); + closers.push(serveEditingRequests(frame, handed)); + return { iframe, frame, handed }; +} + +describe("previewFrameOpaque", () => { + it("is true only for a frame whose window refuses a read", () => { + expect(previewFrameOpaque(document.body.appendChild(document.createElement("iframe")))).toBe( + false, + ); + expect(previewFrameOpaque(document.createElement("iframe"))).toBe(false); + expect(previewFrameOpaque(opaqueFrame().iframe)).toBe(true); + // A window that answers reads is not opaque, even with no document to hand. + const readable = document.createElement("iframe"); + Object.defineProperty(readable, "contentWindow", { value: {} }); + expect(previewFrameOpaque(readable)).toBe(false); + }); +}); + +describe("previewEditingClient", () => { + it("keeps one client per frame load and opens a fresh one after the frame loads again", async () => { + const { iframe, handed } = servedFrame(() => ({ waited: false })); + const first = previewEditingClient(iframe); + expect(previewEditingClient(iframe)).toBe(first); + iframe.dispatchEvent(new Event("load")); + await expect(first.request("wait-for-seek", {})).rejects.toMatchObject({ code: "disposed" }); + const second = previewEditingClient(iframe); + expect(second).not.toBe(first); + await expect(second.request("wait-for-seek", {})).resolves.toEqual({ waited: false }); + expect(handed).toHaveBeenCalledOnce(); + }); + + it("sends an opaque frame's loop start over the port, only when it changes", async () => { + const seen: unknown[] = []; + const { iframe } = servedFrame((_kind, params) => { + seen.push((params as { seconds: unknown }).seconds); + return { applied: true }; + }, true); + setFrameLoopStart(iframe, 2); + setFrameLoopStart(iframe, 2); + setFrameLoopStart(iframe, null); + await vi.waitFor(() => expect(seen).toEqual([2, null])); + }); + + it("calls a scriptable frame's own loop-start hook and never opens a port", async () => { + const { iframe, frame, handed } = servedFrame(() => ({ applied: true })); + const setLoopStart = vi.fn(); + frame.__hf = { setLoopStart }; + setFrameLoopStart(iframe, 2); + setFrameLoopStart(iframe, 2); + expect(setLoopStart.mock.calls).toEqual([[2], [2]]); + await new Promise((resolve) => setTimeout(resolve, 20)); + expect(handed).not.toHaveBeenCalled(); + delete frame.__hf; + expect(() => setFrameLoopStart(iframe, null)).not.toThrow(); + }); + + it("waits for the runtime's seek to paint, and settles anyway when no runtime answers", async () => { + const { iframe, handed } = servedFrame(() => ({ waited: true })); + await previewSeekPainted(iframe); + expect(handed).toHaveBeenCalledWith("wait-for-seek", {}); + const started = Date.now(); + await previewSeekPainted(opaqueFrame().iframe, 15_000); + expect(Date.now() - started).toBeLessThan(AGENT_PROBE_MS + 500); + }); + + it("returns the runtime's manifest only when its reply has the manifest's shape", async () => { + const manifest = { clips: [], durationInFrames: 30, scenes: [] }; + let reply: unknown = { clipManifest: manifest }; + const { iframe } = servedFrame(() => reply); + expect(await previewClipManifest(iframe)).toEqual(manifest); + for (const bad of [ + null, + { clipManifest: "x" }, + { clipManifest: { clips: {}, durationInFrames: 30 } }, + { clipManifest: { clips: [], durationInFrames: "30" } }, + ]) { + reply = bad; + expect(await previewClipManifest(iframe)).toBeNull(); + } + reply = { clipManifest: { clips: [], durationInFrames: Infinity } }; + expect(await previewClipManifest(iframe)).toBeNull(); + }); +}); + +describe("bridge playback for a frame Studio cannot script", () => { + const refs = () => ({ cache: { current: null }, warned: { current: false } }); + + it("drives the runtime with control messages and never reads the frame's window", () => { + const { iframe, posted } = opaqueFrame(); + const adapter = resolveFramePlaybackAdapter(iframe, refs()); + expect(adapter).not.toBeNull(); + adapter!.play(); + adapter!.seek(4); + adapter!.pause(); + expect(posted).toEqual([ + expect.objectContaining({ source: "hf-parent", type: "control", action: "play" }), + expect.objectContaining({ action: "seek", timeSeconds: 4 }), + expect.objectContaining({ action: "pause" }), + ]); + expect(adapter!.getTime()).toBe(4); + expect(adapter!.isPlaying()).toBe(false); + }); + + it("keeps a seek until the runtime confirms it, so a state posted before it cannot undo it", () => { + const { iframe, frame } = opaqueFrame(); + const adapter = bridgePlaybackAdapter(iframe)!; + const clock = vi.spyOn(performance, "now").mockReturnValue(1_000); + try { + adapter.seek(4); + fromFrame(frame, { type: "state", currentTime: 1, isPlaying: false }); + expect(adapter.getTime()).toBe(4); + fromFrame(frame, { type: "state", currentTime: 4, isPlaying: false }); + fromFrame(frame, { type: "state", currentTime: 4.5, isPlaying: true }); + expect(adapter.getTime()).toBe(4.5); + adapter.seek(99); + clock.mockReturnValue(2_000); + fromFrame(frame, { type: "state", currentTime: 12, isPlaying: false }); + expect(adapter.getTime()).toBe(12); + } finally { + clock.mockRestore(); + } + }); + + it("takes a seek the runtime landed on the frame before it, and plays on", () => { + const { iframe, frame } = opaqueFrame(); + const adapter = bridgePlaybackAdapter(iframe)!; + const at30fps = runtimeProtocolMetadata(30); + adapter.seek(4.01); + fromFrame(frame, { type: "state", currentTime: 4, isPlaying: true, ...at30fps }); + expect(adapter.getTime()).toBe(4); + expect(adapter.isPlaying()).toBe(true); + fromFrame(frame, { type: "state", currentTime: 4.3, isPlaying: true, ...at30fps }); + expect(adapter.getTime()).toBe(4.3); + }); + + it("ignores a post from a window it was not built for, or from an origin its frame cannot have", () => { + const opaque = opaqueFrame(); + const adapter = bridgePlaybackAdapter(opaque.iframe)!; + const stranger = opaqueFrame(); + fromFrame(stranger.frame, { type: "state", currentTime: 9, isPlaying: true }); + fromFrame(opaque.frame, { type: "state", currentTime: 8, isPlaying: true }, location.origin); + fromFrame( + opaque.frame, + { type: "state", currentTime: 7, isPlaying: true }, + "https://example.com", + ); + expect(adapter.getTime()).toBe(0); + // A post from a frame nobody registered is dropped, not kept for a later adapter. + expect(bridgePlaybackAdapter(stranger.iframe)!.getTime()).toBe(0); + fromFrame(opaque.frame, { type: "state", currentTime: 2, isPlaying: false }); + expect(adapter.getTime()).toBe(2); + + const sameOrigin = document.createElement("iframe"); + sameOrigin.src = "/preview/index.html"; + const frame = {}; + Object.defineProperty(sameOrigin, "contentWindow", { value: frame }); + watchPreviewFrame(sameOrigin); + const own = bridgePlaybackAdapter(sameOrigin)!; + fromFrame(frame, { type: "state", currentTime: 5, isPlaying: false }, "null"); + expect(own.getTime()).toBe(0); + fromFrame(frame, { type: "state", currentTime: 5, isPlaying: false }, location.origin); + expect(own.getTime()).toBe(5); + }); + + it.each([ + ["srcdoc", (f: HTMLIFrameElement) => f.setAttribute("srcdoc", "

x

"), location.origin], + [ + "about:blank", + (f: HTMLIFrameElement) => f.setAttribute("src", "about:blank"), + location.origin, + ], + ["a data: URL", (f: HTMLIFrameElement) => f.setAttribute("src", "data:text/html,x"), "null"], + [ + "another origin", + (f: HTMLIFrameElement) => f.setAttribute("src", "https://a.test/p"), + "https://a.test", + ], + ])("accepts a frame loading %s only from the origin it has", (_name, load, origin) => { + const iframe = document.createElement("iframe"); + load(iframe); + const frame = {}; + Object.defineProperty(iframe, "contentWindow", { value: frame }); + const adapter = bridgePlaybackAdapter(iframe)!; + const wrong = origin === "null" ? location.origin : "null"; + fromFrame(frame, { type: "state", currentTime: 3, isPlaying: false }, wrong); + expect(adapter.getTime()).toBe(0); + fromFrame(frame, { type: "state", currentTime: 3, isPlaying: false }, origin); + expect(adapter.getTime()).toBe(3); + }); + + it("takes time, play state and duration from its own frame's state and timeline messages", () => { + const { iframe, frame } = opaqueFrame(); + const other = opaqueFrame(); + const adapter = bridgePlaybackAdapter(iframe)!; + fromFrame(frame, { type: "timeline", durationSeconds: 12, clips: [] }); + fromFrame(frame, { type: "state", currentTime: 3.5, isPlaying: true }); + fromFrame(other.frame, { type: "state", currentTime: 9, isPlaying: false }); + fromFrame(frame, { type: "state", currentTime: Number.NaN, isPlaying: true }); + expect(adapter.getDuration()).toBe(12); + expect(adapter.getTime()).toBe(3.5); + expect(adapter.isPlaying()).toBe(true); + }); +}); diff --git a/packages/studio/src/player/lib/previewChannel.ts b/packages/studio/src/player/lib/previewChannel.ts new file mode 100644 index 0000000000..cdc0d2dde1 --- /dev/null +++ b/packages/studio/src/player/lib/previewChannel.ts @@ -0,0 +1,81 @@ +// Runtime-state reads over a preview frame's editing port; an opaque frame's window is out of reach. +import { + createEditingPortClient, + type EditingPortClient, +} from "@hyperframes/core/runtime/editing-port"; +import type { ClipManifest, IframeWindow } from "./playbackTypes"; + +const clients = new WeakMap(); +const loopStarts = new WeakMap(); + +/** A frame whose window refuses Studio's reads; one with no window has nothing to reach either way. */ +export function previewFrameOpaque(iframe: HTMLIFrameElement): boolean { + const win = iframe.contentWindow; + if (!win) return false; + try { + void win.document; + return false; + } catch { + return true; + } +} + +/** One client per frame load: the frame's next load disposes it. */ +export function previewEditingClient(iframe: HTMLIFrameElement): EditingPortClient { + const open = clients.get(iframe); + if (open) return open; + const client = createEditingPortClient({ iframe }); + clients.set(iframe, client); + const onLoad = () => { + client.dispose(); + if (clients.get(iframe) === client) clients.delete(iframe); + }; + iframe.addEventListener("load", onLoad, { once: true }); + return client; +} + +export function setFrameLoopStart(iframe: HTMLIFrameElement, seconds: number | null): void { + if (!previewFrameOpaque(iframe)) { + try { + (iframe.contentWindow as IframeWindow | null)?.__hf?.setLoopStart?.(seconds); + } catch {} + return; + } + const client = previewEditingClient(iframe); + if (loopStarts.has(client) && loopStarts.get(client) === seconds) return; + loopStarts.set(client, seconds); + client.request("set-loop-start", { seconds }).catch(() => { + if (loopStarts.get(client) === seconds) loopStarts.delete(client); + }); +} + +export const AGENT_PROBE_MS = 1_000; + +/** Gives up after AGENT_PROBE_MS with no port: a runtime without the editing agent never answers. */ +export async function previewSeekPainted(iframe: HTMLIFrameElement, timeoutMs?: number) { + const client = previewEditingClient(iframe); + let timer: ReturnType | undefined; + const probe = new Promise( + (resolve) => (timer = setTimeout(resolve, AGENT_PROBE_MS, false)), + ); + const connected = await Promise.race([ + client.ready.then( + () => true, + () => false, + ), + probe, + ]); + clearTimeout(timer); + if (connected) await client.request("wait-for-seek", {}, { timeoutMs }).catch(() => {}); +} + +/** The reply is frame data, so its shape is checked. */ +export async function previewClipManifest(iframe: HTMLIFrameElement): Promise { + const snapshot = await previewEditingClient(iframe).request("runtime-snapshot", {}); + const manifest: unknown = snapshot?.clipManifest; + if (typeof manifest !== "object" || manifest === null) return null; + const { clips, durationInFrames } = manifest as Partial; + if (!Array.isArray(clips) || typeof durationInFrames !== "number") return null; + if (!Number.isFinite(durationInFrames)) return null; + return manifest as ClipManifest; +} diff --git a/packages/studio/src/player/sceneSwap.test.ts b/packages/studio/src/player/sceneSwap.test.ts index 31bae5cee8..e603ff1451 100644 --- a/packages/studio/src/player/sceneSwap.test.ts +++ b/packages/studio/src/player/sceneSwap.test.ts @@ -24,6 +24,18 @@ function swappableIframe(swap: Swap = vi.fn(async () => {})) { } describe("sceneSwapFor", () => { + it("returns no swap for an opaque frame, without reading its window", () => { + const iframe = document.createElement("iframe"); + const blocked = new Proxy({} as Window, { + get: () => { + throw new DOMException("blocked", "SecurityError"); + }, + }); + Object.defineProperty(iframe, "contentWindow", { value: blocked }); + Object.defineProperty(iframe, "contentDocument", { value: null }); + expect(sceneSwapFor(iframe)).toBeNull(); + }); + it("gives up on a preview download that never finishes, so the caller can reload", async () => { vi.useFakeTimers(); vi.spyOn(globalThis, "fetch").mockImplementation( diff --git a/packages/studio/src/player/sceneSwap.ts b/packages/studio/src/player/sceneSwap.ts index dcc003217d..db1cc8aa48 100644 --- a/packages/studio/src/player/sceneSwap.ts +++ b/packages/studio/src/player/sceneSwap.ts @@ -2,6 +2,7 @@ import { studioApiFetch } from "../utils/studioApiFetch"; import { studioFileRevision } from "../utils/studioFileVersion"; import { logReload } from "../utils/reloadDebug"; import { whileScriptWrites } from "./previewReloading"; +import { previewFrameOpaque } from "./lib/previewChannel"; type SwapWindow = Window & { __hfSwapScenes?: (html: string, signal?: AbortSignal) => Promise; @@ -114,6 +115,7 @@ export function swapFreshly( export function sceneSwapFor( iframe: HTMLIFrameElement, ): ((url: string, isCurrent: () => boolean, cancel?: AbortSignal) => Promise) | null { + if (previewFrameOpaque(iframe)) return null; const win = iframe.contentWindow as SwapWindow | null; const swap = win?.__hfSwapScenes; if (typeof swap !== "function") return null;