From de876a4fcce3e77f330ceb59cb38ec481d586f37 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Miguel=20=C3=81ngel?= Date: Sat, 9 May 2026 00:34:43 -0700 Subject: [PATCH] fix(studio): keep notion preview playback moving --- packages/core/src/runtime/init.test.ts | 92 ++++++++++- packages/core/src/runtime/init.ts | 66 ++++++-- .../src/player/components/Player.test.ts | 48 +++++- .../studio/src/player/components/Player.tsx | 17 +- packages/studio/vite.config.ts | 147 ++++++++++-------- .../notion-showcase/image-selection.png | Bin 0 -> 417719 bytes .../notion-showcase/play-button-after.png | Bin 0 -> 515329 bytes .../notion-showcase/text-selection.png | Bin 0 -> 430915 bytes 8 files changed, 286 insertions(+), 84 deletions(-) create mode 100644 qa-artifacts/studio-inspector-selection-ux/notion-showcase/image-selection.png create mode 100644 qa-artifacts/studio-inspector-selection-ux/notion-showcase/play-button-after.png create mode 100644 qa-artifacts/studio-inspector-selection-ux/notion-showcase/text-selection.png diff --git a/packages/core/src/runtime/init.test.ts b/packages/core/src/runtime/init.test.ts index 24c11cb9cf..32e8d0841d 100644 --- a/packages/core/src/runtime/init.test.ts +++ b/packages/core/src/runtime/init.test.ts @@ -1,4 +1,4 @@ -import { afterEach, beforeEach, describe, expect, it } from "vitest"; +import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; import { initSandboxRuntimeModular } from "./init"; import type { RuntimeTimelineLike } from "./types"; @@ -45,6 +45,31 @@ function createPaddableMockTimeline(duration: number): RuntimeTimelineLike { return timeline; } +function createManualRaf() { + let now = 0; + let nextId = 0; + const callbacks = new Map(); + return { + requestAnimationFrame: (callback: FrameRequestCallback) => { + nextId += 1; + callbacks.set(nextId, callback); + return nextId; + }, + cancelAnimationFrame: (id: number) => { + callbacks.delete(id); + }, + step: (milliseconds: number) => { + now += milliseconds; + const pending = Array.from(callbacks.entries()); + callbacks.clear(); + for (const [, callback] of pending) { + callback(now); + } + }, + now: () => now, + }; +} + describe("initSandboxRuntimeModular", () => { const originalRequestAnimationFrame = window.requestAnimationFrame; const originalCancelAnimationFrame = window.cancelAnimationFrame; @@ -67,6 +92,7 @@ describe("initSandboxRuntimeModular", () => { delete (window as Window & { __player?: unknown }).__player; delete (window as Window & { __playerReady?: boolean }).__playerReady; delete (window as Window & { __renderReady?: boolean }).__renderReady; + vi.restoreAllMocks(); window.requestAnimationFrame = originalRequestAnimationFrame; window.cancelAnimationFrame = originalCancelAnimationFrame; }); @@ -282,4 +308,68 @@ describe("initSandboxRuntimeModular", () => { expect(video.paused).toBe(true); expect(video.currentTime).toBe(0); }); + + it("plays scheduled child timelines without a captured root timeline when audio has failed", () => { + const raf = createManualRaf(); + vi.spyOn(performance, "now").mockImplementation(() => raf.now()); + window.requestAnimationFrame = raf.requestAnimationFrame as typeof window.requestAnimationFrame; + window.cancelAnimationFrame = raf.cancelAnimationFrame as typeof window.cancelAnimationFrame; + + const root = document.createElement("div"); + root.setAttribute("data-composition-id", "main"); + root.setAttribute("data-root", "true"); + root.setAttribute("data-start", "0"); + root.setAttribute("data-duration", "4"); + root.setAttribute("data-width", "1920"); + root.setAttribute("data-height", "1080"); + document.body.appendChild(root); + + const child = document.createElement("div"); + child.setAttribute("data-composition-id", "scene"); + child.setAttribute("data-start", "0"); + child.setAttribute("data-duration", "4"); + root.appendChild(child); + + const audio = document.createElement("audio"); + audio.setAttribute("data-start", "0"); + audio.setAttribute("data-duration", "4"); + Object.defineProperty(audio, "error", { + value: { code: 4, message: "format error" }, + configurable: true, + }); + Object.defineProperty(audio, "networkState", { + value: HTMLMediaElement.NETWORK_NO_SOURCE, + configurable: true, + }); + Object.defineProperty(audio, "readyState", { + value: HTMLMediaElement.HAVE_NOTHING, + configurable: true, + }); + Object.defineProperty(audio, "paused", { value: true, configurable: true }); + Object.defineProperty(audio, "currentTime", { value: 0, writable: true, configurable: true }); + audio.load = () => {}; + audio.play = vi.fn(() => Promise.reject(new Error("format error"))); + root.appendChild(audio); + + const childTimeline = createMockTimeline(4); + (window as Window & { __timelines?: Record }).__timelines = { + scene: childTimeline, + }; + + initSandboxRuntimeModular(); + + const player = ( + window as Window & { + __player?: { play: () => void; getTime: () => number; isPlaying: () => boolean }; + } + ).__player; + expect(player).toBeDefined(); + + player?.play(); + raf.step(1_000); + + expect(player?.isPlaying()).toBe(true); + expect(player?.getTime()).toBeCloseTo(1, 1); + expect(childTimeline.time()).toBeCloseTo(1, 1); + }); }); diff --git a/packages/core/src/runtime/init.ts b/packages/core/src/runtime/init.ts index b395198cda..3fb9468aef 100644 --- a/packages/core/src/runtime/init.ts +++ b/packages/core/src/runtime/init.ts @@ -1450,6 +1450,7 @@ export function initSandboxRuntimeModular(): void { .then(() => loadInlineTemplateCompositions(compositionLoaderParams)) .finally(() => { externalCompositionsReady = true; + bindRootTimelineIfAvailable(); runAdapters("discover", state.currentTime); bindMediaMetadataListeners(); installAssetFailureDiagnostics(); @@ -1644,18 +1645,53 @@ export function initSandboxRuntimeModular(): void { let transportTickCount = 0; let inTransportTick = false; + const seekRuntimeTimeline = ( + timeline: RuntimeTimelineLike, + timeSeconds: number, + swallowLabel: string, + ) => { + try { + timeline.pause(); + if (typeof timeline.totalTime === "function") { + timeline.totalTime(timeSeconds, false); + } else { + timeline.seek(timeSeconds, false); + } + } catch (err) { + swallow(swallowLabel, err); + } + }; + + const seekStandaloneRegisteredTimelines = (timeSeconds: number) => { + const timelines = (window.__timelines ?? {}) as Record; + const rootCompositionId = + resolveRootCompositionElement()?.getAttribute("data-composition-id") ?? null; + for (const [compositionId, timeline] of Object.entries(timelines)) { + if (!timeline || compositionId === rootCompositionId) continue; + const node = document.querySelector(`[data-composition-id="${CSS.escape(compositionId)}"]`); + if (!node) continue; + const start = resolveStartForElement(node, 0); + if (!Number.isFinite(start)) continue; + const authoredDuration = resolveDurationForElement(node, { + includeAuthoredTimingAttrs: true, + }); + const timelineDuration = getTimelineDurationSeconds(timeline); + const duration = + authoredDuration != null && authoredDuration > 0 ? authoredDuration : timelineDuration; + const localTime = Math.max( + 0, + duration != null && duration > 0 + ? Math.min(duration, timeSeconds - start) + : timeSeconds - start, + ); + seekRuntimeTimeline(timeline, localTime, "runtime.init.transport.childTimeline"); + } + }; + const seekTimelineAndAdapters = (t: number) => { const tl = state.capturedTimeline; if (tl) { - try { - if (typeof tl.totalTime === "function") { - tl.totalTime(t, false); - } else { - tl.seek(t, false); - } - } catch (err) { - swallow("runtime.init.transport.seek", err); - } + seekRuntimeTimeline(tl, t, "runtime.init.transport.seek"); // Sibling timelines (registered in __timelines but not nested under // the root) are paused alongside the master. We do NOT seek them to // absolute position `t` here — child timelines nested under the root @@ -1663,6 +1699,8 @@ export function initSandboxRuntimeModular(): void { // at absolute `t` would clobber their offset-relative position. // Play/pause propagation for siblings happens in the player.play() // and player.pause() overrides via the adapter layer. + } else { + seekStandaloneRegisteredTimelines(t); } for (const adapter of state.deterministicAdapters) { try { @@ -1741,7 +1779,11 @@ export function initSandboxRuntimeModular(): void { if (!rawEl.paused) { clock.attachAudioSource({ el: rawEl, compositionStart: start, mediaStart }); foundActive = true; - } else if (rawEl.readyState < HTMLMediaElement.HAVE_FUTURE_DATA) { + } else if ( + !rawEl.error && + rawEl.networkState !== HTMLMediaElement.NETWORK_NO_SOURCE && + rawEl.readyState < HTMLMediaElement.HAVE_FUTURE_DATA + ) { // Audio is buffering — freeze visuals at last known position // instead of falling through to monotonic (which runs ahead). clock.attachAudioSource({ currentTimeSeconds: state.currentTime }); @@ -1819,7 +1861,7 @@ export function initSandboxRuntimeModular(): void { // Player methods route through the TransportClock. player.play = () => { const tl = state.capturedTimeline; - if (!tl || clock.isPlaying()) return; + if (clock.isPlaying()) return; const dur = getSafeTimelineDurationSeconds(tl, 0); if (dur > 0) { clock.setDuration(dur); @@ -1829,7 +1871,7 @@ export function initSandboxRuntimeModular(): void { seekTimelineAndAdapters(0); } } - tl.pause(); + if (tl) tl.pause(); if (!clock.play()) return; state.isPlaying = true; state.mediaForceSyncNextTick = true; diff --git a/packages/studio/src/player/components/Player.test.ts b/packages/studio/src/player/components/Player.test.ts index d4879ec1fd..32e1069713 100644 --- a/packages/studio/src/player/components/Player.test.ts +++ b/packages/studio/src/player/components/Player.test.ts @@ -1,5 +1,7 @@ +// @vitest-environment happy-dom + import { describe, expect, it } from "vitest"; -import { shouldShowCompositionLoadingOverlay } from "./Player"; +import { hasUnloadedAssets, shouldShowCompositionLoadingOverlay } from "./Player"; describe("composition loading overlay", () => { it("shows while the composition is loading", () => { @@ -9,4 +11,48 @@ describe("composition loading overlay", () => { it("hides after the composition is ready", () => { expect(shouldShowCompositionLoadingOverlay(false)).toBe(false); }); + + it("keeps the asset overlay up while media is still buffering", () => { + const iframe = document.createElement("iframe"); + document.body.appendChild(iframe); + const audio = iframe.contentDocument?.createElement("audio"); + expect(audio).toBeDefined(); + Object.defineProperty(audio, "readyState", { + value: 0, + configurable: true, + }); + Object.defineProperty(audio, "networkState", { + value: 2, + configurable: true, + }); + iframe.contentDocument?.body.appendChild(audio!); + + expect(hasUnloadedAssets(iframe, false)).toBe(true); + + iframe.remove(); + }); + + it("does not keep the asset overlay stuck on failed media sources", () => { + const iframe = document.createElement("iframe"); + document.body.appendChild(iframe); + const audio = iframe.contentDocument?.createElement("audio"); + expect(audio).toBeDefined(); + Object.defineProperty(audio, "error", { + value: { code: 4, message: "format error" }, + configurable: true, + }); + Object.defineProperty(audio, "readyState", { + value: 0, + configurable: true, + }); + Object.defineProperty(audio, "networkState", { + value: 3, + configurable: true, + }); + iframe.contentDocument?.body.appendChild(audio!); + + expect(hasUnloadedAssets(iframe, false)).toBe(false); + + iframe.remove(); + }); }); diff --git a/packages/studio/src/player/components/Player.tsx b/packages/studio/src/player/components/Player.tsx index 920d442294..f47c044162 100644 --- a/packages/studio/src/player/components/Player.tsx +++ b/packages/studio/src/player/components/Player.tsx @@ -19,6 +19,9 @@ interface HyperframesPlayerElement extends HTMLElement { iframeElement: HTMLIFrameElement; } +const MEDIA_HAVE_FUTURE_DATA = 3; +const MEDIA_NETWORK_NO_SOURCE = 3; + function isRecord(value: unknown): value is Record { return typeof value === "object" && value !== null; } @@ -47,6 +50,11 @@ function enableInteractiveIframe(player: HyperframesPlayerElement): void { iframe?.style.setProperty("pointer-events", "auto"); } +function isPreviewMediaElement(el: Element): el is HTMLMediaElement { + const tagName = el.tagName.toLowerCase(); + return tagName === "video" || tagName === "audio"; +} + // Assets are considered ready when every `