diff --git a/packages/core/src/runtime/mediaSteer.ts b/packages/core/src/runtime/mediaSteer.ts index f9c5b1bb3b..ef72d9ab1e 100644 --- a/packages/core/src/runtime/mediaSteer.ts +++ b/packages/core/src/runtime/mediaSteer.ts @@ -21,9 +21,10 @@ export function steerMedia( offset: number, baseRate: number, settled: boolean, + audioOnly = el.tagName === "AUDIO", ): { rate: number; seek: boolean } { const lag = Math.abs(offset); - if (el.tagName === "AUDIO" && settled && lag > AUDIO_STEER_LIMIT_SECONDS) { + if (audioOnly && settled && lag > AUDIO_STEER_LIMIT_SECONDS) { steering.delete(el); return { rate: baseRate, seek: true }; } @@ -35,7 +36,7 @@ export function steerMedia( return { rate: baseRate, seek: false }; } const step = MEDIA_STEER_STEPS.find(([from]) => lag > from)?.[1] ?? MEDIA_STEER; - const steer = el.tagName === "AUDIO" ? Math.min(step, AUDIO_MAX_STEER) : step; + const steer = audioOnly ? Math.min(step, AUDIO_MAX_STEER) : step; return { rate: baseRate * (1 + direction * steer), seek: false }; } diff --git a/packages/player/src/hyperframes-player.test.ts b/packages/player/src/hyperframes-player.test.ts index 5ab53a4122..c16d9e41e3 100644 --- a/packages/player/src/hyperframes-player.test.ts +++ b/packages/player/src/hyperframes-player.test.ts @@ -147,6 +147,7 @@ describe("HyperframesPlayer parent-frame media", () => { paused: boolean; play: ReturnType; pause: ReturnType; + addEventListener: ReturnType; load: ReturnType; }; @@ -162,6 +163,7 @@ describe("HyperframesPlayer parent-frame media", () => { paused: true, play: vi.fn().mockResolvedValue(undefined), pause: vi.fn(), + addEventListener: vi.fn(), load: vi.fn(), }; @@ -779,206 +781,6 @@ describe("HyperframesPlayer media MutationObserver scoping", () => { }); }); -// ── Parent-proxy time-mirror coalescing ── -// -// `_mirrorParentMediaTime` is the steady-state correction loop that nudges -// every parent-frame audio/video proxy back onto the iframe's timeline. The -// post-`P1-4` contract: a single over-threshold sample (one slow bridge tick, -// one tab-throttled rAF, one GC pause) is absorbed by a per-proxy counter and -// does NOT cost a `currentTime` write. Only a *trending* drift — two -// consecutive samples above the 50 ms threshold — triggers a seek. Forced -// callers (audio-ownership promotion, brand-new proxy initialization) bypass -// the gate so the listener never hears a misaligned sample on cut-over. - -describe("HyperframesPlayer parent-proxy time-mirror coalescing", () => { - type DriftEntry = { - el: { currentTime: number; src: string; pause: () => void }; - start: number; - duration: number; - driftSamples: number; - }; - type PlayerInternal = HTMLElement & { - _parentMedia: DriftEntry[]; - _mirrorParentMediaTime: (timelineSeconds: number, options?: { force?: boolean }) => void; - _promoteToParentProxy?: () => void; - }; - - let player: PlayerInternal; - - beforeEach(async () => { - await import("./hyperframes-player.js"); - player = document.createElement("hyperframes-player") as PlayerInternal; - document.body.appendChild(player); - // No audio-src was set, so `_parentMedia` is empty. Tests push synthetic - // POJO entries — `_mirrorParentMediaTime` only reads/writes - // `el.currentTime`, so a plain object stands in fine for HTMLMediaElement. - }); - - afterEach(() => { - player.remove(); - vi.restoreAllMocks(); - }); - - function makeEntry( - opts: { - currentTime?: number; - start?: number; - duration?: number; - driftSamples?: number; - } = {}, - ): DriftEntry { - // Include `pause`/`src` so `disconnectedCallback`'s teardown loop - // (`m.el.pause(); m.el.src = ""`) doesn't blow up when the player is - // removed at the end of the test — `_mirrorParentMediaTime` itself only - // touches `currentTime`. - const entry: DriftEntry = { - el: { - currentTime: opts.currentTime ?? 0, - src: "", - pause: vi.fn(), - }, - start: opts.start ?? 0, - duration: opts.duration ?? 100, - driftSamples: opts.driftSamples ?? 0, - }; - player._parentMedia.push(entry); - return entry; - } - - it("initializes new parent-media entries with driftSamples=0", () => { - // Mock Audio just for this test so the audio-src bootstrap path produces - // a real entry rather than throwing on construction. - const mockAudio = { - src: "", - preload: "", - muted: false, - playbackRate: 1, - currentTime: 0, - paused: true, - play: vi.fn().mockResolvedValue(undefined), - pause: vi.fn(), - load: vi.fn(), - }; - vi.spyOn(globalThis, "Audio").mockImplementation(function () { - return mockAudio as unknown as HTMLAudioElement; - }); - - const fresh = document.createElement("hyperframes-player") as PlayerInternal; - fresh.setAttribute("audio-src", "https://cdn.example.com/narration.mp3"); - document.body.appendChild(fresh); - - expect(fresh._parentMedia).toHaveLength(1); - expect(fresh._parentMedia[0]?.driftSamples).toBe(0); - fresh.remove(); - }); - - it("does nothing when drift is within the 50 ms threshold", () => { - const m = makeEntry({ currentTime: 5 }); - player._mirrorParentMediaTime(5.04); - expect(m.el.currentTime).toBe(5); - expect(m.driftSamples).toBe(0); - }); - - it("absorbs a single over-threshold spike without writing currentTime", () => { - const m = makeEntry({ currentTime: 5 }); - player._mirrorParentMediaTime(5.5); - expect(m.el.currentTime).toBe(5); - expect(m.driftSamples).toBe(1); - }); - - it("issues a seek on the second consecutive over-threshold sample", () => { - const m = makeEntry({ currentTime: 5 }); - player._mirrorParentMediaTime(5.5); - expect(m.el.currentTime).toBe(5); - expect(m.driftSamples).toBe(1); - // Second sample with the same drift: the gate trips, the write fires, - // and the counter resets so the proxy doesn't re-seek every later tick. - player._mirrorParentMediaTime(5.5); - expect(m.el.currentTime).toBe(mediaElementTime(5.5)); - expect(m.driftSamples).toBe(0); - }); - - it("resets the counter when a sample comes back within threshold", () => { - const m = makeEntry({ currentTime: 5 }); - player._mirrorParentMediaTime(5.5); - expect(m.driftSamples).toBe(1); - // Recovery — counter must clear so a later isolated spike doesn't - // accidentally satisfy the 2-sample gate by piggy-backing on stale state. - player._mirrorParentMediaTime(5.02); - expect(m.driftSamples).toBe(0); - expect(m.el.currentTime).toBe(5); - player._mirrorParentMediaTime(5.5); - expect(m.driftSamples).toBe(1); - expect(m.el.currentTime).toBe(5); - }); - - it("force: true writes immediately on the first over-threshold sample", () => { - const m = makeEntry({ currentTime: 5 }); - player._mirrorParentMediaTime(5.5, { force: true }); - expect(m.el.currentTime).toBe(mediaElementTime(5.5)); - expect(m.driftSamples).toBe(0); - }); - - it("force: true clears any pre-existing drift counter", () => { - const m = makeEntry({ currentTime: 5, driftSamples: 1 }); - player._mirrorParentMediaTime(5.5, { force: true }); - expect(m.el.currentTime).toBe(mediaElementTime(5.5)); - expect(m.driftSamples).toBe(0); - }); - - it("does not seek out-of-range entries and resets their counters", () => { - // Active window [10, 15). currentTime=99 is a sentinel — if the function - // ever writes inside an out-of-range branch the test catches it because - // relTime would be 5 (or 15), not 99. - const m = makeEntry({ - currentTime: 99, - start: 10, - duration: 5, - driftSamples: 5, - }); - player._mirrorParentMediaTime(5); - expect(m.el.currentTime).toBe(99); - expect(m.driftSamples).toBe(0); - // Boundary: relTime === duration → still out of range (the loop uses `>=`). - m.driftSamples = 7; - player._mirrorParentMediaTime(15); - expect(m.el.currentTime).toBe(99); - expect(m.driftSamples).toBe(0); - }); - - it("tracks drift independently across multiple proxies", () => { - // a is drifted; b is aligned. A single tick must increment a's counter - // and reset b's — proving the per-entry state is genuinely per-entry. - const a = makeEntry({ currentTime: 5 }); - const b = makeEntry({ currentTime: 7.01, driftSamples: 1 }); - player._mirrorParentMediaTime(7); - expect(a.el.currentTime).toBe(5); - expect(a.driftSamples).toBe(1); - expect(b.el.currentTime).toBe(7.01); - expect(b.driftSamples).toBe(0); - }); - - it("force: true bypasses the gate for every proxy in a single sweep", () => { - const a = makeEntry({ currentTime: 5 }); - const b = makeEntry({ currentTime: 8 }); - player._mirrorParentMediaTime(7, { force: true }); - expect(a.el.currentTime).toBe(mediaElementTime(7)); - expect(b.el.currentTime).toBe(mediaElementTime(7)); - expect(a.driftSamples).toBe(0); - expect(b.driftSamples).toBe(0); - }); - - it("_promoteToParentProxy invokes _mirrorParentMediaTime with force: true", () => { - // Integration check of the promotion call site — we cannot tolerate even - // ~80 ms of audible drift across an ownership flip, so the call site - // must opt out of the jitter gate. - const spy = vi.spyOn(player, "_mirrorParentMediaTime"); - player._promoteToParentProxy?.(); - const forcedCall = spy.mock.calls.find(([, opts]) => opts?.force === true); - expect(forcedCall).toBeDefined(); - }); -}); - // ── Synchronous seek() with same-origin detection ── // // Studio has long reached past the postMessage bridge and called the runtime's @@ -1021,7 +823,6 @@ describe("HyperframesPlayer seek() sync path", () => { el: { pause: ReturnType; src: string }; start: number; duration: number; - driftSamples: number; source?: HTMLMediaElement | null; }>; }; @@ -1251,14 +1052,12 @@ describe("HyperframesPlayer seek() sync path", () => { el: { pause: slideProxyPause, src: "https://cdn.example.com/slide.mp4" }, start: 0, duration: 5, - driftSamples: 0, source: iframeVideo, }, { el: { pause: globalProxyPause, src: "https://cdn.example.com/background.mp3" }, start: 0, duration: Infinity, - driftSamples: 0, source: null, }, ); @@ -1900,6 +1699,7 @@ describe("HyperframesPlayer volume and mute", () => { load: ReturnType; play: ReturnType; pause: ReturnType; + addEventListener: ReturnType; }; beforeEach(async () => { @@ -1915,6 +1715,7 @@ describe("HyperframesPlayer volume and mute", () => { load: vi.fn(), play: vi.fn().mockResolvedValue(undefined), pause: vi.fn(), + addEventListener: vi.fn(), }; vi.spyOn(globalThis, "Audio").mockImplementation(function () { return mockAudio as unknown as HTMLAudioElement; @@ -2850,6 +2651,7 @@ describe("HyperframesPlayer playback rate", () => { load: ReturnType; play: ReturnType; pause: ReturnType; + addEventListener: ReturnType; }; beforeEach(async () => { @@ -2865,6 +2667,7 @@ describe("HyperframesPlayer playback rate", () => { load: vi.fn(), play: vi.fn().mockResolvedValue(undefined), pause: vi.fn(), + addEventListener: vi.fn(), }; vi.spyOn(globalThis, "Audio").mockImplementation(function () { return mockAudio as unknown as HTMLAudioElement; diff --git a/packages/player/src/parent-media.test.ts b/packages/player/src/parent-media.test.ts index 47514e3c92..2182813c3f 100644 --- a/packages/player/src/parent-media.test.ts +++ b/packages/player/src/parent-media.test.ts @@ -1,10 +1,13 @@ -import { afterEach, describe, it, expect } from "vitest"; +import { afterEach, describe, it, expect, vi } from "vitest"; import { mediaElementTime } from "@hyperframes/core/runtime/media-clock"; import { ParentMediaManager, type ProxyEntry } from "./parent-media"; // A fake media element whose paused state is driven by play()/pause() stubs. -function makeFakeAudio(initiallyPaused: boolean): HTMLMediaElement { - const el = new Audio(); +function makeFakeAudio( + initiallyPaused: boolean, + tag: "audio" | "video" = "audio", +): HTMLMediaElement { + const el = tag === "video" ? document.createElement("video") : new Audio(); let paused = initiallyPaused; Object.defineProperty(el, "paused", { get: () => paused }); el.pause = () => { @@ -71,7 +74,7 @@ describe("ParentMediaManager audio-src proxy lifecycle", () => { el: new Audio(), start: 0, duration: Infinity, - driftSamples: 0, + mediaStart: 0, }; adopted.el.src = "https://example.test/iframe-clip.mp4"; mgr.entries.push(adopted); @@ -93,7 +96,7 @@ describe("ParentMediaManager audio-src proxy lifecycle", () => { it("pauses a proxy once the playhead passes the clip end (trimmed clip)", () => { const mgr = makeManager({ owner: "parent", isPaused: false }); const el = makeFakeAudio(false); // already playing within the clip - mgr.entries.push({ el, start: 0, duration: 5, driftSamples: 0 }); + mgr.entries.push({ el, start: 0, duration: 5, mediaStart: 0 }); mgr.mirrorTime(3); // inside [0, 5) — stays playing expect(el.paused).toBe(false); @@ -111,7 +114,7 @@ describe("ParentMediaManager audio-src proxy lifecycle", () => { document.body.appendChild(source); const el = makeFakeAudio(false); - mgr.entries.push({ el, start: 0, duration: 30, driftSamples: 0, source }); + mgr.entries.push({ el, start: 0, duration: 30, mediaStart: 0, source }); mgr.mirrorTime(20); // within 30 → playing expect(el.paused).toBe(false); @@ -127,13 +130,13 @@ describe("ParentMediaManager audio-src proxy lifecycle", () => { const mgr = makeManager({ owner: "parent" }); const inWin = makeFakeAudio(true); // currently paused — scrub should start it const outWin = makeFakeAudio(false); // currently playing, but outside its window - mgr.entries.push({ el: inWin, start: 0, duration: 5, driftSamples: 0 }); - mgr.entries.push({ el: outWin, start: 10, duration: 5, driftSamples: 0 }); + mgr.entries.push({ el: inWin, start: 0, duration: 5, mediaStart: 0 }); + mgr.entries.push({ el: outWin, start: 10, duration: 5, mediaStart: 0 }); mgr.scrubAll(2); // playhead at 2s // in-window proxy: positioned at rel time and AUDIBLE (the point of scrub-audio) - expect(inWin.currentTime).toBe(mediaElementTime(2)); + expect(inWin.currentTime).toBeCloseTo(mediaElementTime(2), 9); expect(inWin.paused).toBe(false); // out-of-window proxy: paused, not blipped expect(outWin.paused).toBe(true); @@ -146,7 +149,7 @@ describe("ParentMediaManager audio-src proxy lifecycle", () => { el: new Audio(), start: 0, duration: Infinity, - driftSamples: 0, + mediaStart: 0, }; adopted.el.src = "https://example.test/shared.mp3"; mgr.entries.push(adopted); @@ -266,7 +269,7 @@ describe("ParentMediaManager following its clips", () => { expect(mgr.entries.map((m) => m.el.src)).toEqual([copy]); }); - it("keeps one proxy per file when a clip is re-pointed at another clip's file", async () => { + it("gives a clip re-pointed at another clip's file its own proxy", async () => { const mgr = makeManager(); adoptedClip(mgr, "https://example.test/a.mov"); const b = adoptedClip(mgr, "https://example.test/b.mov"); @@ -274,19 +277,10 @@ describe("ParentMediaManager following its clips", () => { b.setAttribute("src", "https://example.test/a.mov"); await flushObserver(); - expect(mgr.entries.map((m) => m.el.src)).toEqual(["https://example.test/a.mov"]); - }); - - it("keeps one proxy per file when the parent takes over after such a re-point", async () => { - const mgr = makeManager(); - adoptedClip(mgr, "https://example.test/a.mov"); - const b = adoptedClip(mgr, "https://example.test/b.mov"); - b.setAttribute("src", "https://example.test/a.mov"); - await flushObserver(); - - mgr.promoteToParentProxy(document); - - expect(mgr.entries.map((m) => m.el.src)).toEqual(["https://example.test/a.mov"]); + expect(mgr.entries.map((m) => m.el.src)).toEqual([ + "https://example.test/a.mov", + "https://example.test/a.mov", + ]); }); const files = (mgr: ParentMediaManager) => mgr.entries.map((m) => m.el.src).sort(); @@ -318,25 +312,12 @@ describe("ParentMediaManager following its clips", () => { it("keeps the proxy of a clip when another clip on the same file leaves", async () => { const mgr = makeManager(); - adoptedClip(mgr, "https://example.test/a.mov"); + const first = adoptedClip(mgr, "https://example.test/a.mov"); const twin = adoptedClip(mgr, "https://example.test/a.mov"); twin.remove(); await flushObserver(); - expect(files(mgr)).toEqual(["https://example.test/a.mov"]); - }); - - it("gives a clip that replaces another on the same file its own proxy", async () => { - const mgr = makeManager(); - const old = adoptedClip(mgr, "https://example.test/a.mov"); - const replacement = document.createElement("video"); - replacement.setAttribute("src", "https://example.test/a.mov"); - replacement.setAttribute("data-start", "0"); - replacement.preload = "auto"; - document.body.replaceChild(replacement, old); - await flushObserver(); - - expect(mgr.entries.map((m) => m.source)).toEqual([replacement]); + expect(mgr.entries.map((m) => m.source)).toEqual([first]); }); it("drops the proxy of a re-pointed clip when the clip leaves", async () => { @@ -356,7 +337,7 @@ describe("ParentMediaManager clip window", () => { it("plays a proxy inside its clip window and pauses it at the clip end instant", () => { const mgr = makeManager({ isPaused: false, owner: "parent" }); const el = makeFakeAudio(false); - mgr.entries.push({ el, start: 1, duration: 2, driftSamples: 0 }); + mgr.entries.push({ el, start: 1, duration: 2, mediaStart: 0 }); mgr.mirrorTime(2.999); expect(el.paused).toBe(false); @@ -364,3 +345,255 @@ describe("ParentMediaManager clip window", () => { expect(el.paused).toBe(true); }); }); + +describe("ParentMediaManager takes cut from one joined file", () => { + afterEach(() => document.body.replaceChildren()); + + const FILE = "https://example.test/joined.mp4"; + // [data-start, data-duration, data-media-start] of each take: the picture cuts hard, the audio window + // starts a little before the cut and ends a little after it. + const TAKES = [ + [0, 8.2, 3], + [8, 8.2, 20], + [16, 8.8, 31.5], + ]; + + function mountTakes() { + const audios: HTMLAudioElement[] = []; + for (const [start, duration, mediaStart] of TAKES) { + const video = document.createElement("video"); + video.setAttribute("src", FILE); + video.setAttribute("muted", ""); + video.setAttribute("data-start", String(start)); + video.setAttribute("data-duration", String(duration)); + video.setAttribute("data-media-start", String(mediaStart)); + const audio = document.createElement("audio"); + audio.setAttribute("src", FILE); + audio.setAttribute("data-start", String(start)); + audio.setAttribute("data-duration", String(duration)); + audio.setAttribute("data-media-start", String(mediaStart)); + document.body.append(video, audio); + audios.push(audio); + } + return audios; + } + + // Count writes to currentTime the way a seek storm shows up in a browser. + function trackSeeks(el: HTMLMediaElement) { + let time = 0; + const seeks: number[] = []; + Object.defineProperty(el, "currentTime", { + get: () => time, + set: (v: number) => { + time = v; + seeks.push(v); + }, + }); + let paused = true; + Object.defineProperty(el, "paused", { get: () => paused }); + el.play = () => { + paused = false; + return Promise.resolve(); + }; + el.pause = () => { + paused = true; + }; + return { + seeks, + advance: (dt: number) => { + if (!paused) time += dt; + }, + }; + } + + it("plays every take from its own media start for its own window, with one seek per take", () => { + const audios = mountTakes(); + const mgr = makeManager({ isPaused: false }); + mgr.setupFromIframe(document); + expect(mgr.entries).toHaveLength(audios.length); + const tracks = mgr.entries.map((m) => trackSeeks(m.el)); + mgr.promoteToParentProxy(document); + + let t = 0; + const playing: boolean[][] = []; + for (; t <= 25; t += 0.1) { + mgr.mirrorTime(t); + playing.push(mgr.entries.map((m) => !m.el.paused)); + for (const track of tracks) track.advance(0.1); + } + + // each take is heard, and from the cut footage's own place in the file + TAKES.forEach(([start, duration, mediaStart], i) => { + const first = tracks[i].seeks[0]; + expect(first).toBeGreaterThanOrEqual(mediaStart); + expect(first).toBeLessThanOrEqual(mediaStart + 0.1); + expect(playing.some((row) => row[i])).toBe(true); + expect(mgr.entries[i].start).toBe(start); + expect(mgr.entries[i].duration).toBe(duration); + // at most the entry seek per take window (promotion adds one aligned seek at most) + expect(tracks[i].seeks.length).toBeLessThanOrEqual(2); + }); + // a muted video is picture only: no proxy plays its file's audio on top of its take's audio + expect(mgr.entries.every((m) => m.source?.tagName === "AUDIO")).toBe(true); + }); + + it("keeps every take silent outside its window", () => { + mountTakes(); + const mgr = makeManager({ isPaused: false }); + mgr.setupFromIframe(document); + mgr.entries.forEach((m) => trackSeeks(m.el)); + mgr.promoteToParentProxy(document); + + mgr.mirrorTime(12); + expect(mgr.entries.map((m) => !m.el.paused)).toEqual([false, true, false]); + mgr.mirrorTime(24.5); + expect(mgr.entries.map((m) => !m.el.paused)).toEqual([false, false, true]); + }); + + it("seeks a take to its media start plus the time into its window on a user seek", () => { + mountTakes(); + const mgr = makeManager(); + mgr.setupFromIframe(document); + const tracks = mgr.entries.map((m) => trackSeeks(m.el)); + + mgr.seekAll(10); + + expect(tracks[1].seeks.map(Math.round)).toEqual([22]); + expect(tracks[0].seeks).toEqual([]); + }); + + it("positions an upcoming take at its media start before its window opens", () => { + mountTakes(); + const mgr = makeManager({ isPaused: false }); + mgr.setupFromIframe(document); + const tracks = mgr.entries.map((m) => trackSeeks(m.el)); + mgr.promoteToParentProxy(document); + + mgr.mirrorTime(6.4); // inside take 1's window, 1.6 s before take 2 + expect(tracks[1].seeks.at(-1)).toBeUndefined(); + mgr.mirrorTime(7.0); + expect(tracks[1].seeks.map(Math.round)).toEqual([20]); + expect(mgr.entries[1].el.paused).toBe(true); + + // at the cut it only plays: no second seek + mgr.mirrorTime(8.0); + expect(mgr.entries[1].el.paused).toBe(false); + expect(tracks[1].seeks.map(Math.round)).toEqual([20]); + }); +}); + +describe("ParentMediaManager drift policy", () => { + afterEach(() => { + document.body.replaceChildren(); + vi.restoreAllMocks(); + }); + + // A playing proxy `lag` seconds behind a timeline at `t`, a clip that starts at 0 and plays from media 0. + function playingProxy(lag: number, t = 5, tag: "audio" | "video" = "audio") { + const mgr = makeManager({ isPaused: false }); + const el = makeFakeAudio(false, tag); + const writes: number[] = []; + let time = t - lag; + Object.defineProperty(el, "currentTime", { + get: () => time, + set: (v: number) => { + time = v; + writes.push(v); + }, + }); + mgr.entries.push({ el, start: 0, duration: 30, mediaStart: 0 }); + return { mgr, el, writes, setTime: (v: number) => (time = v) }; + } + + it("leaves a proxy inside the sync tolerance alone", () => { + const { mgr, el, writes, setTime } = playingProxy(0.03); + mgr.mirrorTime(5); + setTime(5.07); + mgr.mirrorTime(5.1); + expect(writes).toEqual([]); + expect(el.playbackRate).toBe(1); + }); + + it("steers a proxy that lags a little by rate, capped at 10%, without seeking", () => { + const { mgr, el, writes } = playingProxy(0.07); + mgr.mirrorTime(5); + mgr.mirrorTime(5.08); + expect(writes).toEqual([]); + expect(el.playbackRate).toBeGreaterThan(1); + expect(el.playbackRate).toBeLessThanOrEqual(1.1); + }); + + it("returns a steered proxy to its authored rate once it has caught up", () => { + const { mgr, el, setTime } = playingProxy(0.07); + mgr.mirrorTime(5); + setTime(5.1); + mgr.mirrorTime(5.1); + expect(el.playbackRate).toBe(1); + }); + + it("seeks a proxy that holds a lag past 100 ms, once", () => { + const { mgr, writes, setTime } = playingProxy(0.25); + mgr.mirrorTime(5); // first sample: offset not yet known to hold + setTime(4.85); // the proxy advanced with the timeline: the lag held + mgr.mirrorTime(5.1); + expect(writes).toHaveLength(1); + expect(writes[0]).toBeCloseTo(mediaElementTime(5.1), 9); + }); + + it("seeks a video proxy that holds a 1 s lag: a proxy is heard, whatever its tag", () => { + const { mgr, el, writes, setTime } = playingProxy(1, 5, "video"); + mgr.mirrorTime(5); + setTime(4.1); + mgr.mirrorTime(5.1); + expect(writes).toHaveLength(1); + expect(el.playbackRate).toBe(1); + }); + + it("steers, never seeks, a proxy whose lag keeps growing because it is stalled", () => { + const { mgr, el, writes } = playingProxy(0.25); + mgr.mirrorTime(5); + mgr.mirrorTime(5.1); // currentTime did not move: lag grew by 100 ms + expect(writes).toEqual([]); + expect(el.playbackRate).toBeLessThanOrEqual(1.1); + }); + + it("seeks at once on a forced alignment when the proxy is off the playhead", () => { + const { mgr, writes } = playingProxy(2); + mgr.mirrorTime(5, { force: true }); + expect(writes).toHaveLength(1); + expect(writes[0]).toBeCloseTo(mediaElementTime(5), 9); + }); + + it("aims a seek of a playing proxy ahead by the time its last seek took to land", () => { + let now = 1000; + vi.spyOn(performance, "now").mockImplementation(() => now); + const mgr = makeManager({ isPaused: false }); + mgr.setupFromUrl("https://example.test/take.mp3"); + const el = mgr.entries[0].el; + const writes: number[] = []; + let time = 4.75; + Object.defineProperty(el, "currentTime", { + get: () => time, + set: (v: number) => { + time = v; + writes.push(v); + }, + }); + Object.defineProperty(el, "paused", { get: () => false }); + mgr.promoteToParentProxy(null, () => {}); // owner parent without the alignment seek + + mgr.mirrorTime(5); // 250 ms behind + time = 4.85; + mgr.mirrorTime(5.1); // the lag held: seek, no landing time known yet + now = 1230; + el.dispatchEvent(new Event("seeked")); // it took 230 ms to land + time = 5.0; // landed 230 ms late, lagging the playhead again + mgr.mirrorTime(5.33); + time = 5.1; + mgr.mirrorTime(5.43); + + expect(writes).toHaveLength(2); + expect(writes[0]).toBeCloseTo(mediaElementTime(5.1), 9); + expect(writes[1]).toBeCloseTo(mediaElementTime(5.43 + 0.23), 9); + }); +}); diff --git a/packages/player/src/parent-media.ts b/packages/player/src/parent-media.ts index c8c9645faa..3781b341a9 100644 --- a/packages/player/src/parent-media.ts +++ b/packages/player/src/parent-media.ts @@ -12,40 +12,42 @@ import { selectMediaObserverTargets } from "./mediaObserverScope.js"; import { isRealmElement, isRealmHtmlMediaElement } from "./media-element-guards.js"; import { isInClipWindow } from "@hyperframes/core/runtime/clip-window"; -import { mediaElementTime } from "@hyperframes/core/runtime/media-clock"; import { readClipTiming } from "@hyperframes/core/composition-contract"; +import { readLinkTiming } from "@hyperframes/core/media-link"; +import { mediaElementTime } from "@hyperframes/core/runtime/media-clock"; +import { + MEDIA_SYNC_TOLERANCE_SECONDS, + forgetSteering, + steerMedia, +} from "@hyperframes/core/runtime/media-steer"; -/** Minimum absolute drift before a currentTime correction is attempted. */ -const MIRROR_DRIFT_THRESHOLD_SECONDS = 0.05; +/** An offset that moved less than this since the last sample is not growing: the element is not stalled. */ +const SETTLED_OFFSET_DELTA_SECONDS = 0.004; /** - * How many *consecutive* over-threshold samples are required before issuing a - * `currentTime` write. Absorbs single-sample jitter (GC pause, slow bridge - * tick) without thrashing. Forced calls bypass this gate. - * - * Worst-case correction latency ≈ this × bridgeMaxPostIntervalMs (80 ms in - * core/runtime/state.ts) = 160 ms — well under human A/V re-sync tolerance. + * How far ahead of its window a paused proxy is positioned at its media start, so the cut + * only has to `play()`: a seek into unbuffered media lands about 80 ms late. */ -const MIRROR_REQUIRED_CONSECUTIVE_DRIFT_SAMPLES = 2; +const PRESEEK_SECONDS = 1.5; export interface ProxyEntry { el: HTMLMediaElement; start: number; duration: number; + /** Where in the file the clip begins (`data-media-start`); the proxy plays `t - start + mediaStart`. */ + mediaStart: number; /** * The iframe media element this proxy mirrors, when adopted from the DOM. * Its `data-start`/`data-duration` are re-read each tick so live timeline * edits (trim/move) bound the proxy correctly. Null for URL-driven proxies. */ source?: HTMLMediaElement | null; - /** - * Count of consecutive steady-state samples in which the proxy's - * `currentTime` was found drifted beyond `MIRROR_DRIFT_THRESHOLD_SECONDS`. - * Reset on every in-threshold sample. A write is only issued once this - * reaches `MIRROR_REQUIRED_CONSECUTIVE_DRIFT_SAMPLES`, absorbing - * single-sample jitter without thrashing. - */ - driftSamples: number; + /** The offset behind the timeline (+) at the last drift sample; steering is only trusted when it stopped growing. */ + lastOffset?: number; + /** Seconds a seek of this playing proxy takes to land (smoothed); a seek aims this far ahead so it lands on time. */ + seekLatency?: number; + /** When the seek still landing was issued (ms); unset when none is pending. */ + seekIssuedAt?: number; } export class ParentMediaManager { @@ -157,6 +159,7 @@ export class ParentMediaManager { // gate never closes and the proxy plays past its clip end. const timing = readClipTiming(m.source); m.start = timing.start ?? 0; + m.mediaStart = readLinkTiming(m.source).mediaStart; m.duration = timing.duration != null && timing.duration > 0 ? timing.duration : Number.POSITIVE_INFINITY; } @@ -167,10 +170,45 @@ export class ParentMediaManager { return isInClipWindow(timeSeconds, m.start, m.start + m.duration); } + private _preseekUpcoming(m: ProxyEntry, timelineSeconds: number): void { + const ahead = m.start - timelineSeconds; + if (ahead <= 0 || ahead > PRESEEK_SECONDS || !m.el.paused) return; + if (Math.abs(m.el.currentTime - m.mediaStart) > MEDIA_SYNC_TOLERANCE_SECONDS) { + this._seekTo(m, m.mediaStart); + } + } + + // The one writer of a proxy's position: every seek lands here. A seek of a playing proxy aims ahead by + // the time its seeks take to land: aimed at the playhead it lands behind it and the next tick seeks again. + private _seekTo(m: ProxyEntry, mediaSeconds: number, playing = false): void { + const lead = playing ? (m.seekLatency ?? 0) * this._getPlaybackRate() : 0; + m.seekIssuedAt = playing ? performance.now() : undefined; + m.el.currentTime = mediaElementTime(mediaSeconds + lead); + } + + private _noteSeekLanded(m: ProxyEntry): void { + if (m.seekIssuedAt === undefined) return; + const landed = (performance.now() - m.seekIssuedAt) / 1000; + m.seekLatency = m.seekLatency === undefined ? landed : (m.seekLatency + landed) / 2; + m.seekIssuedAt = undefined; + } + + // Back to the authored rate with no drift history: the proxy was seeked, paused or left its window. + private _releaseSteering(m: ProxyEntry): void { + forgetSteering(m.el); + m.lastOffset = undefined; + const rate = this._getPlaybackRate(); + if (m.el.playbackRate !== rate) m.el.playbackRate = rate; + } + + private _mediaTime(m: ProxyEntry, timelineSeconds: number): number { + return timelineSeconds - m.start + m.mediaStart; + } + private _gateEntryPlayback(m: ProxyEntry, timeSeconds: number): boolean { if (!this._inWindow(m, timeSeconds)) { if (!m.el.paused) m.el.pause(); - m.driftSamples = 0; + this._releaseSteering(m); return false; } if (this._audioOwner === "parent" && !this._isPaused() && m.el.paused) this._playEntry(m); @@ -196,8 +234,7 @@ export class ParentMediaManager { // Re-read live bounds so a trim/move just before a paused scrub gates and // positions against the current clip window, not the adopt-time one. this._refreshEntryBounds(m); - if (this._inWindow(m, timeInSeconds)) - m.el.currentTime = mediaElementTime(timeInSeconds - m.start); + if (this._inWindow(m, timeInSeconds)) this._jumpTo(m, this._mediaTime(m, timeInSeconds)); } } @@ -212,7 +249,7 @@ export class ParentMediaManager { for (const m of this._entries) { this._refreshEntryBounds(m); if (this._inWindow(m, timeInSeconds)) { - m.el.currentTime = mediaElementTime(timeInSeconds - m.start); + this._jumpTo(m, this._mediaTime(m, timeInSeconds)); this._playEntry(m); } else if (!m.el.paused) { m.el.pause(); @@ -221,29 +258,43 @@ export class ParentMediaManager { } /** - * Mirror parent-proxy `currentTime` to the iframe timeline, with optional - * jitter-coalescing. Pass `{ force: true }` for alignment moments (ownership - * promotion, new proxy initialization) where drift must be corrected - * immediately. + * Mirror parent-proxy `currentTime` to the iframe timeline. Drift is steered by rate and + * seeked only past the player's one drift policy (`steerMedia`). Pass `{ force: true }` + * for alignment moments (ownership promotion, new proxy initialization) where a proxy off + * the playhead must be seeked immediately. */ mirrorTime(timelineSeconds: number, options?: { force?: boolean }): void { const force = options?.force === true; for (const m of this._entries) { this._refreshEntryBounds(m); + this._preseekUpcoming(m, timelineSeconds); + // A proxy entering its window that was not pre-positioned (a user seek) is seeked once, before it plays. + const entering = m.el.paused; if (!this._gateEntryPlayback(m, timelineSeconds)) continue; - const relTime = timelineSeconds - m.start; - if (Math.abs(m.el.currentTime - relTime) > MIRROR_DRIFT_THRESHOLD_SECONDS) { - m.driftSamples += 1; - if (force || m.driftSamples >= MIRROR_REQUIRED_CONSECUTIVE_DRIFT_SAMPLES) { - m.el.currentTime = mediaElementTime(relTime); - m.driftSamples = 0; - } - } else { - m.driftSamples = 0; + const target = this._mediaTime(m, timelineSeconds); + const offset = target - m.el.currentTime; + if (force || entering) { + if (Math.abs(offset) > MEDIA_SYNC_TOLERANCE_SECONDS) this._jumpTo(m, target); + continue; } + this._steer(m, target, offset); } } + private _steer(m: ProxyEntry, target: number, offset: number): void { + const settled = + m.lastOffset !== undefined && Math.abs(offset - m.lastOffset) < SETTLED_OFFSET_DELTA_SECONDS; + m.lastOffset = offset; + const { rate, seek } = steerMedia(m.el, offset, this._getPlaybackRate(), settled, true); + if (seek) this._jumpTo(m, target, true); + else if (Math.abs(m.el.playbackRate - rate) > 1e-6) m.el.playbackRate = rate; + } + + private _jumpTo(m: ProxyEntry, mediaSeconds: number, playing = false): void { + this._seekTo(m, mediaSeconds, playing); + this._releaseSteering(m); + } + /** * Take ownership of audible playback in response to the runtime's * `media-autoplay-blocked` signal. Idempotent. @@ -347,8 +398,9 @@ export class ParentMediaManager { } /** - * Create a parent-frame media element and start preloading it. Returns the - * new entry, or `null` if a proxy for this src already exists (dedup). + * Create a parent-frame media element and start preloading it. A URL-driven + * proxy returns `null` when the file already has one; a clip always gets its + * own, since its window and media offset are its own. */ private _createEntry( src: string, @@ -357,7 +409,7 @@ export class ParentMediaManager { duration: number, source?: HTMLMediaElement | null, ): ProxyEntry | null { - if (this._urlTaken(src)) return null; + if (!source && this._urlTaken(src)) return null; const el = tag === "video" ? document.createElement("video") : new Audio(); el.preload = "auto"; @@ -368,7 +420,14 @@ export class ParentMediaManager { const rate = this._getPlaybackRate(); if (rate !== 1) el.playbackRate = rate; - const entry: ProxyEntry = { el, start, duration, driftSamples: 0, source }; + const entry: ProxyEntry = { + el, + start, + duration, + mediaStart: source ? readLinkTiming(source).mediaStart : 0, + source, + }; + el.addEventListener("seeked", () => this._noteSeekLanded(entry)); this._entries.push(entry); return entry; } @@ -386,6 +445,11 @@ export class ParentMediaManager { // "auto" and drops its proxy, which fetches the whole file, when it turns back. if (iframeEl.preload === "metadata" || iframeEl.preload === "none") return; + if (this._entries.some((m) => m.source === iframeEl)) return; + + // A video authored muted is picture only: its file's audio belongs to the clip that plays it. + if (iframeEl.tagName === "VIDEO" && iframeEl.hasAttribute("muted")) return; + const src = this._resolveIframeMediaSrc(iframeEl); if (!src) return; @@ -414,11 +478,11 @@ export class ParentMediaManager { if (!this._isPaused()) this._playEntryIfActive(entry); } - /** One proxy per file: a second clip on the same file is heard through the first one's proxy. - * A proxy's file is its clip's current one (a clip that left the page owns none). */ - private _urlTaken(src: string, except?: ProxyEntry): boolean { + /** Whether a clip's proxy already plays this file. A proxy's file is its clip's current one + * (a clip that left the page owns none). */ + private _urlTaken(src: string): boolean { return this._entries.some((m) => { - if (m === except || (m.source && !m.source.isConnected)) return false; + if (m.source && !m.source.isConnected) return false; return ((m.source && this._resolveIframeMediaSrc(m.source)) || m.el.src) === src; }); } @@ -426,10 +490,6 @@ export class ParentMediaManager { private _repointToSource(entry: ProxyEntry): boolean { const src = entry.source ? this._resolveIframeMediaSrc(entry.source) : null; if (!src || entry.el.src === src) return false; - if (this._urlTaken(src, entry)) { - this._removeEntry(entry); - return false; - } entry.el.src = src; entry.el.load(); return true;