ext/hyperframes
2
0
Fork 0
mirror of https://github.com/heygen-com/hyperframes.git synced 2026-10-30 09:16:55 +00:00
Code Issues Packages Projects Releases Wiki Activity

fix(player): each take plays its own audio when the browser blocks autoplay (#6417)

Browse Source 
* fix(player): each take plays its own audio when the browser blocks autoplay

* fix(player): a parent audio proxy of a video clip is seeked when its lag holds, not rate-chased

* style(core): keep the steering policy's comment share where it was
This commit is contained in:
Miguel Ángel
2026-10-27 01:49:09 +00:00
committed by GitHub
parent 63d9435625
commit 736f38ff4a
4 changed files with 389 additions and 292 deletions
Download Patch File Download Diff File Expand all files Collapse all files
packages/core/src/runtime/mediaSteer.ts
+3 -2
View File
@@ -21,9 +21,10 @@ export function steerMedia(
offset: number,
baseRate: number,
settled: boolean,
audioOnly = el.tagName === "AUDIO",
): { rate: number; seek: boolean } {
const lag = Math.abs(offset);
if (el.tagName === "AUDIO" && settled && lag > AUDIO_STEER_LIMIT_SECONDS) {
if (audioOnly && settled && lag > AUDIO_STEER_LIMIT_SECONDS) {
steering.delete(el);
return { rate: baseRate, seek: true };
}
@@ -35,7 +36,7 @@ export function steerMedia(
return { rate: baseRate, seek: false };
}
const step = MEDIA_STEER_STEPS.find(([from]) => lag > from)?.[1] ?? MEDIA_STEER;
const steer = el.tagName === "AUDIO" ? Math.min(step, AUDIO_MAX_STEER) : step;
const steer = audioOnly ? Math.min(step, AUDIO_MAX_STEER) : step;
return { rate: baseRate * (1 + direction * steer), seek: false };
}
packages/player/src/hyperframes-player.test.ts
+6 -203
View File
@@ -147,6 +147,7 @@ describe("HyperframesPlayer parent-frame media", () => {
paused: boolean;
play: ReturnType<typeof vi.fn>;
pause: ReturnType<typeof vi.fn>;
addEventListener: ReturnType<typeof vi.fn>;
load: ReturnType<typeof vi.fn>;
};
@@ -162,6 +163,7 @@ describe("HyperframesPlayer parent-frame media", () => {
paused: true,
play: vi.fn().mockResolvedValue(undefined),
pause: vi.fn(),
addEventListener: vi.fn(),
load: vi.fn(),
};
@@ -779,206 +781,6 @@ describe("HyperframesPlayer media MutationObserver scoping", () => {
});
});
// ── Parent-proxy time-mirror coalescing ──
//
// `_mirrorParentMediaTime` is the steady-state correction loop that nudges
// every parent-frame audio/video proxy back onto the iframe's timeline. The
// post-`P1-4` contract: a single over-threshold sample (one slow bridge tick,
// one tab-throttled rAF, one GC pause) is absorbed by a per-proxy counter and
// does NOT cost a `currentTime` write. Only a *trending* drift — two
// consecutive samples above the 50 ms threshold — triggers a seek. Forced
// callers (audio-ownership promotion, brand-new proxy initialization) bypass
// the gate so the listener never hears a misaligned sample on cut-over.
describe("HyperframesPlayer parent-proxy time-mirror coalescing", () => {
type DriftEntry = {
el: { currentTime: number; src: string; pause: () => void };
start: number;
duration: number;
driftSamples: number;
};
type PlayerInternal = HTMLElement & {
_parentMedia: DriftEntry[];
_mirrorParentMediaTime: (timelineSeconds: number, options?: { force?: boolean }) => void;
_promoteToParentProxy?: () => void;
};
let player: PlayerInternal;
beforeEach(async () => {
await import("./hyperframes-player.js");
player = document.createElement("hyperframes-player") as PlayerInternal;
document.body.appendChild(player);
// No audio-src was set, so `_parentMedia` is empty. Tests push synthetic
// POJO entries — `_mirrorParentMediaTime` only reads/writes
// `el.currentTime`, so a plain object stands in fine for HTMLMediaElement.
});
afterEach(() => {
player.remove();
vi.restoreAllMocks();
});
function makeEntry(
opts: {
currentTime?: number;
start?: number;
duration?: number;
driftSamples?: number;
} = {},
): DriftEntry {
// Include `pause`/`src` so `disconnectedCallback`'s teardown loop
// (`m.el.pause(); m.el.src = ""`) doesn't blow up when the player is
// removed at the end of the test — `_mirrorParentMediaTime` itself only
// touches `currentTime`.
const entry: DriftEntry = {
el: {
currentTime: opts.currentTime ?? 0,
src: "",
pause: vi.fn(),
},
start: opts.start ?? 0,
duration: opts.duration ?? 100,
driftSamples: opts.driftSamples ?? 0,
};
player._parentMedia.push(entry);
return entry;
}
it("initializes new parent-media entries with driftSamples=0", () => {
// Mock Audio just for this test so the audio-src bootstrap path produces
// a real entry rather than throwing on construction.
const mockAudio = {
src: "",
preload: "",
muted: false,
playbackRate: 1,
currentTime: 0,
paused: true,
play: vi.fn().mockResolvedValue(undefined),
pause: vi.fn(),
load: vi.fn(),
};
vi.spyOn(globalThis, "Audio").mockImplementation(function () {
return mockAudio as unknown as HTMLAudioElement;
});
const fresh = document.createElement("hyperframes-player") as PlayerInternal;
fresh.setAttribute("audio-src", "https://cdn.example.com/narration.mp3");
document.body.appendChild(fresh);
expect(fresh._parentMedia).toHaveLength(1);
expect(fresh._parentMedia[0]?.driftSamples).toBe(0);
fresh.remove();
});
it("does nothing when drift is within the 50 ms threshold", () => {
const m = makeEntry({ currentTime: 5 });
player._mirrorParentMediaTime(5.04);
expect(m.el.currentTime).toBe(5);
expect(m.driftSamples).toBe(0);
});
it("absorbs a single over-threshold spike without writing currentTime", () => {
const m = makeEntry({ currentTime: 5 });
player._mirrorParentMediaTime(5.5);
expect(m.el.currentTime).toBe(5);
expect(m.driftSamples).toBe(1);
});
it("issues a seek on the second consecutive over-threshold sample", () => {
const m = makeEntry({ currentTime: 5 });
player._mirrorParentMediaTime(5.5);
expect(m.el.currentTime).toBe(5);
expect(m.driftSamples).toBe(1);
// Second sample with the same drift: the gate trips, the write fires,
// and the counter resets so the proxy doesn't re-seek every later tick.
player._mirrorParentMediaTime(5.5);
expect(m.el.currentTime).toBe(mediaElementTime(5.5));
expect(m.driftSamples).toBe(0);
});
it("resets the counter when a sample comes back within threshold", () => {
const m = makeEntry({ currentTime: 5 });
player._mirrorParentMediaTime(5.5);
expect(m.driftSamples).toBe(1);
// Recovery — counter must clear so a later isolated spike doesn't
// accidentally satisfy the 2-sample gate by piggy-backing on stale state.
player._mirrorParentMediaTime(5.02);
expect(m.driftSamples).toBe(0);
expect(m.el.currentTime).toBe(5);
player._mirrorParentMediaTime(5.5);
expect(m.driftSamples).toBe(1);
expect(m.el.currentTime).toBe(5);
});
it("force: true writes immediately on the first over-threshold sample", () => {
const m = makeEntry({ currentTime: 5 });
player._mirrorParentMediaTime(5.5, { force: true });
expect(m.el.currentTime).toBe(mediaElementTime(5.5));
expect(m.driftSamples).toBe(0);
});
it("force: true clears any pre-existing drift counter", () => {
const m = makeEntry({ currentTime: 5, driftSamples: 1 });
player._mirrorParentMediaTime(5.5, { force: true });
expect(m.el.currentTime).toBe(mediaElementTime(5.5));
expect(m.driftSamples).toBe(0);
});
it("does not seek out-of-range entries and resets their counters", () => {
// Active window [10, 15). currentTime=99 is a sentinel — if the function
// ever writes inside an out-of-range branch the test catches it because
// relTime would be 5 (or 15), not 99.
const m = makeEntry({
currentTime: 99,
start: 10,
duration: 5,
driftSamples: 5,
});
player._mirrorParentMediaTime(5);
expect(m.el.currentTime).toBe(99);
expect(m.driftSamples).toBe(0);
// Boundary: relTime === duration → still out of range (the loop uses `>=`).
m.driftSamples = 7;
player._mirrorParentMediaTime(15);
expect(m.el.currentTime).toBe(99);
expect(m.driftSamples).toBe(0);
});
it("tracks drift independently across multiple proxies", () => {
// a is drifted; b is aligned. A single tick must increment a's counter
// and reset b's — proving the per-entry state is genuinely per-entry.
const a = makeEntry({ currentTime: 5 });
const b = makeEntry({ currentTime: 7.01, driftSamples: 1 });
player._mirrorParentMediaTime(7);
expect(a.el.currentTime).toBe(5);
expect(a.driftSamples).toBe(1);
expect(b.el.currentTime).toBe(7.01);
expect(b.driftSamples).toBe(0);
});
it("force: true bypasses the gate for every proxy in a single sweep", () => {
const a = makeEntry({ currentTime: 5 });
const b = makeEntry({ currentTime: 8 });
player._mirrorParentMediaTime(7, { force: true });
expect(a.el.currentTime).toBe(mediaElementTime(7));
expect(b.el.currentTime).toBe(mediaElementTime(7));
expect(a.driftSamples).toBe(0);
expect(b.driftSamples).toBe(0);
});
it("_promoteToParentProxy invokes _mirrorParentMediaTime with force: true", () => {
// Integration check of the promotion call site — we cannot tolerate even
// ~80 ms of audible drift across an ownership flip, so the call site
// must opt out of the jitter gate.
const spy = vi.spyOn(player, "_mirrorParentMediaTime");
player._promoteToParentProxy?.();
const forcedCall = spy.mock.calls.find(([, opts]) => opts?.force === true);
expect(forcedCall).toBeDefined();
});
});
// ── Synchronous seek() with same-origin detection ──
//
// Studio has long reached past the postMessage bridge and called the runtime's
@@ -1021,7 +823,6 @@ describe("HyperframesPlayer seek() sync path", () => {
el: { pause: ReturnType<typeof vi.fn>; src: string };
start: number;
duration: number;
driftSamples: number;
source?: HTMLMediaElement | null;
}>;
};
@@ -1251,14 +1052,12 @@ describe("HyperframesPlayer seek() sync path", () => {
el: { pause: slideProxyPause, src: "https://cdn.example.com/slide.mp4" },
start: 0,
duration: 5,
driftSamples: 0,
source: iframeVideo,
},
{
el: { pause: globalProxyPause, src: "https://cdn.example.com/background.mp3" },
start: 0,
duration: Infinity,
driftSamples: 0,
source: null,
},
);
@@ -1900,6 +1699,7 @@ describe("HyperframesPlayer volume and mute", () => {
load: ReturnType<typeof vi.fn>;
play: ReturnType<typeof vi.fn>;
pause: ReturnType<typeof vi.fn>;
addEventListener: ReturnType<typeof vi.fn>;
};
beforeEach(async () => {
@@ -1915,6 +1715,7 @@ describe("HyperframesPlayer volume and mute", () => {
load: vi.fn(),
play: vi.fn().mockResolvedValue(undefined),
pause: vi.fn(),
addEventListener: vi.fn(),
};
vi.spyOn(globalThis, "Audio").mockImplementation(function () {
return mockAudio as unknown as HTMLAudioElement;
@@ -2850,6 +2651,7 @@ describe("HyperframesPlayer playback rate", () => {
load: ReturnType<typeof vi.fn>;
play: ReturnType<typeof vi.fn>;
pause: ReturnType<typeof vi.fn>;
addEventListener: ReturnType<typeof vi.fn>;
};
beforeEach(async () => {
@@ -2865,6 +2667,7 @@ describe("HyperframesPlayer playback rate", () => {
load: vi.fn(),
play: vi.fn().mockResolvedValue(undefined),
pause: vi.fn(),
addEventListener: vi.fn(),
};
vi.spyOn(globalThis, "Audio").mockImplementation(function () {
return mockAudio as unknown as HTMLAudioElement;
packages/player/src/parent-media.test.ts
+273 -40
View File
@@ -1,10 +1,13 @@
import { afterEach, describe, it, expect } from "vitest";
import { afterEach, describe, it, expect, vi } from "vitest";
import { mediaElementTime } from "@hyperframes/core/runtime/media-clock";
import { ParentMediaManager, type ProxyEntry } from "./parent-media";
// A fake media element whose paused state is driven by play()/pause() stubs.
function makeFakeAudio(initiallyPaused: boolean): HTMLMediaElement {
const el = new Audio();
function makeFakeAudio(
initiallyPaused: boolean,
tag: "audio" | "video" = "audio",
): HTMLMediaElement {
const el = tag === "video" ? document.createElement("video") : new Audio();
let paused = initiallyPaused;
Object.defineProperty(el, "paused", { get: () => paused });
el.pause = () => {
@@ -71,7 +74,7 @@ describe("ParentMediaManager audio-src proxy lifecycle", () => {
el: new Audio(),
start: 0,
duration: Infinity,
driftSamples: 0,
mediaStart: 0,
};
adopted.el.src = "https://example.test/iframe-clip.mp4";
mgr.entries.push(adopted);
@@ -93,7 +96,7 @@ describe("ParentMediaManager audio-src proxy lifecycle", () => {
it("pauses a proxy once the playhead passes the clip end (trimmed clip)", () => {
const mgr = makeManager({ owner: "parent", isPaused: false });
const el = makeFakeAudio(false); // already playing within the clip
mgr.entries.push({ el, start: 0, duration: 5, driftSamples: 0 });
mgr.entries.push({ el, start: 0, duration: 5, mediaStart: 0 });
mgr.mirrorTime(3); // inside [0, 5) — stays playing
expect(el.paused).toBe(false);
@@ -111,7 +114,7 @@ describe("ParentMediaManager audio-src proxy lifecycle", () => {
document.body.appendChild(source);
const el = makeFakeAudio(false);
mgr.entries.push({ el, start: 0, duration: 30, driftSamples: 0, source });
mgr.entries.push({ el, start: 0, duration: 30, mediaStart: 0, source });
mgr.mirrorTime(20); // within 30 → playing
expect(el.paused).toBe(false);
@@ -127,13 +130,13 @@ describe("ParentMediaManager audio-src proxy lifecycle", () => {
const mgr = makeManager({ owner: "parent" });
const inWin = makeFakeAudio(true); // currently paused — scrub should start it
const outWin = makeFakeAudio(false); // currently playing, but outside its window
mgr.entries.push({ el: inWin, start: 0, duration: 5, driftSamples: 0 });
mgr.entries.push({ el: outWin, start: 10, duration: 5, driftSamples: 0 });
mgr.entries.push({ el: inWin, start: 0, duration: 5, mediaStart: 0 });
mgr.entries.push({ el: outWin, start: 10, duration: 5, mediaStart: 0 });
mgr.scrubAll(2); // playhead at 2s
// in-window proxy: positioned at rel time and AUDIBLE (the point of scrub-audio)
expect(inWin.currentTime).toBe(mediaElementTime(2));
expect(inWin.currentTime).toBeCloseTo(mediaElementTime(2), 9);
expect(inWin.paused).toBe(false);
// out-of-window proxy: paused, not blipped
expect(outWin.paused).toBe(true);
@@ -146,7 +149,7 @@ describe("ParentMediaManager audio-src proxy lifecycle", () => {
el: new Audio(),
start: 0,
duration: Infinity,
driftSamples: 0,
mediaStart: 0,
};
adopted.el.src = "https://example.test/shared.mp3";
mgr.entries.push(adopted);
@@ -266,7 +269,7 @@ describe("ParentMediaManager following its clips", () => {
expect(mgr.entries.map((m) => m.el.src)).toEqual([copy]);
});
it("keeps one proxy per file when a clip is re-pointed at another clip's file", async () => {
it("gives a clip re-pointed at another clip's file its own proxy", async () => {
const mgr = makeManager();
adoptedClip(mgr, "https://example.test/a.mov");
const b = adoptedClip(mgr, "https://example.test/b.mov");
@@ -274,19 +277,10 @@ describe("ParentMediaManager following its clips", () => {
b.setAttribute("src", "https://example.test/a.mov");
await flushObserver();
expect(mgr.entries.map((m) => m.el.src)).toEqual(["https://example.test/a.mov"]);
});
it("keeps one proxy per file when the parent takes over after such a re-point", async () => {
const mgr = makeManager();
adoptedClip(mgr, "https://example.test/a.mov");
const b = adoptedClip(mgr, "https://example.test/b.mov");
b.setAttribute("src", "https://example.test/a.mov");
await flushObserver();
mgr.promoteToParentProxy(document);
expect(mgr.entries.map((m) => m.el.src)).toEqual(["https://example.test/a.mov"]);
expect(mgr.entries.map((m) => m.el.src)).toEqual([
"https://example.test/a.mov",
"https://example.test/a.mov",
]);
});
const files = (mgr: ParentMediaManager) => mgr.entries.map((m) => m.el.src).sort();
@@ -318,25 +312,12 @@ describe("ParentMediaManager following its clips", () => {
it("keeps the proxy of a clip when another clip on the same file leaves", async () => {
const mgr = makeManager();
adoptedClip(mgr, "https://example.test/a.mov");
const first = adoptedClip(mgr, "https://example.test/a.mov");
const twin = adoptedClip(mgr, "https://example.test/a.mov");
twin.remove();
await flushObserver();
expect(files(mgr)).toEqual(["https://example.test/a.mov"]);
});
it("gives a clip that replaces another on the same file its own proxy", async () => {
const mgr = makeManager();
const old = adoptedClip(mgr, "https://example.test/a.mov");
const replacement = document.createElement("video");
replacement.setAttribute("src", "https://example.test/a.mov");
replacement.setAttribute("data-start", "0");
replacement.preload = "auto";
document.body.replaceChild(replacement, old);
await flushObserver();
expect(mgr.entries.map((m) => m.source)).toEqual([replacement]);
expect(mgr.entries.map((m) => m.source)).toEqual([first]);
});
it("drops the proxy of a re-pointed clip when the clip leaves", async () => {
@@ -356,7 +337,7 @@ describe("ParentMediaManager clip window", () => {
it("plays a proxy inside its clip window and pauses it at the clip end instant", () => {
const mgr = makeManager({ isPaused: false, owner: "parent" });
const el = makeFakeAudio(false);
mgr.entries.push({ el, start: 1, duration: 2, driftSamples: 0 });
mgr.entries.push({ el, start: 1, duration: 2, mediaStart: 0 });
mgr.mirrorTime(2.999);
expect(el.paused).toBe(false);
@@ -364,3 +345,255 @@ describe("ParentMediaManager clip window", () => {
expect(el.paused).toBe(true);
});
});
describe("ParentMediaManager takes cut from one joined file", () => {
afterEach(() => document.body.replaceChildren());
const FILE = "https://example.test/joined.mp4";
// [data-start, data-duration, data-media-start] of each take: the picture cuts hard, the audio window
// starts a little before the cut and ends a little after it.
const TAKES = [
[0, 8.2, 3],
[8, 8.2, 20],
[16, 8.8, 31.5],
];
function mountTakes() {
const audios: HTMLAudioElement[] = [];
for (const [start, duration, mediaStart] of TAKES) {
const video = document.createElement("video");
video.setAttribute("src", FILE);
video.setAttribute("muted", "");
video.setAttribute("data-start", String(start));
video.setAttribute("data-duration", String(duration));
video.setAttribute("data-media-start", String(mediaStart));
const audio = document.createElement("audio");
audio.setAttribute("src", FILE);
audio.setAttribute("data-start", String(start));
audio.setAttribute("data-duration", String(duration));
audio.setAttribute("data-media-start", String(mediaStart));
document.body.append(video, audio);
audios.push(audio);
}
return audios;
}
// Count writes to currentTime the way a seek storm shows up in a browser.
function trackSeeks(el: HTMLMediaElement) {
let time = 0;
const seeks: number[] = [];
Object.defineProperty(el, "currentTime", {
get: () => time,
set: (v: number) => {
time = v;
seeks.push(v);
},
});
let paused = true;
Object.defineProperty(el, "paused", { get: () => paused });
el.play = () => {
paused = false;
return Promise.resolve();
};
el.pause = () => {
paused = true;
};
return {
seeks,
advance: (dt: number) => {
if (!paused) time += dt;
},
};
}
it("plays every take from its own media start for its own window, with one seek per take", () => {
const audios = mountTakes();
const mgr = makeManager({ isPaused: false });
mgr.setupFromIframe(document);
expect(mgr.entries).toHaveLength(audios.length);
const tracks = mgr.entries.map((m) => trackSeeks(m.el));
mgr.promoteToParentProxy(document);
let t = 0;
const playing: boolean[][] = [];
for (; t <= 25; t += 0.1) {
mgr.mirrorTime(t);
playing.push(mgr.entries.map((m) => !m.el.paused));
for (const track of tracks) track.advance(0.1);
}
// each take is heard, and from the cut footage's own place in the file
TAKES.forEach(([start, duration, mediaStart], i) => {
const first = tracks[i].seeks[0];
expect(first).toBeGreaterThanOrEqual(mediaStart);
expect(first).toBeLessThanOrEqual(mediaStart + 0.1);
expect(playing.some((row) => row[i])).toBe(true);
expect(mgr.entries[i].start).toBe(start);
expect(mgr.entries[i].duration).toBe(duration);
// at most the entry seek per take window (promotion adds one aligned seek at most)
expect(tracks[i].seeks.length).toBeLessThanOrEqual(2);
});
// a muted video is picture only: no proxy plays its file's audio on top of its take's audio
expect(mgr.entries.every((m) => m.source?.tagName === "AUDIO")).toBe(true);
});
it("keeps every take silent outside its window", () => {
mountTakes();
const mgr = makeManager({ isPaused: false });
mgr.setupFromIframe(document);
mgr.entries.forEach((m) => trackSeeks(m.el));
mgr.promoteToParentProxy(document);
mgr.mirrorTime(12);
expect(mgr.entries.map((m) => !m.el.paused)).toEqual([false, true, false]);
mgr.mirrorTime(24.5);
expect(mgr.entries.map((m) => !m.el.paused)).toEqual([false, false, true]);
});
it("seeks a take to its media start plus the time into its window on a user seek", () => {
mountTakes();
const mgr = makeManager();
mgr.setupFromIframe(document);
const tracks = mgr.entries.map((m) => trackSeeks(m.el));
mgr.seekAll(10);
expect(tracks[1].seeks.map(Math.round)).toEqual([22]);
expect(tracks[0].seeks).toEqual([]);
});
it("positions an upcoming take at its media start before its window opens", () => {
mountTakes();
const mgr = makeManager({ isPaused: false });
mgr.setupFromIframe(document);
const tracks = mgr.entries.map((m) => trackSeeks(m.el));
mgr.promoteToParentProxy(document);
mgr.mirrorTime(6.4); // inside take 1's window, 1.6 s before take 2
expect(tracks[1].seeks.at(-1)).toBeUndefined();
mgr.mirrorTime(7.0);
expect(tracks[1].seeks.map(Math.round)).toEqual([20]);
expect(mgr.entries[1].el.paused).toBe(true);
// at the cut it only plays: no second seek
mgr.mirrorTime(8.0);
expect(mgr.entries[1].el.paused).toBe(false);
expect(tracks[1].seeks.map(Math.round)).toEqual([20]);
});
});
describe("ParentMediaManager drift policy", () => {
afterEach(() => {
document.body.replaceChildren();
vi.restoreAllMocks();
});
// A playing proxy `lag` seconds behind a timeline at `t`, a clip that starts at 0 and plays from media 0.
function playingProxy(lag: number, t = 5, tag: "audio" | "video" = "audio") {
const mgr = makeManager({ isPaused: false });
const el = makeFakeAudio(false, tag);
const writes: number[] = [];
let time = t - lag;
Object.defineProperty(el, "currentTime", {
get: () => time,
set: (v: number) => {
time = v;
writes.push(v);
},
});
mgr.entries.push({ el, start: 0, duration: 30, mediaStart: 0 });
return { mgr, el, writes, setTime: (v: number) => (time = v) };
}
it("leaves a proxy inside the sync tolerance alone", () => {
const { mgr, el, writes, setTime } = playingProxy(0.03);
mgr.mirrorTime(5);
setTime(5.07);
mgr.mirrorTime(5.1);
expect(writes).toEqual([]);
expect(el.playbackRate).toBe(1);
});
it("steers a proxy that lags a little by rate, capped at 10%, without seeking", () => {
const { mgr, el, writes } = playingProxy(0.07);
mgr.mirrorTime(5);
mgr.mirrorTime(5.08);
expect(writes).toEqual([]);
expect(el.playbackRate).toBeGreaterThan(1);
expect(el.playbackRate).toBeLessThanOrEqual(1.1);
});
it("returns a steered proxy to its authored rate once it has caught up", () => {
const { mgr, el, setTime } = playingProxy(0.07);
mgr.mirrorTime(5);
setTime(5.1);
mgr.mirrorTime(5.1);
expect(el.playbackRate).toBe(1);
});
it("seeks a proxy that holds a lag past 100 ms, once", () => {
const { mgr, writes, setTime } = playingProxy(0.25);
mgr.mirrorTime(5); // first sample: offset not yet known to hold
setTime(4.85); // the proxy advanced with the timeline: the lag held
mgr.mirrorTime(5.1);
expect(writes).toHaveLength(1);
expect(writes[0]).toBeCloseTo(mediaElementTime(5.1), 9);
});
it("seeks a video proxy that holds a 1 s lag: a proxy is heard, whatever its tag", () => {
const { mgr, el, writes, setTime } = playingProxy(1, 5, "video");
mgr.mirrorTime(5);
setTime(4.1);
mgr.mirrorTime(5.1);
expect(writes).toHaveLength(1);
expect(el.playbackRate).toBe(1);
});
it("steers, never seeks, a proxy whose lag keeps growing because it is stalled", () => {
const { mgr, el, writes } = playingProxy(0.25);
mgr.mirrorTime(5);
mgr.mirrorTime(5.1); // currentTime did not move: lag grew by 100 ms
expect(writes).toEqual([]);
expect(el.playbackRate).toBeLessThanOrEqual(1.1);
});
it("seeks at once on a forced alignment when the proxy is off the playhead", () => {
const { mgr, writes } = playingProxy(2);
mgr.mirrorTime(5, { force: true });
expect(writes).toHaveLength(1);
expect(writes[0]).toBeCloseTo(mediaElementTime(5), 9);
});
it("aims a seek of a playing proxy ahead by the time its last seek took to land", () => {
let now = 1000;
vi.spyOn(performance, "now").mockImplementation(() => now);
const mgr = makeManager({ isPaused: false });
mgr.setupFromUrl("https://example.test/take.mp3");
const el = mgr.entries[0].el;
const writes: number[] = [];
let time = 4.75;
Object.defineProperty(el, "currentTime", {
get: () => time,
set: (v: number) => {
time = v;
writes.push(v);
},
});
Object.defineProperty(el, "paused", { get: () => false });
mgr.promoteToParentProxy(null, () => {}); // owner parent without the alignment seek
mgr.mirrorTime(5); // 250 ms behind
time = 4.85;
mgr.mirrorTime(5.1); // the lag held: seek, no landing time known yet
now = 1230;
el.dispatchEvent(new Event("seeked")); // it took 230 ms to land
time = 5.0; // landed 230 ms late, lagging the playhead again
mgr.mirrorTime(5.33);
time = 5.1;
mgr.mirrorTime(5.43);
expect(writes).toHaveLength(2);
expect(writes[0]).toBeCloseTo(mediaElementTime(5.1), 9);
expect(writes[1]).toBeCloseTo(mediaElementTime(5.43 + 0.23), 9);
});
});
packages/player/src/parent-media.ts
+107 -47
View File
@@ -12,40 +12,42 @@
import { selectMediaObserverTargets } from "./mediaObserverScope.js";
import { isRealmElement, isRealmHtmlMediaElement } from "./media-element-guards.js";
import { isInClipWindow } from "@hyperframes/core/runtime/clip-window";
import { mediaElementTime } from "@hyperframes/core/runtime/media-clock";
import { readClipTiming } from "@hyperframes/core/composition-contract";
import { readLinkTiming } from "@hyperframes/core/media-link";
import { mediaElementTime } from "@hyperframes/core/runtime/media-clock";
import {
MEDIA_SYNC_TOLERANCE_SECONDS,
forgetSteering,
steerMedia,
} from "@hyperframes/core/runtime/media-steer";
/** Minimum absolute drift before a currentTime correction is attempted. */
const MIRROR_DRIFT_THRESHOLD_SECONDS = 0.05;
/** An offset that moved less than this since the last sample is not growing: the element is not stalled. */
const SETTLED_OFFSET_DELTA_SECONDS = 0.004;
/**
* How many *consecutive* over-threshold samples are required before issuing a
* `currentTime` write. Absorbs single-sample jitter (GC pause, slow bridge
* tick) without thrashing. Forced calls bypass this gate.
*
* Worst-case correction latency ≈ this × bridgeMaxPostIntervalMs (80 ms in
* core/runtime/state.ts) = 160 ms — well under human A/V re-sync tolerance.
* How far ahead of its window a paused proxy is positioned at its media start, so the cut
* only has to `play()`: a seek into unbuffered media lands about 80 ms late.
*/
const MIRROR_REQUIRED_CONSECUTIVE_DRIFT_SAMPLES = 2;
const PRESEEK_SECONDS = 1.5;
export interface ProxyEntry {
el: HTMLMediaElement;
start: number;
duration: number;
/** Where in the file the clip begins (`data-media-start`); the proxy plays `t - start + mediaStart`. */
mediaStart: number;
/**
* The iframe media element this proxy mirrors, when adopted from the DOM.
* Its `data-start`/`data-duration` are re-read each tick so live timeline
* edits (trim/move) bound the proxy correctly. Null for URL-driven proxies.
*/
source?: HTMLMediaElement | null;
/**
* Count of consecutive steady-state samples in which the proxy's
* `currentTime` was found drifted beyond `MIRROR_DRIFT_THRESHOLD_SECONDS`.
* Reset on every in-threshold sample. A write is only issued once this
* reaches `MIRROR_REQUIRED_CONSECUTIVE_DRIFT_SAMPLES`, absorbing
* single-sample jitter without thrashing.
*/
driftSamples: number;
/** The offset behind the timeline (+) at the last drift sample; steering is only trusted when it stopped growing. */
lastOffset?: number;
/** Seconds a seek of this playing proxy takes to land (smoothed); a seek aims this far ahead so it lands on time. */
seekLatency?: number;
/** When the seek still landing was issued (ms); unset when none is pending. */
seekIssuedAt?: number;
}
export class ParentMediaManager {
@@ -157,6 +159,7 @@ export class ParentMediaManager {
// gate never closes and the proxy plays past its clip end.
const timing = readClipTiming(m.source);
m.start = timing.start ?? 0;
m.mediaStart = readLinkTiming(m.source).mediaStart;
m.duration =
timing.duration != null && timing.duration > 0 ? timing.duration : Number.POSITIVE_INFINITY;
}
@@ -167,10 +170,45 @@ export class ParentMediaManager {
return isInClipWindow(timeSeconds, m.start, m.start + m.duration);
}
private _preseekUpcoming(m: ProxyEntry, timelineSeconds: number): void {
const ahead = m.start - timelineSeconds;
if (ahead <= 0 || ahead > PRESEEK_SECONDS || !m.el.paused) return;
if (Math.abs(m.el.currentTime - m.mediaStart) > MEDIA_SYNC_TOLERANCE_SECONDS) {
this._seekTo(m, m.mediaStart);
}
}
// The one writer of a proxy's position: every seek lands here. A seek of a playing proxy aims ahead by
// the time its seeks take to land: aimed at the playhead it lands behind it and the next tick seeks again.
private _seekTo(m: ProxyEntry, mediaSeconds: number, playing = false): void {
const lead = playing ? (m.seekLatency ?? 0) * this._getPlaybackRate() : 0;
m.seekIssuedAt = playing ? performance.now() : undefined;
m.el.currentTime = mediaElementTime(mediaSeconds + lead);
}
private _noteSeekLanded(m: ProxyEntry): void {
if (m.seekIssuedAt === undefined) return;
const landed = (performance.now() - m.seekIssuedAt) / 1000;
m.seekLatency = m.seekLatency === undefined ? landed : (m.seekLatency + landed) / 2;
m.seekIssuedAt = undefined;
}
// Back to the authored rate with no drift history: the proxy was seeked, paused or left its window.
private _releaseSteering(m: ProxyEntry): void {
forgetSteering(m.el);
m.lastOffset = undefined;
const rate = this._getPlaybackRate();
if (m.el.playbackRate !== rate) m.el.playbackRate = rate;
}
private _mediaTime(m: ProxyEntry, timelineSeconds: number): number {
return timelineSeconds - m.start + m.mediaStart;
}
private _gateEntryPlayback(m: ProxyEntry, timeSeconds: number): boolean {
if (!this._inWindow(m, timeSeconds)) {
if (!m.el.paused) m.el.pause();
m.driftSamples = 0;
this._releaseSteering(m);
return false;
}
if (this._audioOwner === "parent" && !this._isPaused() && m.el.paused) this._playEntry(m);
@@ -196,8 +234,7 @@ export class ParentMediaManager {
// Re-read live bounds so a trim/move just before a paused scrub gates and
// positions against the current clip window, not the adopt-time one.
this._refreshEntryBounds(m);
if (this._inWindow(m, timeInSeconds))
m.el.currentTime = mediaElementTime(timeInSeconds - m.start);
if (this._inWindow(m, timeInSeconds)) this._jumpTo(m, this._mediaTime(m, timeInSeconds));
}
}
@@ -212,7 +249,7 @@ export class ParentMediaManager {
for (const m of this._entries) {
this._refreshEntryBounds(m);
if (this._inWindow(m, timeInSeconds)) {
m.el.currentTime = mediaElementTime(timeInSeconds - m.start);
this._jumpTo(m, this._mediaTime(m, timeInSeconds));
this._playEntry(m);
} else if (!m.el.paused) {
m.el.pause();
@@ -221,29 +258,43 @@ export class ParentMediaManager {
}
/**
* Mirror parent-proxy `currentTime` to the iframe timeline, with optional
* jitter-coalescing. Pass `{ force: true }` for alignment moments (ownership
* promotion, new proxy initialization) where drift must be corrected
* immediately.
* Mirror parent-proxy `currentTime` to the iframe timeline. Drift is steered by rate and
* seeked only past the player's one drift policy (`steerMedia`). Pass `{ force: true }`
* for alignment moments (ownership promotion, new proxy initialization) where a proxy off
* the playhead must be seeked immediately.
*/
mirrorTime(timelineSeconds: number, options?: { force?: boolean }): void {
const force = options?.force === true;
for (const m of this._entries) {
this._refreshEntryBounds(m);
this._preseekUpcoming(m, timelineSeconds);
// A proxy entering its window that was not pre-positioned (a user seek) is seeked once, before it plays.
const entering = m.el.paused;
if (!this._gateEntryPlayback(m, timelineSeconds)) continue;
const relTime = timelineSeconds - m.start;
if (Math.abs(m.el.currentTime - relTime) > MIRROR_DRIFT_THRESHOLD_SECONDS) {
m.driftSamples += 1;
if (force || m.driftSamples >= MIRROR_REQUIRED_CONSECUTIVE_DRIFT_SAMPLES) {
m.el.currentTime = mediaElementTime(relTime);
m.driftSamples = 0;
}
} else {
m.driftSamples = 0;
const target = this._mediaTime(m, timelineSeconds);
const offset = target - m.el.currentTime;
if (force || entering) {
if (Math.abs(offset) > MEDIA_SYNC_TOLERANCE_SECONDS) this._jumpTo(m, target);
continue;
}
this._steer(m, target, offset);
}
}
private _steer(m: ProxyEntry, target: number, offset: number): void {
const settled =
m.lastOffset !== undefined && Math.abs(offset - m.lastOffset) < SETTLED_OFFSET_DELTA_SECONDS;
m.lastOffset = offset;
const { rate, seek } = steerMedia(m.el, offset, this._getPlaybackRate(), settled, true);
if (seek) this._jumpTo(m, target, true);
else if (Math.abs(m.el.playbackRate - rate) > 1e-6) m.el.playbackRate = rate;
}
private _jumpTo(m: ProxyEntry, mediaSeconds: number, playing = false): void {
this._seekTo(m, mediaSeconds, playing);
this._releaseSteering(m);
}
/**
* Take ownership of audible playback in response to the runtime's
* `media-autoplay-blocked` signal. Idempotent.
@@ -347,8 +398,9 @@ export class ParentMediaManager {
}
/**
* Create a parent-frame media element and start preloading it. Returns the
* new entry, or `null` if a proxy for this src already exists (dedup).
* Create a parent-frame media element and start preloading it. A URL-driven
* proxy returns `null` when the file already has one; a clip always gets its
* own, since its window and media offset are its own.
*/
private _createEntry(
src: string,
@@ -357,7 +409,7 @@ export class ParentMediaManager {
duration: number,
source?: HTMLMediaElement | null,
): ProxyEntry | null {
if (this._urlTaken(src)) return null;
if (!source && this._urlTaken(src)) return null;
const el = tag === "video" ? document.createElement("video") : new Audio();
el.preload = "auto";
@@ -368,7 +420,14 @@ export class ParentMediaManager {
const rate = this._getPlaybackRate();
if (rate !== 1) el.playbackRate = rate;
const entry: ProxyEntry = { el, start, duration, driftSamples: 0, source };
const entry: ProxyEntry = {
el,
start,
duration,
mediaStart: source ? readLinkTiming(source).mediaStart : 0,
source,
};
el.addEventListener("seeked", () => this._noteSeekLanded(entry));
this._entries.push(entry);
return entry;
}
@@ -386,6 +445,11 @@ export class ParentMediaManager {
// "auto" and drops its proxy, which fetches the whole file, when it turns back.
if (iframeEl.preload === "metadata" || iframeEl.preload === "none") return;
if (this._entries.some((m) => m.source === iframeEl)) return;
// A video authored muted is picture only: its file's audio belongs to the clip that plays it.
if (iframeEl.tagName === "VIDEO" && iframeEl.hasAttribute("muted")) return;
const src = this._resolveIframeMediaSrc(iframeEl);
if (!src) return;
@@ -414,11 +478,11 @@ export class ParentMediaManager {
if (!this._isPaused()) this._playEntryIfActive(entry);
}
/** One proxy per file: a second clip on the same file is heard through the first one's proxy.
* A proxy's file is its clip's current one (a clip that left the page owns none). */
private _urlTaken(src: string, except?: ProxyEntry): boolean {
/** Whether a clip's proxy already plays this file. A proxy's file is its clip's current one
* (a clip that left the page owns none). */
private _urlTaken(src: string): boolean {
return this._entries.some((m) => {
if (m === except || (m.source && !m.source.isConnected)) return false;
if (m.source && !m.source.isConnected) return false;
return ((m.source && this._resolveIframeMediaSrc(m.source)) || m.el.src) === src;
});
}
@@ -426,10 +490,6 @@ export class ParentMediaManager {
private _repointToSource(entry: ProxyEntry): boolean {
const src = entry.source ? this._resolveIframeMediaSrc(entry.source) : null;
if (!src || entry.el.src === src) return false;
if (this._urlTaken(src, entry)) {
this._removeEntry(entry);
return false;
}
entry.el.src = src;
entry.el.load();
return true;