mirror of
https://github.com/heygen-com/hyperframes.git
synced 2026-09-03 04:38:33 +00:00
perf(player): coalesce _mirrorParentMediaTime writes (#396)
## Summary Coalesce writes to `el.currentTime` inside `_mirrorParentMediaTime` so a single jitter sample no longer triggers a parent-media seek. A drift correction now requires **two consecutive samples** above the threshold (~`MIRROR_DRIFT_THRESHOLD_SECONDS`) before the player writes back. One-shot alignment paths (`promoteToParentProxy`, `_onIframeMediaAdded`) opt out via `force: true` so initial alignment stays immediate. ## Why Step `P1-4` of the player perf proposal. `_mirrorParentMediaTime` is called every animation frame on parent media proxies. Even without true drift, browser internals report tiny jitter on `currentTime` reads — typically below 30 ms but occasionally crossing the threshold for a frame. Writing to `currentTime` triggers a seek, which is expensive *and* invalidates pipeline buffers, which causes the next frame's reading to jitter further. The result was unnecessary seek thrash on otherwise-aligned media. By requiring two consecutive over-threshold samples, transient jitter is filtered out while real drift (a sustained offset) still corrects within ~1 frame of latency. This eliminates the most common cause of dropped frames on the studio thumbnail grid. ## What changed - Each `_parentMedia` entry gains a `driftSamples` counter that increments while the absolute drift is above `MIRROR_DRIFT_THRESHOLD_SECONDS` and resets to 0 on the first sample below. - `_mirrorParentMediaTime(el, opts)` only writes back when `driftSamples >= 2`, except when `opts.force === true`. - `promoteToParentProxy` and `_onIframeMediaAdded` pass `force: true` so the first alignment after registration is still immediate (these are user-visible state transitions, not steady-state telemetry). ## Test plan - [x] 11 new unit/integration tests in `hyperframes-player.test.ts` covering: - Single-sample jitter does not trigger a write. - Two-sample sustained drift does trigger a write. - Trending drift correction (gradually increasing offset) is detected within 2 samples. - `force: true` override bypasses the sample requirement. - Out-of-range proxies (proxies whose source has been removed) do not panic. - Multiple proxies maintain independent counters — drift on one does not affect the other. - `_promoteToParentProxy` alignment is immediate. ## Stack Step `P1-4` of the player perf proposal. Builds on `P1-1` (shared adopted stylesheets) and `P1-2` (scoped media observer). Together these three target the studio multi-player render path — `P0-1*` perf gate scenarios will pick up the wins automatically.
This commit is contained in:
@@ -30,6 +30,27 @@ export type RuntimeState = {
|
||||
bridgeLastPostedAt: number;
|
||||
bridgeLastPostedPlaying: boolean;
|
||||
bridgeLastPostedMuted: boolean;
|
||||
/**
|
||||
* Max interval (ms) between outbound timeline samples on the parent-frame
|
||||
* control bridge. The bridge posts on every changed frame, but also at
|
||||
* least once per this interval so a paused/idle timeline still confirms
|
||||
* its position to any listener.
|
||||
*
|
||||
* **Cross-reference (do not change in isolation)**: the parent-frame
|
||||
* audio-mirror loop in `<hyperframes-player>` waits for
|
||||
* `MIRROR_REQUIRED_CONSECUTIVE_DRIFT_SAMPLES` consecutive over-threshold
|
||||
* samples before issuing a `currentTime` correction. The product of
|
||||
* those two constants is the worst-case A/V re-sync latency:
|
||||
*
|
||||
* worst_case_correction_latency_ms
|
||||
* ≈ MIRROR_REQUIRED_CONSECUTIVE_DRIFT_SAMPLES × bridgeMaxPostIntervalMs
|
||||
*
|
||||
* Today: `2 × 80 ms = 160 ms`, which sits comfortably under the
|
||||
* perceptual A/V re-sync tolerance. If you raise this interval, audit
|
||||
* `MIRROR_REQUIRED_CONSECUTIVE_DRIFT_SAMPLES` in
|
||||
* `packages/player/src/hyperframes-player.ts` — leaving it at `2` will
|
||||
* silently push correction latency past the tolerance budget.
|
||||
*/
|
||||
bridgeMaxPostIntervalMs: number;
|
||||
timelinePollIntervalId: ReturnType<typeof setInterval> | null;
|
||||
controlBridgeHandler: ((event: MessageEvent) => void) | null;
|
||||
|
||||
@@ -440,3 +440,203 @@ describe("HyperframesPlayer media MutationObserver scoping", () => {
|
||||
expect(observeSpy.mock.calls[0]?.[0]).toBe(fakeDoc.body);
|
||||
});
|
||||
});
|
||||
|
||||
// ── Parent-proxy time-mirror coalescing ──
|
||||
//
|
||||
// `_mirrorParentMediaTime` is the steady-state correction loop that nudges
|
||||
// every parent-frame audio/video proxy back onto the iframe's timeline. The
|
||||
// post-`P1-4` contract: a single over-threshold sample (one slow bridge tick,
|
||||
// one tab-throttled rAF, one GC pause) is absorbed by a per-proxy counter and
|
||||
// does NOT cost a `currentTime` write. Only a *trending* drift — two
|
||||
// consecutive samples above the 50 ms threshold — triggers a seek. Forced
|
||||
// callers (audio-ownership promotion, brand-new proxy initialization) bypass
|
||||
// the gate so the listener never hears a misaligned sample on cut-over.
|
||||
|
||||
describe("HyperframesPlayer parent-proxy time-mirror coalescing", () => {
|
||||
type DriftEntry = {
|
||||
el: { currentTime: number; src: string; pause: () => void };
|
||||
start: number;
|
||||
duration: number;
|
||||
driftSamples: number;
|
||||
};
|
||||
type PlayerInternal = HTMLElement & {
|
||||
_parentMedia: DriftEntry[];
|
||||
_mirrorParentMediaTime: (timelineSeconds: number, options?: { force?: boolean }) => void;
|
||||
_promoteToParentProxy?: () => void;
|
||||
};
|
||||
|
||||
let player: PlayerInternal;
|
||||
|
||||
beforeEach(async () => {
|
||||
await import("./hyperframes-player.js");
|
||||
player = document.createElement("hyperframes-player") as PlayerInternal;
|
||||
document.body.appendChild(player);
|
||||
// No audio-src was set, so `_parentMedia` is empty. Tests push synthetic
|
||||
// POJO entries — `_mirrorParentMediaTime` only reads/writes
|
||||
// `el.currentTime`, so a plain object stands in fine for HTMLMediaElement.
|
||||
});
|
||||
|
||||
afterEach(() => {
|
||||
player.remove();
|
||||
vi.restoreAllMocks();
|
||||
});
|
||||
|
||||
function makeEntry(
|
||||
opts: {
|
||||
currentTime?: number;
|
||||
start?: number;
|
||||
duration?: number;
|
||||
driftSamples?: number;
|
||||
} = {},
|
||||
): DriftEntry {
|
||||
// Include `pause`/`src` so `disconnectedCallback`'s teardown loop
|
||||
// (`m.el.pause(); m.el.src = ""`) doesn't blow up when the player is
|
||||
// removed at the end of the test — `_mirrorParentMediaTime` itself only
|
||||
// touches `currentTime`.
|
||||
const entry: DriftEntry = {
|
||||
el: {
|
||||
currentTime: opts.currentTime ?? 0,
|
||||
src: "",
|
||||
pause: vi.fn(),
|
||||
},
|
||||
start: opts.start ?? 0,
|
||||
duration: opts.duration ?? 100,
|
||||
driftSamples: opts.driftSamples ?? 0,
|
||||
};
|
||||
player._parentMedia.push(entry);
|
||||
return entry;
|
||||
}
|
||||
|
||||
it("initializes new parent-media entries with driftSamples=0", () => {
|
||||
// Mock Audio just for this test so the audio-src bootstrap path produces
|
||||
// a real entry rather than throwing on construction.
|
||||
const mockAudio = {
|
||||
src: "",
|
||||
preload: "",
|
||||
muted: false,
|
||||
playbackRate: 1,
|
||||
currentTime: 0,
|
||||
paused: true,
|
||||
play: vi.fn().mockResolvedValue(undefined),
|
||||
pause: vi.fn(),
|
||||
load: vi.fn(),
|
||||
};
|
||||
vi.spyOn(globalThis, "Audio").mockImplementation(
|
||||
() => mockAudio as unknown as HTMLAudioElement,
|
||||
);
|
||||
|
||||
const fresh = document.createElement("hyperframes-player") as PlayerInternal;
|
||||
fresh.setAttribute("audio-src", "https://cdn.example.com/narration.mp3");
|
||||
document.body.appendChild(fresh);
|
||||
|
||||
expect(fresh._parentMedia).toHaveLength(1);
|
||||
expect(fresh._parentMedia[0]?.driftSamples).toBe(0);
|
||||
fresh.remove();
|
||||
});
|
||||
|
||||
it("does nothing when drift is within the 50 ms threshold", () => {
|
||||
const m = makeEntry({ currentTime: 5 });
|
||||
player._mirrorParentMediaTime(5.04);
|
||||
expect(m.el.currentTime).toBe(5);
|
||||
expect(m.driftSamples).toBe(0);
|
||||
});
|
||||
|
||||
it("absorbs a single over-threshold spike without writing currentTime", () => {
|
||||
const m = makeEntry({ currentTime: 5 });
|
||||
player._mirrorParentMediaTime(5.5);
|
||||
expect(m.el.currentTime).toBe(5);
|
||||
expect(m.driftSamples).toBe(1);
|
||||
});
|
||||
|
||||
it("issues a seek on the second consecutive over-threshold sample", () => {
|
||||
const m = makeEntry({ currentTime: 5 });
|
||||
player._mirrorParentMediaTime(5.5);
|
||||
expect(m.el.currentTime).toBe(5);
|
||||
expect(m.driftSamples).toBe(1);
|
||||
// Second sample with the same drift: the gate trips, the write fires,
|
||||
// and the counter resets so the proxy doesn't re-seek every later tick.
|
||||
player._mirrorParentMediaTime(5.5);
|
||||
expect(m.el.currentTime).toBe(5.5);
|
||||
expect(m.driftSamples).toBe(0);
|
||||
});
|
||||
|
||||
it("resets the counter when a sample comes back within threshold", () => {
|
||||
const m = makeEntry({ currentTime: 5 });
|
||||
player._mirrorParentMediaTime(5.5);
|
||||
expect(m.driftSamples).toBe(1);
|
||||
// Recovery — counter must clear so a later isolated spike doesn't
|
||||
// accidentally satisfy the 2-sample gate by piggy-backing on stale state.
|
||||
player._mirrorParentMediaTime(5.02);
|
||||
expect(m.driftSamples).toBe(0);
|
||||
expect(m.el.currentTime).toBe(5);
|
||||
player._mirrorParentMediaTime(5.5);
|
||||
expect(m.driftSamples).toBe(1);
|
||||
expect(m.el.currentTime).toBe(5);
|
||||
});
|
||||
|
||||
it("force: true writes immediately on the first over-threshold sample", () => {
|
||||
const m = makeEntry({ currentTime: 5 });
|
||||
player._mirrorParentMediaTime(5.5, { force: true });
|
||||
expect(m.el.currentTime).toBe(5.5);
|
||||
expect(m.driftSamples).toBe(0);
|
||||
});
|
||||
|
||||
it("force: true clears any pre-existing drift counter", () => {
|
||||
const m = makeEntry({ currentTime: 5, driftSamples: 1 });
|
||||
player._mirrorParentMediaTime(5.5, { force: true });
|
||||
expect(m.el.currentTime).toBe(5.5);
|
||||
expect(m.driftSamples).toBe(0);
|
||||
});
|
||||
|
||||
it("does not seek out-of-range entries and resets their counters", () => {
|
||||
// Active window [10, 15). currentTime=99 is a sentinel — if the function
|
||||
// ever writes inside an out-of-range branch the test catches it because
|
||||
// relTime would be 5 (or 15), not 99.
|
||||
const m = makeEntry({
|
||||
currentTime: 99,
|
||||
start: 10,
|
||||
duration: 5,
|
||||
driftSamples: 5,
|
||||
});
|
||||
player._mirrorParentMediaTime(5);
|
||||
expect(m.el.currentTime).toBe(99);
|
||||
expect(m.driftSamples).toBe(0);
|
||||
// Boundary: relTime === duration → still out of range (the loop uses `>=`).
|
||||
m.driftSamples = 7;
|
||||
player._mirrorParentMediaTime(15);
|
||||
expect(m.el.currentTime).toBe(99);
|
||||
expect(m.driftSamples).toBe(0);
|
||||
});
|
||||
|
||||
it("tracks drift independently across multiple proxies", () => {
|
||||
// a is drifted; b is aligned. A single tick must increment a's counter
|
||||
// and reset b's — proving the per-entry state is genuinely per-entry.
|
||||
const a = makeEntry({ currentTime: 5 });
|
||||
const b = makeEntry({ currentTime: 7.01, driftSamples: 1 });
|
||||
player._mirrorParentMediaTime(7);
|
||||
expect(a.el.currentTime).toBe(5);
|
||||
expect(a.driftSamples).toBe(1);
|
||||
expect(b.el.currentTime).toBe(7.01);
|
||||
expect(b.driftSamples).toBe(0);
|
||||
});
|
||||
|
||||
it("force: true bypasses the gate for every proxy in a single sweep", () => {
|
||||
const a = makeEntry({ currentTime: 5 });
|
||||
const b = makeEntry({ currentTime: 8 });
|
||||
player._mirrorParentMediaTime(7, { force: true });
|
||||
expect(a.el.currentTime).toBe(7);
|
||||
expect(b.el.currentTime).toBe(7);
|
||||
expect(a.driftSamples).toBe(0);
|
||||
expect(b.driftSamples).toBe(0);
|
||||
});
|
||||
|
||||
it("_promoteToParentProxy invokes _mirrorParentMediaTime with force: true", () => {
|
||||
// Integration check of the promotion call site — we cannot tolerate even
|
||||
// ~80 ms of audible drift across an ownership flip, so the call site
|
||||
// must opt out of the jitter gate.
|
||||
const spy = vi.spyOn(player, "_mirrorParentMediaTime");
|
||||
player._promoteToParentProxy?.();
|
||||
const forcedCall = spy.mock.calls.find(([, opts]) => opts?.force === true);
|
||||
expect(forcedCall).toBeDefined();
|
||||
});
|
||||
});
|
||||
|
||||
@@ -60,6 +60,16 @@ class HyperframesPlayer extends HTMLElement {
|
||||
el: HTMLMediaElement;
|
||||
start: number;
|
||||
duration: number;
|
||||
/**
|
||||
* Count of consecutive steady-state samples in which the proxy's
|
||||
* `currentTime` was found drifted beyond `MIRROR_DRIFT_THRESHOLD_SECONDS`.
|
||||
* Reset on every in-threshold sample. `_mirrorParentMediaTime` only
|
||||
* issues a write once this passes `MIRROR_REQUIRED_CONSECUTIVE_DRIFT_SAMPLES`,
|
||||
* which absorbs single-sample jitter (e.g. one slow bridge tick) without
|
||||
* thrashing the media element with seeks. Forced calls (promotion,
|
||||
* media-added) bypass the gate and reset the counter.
|
||||
*/
|
||||
driftSamples: number;
|
||||
}> = [];
|
||||
|
||||
/**
|
||||
@@ -631,12 +641,62 @@ class HyperframesPlayer extends HTMLElement {
|
||||
*/
|
||||
private static readonly MIRROR_DRIFT_THRESHOLD_SECONDS = 0.05;
|
||||
|
||||
private _mirrorParentMediaTime(timelineSeconds: number) {
|
||||
/**
|
||||
* How many *consecutive* over-threshold steady-state samples we wait for
|
||||
* before issuing a `currentTime` write. A value of 2 means a single
|
||||
* spike (one slow bridge tick, one tab-throttled rAF batch, one GC pause)
|
||||
* is absorbed without a seek; sustained drift still corrects on the very
|
||||
* next tick after the threshold is crossed twice in a row.
|
||||
*
|
||||
* **Coupling with the timeline-control bridge** — read before changing:
|
||||
* worst_case_correction_latency_ms
|
||||
* ≈ MIRROR_REQUIRED_CONSECUTIVE_DRIFT_SAMPLES × bridgeMaxPostIntervalMs
|
||||
*
|
||||
* `bridgeMaxPostIntervalMs` (currently `80`) lives at
|
||||
* `packages/core/src/runtime/state.ts` (field on `RuntimeState`). At
|
||||
* today's values, worst-case is `2 × 80 ms = 160 ms` — still well under
|
||||
* the human shot-change tolerance for A/V re-sync. If you bump bridge
|
||||
* cadence (raising `bridgeMaxPostIntervalMs`) you may need to drop this
|
||||
* constant to `1` to keep the product under ~150 ms; if you tighten
|
||||
* cadence you can raise this to absorb more jitter without perceptual
|
||||
* cost. There is a back-reference in `state.ts` next to
|
||||
* `bridgeMaxPostIntervalMs` so a change to either side surfaces the
|
||||
* coupling.
|
||||
*/
|
||||
private static readonly MIRROR_REQUIRED_CONSECUTIVE_DRIFT_SAMPLES = 2;
|
||||
|
||||
/**
|
||||
* Mirror parent-proxy `currentTime` to the iframe timeline. Defaults to
|
||||
* the *coalesced* path: a single over-threshold sample is treated as
|
||||
* jitter and merely increments a per-proxy counter; the actual seek only
|
||||
* fires once `MIRROR_REQUIRED_CONSECUTIVE_DRIFT_SAMPLES` consecutive
|
||||
* samples agree. Pass `{ force: true }` for one-shot alignment moments
|
||||
* (audio-ownership promotion, brand-new proxy initialization) where we
|
||||
* cannot tolerate even ~80 ms of misaligned audible playback.
|
||||
*
|
||||
* The counter is also reset on any in-threshold sample and on any
|
||||
* out-of-range timeline position, so a proxy that drops back into a
|
||||
* scene later starts fresh rather than carrying stale samples from the
|
||||
* last time it was active.
|
||||
*/
|
||||
private _mirrorParentMediaTime(timelineSeconds: number, options?: { force?: boolean }) {
|
||||
const force = options?.force === true;
|
||||
const requiredSamples = HyperframesPlayer.MIRROR_REQUIRED_CONSECUTIVE_DRIFT_SAMPLES;
|
||||
const threshold = HyperframesPlayer.MIRROR_DRIFT_THRESHOLD_SECONDS;
|
||||
for (const m of this._parentMedia) {
|
||||
const relTime = timelineSeconds - m.start;
|
||||
if (relTime < 0 || relTime >= m.duration) continue;
|
||||
if (Math.abs(m.el.currentTime - relTime) > HyperframesPlayer.MIRROR_DRIFT_THRESHOLD_SECONDS) {
|
||||
m.el.currentTime = relTime;
|
||||
if (relTime < 0 || relTime >= m.duration) {
|
||||
m.driftSamples = 0;
|
||||
continue;
|
||||
}
|
||||
if (Math.abs(m.el.currentTime - relTime) > threshold) {
|
||||
m.driftSamples += 1;
|
||||
if (force || m.driftSamples >= requiredSamples) {
|
||||
m.el.currentTime = relTime;
|
||||
m.driftSamples = 0;
|
||||
}
|
||||
} else {
|
||||
m.driftSamples = 0;
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -668,7 +728,10 @@ class HyperframesPlayer extends HTMLElement {
|
||||
// precisely because the scenario that triggered promotion is
|
||||
// "autoplay blocked" — the iframe can't make noise on its own.
|
||||
this._sendControl("set-media-output-muted", { muted: true });
|
||||
this._mirrorParentMediaTime(this._currentTime);
|
||||
// One-shot alignment: a brand-new proxy must pick up the iframe's exact
|
||||
// timeline position immediately to avoid an audible jump. Bypass the
|
||||
// jitter-coalescing gate.
|
||||
this._mirrorParentMediaTime(this._currentTime, { force: true });
|
||||
if (!this._paused) this._playParentMedia();
|
||||
this.dispatchEvent(
|
||||
new CustomEvent("audioownershipchange", {
|
||||
@@ -688,7 +751,7 @@ class HyperframesPlayer extends HTMLElement {
|
||||
tag: "audio" | "video",
|
||||
start: number,
|
||||
duration: number,
|
||||
): { el: HTMLMediaElement; start: number; duration: number } | null {
|
||||
): { el: HTMLMediaElement; start: number; duration: number; driftSamples: number } | null {
|
||||
// Deduplicate — browsers normalize URLs so we compare on the element after assignment
|
||||
if (this._parentMedia.some((m) => m.el.src === src)) return null;
|
||||
|
||||
@@ -699,7 +762,7 @@ class HyperframesPlayer extends HTMLElement {
|
||||
el.muted = this.muted;
|
||||
if (this.playbackRate !== 1) el.playbackRate = this.playbackRate;
|
||||
|
||||
const entry = { el, start, duration };
|
||||
const entry = { el, start, duration, driftSamples: 0 };
|
||||
this._parentMedia.push(entry);
|
||||
return entry;
|
||||
}
|
||||
@@ -778,7 +841,10 @@ class HyperframesPlayer extends HTMLElement {
|
||||
// start producing audio right away — otherwise it sits silent through
|
||||
// the next several hundred ms until the next runtime state message.
|
||||
if (created && this._audioOwner === "parent") {
|
||||
this._mirrorParentMediaTime(this._currentTime);
|
||||
// One-shot alignment: a freshly-created proxy must catch up to the
|
||||
// current timeline position on the very first sample, so bypass the
|
||||
// jitter-coalescing gate.
|
||||
this._mirrorParentMediaTime(this._currentTime, { force: true });
|
||||
if (!this._paused && created.el.src) {
|
||||
created.el.play().catch((err: unknown) => this._reportPlaybackError(err));
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user