mirror of
https://github.com/heygen-com/hyperframes.git
synced 2026-11-18 07:50:13 +00:00
fix(core): preview playback moves every frame between an audio track's time updates (#7373)
This commit is contained in:
@@ -0,0 +1,108 @@
|
||||
import { afterEach, beforeEach, describe, expect, it, vi } from "vitest";
|
||||
import { TransportClock } from "./clock";
|
||||
import { syncRuntimeMedia, type RuntimeMediaClip } from "./media";
|
||||
|
||||
// Chromium moves an element's currentTime in steps; the element plays at whatever rate it is given.
|
||||
function steppingAudio(stepMs: number) {
|
||||
const el = document.createElement("audio");
|
||||
document.body.appendChild(el);
|
||||
let position = 0;
|
||||
let rate = 1;
|
||||
let rateWrites = 0;
|
||||
let seeks = 0;
|
||||
Object.defineProperties(el, {
|
||||
paused: { configurable: true, get: () => false },
|
||||
seeking: { configurable: true, get: () => false },
|
||||
readyState: { configurable: true, get: () => 4 },
|
||||
duration: { configurable: true, get: () => 60 },
|
||||
playbackRate: {
|
||||
configurable: true,
|
||||
get: () => rate,
|
||||
set: (v: number) => {
|
||||
rateWrites += 1;
|
||||
rate = v;
|
||||
},
|
||||
},
|
||||
currentTime: {
|
||||
configurable: true,
|
||||
get: () => (Math.floor((position * 1000) / stepMs) * stepMs) / 1000,
|
||||
set: (v: number) => {
|
||||
seeks += 1;
|
||||
position = v;
|
||||
},
|
||||
},
|
||||
});
|
||||
el.play = vi.fn(() => Promise.resolve());
|
||||
el.pause = vi.fn();
|
||||
return {
|
||||
el,
|
||||
advance: (dtMs: number) => (position += (dtMs / 1000) * rate),
|
||||
rateWrites: () => rateWrites,
|
||||
seeks: () => seeks,
|
||||
};
|
||||
}
|
||||
|
||||
/** Plays one audio clip the clock follows for `seconds` of `hz` display frames, syncing media as the runtime does. */
|
||||
function play(stepMs: number, hz: number, seconds = 10) {
|
||||
let ms = 0;
|
||||
const audio = steppingAudio(stepMs);
|
||||
const clock = new TransportClock({ nowMs: () => ms, duration: 60 });
|
||||
const clip: RuntimeMediaClip = {
|
||||
el: audio.el,
|
||||
start: 0,
|
||||
mediaStart: 0,
|
||||
duration: 60,
|
||||
end: 60,
|
||||
volume: null,
|
||||
playbackRate: 1,
|
||||
loop: false,
|
||||
sourceDuration: null,
|
||||
};
|
||||
clock.play();
|
||||
const tickMs = 1000 / hz;
|
||||
for (let tick = 1; tick <= seconds * hz; tick++) {
|
||||
ms = tick * tickMs;
|
||||
audio.advance(tickMs);
|
||||
clock.attachAudioSource({ el: audio.el, compositionStart: 0, mediaStart: 0 });
|
||||
syncRuntimeMedia({
|
||||
clips: [clip],
|
||||
timeSeconds: clock.now(),
|
||||
playing: true,
|
||||
playbackRate: 1,
|
||||
clockAhead: (el) => clock.aheadOf(el),
|
||||
getCompositionDuration: () => 60,
|
||||
});
|
||||
}
|
||||
return { rateWrites: audio.rateWrites(), seeks: audio.seeks(), filmSpeed: clock.now() / seconds };
|
||||
}
|
||||
|
||||
describe("a clock that runs between the steps of the audio it follows, as Chromium reports it", () => {
|
||||
beforeEach(() => {
|
||||
vi.spyOn(navigator, "userAgent", "get").mockReturnValue(
|
||||
"Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/140.0.0.0 Safari/537.36",
|
||||
);
|
||||
});
|
||||
|
||||
afterEach(() => {
|
||||
vi.restoreAllMocks();
|
||||
document.body.innerHTML = "";
|
||||
});
|
||||
|
||||
it("does not steer that audio against its own lead, so the film keeps its speed", () => {
|
||||
for (const [stepMs, hz] of [
|
||||
[45, 60],
|
||||
[50, 144],
|
||||
[60, 60],
|
||||
[70, 60],
|
||||
[90, 120],
|
||||
]) {
|
||||
const run = play(stepMs, hz);
|
||||
expect(run.rateWrites, `${stepMs} ms steps at ${hz} Hz: rate writes`).toBe(0);
|
||||
expect(run.seeks, `${stepMs} ms steps at ${hz} Hz: seeks`).toBe(0);
|
||||
expect(
|
||||
Math.abs(run.filmSpeed - 1),
|
||||
`${stepMs} ms steps at ${hz} Hz: film speed`,
|
||||
).toBeLessThan(0.001);
|
||||
}
|
||||
});
|
||||
});
|
||||
@@ -284,6 +284,63 @@ describe("TransportClock", () => {
|
||||
expect(clock.getSource()).toBe("audio");
|
||||
});
|
||||
|
||||
/**
|
||||
* Plays 1/60 s ticks against an element whose time moves in `stepMs` steps at `speed`; frames are 30 fps. Frames
|
||||
* count from the 7th tick: until the element's second update the clock does not know its step.
|
||||
*/
|
||||
function playStepping(ticks: number, stepMs: number, speed = 1, stallAfterMs = Infinity) {
|
||||
const { clock, advance, getMs } = createClock({ duration: 60 });
|
||||
const el = createMockAudioEl(0, false);
|
||||
clock.play();
|
||||
clock.attachAudioSource({ el, compositionStart: 0, mediaStart: 0 });
|
||||
let last = 0;
|
||||
let skipped = 0;
|
||||
let heldTicks = 0;
|
||||
let longestHold = 0;
|
||||
let mostAhead = 0;
|
||||
let back = 0;
|
||||
let previous = 0;
|
||||
for (let tick = 1; tick <= ticks; tick++) {
|
||||
advance(1000 / 60);
|
||||
const played = Math.min(getMs(), stallAfterMs) * speed;
|
||||
el.currentTime = (Math.floor(played / stepMs) * stepMs) / 1000;
|
||||
const now = clock.now();
|
||||
if (now < previous) back += 1;
|
||||
previous = now;
|
||||
mostAhead = Math.max(mostAhead, now - el.currentTime);
|
||||
const frame = Math.floor(now * 30 + 1e-6);
|
||||
if (tick > 6 && frame > last + 1) skipped += 1;
|
||||
heldTicks = frame === last ? heldTicks + 1 : 0;
|
||||
if (tick > 6) longestHold = Math.max(longestHold, heldTicks);
|
||||
last = frame;
|
||||
}
|
||||
return { skipped, longestHold, mostAhead, back, end: clock.now() };
|
||||
}
|
||||
|
||||
it("moves on every display frame between the element's time updates, never skipping a film frame", () => {
|
||||
for (const stepMs of [30, 40, 50]) {
|
||||
const run = playStepping(240, stepMs);
|
||||
expect(run.skipped, `${stepMs} ms steps: film frames skipped`).toBe(0);
|
||||
// A 30 fps frame spans two 60 Hz ticks: held for one extra tick, never two.
|
||||
expect(run.longestHold, `${stepMs} ms steps: ticks a frame is held`).toBeLessThanOrEqual(1);
|
||||
expect(run.back, `${stepMs} ms steps: reads that went back`).toBe(0);
|
||||
expect(run.mostAhead, `${stepMs} ms steps: lead over the element`).toBeLessThanOrEqual(
|
||||
stepMs / 1000 + 1e-9,
|
||||
);
|
||||
}
|
||||
});
|
||||
|
||||
it("stops within one of the element's own updates when the element stops moving", () => {
|
||||
const run = playStepping(120, 40, 1, 1000);
|
||||
expect(run.end).toBeLessThanOrEqual(1 + 0.04 + 1e-9);
|
||||
});
|
||||
|
||||
it("stays within one update of an element that plays slightly slower than the wall clock", () => {
|
||||
const run = playStepping(600, 40, 0.98);
|
||||
expect(run.mostAhead).toBeLessThanOrEqual(0.04 + 1e-9);
|
||||
expect(run.back).toBe(0);
|
||||
});
|
||||
|
||||
it("names the clock it follows: an audio element, WebAudio, or neither", () => {
|
||||
const { clock } = createClock({ duration: 10 });
|
||||
expect(clock.followedClock()).toBe("monotonic");
|
||||
|
||||
@@ -28,8 +28,27 @@ const STALL_ADJUSTED_LAG_MS = 33;
|
||||
const HAVE_FUTURE_DATA = 3;
|
||||
const AUDIO_START_WAIT_MS = 1500;
|
||||
const MEDIA_SYNC_STEER_TOLERANCE_SECONDS = 0.04;
|
||||
const MAX_MEDIA_STEP_SECONDS = 0.25;
|
||||
const MEDIA_STEPS_KEPT = 8;
|
||||
|
||||
type PendingAudioStart = { sinceMs: number; at: number; from: number | null };
|
||||
type MediaReading = {
|
||||
el: HTMLMediaElement;
|
||||
time: number;
|
||||
steps: number[];
|
||||
from: number;
|
||||
fromMs: number;
|
||||
};
|
||||
|
||||
const furthestAhead = (reading: MediaReading | null) =>
|
||||
reading ? Math.max(0, ...reading.steps) : 0;
|
||||
|
||||
function noteStep(reading: MediaReading, time: number): void {
|
||||
const step = time - reading.time;
|
||||
if (step > 0 && step <= MAX_MEDIA_STEP_SECONDS)
|
||||
reading.steps = [...reading.steps.slice(1 - MEDIA_STEPS_KEPT), step];
|
||||
reading.time = time;
|
||||
}
|
||||
|
||||
export class TransportClock {
|
||||
private _baseTime = 0;
|
||||
@@ -45,6 +64,8 @@ export class TransportClock {
|
||||
private _lastReadMs: number | null = null;
|
||||
private _lastNow = 0;
|
||||
private _audioStart: PendingAudioStart | null = null;
|
||||
private _reading: MediaReading | null = null;
|
||||
private _ahead: { el: HTMLMediaElement; seconds: number } | null = null;
|
||||
|
||||
constructor(opts?: {
|
||||
initialTime?: number;
|
||||
@@ -59,6 +80,7 @@ export class TransportClock {
|
||||
}
|
||||
|
||||
now(): number {
|
||||
this._ahead = null;
|
||||
if (this._playStartMs === null) return this._baseTime;
|
||||
|
||||
// Audio-master: drift is impossible because audio IS the clock. Clearing
|
||||
@@ -97,12 +119,43 @@ export class TransportClock {
|
||||
const spec = rate ?? 1;
|
||||
const time = timeAtSourceTime(spec, el.currentTime - mediaStart) + compositionStart;
|
||||
const held = this._heldUntilAudioOutputStarts(el.seeking, time);
|
||||
if (held !== undefined) return held;
|
||||
if (time >= this._lastNow) return time;
|
||||
if (held !== undefined) {
|
||||
this._reading = null;
|
||||
return held;
|
||||
}
|
||||
const between = this._betweenUpdates(el, time);
|
||||
if (between !== null) return between;
|
||||
const wrapped = el.loop && this._lastNow - time > timeAtSourceTime(spec, el.duration) / 2;
|
||||
return !wrapped && el.readyState < HAVE_FUTURE_DATA ? this._lastNow : null;
|
||||
}
|
||||
|
||||
// Chromium moves currentTime in 10-50 ms steps, so reading it as is holds a frame, then jumps. Run on at the play
|
||||
// rate, kept within [element time, element time + its largest recent step], never back; null when time is behind
|
||||
// the playhead by more than that step (a late clip, not a step).
|
||||
private _betweenUpdates(el: HTMLMediaElement, time: number): number | null {
|
||||
const nowMs = this._nowMs();
|
||||
const last = this._reading?.el === el ? this._reading : null;
|
||||
if (time < this._lastNow - furthestAhead(last)) return null;
|
||||
if (!last) {
|
||||
this._reading = { el, time, steps: [], from: time, fromMs: nowMs };
|
||||
return this._runAhead(el, time, Math.max(time, this._lastNow));
|
||||
}
|
||||
noteStep(last, time);
|
||||
const run = last.from + ((nowMs - last.fromMs) / 1000) * this._rate;
|
||||
const held = Math.min(Math.max(run, time), time + furthestAhead(last));
|
||||
if (held !== run) Object.assign(last, { from: held, fromMs: nowMs });
|
||||
return this._runAhead(el, time, Math.max(held, this._lastNow));
|
||||
}
|
||||
|
||||
private _runAhead(el: HTMLMediaElement, time: number, now: number): number {
|
||||
this._ahead = { el, seconds: now - time };
|
||||
return now;
|
||||
}
|
||||
|
||||
aheadOf(el: HTMLMediaElement): number {
|
||||
return this._ahead?.el === el ? this._ahead.seconds : 0;
|
||||
}
|
||||
|
||||
private _heldUntilAudioOutputStarts(seeking: boolean, time: number): number | null | undefined {
|
||||
const pending = this._pendingAudioStart();
|
||||
if (seeking) return pending ? this._lastNow : null;
|
||||
@@ -139,6 +192,7 @@ export class TransportClock {
|
||||
this._playStartMs = this._nowMs();
|
||||
this._audioStart = { sinceMs: this._playStartMs, at: this._baseTime, from: null };
|
||||
this._lastNow = this._baseTime;
|
||||
this._reading = null;
|
||||
// Not a stall: the gap since the clock was last read (possibly a long
|
||||
// paused idle) says nothing about lost playback time, since none was
|
||||
// playing. `_applyStallCorrection` treats null as "nothing to compare
|
||||
@@ -151,6 +205,7 @@ export class TransportClock {
|
||||
if (this._playStartMs === null) return false;
|
||||
this._baseTime = this.now();
|
||||
this._playStartMs = null;
|
||||
this._reading = null;
|
||||
return true;
|
||||
}
|
||||
|
||||
@@ -160,6 +215,7 @@ export class TransportClock {
|
||||
: Math.max(0, timeSeconds);
|
||||
this._baseTime = clamped;
|
||||
this._lastNow = clamped;
|
||||
this._reading = null;
|
||||
if (this._playStartMs !== null) {
|
||||
this._playStartMs = this._nowMs();
|
||||
this._audioStart = { sinceMs: this._playStartMs, at: clamped, from: null };
|
||||
@@ -175,6 +231,7 @@ export class TransportClock {
|
||||
|
||||
setRate(rate: number): void {
|
||||
const safe = Number.isFinite(rate) && rate > 0 ? Math.max(0.1, Math.min(5, rate)) : 1;
|
||||
this._reading = null;
|
||||
if (this._playStartMs !== null) {
|
||||
this._baseTime = this.now();
|
||||
this._playStartMs = this._nowMs();
|
||||
@@ -230,6 +287,7 @@ export class TransportClock {
|
||||
this._lastReadMs = null;
|
||||
}
|
||||
this._audioSource = null;
|
||||
this._reading = null;
|
||||
}
|
||||
|
||||
/** The element the playhead currently follows, if the source is one. */
|
||||
|
||||
@@ -3826,6 +3826,7 @@ export function initSandboxRuntimeModular(): void {
|
||||
forceSync,
|
||||
keepLateMedia,
|
||||
cueAheadSeconds,
|
||||
clockAhead: (el) => clock.aheadOf(el),
|
||||
clockAdvancing:
|
||||
clock.getSource() !== "audio" ||
|
||||
lastSyncedMediaTimeSeconds === null ||
|
||||
|
||||
@@ -499,6 +499,7 @@ export function syncRuntimeMedia(params: {
|
||||
* outbound message; further invocations are suppressed by the caller.
|
||||
*/
|
||||
onAutoplayBlocked?: () => void;
|
||||
clockAhead?: (el: HTMLMediaElement) => number;
|
||||
/** `clipGain` is `data-volume` for a routed clip, played ahead of its FX; 1 otherwise. */
|
||||
onElementVolume?: (
|
||||
el: HTMLMediaElement,
|
||||
@@ -557,7 +558,7 @@ export function syncRuntimeMedia(params: {
|
||||
!inWindow &&
|
||||
(params.timeSeconds >= clip.end || sameInstant(params.timeSeconds, clip.end)) &&
|
||||
isClipVisibleAt(params.timeSeconds, clip.start, clip.end, params.getCompositionDuration());
|
||||
const relTime = mediaSourceTimeAt(clip, params.timeSeconds);
|
||||
const relTime = mediaSourceTimeAt(clip, params.timeSeconds - (params.clockAhead?.(el) ?? 0));
|
||||
const isHeldVideoTail =
|
||||
isTerminalVideo ||
|
||||
(isNonLoopVideo && clip.sourceDuration != null && relTime >= clip.sourceDuration && inWindow);
|
||||
|
||||
Reference in New Issue
Block a user