fix(core): preview playback moves every frame between an audio track's time updates (#7373)

This commit is contained in:
Miguel Ángel
2026-11-07 22:50:52 +00:00
committed by GitHub
parent 841e90ffe2
commit 942893eb2c
5 changed files with 228 additions and 3 deletions
@@ -0,0 +1,108 @@
import { afterEach, beforeEach, describe, expect, it, vi } from "vitest";
import { TransportClock } from "./clock";
import { syncRuntimeMedia, type RuntimeMediaClip } from "./media";
// Chromium moves an element's currentTime in steps; the element plays at whatever rate it is given.
function steppingAudio(stepMs: number) {
const el = document.createElement("audio");
document.body.appendChild(el);
let position = 0;
let rate = 1;
let rateWrites = 0;
let seeks = 0;
Object.defineProperties(el, {
paused: { configurable: true, get: () => false },
seeking: { configurable: true, get: () => false },
readyState: { configurable: true, get: () => 4 },
duration: { configurable: true, get: () => 60 },
playbackRate: {
configurable: true,
get: () => rate,
set: (v: number) => {
rateWrites += 1;
rate = v;
},
},
currentTime: {
configurable: true,
get: () => (Math.floor((position * 1000) / stepMs) * stepMs) / 1000,
set: (v: number) => {
seeks += 1;
position = v;
},
},
});
el.play = vi.fn(() => Promise.resolve());
el.pause = vi.fn();
return {
el,
advance: (dtMs: number) => (position += (dtMs / 1000) * rate),
rateWrites: () => rateWrites,
seeks: () => seeks,
};
}
/** Plays one audio clip the clock follows for `seconds` of `hz` display frames, syncing media as the runtime does. */
function play(stepMs: number, hz: number, seconds = 10) {
let ms = 0;
const audio = steppingAudio(stepMs);
const clock = new TransportClock({ nowMs: () => ms, duration: 60 });
const clip: RuntimeMediaClip = {
el: audio.el,
start: 0,
mediaStart: 0,
duration: 60,
end: 60,
volume: null,
playbackRate: 1,
loop: false,
sourceDuration: null,
};
clock.play();
const tickMs = 1000 / hz;
for (let tick = 1; tick <= seconds * hz; tick++) {
ms = tick * tickMs;
audio.advance(tickMs);
clock.attachAudioSource({ el: audio.el, compositionStart: 0, mediaStart: 0 });
syncRuntimeMedia({
clips: [clip],
timeSeconds: clock.now(),
playing: true,
playbackRate: 1,
clockAhead: (el) => clock.aheadOf(el),
getCompositionDuration: () => 60,
});
}
return { rateWrites: audio.rateWrites(), seeks: audio.seeks(), filmSpeed: clock.now() / seconds };
}
describe("a clock that runs between the steps of the audio it follows, as Chromium reports it", () => {
beforeEach(() => {
vi.spyOn(navigator, "userAgent", "get").mockReturnValue(
"Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/140.0.0.0 Safari/537.36",
);
});
afterEach(() => {
vi.restoreAllMocks();
document.body.innerHTML = "";
});
it("does not steer that audio against its own lead, so the film keeps its speed", () => {
for (const [stepMs, hz] of [
[45, 60],
[50, 144],
[60, 60],
[70, 60],
[90, 120],
]) {
const run = play(stepMs, hz);
expect(run.rateWrites, `${stepMs} ms steps at ${hz} Hz: rate writes`).toBe(0);
expect(run.seeks, `${stepMs} ms steps at ${hz} Hz: seeks`).toBe(0);
expect(
Math.abs(run.filmSpeed - 1),
`${stepMs} ms steps at ${hz} Hz: film speed`,
).toBeLessThan(0.001);
}
});
});
+57
View File
@@ -284,6 +284,63 @@ describe("TransportClock", () => {
expect(clock.getSource()).toBe("audio");
});
/**
* Plays 1/60 s ticks against an element whose time moves in `stepMs` steps at `speed`; frames are 30 fps. Frames
* count from the 7th tick: until the element's second update the clock does not know its step.
*/
function playStepping(ticks: number, stepMs: number, speed = 1, stallAfterMs = Infinity) {
const { clock, advance, getMs } = createClock({ duration: 60 });
const el = createMockAudioEl(0, false);
clock.play();
clock.attachAudioSource({ el, compositionStart: 0, mediaStart: 0 });
let last = 0;
let skipped = 0;
let heldTicks = 0;
let longestHold = 0;
let mostAhead = 0;
let back = 0;
let previous = 0;
for (let tick = 1; tick <= ticks; tick++) {
advance(1000 / 60);
const played = Math.min(getMs(), stallAfterMs) * speed;
el.currentTime = (Math.floor(played / stepMs) * stepMs) / 1000;
const now = clock.now();
if (now < previous) back += 1;
previous = now;
mostAhead = Math.max(mostAhead, now - el.currentTime);
const frame = Math.floor(now * 30 + 1e-6);
if (tick > 6 && frame > last + 1) skipped += 1;
heldTicks = frame === last ? heldTicks + 1 : 0;
if (tick > 6) longestHold = Math.max(longestHold, heldTicks);
last = frame;
}
return { skipped, longestHold, mostAhead, back, end: clock.now() };
}
it("moves on every display frame between the element's time updates, never skipping a film frame", () => {
for (const stepMs of [30, 40, 50]) {
const run = playStepping(240, stepMs);
expect(run.skipped, `${stepMs} ms steps: film frames skipped`).toBe(0);
// A 30 fps frame spans two 60 Hz ticks: held for one extra tick, never two.
expect(run.longestHold, `${stepMs} ms steps: ticks a frame is held`).toBeLessThanOrEqual(1);
expect(run.back, `${stepMs} ms steps: reads that went back`).toBe(0);
expect(run.mostAhead, `${stepMs} ms steps: lead over the element`).toBeLessThanOrEqual(
stepMs / 1000 + 1e-9,
);
}
});
it("stops within one of the element's own updates when the element stops moving", () => {
const run = playStepping(120, 40, 1, 1000);
expect(run.end).toBeLessThanOrEqual(1 + 0.04 + 1e-9);
});
it("stays within one update of an element that plays slightly slower than the wall clock", () => {
const run = playStepping(600, 40, 0.98);
expect(run.mostAhead).toBeLessThanOrEqual(0.04 + 1e-9);
expect(run.back).toBe(0);
});
it("names the clock it follows: an audio element, WebAudio, or neither", () => {
const { clock } = createClock({ duration: 10 });
expect(clock.followedClock()).toBe("monotonic");
+60 -2
View File
@@ -28,8 +28,27 @@ const STALL_ADJUSTED_LAG_MS = 33;
const HAVE_FUTURE_DATA = 3;
const AUDIO_START_WAIT_MS = 1500;
const MEDIA_SYNC_STEER_TOLERANCE_SECONDS = 0.04;
const MAX_MEDIA_STEP_SECONDS = 0.25;
const MEDIA_STEPS_KEPT = 8;
type PendingAudioStart = { sinceMs: number; at: number; from: number | null };
type MediaReading = {
el: HTMLMediaElement;
time: number;
steps: number[];
from: number;
fromMs: number;
};
const furthestAhead = (reading: MediaReading | null) =>
reading ? Math.max(0, ...reading.steps) : 0;
function noteStep(reading: MediaReading, time: number): void {
const step = time - reading.time;
if (step > 0 && step <= MAX_MEDIA_STEP_SECONDS)
reading.steps = [...reading.steps.slice(1 - MEDIA_STEPS_KEPT), step];
reading.time = time;
}
export class TransportClock {
private _baseTime = 0;
@@ -45,6 +64,8 @@ export class TransportClock {
private _lastReadMs: number | null = null;
private _lastNow = 0;
private _audioStart: PendingAudioStart | null = null;
private _reading: MediaReading | null = null;
private _ahead: { el: HTMLMediaElement; seconds: number } | null = null;
constructor(opts?: {
initialTime?: number;
@@ -59,6 +80,7 @@ export class TransportClock {
}
now(): number {
this._ahead = null;
if (this._playStartMs === null) return this._baseTime;
// Audio-master: drift is impossible because audio IS the clock. Clearing
@@ -97,12 +119,43 @@ export class TransportClock {
const spec = rate ?? 1;
const time = timeAtSourceTime(spec, el.currentTime - mediaStart) + compositionStart;
const held = this._heldUntilAudioOutputStarts(el.seeking, time);
if (held !== undefined) return held;
if (time >= this._lastNow) return time;
if (held !== undefined) {
this._reading = null;
return held;
}
const between = this._betweenUpdates(el, time);
if (between !== null) return between;
const wrapped = el.loop && this._lastNow - time > timeAtSourceTime(spec, el.duration) / 2;
return !wrapped && el.readyState < HAVE_FUTURE_DATA ? this._lastNow : null;
}
// Chromium moves currentTime in 10-50 ms steps, so reading it as is holds a frame, then jumps. Run on at the play
// rate, kept within [element time, element time + its largest recent step], never back; null when time is behind
// the playhead by more than that step (a late clip, not a step).
private _betweenUpdates(el: HTMLMediaElement, time: number): number | null {
const nowMs = this._nowMs();
const last = this._reading?.el === el ? this._reading : null;
if (time < this._lastNow - furthestAhead(last)) return null;
if (!last) {
this._reading = { el, time, steps: [], from: time, fromMs: nowMs };
return this._runAhead(el, time, Math.max(time, this._lastNow));
}
noteStep(last, time);
const run = last.from + ((nowMs - last.fromMs) / 1000) * this._rate;
const held = Math.min(Math.max(run, time), time + furthestAhead(last));
if (held !== run) Object.assign(last, { from: held, fromMs: nowMs });
return this._runAhead(el, time, Math.max(held, this._lastNow));
}
private _runAhead(el: HTMLMediaElement, time: number, now: number): number {
this._ahead = { el, seconds: now - time };
return now;
}
aheadOf(el: HTMLMediaElement): number {
return this._ahead?.el === el ? this._ahead.seconds : 0;
}
private _heldUntilAudioOutputStarts(seeking: boolean, time: number): number | null | undefined {
const pending = this._pendingAudioStart();
if (seeking) return pending ? this._lastNow : null;
@@ -139,6 +192,7 @@ export class TransportClock {
this._playStartMs = this._nowMs();
this._audioStart = { sinceMs: this._playStartMs, at: this._baseTime, from: null };
this._lastNow = this._baseTime;
this._reading = null;
// Not a stall: the gap since the clock was last read (possibly a long
// paused idle) says nothing about lost playback time, since none was
// playing. `_applyStallCorrection` treats null as "nothing to compare
@@ -151,6 +205,7 @@ export class TransportClock {
if (this._playStartMs === null) return false;
this._baseTime = this.now();
this._playStartMs = null;
this._reading = null;
return true;
}
@@ -160,6 +215,7 @@ export class TransportClock {
: Math.max(0, timeSeconds);
this._baseTime = clamped;
this._lastNow = clamped;
this._reading = null;
if (this._playStartMs !== null) {
this._playStartMs = this._nowMs();
this._audioStart = { sinceMs: this._playStartMs, at: clamped, from: null };
@@ -175,6 +231,7 @@ export class TransportClock {
setRate(rate: number): void {
const safe = Number.isFinite(rate) && rate > 0 ? Math.max(0.1, Math.min(5, rate)) : 1;
this._reading = null;
if (this._playStartMs !== null) {
this._baseTime = this.now();
this._playStartMs = this._nowMs();
@@ -230,6 +287,7 @@ export class TransportClock {
this._lastReadMs = null;
}
this._audioSource = null;
this._reading = null;
}
/** The element the playhead currently follows, if the source is one. */
+1
View File
@@ -3826,6 +3826,7 @@ export function initSandboxRuntimeModular(): void {
forceSync,
keepLateMedia,
cueAheadSeconds,
clockAhead: (el) => clock.aheadOf(el),
clockAdvancing:
clock.getSource() !== "audio" ||
lastSyncedMediaTimeSeconds === null ||
+2 -1
View File
@@ -499,6 +499,7 @@ export function syncRuntimeMedia(params: {
* outbound message; further invocations are suppressed by the caller.
*/
onAutoplayBlocked?: () => void;
clockAhead?: (el: HTMLMediaElement) => number;
/** `clipGain` is `data-volume` for a routed clip, played ahead of its FX; 1 otherwise. */
onElementVolume?: (
el: HTMLMediaElement,
@@ -557,7 +558,7 @@ export function syncRuntimeMedia(params: {
!inWindow &&
(params.timeSeconds >= clip.end || sameInstant(params.timeSeconds, clip.end)) &&
isClipVisibleAt(params.timeSeconds, clip.start, clip.end, params.getCompositionDuration());
const relTime = mediaSourceTimeAt(clip, params.timeSeconds);
const relTime = mediaSourceTimeAt(clip, params.timeSeconds - (params.clockAhead?.(el) ?? 0));
const isHeldVideoTail =
isTerminalVideo ||
(isNonLoopVideo && clip.sourceDuration != null && relTime >= clip.sourceDuration && inWindow);