diff --git a/packages/core/src/runtime/clock.steer.test.ts b/packages/core/src/runtime/clock.steer.test.ts new file mode 100644 index 0000000000..ff88d85dad --- /dev/null +++ b/packages/core/src/runtime/clock.steer.test.ts @@ -0,0 +1,108 @@ +import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; +import { TransportClock } from "./clock"; +import { syncRuntimeMedia, type RuntimeMediaClip } from "./media"; + +// Chromium moves an element's currentTime in steps; the element plays at whatever rate it is given. +function steppingAudio(stepMs: number) { + const el = document.createElement("audio"); + document.body.appendChild(el); + let position = 0; + let rate = 1; + let rateWrites = 0; + let seeks = 0; + Object.defineProperties(el, { + paused: { configurable: true, get: () => false }, + seeking: { configurable: true, get: () => false }, + readyState: { configurable: true, get: () => 4 }, + duration: { configurable: true, get: () => 60 }, + playbackRate: { + configurable: true, + get: () => rate, + set: (v: number) => { + rateWrites += 1; + rate = v; + }, + }, + currentTime: { + configurable: true, + get: () => (Math.floor((position * 1000) / stepMs) * stepMs) / 1000, + set: (v: number) => { + seeks += 1; + position = v; + }, + }, + }); + el.play = vi.fn(() => Promise.resolve()); + el.pause = vi.fn(); + return { + el, + advance: (dtMs: number) => (position += (dtMs / 1000) * rate), + rateWrites: () => rateWrites, + seeks: () => seeks, + }; +} + +/** Plays one audio clip the clock follows for `seconds` of `hz` display frames, syncing media as the runtime does. */ +function play(stepMs: number, hz: number, seconds = 10) { + let ms = 0; + const audio = steppingAudio(stepMs); + const clock = new TransportClock({ nowMs: () => ms, duration: 60 }); + const clip: RuntimeMediaClip = { + el: audio.el, + start: 0, + mediaStart: 0, + duration: 60, + end: 60, + volume: null, + playbackRate: 1, + loop: false, + sourceDuration: null, + }; + clock.play(); + const tickMs = 1000 / hz; + for (let tick = 1; tick <= seconds * hz; tick++) { + ms = tick * tickMs; + audio.advance(tickMs); + clock.attachAudioSource({ el: audio.el, compositionStart: 0, mediaStart: 0 }); + syncRuntimeMedia({ + clips: [clip], + timeSeconds: clock.now(), + playing: true, + playbackRate: 1, + clockAhead: (el) => clock.aheadOf(el), + getCompositionDuration: () => 60, + }); + } + return { rateWrites: audio.rateWrites(), seeks: audio.seeks(), filmSpeed: clock.now() / seconds }; +} + +describe("a clock that runs between the steps of the audio it follows, as Chromium reports it", () => { + beforeEach(() => { + vi.spyOn(navigator, "userAgent", "get").mockReturnValue( + "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/140.0.0.0 Safari/537.36", + ); + }); + + afterEach(() => { + vi.restoreAllMocks(); + document.body.innerHTML = ""; + }); + + it("does not steer that audio against its own lead, so the film keeps its speed", () => { + for (const [stepMs, hz] of [ + [45, 60], + [50, 144], + [60, 60], + [70, 60], + [90, 120], + ]) { + const run = play(stepMs, hz); + expect(run.rateWrites, `${stepMs} ms steps at ${hz} Hz: rate writes`).toBe(0); + expect(run.seeks, `${stepMs} ms steps at ${hz} Hz: seeks`).toBe(0); + expect( + Math.abs(run.filmSpeed - 1), + `${stepMs} ms steps at ${hz} Hz: film speed`, + ).toBeLessThan(0.001); + } + }); +}); diff --git a/packages/core/src/runtime/clock.test.ts b/packages/core/src/runtime/clock.test.ts index 1d497c0c66..431acc8027 100644 --- a/packages/core/src/runtime/clock.test.ts +++ b/packages/core/src/runtime/clock.test.ts @@ -284,6 +284,63 @@ describe("TransportClock", () => { expect(clock.getSource()).toBe("audio"); }); + /** + * Plays 1/60 s ticks against an element whose time moves in `stepMs` steps at `speed`; frames are 30 fps. Frames + * count from the 7th tick: until the element's second update the clock does not know its step. + */ + function playStepping(ticks: number, stepMs: number, speed = 1, stallAfterMs = Infinity) { + const { clock, advance, getMs } = createClock({ duration: 60 }); + const el = createMockAudioEl(0, false); + clock.play(); + clock.attachAudioSource({ el, compositionStart: 0, mediaStart: 0 }); + let last = 0; + let skipped = 0; + let heldTicks = 0; + let longestHold = 0; + let mostAhead = 0; + let back = 0; + let previous = 0; + for (let tick = 1; tick <= ticks; tick++) { + advance(1000 / 60); + const played = Math.min(getMs(), stallAfterMs) * speed; + el.currentTime = (Math.floor(played / stepMs) * stepMs) / 1000; + const now = clock.now(); + if (now < previous) back += 1; + previous = now; + mostAhead = Math.max(mostAhead, now - el.currentTime); + const frame = Math.floor(now * 30 + 1e-6); + if (tick > 6 && frame > last + 1) skipped += 1; + heldTicks = frame === last ? heldTicks + 1 : 0; + if (tick > 6) longestHold = Math.max(longestHold, heldTicks); + last = frame; + } + return { skipped, longestHold, mostAhead, back, end: clock.now() }; + } + + it("moves on every display frame between the element's time updates, never skipping a film frame", () => { + for (const stepMs of [30, 40, 50]) { + const run = playStepping(240, stepMs); + expect(run.skipped, `${stepMs} ms steps: film frames skipped`).toBe(0); + // A 30 fps frame spans two 60 Hz ticks: held for one extra tick, never two. + expect(run.longestHold, `${stepMs} ms steps: ticks a frame is held`).toBeLessThanOrEqual(1); + expect(run.back, `${stepMs} ms steps: reads that went back`).toBe(0); + expect(run.mostAhead, `${stepMs} ms steps: lead over the element`).toBeLessThanOrEqual( + stepMs / 1000 + 1e-9, + ); + } + }); + + it("stops within one of the element's own updates when the element stops moving", () => { + const run = playStepping(120, 40, 1, 1000); + expect(run.end).toBeLessThanOrEqual(1 + 0.04 + 1e-9); + }); + + it("stays within one update of an element that plays slightly slower than the wall clock", () => { + const run = playStepping(600, 40, 0.98); + expect(run.mostAhead).toBeLessThanOrEqual(0.04 + 1e-9); + expect(run.back).toBe(0); + }); + it("names the clock it follows: an audio element, WebAudio, or neither", () => { const { clock } = createClock({ duration: 10 }); expect(clock.followedClock()).toBe("monotonic"); diff --git a/packages/core/src/runtime/clock.ts b/packages/core/src/runtime/clock.ts index cacf384686..c7d556606d 100644 --- a/packages/core/src/runtime/clock.ts +++ b/packages/core/src/runtime/clock.ts @@ -28,8 +28,27 @@ const STALL_ADJUSTED_LAG_MS = 33; const HAVE_FUTURE_DATA = 3; const AUDIO_START_WAIT_MS = 1500; const MEDIA_SYNC_STEER_TOLERANCE_SECONDS = 0.04; +const MAX_MEDIA_STEP_SECONDS = 0.25; +const MEDIA_STEPS_KEPT = 8; type PendingAudioStart = { sinceMs: number; at: number; from: number | null }; +type MediaReading = { + el: HTMLMediaElement; + time: number; + steps: number[]; + from: number; + fromMs: number; +}; + +const furthestAhead = (reading: MediaReading | null) => + reading ? Math.max(0, ...reading.steps) : 0; + +function noteStep(reading: MediaReading, time: number): void { + const step = time - reading.time; + if (step > 0 && step <= MAX_MEDIA_STEP_SECONDS) + reading.steps = [...reading.steps.slice(1 - MEDIA_STEPS_KEPT), step]; + reading.time = time; +} export class TransportClock { private _baseTime = 0; @@ -45,6 +64,8 @@ export class TransportClock { private _lastReadMs: number | null = null; private _lastNow = 0; private _audioStart: PendingAudioStart | null = null; + private _reading: MediaReading | null = null; + private _ahead: { el: HTMLMediaElement; seconds: number } | null = null; constructor(opts?: { initialTime?: number; @@ -59,6 +80,7 @@ export class TransportClock { } now(): number { + this._ahead = null; if (this._playStartMs === null) return this._baseTime; // Audio-master: drift is impossible because audio IS the clock. Clearing @@ -97,12 +119,43 @@ export class TransportClock { const spec = rate ?? 1; const time = timeAtSourceTime(spec, el.currentTime - mediaStart) + compositionStart; const held = this._heldUntilAudioOutputStarts(el.seeking, time); - if (held !== undefined) return held; - if (time >= this._lastNow) return time; + if (held !== undefined) { + this._reading = null; + return held; + } + const between = this._betweenUpdates(el, time); + if (between !== null) return between; const wrapped = el.loop && this._lastNow - time > timeAtSourceTime(spec, el.duration) / 2; return !wrapped && el.readyState < HAVE_FUTURE_DATA ? this._lastNow : null; } + // Chromium moves currentTime in 10-50 ms steps, so reading it as is holds a frame, then jumps. Run on at the play + // rate, kept within [element time, element time + its largest recent step], never back; null when time is behind + // the playhead by more than that step (a late clip, not a step). + private _betweenUpdates(el: HTMLMediaElement, time: number): number | null { + const nowMs = this._nowMs(); + const last = this._reading?.el === el ? this._reading : null; + if (time < this._lastNow - furthestAhead(last)) return null; + if (!last) { + this._reading = { el, time, steps: [], from: time, fromMs: nowMs }; + return this._runAhead(el, time, Math.max(time, this._lastNow)); + } + noteStep(last, time); + const run = last.from + ((nowMs - last.fromMs) / 1000) * this._rate; + const held = Math.min(Math.max(run, time), time + furthestAhead(last)); + if (held !== run) Object.assign(last, { from: held, fromMs: nowMs }); + return this._runAhead(el, time, Math.max(held, this._lastNow)); + } + + private _runAhead(el: HTMLMediaElement, time: number, now: number): number { + this._ahead = { el, seconds: now - time }; + return now; + } + + aheadOf(el: HTMLMediaElement): number { + return this._ahead?.el === el ? this._ahead.seconds : 0; + } + private _heldUntilAudioOutputStarts(seeking: boolean, time: number): number | null | undefined { const pending = this._pendingAudioStart(); if (seeking) return pending ? this._lastNow : null; @@ -139,6 +192,7 @@ export class TransportClock { this._playStartMs = this._nowMs(); this._audioStart = { sinceMs: this._playStartMs, at: this._baseTime, from: null }; this._lastNow = this._baseTime; + this._reading = null; // Not a stall: the gap since the clock was last read (possibly a long // paused idle) says nothing about lost playback time, since none was // playing. `_applyStallCorrection` treats null as "nothing to compare @@ -151,6 +205,7 @@ export class TransportClock { if (this._playStartMs === null) return false; this._baseTime = this.now(); this._playStartMs = null; + this._reading = null; return true; } @@ -160,6 +215,7 @@ export class TransportClock { : Math.max(0, timeSeconds); this._baseTime = clamped; this._lastNow = clamped; + this._reading = null; if (this._playStartMs !== null) { this._playStartMs = this._nowMs(); this._audioStart = { sinceMs: this._playStartMs, at: clamped, from: null }; @@ -175,6 +231,7 @@ export class TransportClock { setRate(rate: number): void { const safe = Number.isFinite(rate) && rate > 0 ? Math.max(0.1, Math.min(5, rate)) : 1; + this._reading = null; if (this._playStartMs !== null) { this._baseTime = this.now(); this._playStartMs = this._nowMs(); @@ -230,6 +287,7 @@ export class TransportClock { this._lastReadMs = null; } this._audioSource = null; + this._reading = null; } /** The element the playhead currently follows, if the source is one. */ diff --git a/packages/core/src/runtime/init.ts b/packages/core/src/runtime/init.ts index 981f5c51f1..614cb777a3 100644 --- a/packages/core/src/runtime/init.ts +++ b/packages/core/src/runtime/init.ts @@ -3826,6 +3826,7 @@ export function initSandboxRuntimeModular(): void { forceSync, keepLateMedia, cueAheadSeconds, + clockAhead: (el) => clock.aheadOf(el), clockAdvancing: clock.getSource() !== "audio" || lastSyncedMediaTimeSeconds === null || diff --git a/packages/core/src/runtime/media.ts b/packages/core/src/runtime/media.ts index 1e2721cf97..9ce52a3d64 100644 --- a/packages/core/src/runtime/media.ts +++ b/packages/core/src/runtime/media.ts @@ -499,6 +499,7 @@ export function syncRuntimeMedia(params: { * outbound message; further invocations are suppressed by the caller. */ onAutoplayBlocked?: () => void; + clockAhead?: (el: HTMLMediaElement) => number; /** `clipGain` is `data-volume` for a routed clip, played ahead of its FX; 1 otherwise. */ onElementVolume?: ( el: HTMLMediaElement, @@ -557,7 +558,7 @@ export function syncRuntimeMedia(params: { !inWindow && (params.timeSeconds >= clip.end || sameInstant(params.timeSeconds, clip.end)) && isClipVisibleAt(params.timeSeconds, clip.start, clip.end, params.getCompositionDuration()); - const relTime = mediaSourceTimeAt(clip, params.timeSeconds); + const relTime = mediaSourceTimeAt(clip, params.timeSeconds - (params.clockAhead?.(el) ?? 0)); const isHeldVideoTail = isTerminalVideo || (isNonLoopVideo && clip.sourceDuration != null && relTime >= clip.sourceDuration && inWindow);