diff --git a/packages/core/src/runtime/media.test.ts b/packages/core/src/runtime/media.test.ts index 460069ab77..cc9b9e1463 100644 --- a/packages/core/src/runtime/media.test.ts +++ b/packages/core/src/runtime/media.test.ts @@ -7,6 +7,7 @@ import { readElementPlaybackRate, readElementPlaybackStart, refreshRuntimeMediaCache, + resetVideoStartLatency, resolveRuntimeMediaClipDuration, syncRuntimeMedia as syncRuntimeMediaWithDuration, } from "./media"; @@ -2052,6 +2053,105 @@ describe("syncRuntimeMedia", () => { } expect(el.pause).not.toHaveBeenCalled(); }); + + describe("start latency", () => { + afterEach(() => resetVideoStartLatency()); + + // Plays `clip` at `playAt`, then has its element `lag` s behind the playhead at `sampleAt`. + const startWithLag = ( + clip: RuntimeMediaClip, + playAt: number, + sampleAt: number, + lag: number, + ) => { + tick(clip, playAt); + expect(clip.el.play).toHaveBeenCalledTimes(1); + Object.defineProperty(clip.el, "paused", { + value: false, + writable: true, + configurable: true, + }); + clip.el.currentTime = clip.mediaStart + (sampleAt - clip.start) - lag; + tick(clip, sampleAt); + }; + + it("does not start an upcoming video early before any start is measured", () => { + const clip = videoClip({ start: 10, end: 20, mediaStart: 0 }); + tick(clip, 9.99); + expect(clip.el.play).not.toHaveBeenCalled(); + }); + + it("starts an upcoming video early by the start latency measured on earlier ones", () => { + startWithLag(videoClip({ start: 0, end: 10, mediaStart: 0 }), 0, 0.3, 0.06); + const due50ms = videoClip({ start: 10, end: 20, mediaStart: 0 }); + tick(due50ms, 9.95); + expect(due50ms.el.play).toHaveBeenCalledTimes(1); + const due70ms = videoClip({ start: 10, end: 20, mediaStart: 0 }); + tick(due70ms, 9.93); + expect(due70ms.el.play).not.toHaveBeenCalled(); + }); + + it("starts an upcoming video on the tick nearest its lead", () => { + startWithLag(videoClip({ start: 0, end: 10, mediaStart: 0 }), 0, 0.3, 0.06); + const due75ms = videoClip({ start: 10, end: 20, mediaStart: 0 }); + // Now is 15 ms earlier than the 60 ms lead; the next tick, 40 ms on, would be 25 ms late. + syncRuntimeMedia({ + clips: [due75ms], + timeSeconds: 9.925, + playing: true, + playbackRate: 1, + cueAheadSeconds: 0.04, + getCompositionDuration: () => 30, + }); + expect(due75ms.el.play).toHaveBeenCalledTimes(1); + }); + + it("learns from a video seeked and started in the same tick", () => { + const seeked = videoClip({ start: 0, end: 10, mediaStart: 0 }); + seeked.el.currentTime = 0; + startWithLag(seeked, 5, 5.3, 0.06); + const due25ms = videoClip({ start: 10, end: 20, mediaStart: 0 }); + tick(due25ms, 9.975); + expect(due25ms.el.play).toHaveBeenCalledTimes(1); + }); + + it("keeps its lead when an early start lands on time", () => { + startWithLag(videoClip({ start: 0, end: 10, mediaStart: 0 }), 0, 0.3, 0.06); + startWithLag(videoClip({ start: 10, end: 20, mediaStart: 0 }), 9.945, 10.3, 0.005); + const due58ms = videoClip({ start: 20, end: 30, mediaStart: 0 }); + tick(due58ms, 19.942); + expect(due58ms.el.play).toHaveBeenCalledTimes(1); + }); + + it("caps the learned start delay at 100 ms", () => { + startWithLag(videoClip({ start: 0, end: 10, mediaStart: 0 }), 0, 0.3, 0.3); + const due150ms = videoClip({ start: 10, end: 20, mediaStart: 0 }); + tick(due150ms, 9.85); + expect(due150ms.el.play).not.toHaveBeenCalled(); + }); + + it("does not learn from a looping video sampled across a lap wrap", () => { + startWithLag(videoClip({ start: 0, end: 10, mediaStart: 0 }), 0, 0.3, 0.06); + const loop = videoClip({ + start: 10, + end: 20, + mediaStart: 0, + loop: true, + sourceDuration: 0.2, + }); + tick(loop, 10); + Object.defineProperty(loop.el, "paused", { + value: false, + writable: true, + configurable: true, + }); + loop.el.currentTime = 0.19; // the playhead has wrapped to 0.06, the element not yet + tick(loop, 10.26); + const due50ms = videoClip({ start: 30, end: 40, mediaStart: 0 }); + tick(due50ms, 29.95); + expect(due50ms.el.play).toHaveBeenCalledTimes(1); + }); + }); }); describe("playing media drift", () => { diff --git a/packages/core/src/runtime/media.ts b/packages/core/src/runtime/media.ts index 36796f080c..f7ab1f0d90 100644 --- a/packages/core/src/runtime/media.ts +++ b/packages/core/src/runtime/media.ts @@ -236,6 +236,7 @@ export function evictMediaSyncState(el: HTMLMediaElement): void { lastRuntimeAppliedVolume.delete(el); forgetSteering(el); heldAhead.delete(el); + videoStarts.delete(el); } /** Test-only seam: whether any per-source sync state is still tracked for `el`. */ @@ -302,6 +303,26 @@ const MIN_NATIVE_PLAYBACK_RATE = 1 / 16; const VIDEO_LEAD_HOLD_SECONDS = 0.1; const heldAhead = new WeakSet(); +// A muted video's clock is its audio track, which starts tens of ms after play(), so an upcoming +// video starts that early, hidden, by the delay learned from earlier starts. +let videoStartLatency: number | null = null; +const videoStarts = new WeakMap(); + +/** Test-only seam. */ +export function resetVideoStartLatency(): void { + videoStartLatency = null; +} + +/** A quarter second in: its lag plus how early it started, capped at 0.1 s. */ +function learnVideoStartLatency(el: HTMLMediaElement, time: number, offset: number, rate: number) { + const start = videoStarts.get(el); + if (!start || time < start.at) return; + videoStarts.delete(el); + const sample = (offset - start.offset) / rate + start.lead; + const averaged = videoStartLatency === null ? sample : (videoStartLatency + sample) / 2; + videoStartLatency = Math.min(0.1, Math.max(0, averaged)); +} + // fallow-ignore-next-line complexity export function syncRuntimeMedia(params: { clips: RuntimeMediaClip[]; @@ -364,15 +385,18 @@ export function syncRuntimeMedia(params: { const isNonLoopVideo = el.tagName === "VIDEO" && !clip.loop; const inWindow = isInClipWindow(params.timeSeconds, clip.start, clip.end); const dueIn = clip.start - params.timeSeconds; + const cueAhead = + el.tagName === "AUDIO" + ? (params.cueAheadSeconds ?? 0) + : params.playing && videoStartLatency !== null + ? // Half a tick: it starts on the tick nearest its lead. + videoStartLatency * params.playbackRate + (params.cueAheadSeconds ?? 0) / 2 + : 0; const startsEarly = - el.tagName === "AUDIO" && !inWindow && dueIn > 0 && dueIn * Math.max(1, rateAt(clipRate, clip.start - origin)) <= - Math.max( - params.cueAheadSeconds ?? 0, - startedEarly.has(el) ? MEDIA_SYNC_TOLERANCE_SECONDS : 0, - ); + Math.max(cueAhead, startedEarly.has(el) ? MEDIA_SYNC_TOLERANCE_SECONDS : 0); if (startsEarly) startedEarly.add(el); else startedEarly.delete(el); // A video that runs to the composition end stays the visible frame at and past it, so it @@ -569,6 +593,7 @@ export function syncRuntimeMedia(params: { } const holdingAhead = heldAhead.has(el); const isPlayingMedia = !el.paused && playing; + if (isPlayingMedia) learnVideoStartLatency(el, params.timeSeconds, offset, baseRate); // Only apply strict sync when offset has stabilized (not growing). // During initial buffering, offset grows ~16ms/tick as the timeline // advances while media stays at 0. Accumulated drift from pause/play @@ -641,6 +666,7 @@ export function syncRuntimeMedia(params: { } playRequested.delete(el); heldAhead.delete(el); + videoStarts.delete(el); } else if (!playing) { holdSeekBarrierUntilVideoLands(el); } @@ -666,6 +692,13 @@ export function syncRuntimeMedia(params: { // fire between `load()` and `addEventListener` attachment, wedging // the element waiting for a callback that never came. markPlayRequested(el); + // Offset read after any seek above; a loop could wrap a lap before the sample. + if (isNonLoopVideo) + videoStarts.set(el, { + offset: relTime - (el.currentTime || 0), + lead: Math.max(0, clip.start - params.timeSeconds) / params.playbackRate, + at: Math.max(params.timeSeconds, clip.start) + 0.25 * params.playbackRate, + }); void el.play().catch((err: unknown) => { // If play() rejects — e.g. autoplay blocked, element removed // mid-flight — drop the in-flight flag so a future sync tick can diff --git a/packages/player/tests/playbench/analyze.ts b/packages/player/tests/playbench/analyze.ts index 9012f9d08f..af17a86926 100644 --- a/packages/player/tests/playbench/analyze.ts +++ b/packages/player/tests/playbench/analyze.ts @@ -46,6 +46,9 @@ export type Metrics = { /** Problem 4 */ secondFrameGapMedianMs: number | null; secondFrameGapMaxMs: number | null; + /** How far the incoming picture sits behind the playhead over its take's first 0.5 s, median per cut. */ + cutPictureLagMedianMs: number | null; + cutPictureLagMaxMs: number | null; playheadHoldMedianMs: number | null; playheadHoldP90Ms: number | null; holdsOver100: number; @@ -222,7 +225,13 @@ export function analyze(run: Run): Metrics { // ---- Problem 4: what a cut costs const gaps: number[] = []; const holds: number[] = []; + const lags: number[] = []; for (const t of takes.slice(1)) { + const opening = byId(samples, `v${t.index}`).filter( + (s) => s[3] === 0 && s[7] !== null && s[7] >= t.start && s[7] <= t.start + 0.5, + ); + const lag = median(opening.map((s) => (t.inPoint + s[7]! - t.start - s[2]) * 1000)); + if (lag !== null) lags.push(lag); const fr = comp.frames .filter((f) => f[1] === `v${t.index}` && f[4] === 1) .sort((a, b) => a[0] - b[0]); @@ -234,6 +243,8 @@ export function analyze(run: Run): Metrics { m.cuts = takes.length - 1; m.secondFrameGapMedianMs = round(median(gaps)); m.secondFrameGapMaxMs = round(gaps.length ? Math.max(...gaps) : null); + m.cutPictureLagMedianMs = round(median(lags)); + m.cutPictureLagMaxMs = round(lags.length ? Math.max(...lags) : null); m.playheadHoldMedianMs = round(median(holds)); m.playheadHoldP90Ms = round(pct(holds, 0.9)); m.holdsOver100 = holds.filter((h) => h > 100).length; diff --git a/packages/player/tests/playbench/playbench.test.ts b/packages/player/tests/playbench/playbench.test.ts index 2f1ecdc079..a5e1e089d0 100644 --- a/packages/player/tests/playbench/playbench.test.ts +++ b/packages/player/tests/playbench/playbench.test.ts @@ -121,6 +121,27 @@ describe("analyze", () => { expect(analyze(run).playheadHoldMedianMs).toBeCloseTo(30, 0); }); + it("reports an incoming picture that runs 50 ms behind the playhead after its cut", () => { + const run = base(); + for (const t of edit.takes.slice(1)) + for (let i = 0; i <= 50; i++) { + const ph = t.start + i * 0.01; + run.comp.samples.push([ + 1000 + ph * 1000, + `v${t.index}`, + t.inPoint + ph - t.start - 0.05, + 0, + 0, + 1, + 1, + ph, + ]); + } + const m = analyze(run); + expect(m.cutPictureLagMedianMs).toBeCloseTo(50, 0); + expect(m.cutPictureLagMaxMs).toBeCloseTo(50, 0); + }); + it("counts a take as heard only when the audio reaching the output comes from that take's stretch of the file", () => { const run = base(); run.ownership = [[1000, "parent"]]; diff --git a/packages/player/tests/playbench/run.ts b/packages/player/tests/playbench/run.ts index 01d7c93e63..ffb7a6bd8d 100644 --- a/packages/player/tests/playbench/run.ts +++ b/packages/player/tests/playbench/run.ts @@ -229,6 +229,16 @@ const ROWS: [string, string, (m: Metrics) => number | boolean | null][] = [ ["4", "cuts", (m) => m.cuts], ["4", "incoming second-frame gap median (ms)", (m) => m.secondFrameGapMedianMs], ["4", "incoming second-frame gap max (ms)", (m) => m.secondFrameGapMaxMs], + [ + "4", + "incoming picture behind the playhead, first 0.5 s, median over cuts (ms)", + (m) => m.cutPictureLagMedianMs, + ], + [ + "4", + "incoming picture behind the playhead, first 0.5 s, worst cut (ms)", + (m) => m.cutPictureLagMaxMs, + ], ["4", "playhead hold per cut median (ms)", (m) => m.playheadHoldMedianMs], ["4", "playhead hold per cut p90 (ms)", (m) => m.playheadHoldP90Ms], ["4", "cuts with a hold over 100 ms", (m) => m.holdsOver100],