fix(core): a video's voice keeps pace with the playhead instead of trailing it (#4511)

* fix(core): a video's voice keeps pace with the playhead instead of trailing it A playing video is exempt from seek-based drift correction, because seeking resets its decoder and stutters. Nothing else corrected it, so a talking-head video that started a few frames late (92 ms measured in Studio) kept its voice behind the playhead for the whole clip, and further behind after each loop. Media sync now runs a video that drifts past the 40 ms sync tolerance 3% fast or slow (pitch preserved) until it is within 10 ms, then returns it to the authored rate. The rate is written only when steering starts or stops, since every rate change costs Chrome a frame; large jumps still hard-seek. Measured in Studio: 92 ms constant on main, about 10 ms steady with the fix. * fix(core): video steering holds through speed ramps and hard syncs Review follow-ups: steering inferred itself from the element's current rate, which a speed ramp moves every tick, so a ramped video stopped steering at the 40 ms tolerance instead of 10 ms; the direction is now kept per element. A hard-sync tick plays the authored rate, since its pre-seek offset is stale, and a rate that reads back at lower precision is not rewritten every tick. * test(core): pin clearing and write-once for video steering Review follow-up: nothing failed when the steering state was left uncleared on eviction or pause, or when the rate compare went back to strict equality. Tests now cover all three, and the sync-state test seam reports the steering map so the proxy's leak test sees it. * chore(core): format the video steering and keep its tolerance module-private * chore(core): shorten the video steering comment
2026-10-30 09:16:55 +00:00 · 2026-09-26 05:02:44 +00:00
parent b667398207
commit f2c63ca424
2 changed files with 162 additions and 12 deletions
@@ -1,5 +1,7 @@
 import { describe, it, expect, vi, afterEach } from "vitest";
 import {
+  evictMediaSyncState,
+  hasMediaSyncStateForTest,
  readElementPlaybackRate,
  readElementPlaybackStart,
  refreshRuntimeMediaCache,
@@ -1741,6 +1743,124 @@ describe("syncRuntimeMedia", () => {
    expect(clip.el.currentTime).toBe(5);
  });

+  describe("playing video drift", () => {
+    function playingVideoAt(currentTime: number, playbackRate = 1) {
+      const clip = createMockClip({ start: 0, end: 20, duration: 20 });
+      Object.defineProperty(clip.el, "paused", { value: false, writable: true });
+      Object.defineProperty(clip.el, "currentTime", { value: currentTime, writable: true });
+      clip.el.playbackRate = playbackRate;
+      return clip;
+    }
+    const tick = (clip: RuntimeMediaClip, timeSeconds: number, playbackRate = 1) =>
+      syncRuntimeMedia({
+        clips: [clip],
+        timeSeconds,
+        playing: true,
+        playbackRate,
+        getCompositionDuration: () => 20,
+      });
+
+    it("runs a video lagging past the sync tolerance 3% fast instead of seeking it", () => {
+      const clip = playingVideoAt(4.908);
+      tick(clip, 5);
+      tick(clip, 5);
+      expect(clip.el.playbackRate).toBeCloseTo(1.03, 9);
+      expect(clip.el.currentTime).toBe(4.908);
+    });
+
+    it("runs a video that is ahead 3% slow", () => {
+      const clip = playingVideoAt(5.3);
+      tick(clip, 5);
+      tick(clip, 5);
+      expect(clip.el.playbackRate).toBeCloseTo(0.97, 9);
+    });
+
+    it("keeps steering until the video is nearly back, then returns to the authored rate", () => {
+      const clip = playingVideoAt(4.9);
+      tick(clip, 5);
+      tick(clip, 5);
+      clip.el.currentTime = 4.98;
+      tick(clip, 5);
+      expect(clip.el.playbackRate).toBeCloseTo(1.03, 9);
+      clip.el.currentTime = 4.995;
+      tick(clip, 5);
+      expect(clip.el.playbackRate).toBe(1);
+    });
+
+    it("keeps steering through a speed ramp, whose base rate moves every tick", () => {
+      const clip = playingVideoAt(4.9);
+      tick(clip, 5, 1);
+      tick(clip, 5, 1);
+      clip.el.currentTime = 4.98;
+      tick(clip, 5, 1.2);
+      expect(clip.el.playbackRate).toBeCloseTo(1.2 * 1.03, 9);
+    });
+
+    it("plays a hard-synced video at its authored rate on the tick it is seeked", () => {
+      const clip = playingVideoAt(4.9);
+      tick(clip, 5);
+      tick(clip, 5);
+      expect(clip.el.playbackRate).toBeCloseTo(1.03, 9);
+      tick(clip, 9); // a jump past the hard-sync threshold
+      expect(clip.el.currentTime).toBe(9);
+      expect(clip.el.playbackRate).toBe(1);
+    });
+
+    it("does not start steering inside the sync tolerance", () => {
+      const clip = playingVideoAt(4.97, 2);
+      tick(clip, 5, 2);
+      tick(clip, 5, 2);
+      expect(clip.el.playbackRate).toBe(2);
+    });
+
+    it("returns a steered video to its authored rate when the transport pauses", () => {
+      const clip = playingVideoAt(4.9);
+      tick(clip, 5);
+      tick(clip, 5);
+      syncRuntimeMedia({
+        clips: [clip],
+        timeSeconds: 5,
+        playing: false,
+        playbackRate: 1,
+        getCompositionDuration: () => 20,
+      });
+      clip.el.currentTime = 4.98; // inside the tolerance: must not resume steering
+      tick(clip, 5);
+      expect(clip.el.playbackRate).toBe(1);
+    });
+
+    it("forgets the steering when the element's sync state is evicted", () => {
+      const clip = playingVideoAt(4.9);
+      tick(clip, 5);
+      tick(clip, 5);
+      evictMediaSyncState(clip.el);
+      expect(hasMediaSyncStateForTest(clip.el)).toBe(false);
+    });
+
+    it("does not rewrite a steered rate that reads back at lower precision", () => {
+      const clip = playingVideoAt(4.9);
+      let stored = 1;
+      let writes = 0;
+      Object.defineProperty(clip.el, "playbackRate", {
+        configurable: true,
+        get: () => stored,
+        set: (v: number) => {
+          writes += 1;
+          stored = Math.fround(v);
+        },
+      });
+      for (let i = 0; i < 5; i++) tick(clip, 5);
+      expect(writes).toBe(1);
+    });
+
+    it("scales the steer with the transport rate", () => {
+      const clip = playingVideoAt(4.908, 2);
+      tick(clip, 5, 2);
+      tick(clip, 5, 2);
+      expect(clip.el.playbackRate).toBeCloseTo(2.06, 9);
+    });
+  });
+
  // A seek while playing pauses and syncs in one pass, before the video element has paused.
  it("a seek that pauses mid-playback lands a lagging playing video on the new time", () => {
    const clip = createMockClip({ start: 3.85, end: 5.6, duration: 1.75 });
@@ -234,6 +234,7 @@ export function evictMediaSyncState(el: HTMLMediaElement): void {
  strictDriftSamples.delete(el);
  seekLoadRetried.delete(el);
  lastRuntimeAppliedVolume.delete(el);
+  videoSteering.delete(el);
 }

 /** Test-only seam: whether any per-source sync state is still tracked for `el`. */
@@ -243,10 +244,34 @@ export function hasMediaSyncStateForTest(el: HTMLMediaElement): boolean {
    lastRelativeTime.has(el) ||
    strictDriftSamples.has(el) ||
    seekLoadRetried.has(el) ||
-    lastRuntimeAppliedVolume.has(el)
+    lastRuntimeAppliedVolume.has(el) ||
+    videoSteering.has(el)
  );
 }

+/** Drift a playing audio element may carry before sync pulls it back onto the playhead. */
+const MEDIA_SYNC_TOLERANCE_SECONDS = 0.04;
+
+// A playing video is steered back by rate, not seeked (a seek resets its decoder).
+// Its rate is written only when steering starts or stops: every write costs a frame.
+const VIDEO_STEER = 0.03;
+const VIDEO_STEER_RELEASE_SECONDS = 0.01;
+
+/** Direction (+1 fast, -1 slow) a video is being steered in; absent when it plays at its authored rate. */
+const videoSteering = new WeakMap<HTMLMediaElement, number>();
+
+/** Rate for a playing video `offset` seconds behind (+) or ahead (-) of the playhead. */
+function steeredVideoRate(el: HTMLMediaElement, offset: number, baseRate: number): number {
+  const direction = Math.sign(offset);
+  const steering = videoSteering.get(el);
+  if (Math.abs(offset) > MEDIA_SYNC_TOLERANCE_SECONDS) videoSteering.set(el, direction);
+  else if (steering !== direction || Math.abs(offset) <= VIDEO_STEER_RELEASE_SECONDS) {
+    videoSteering.delete(el);
+    return baseRate;
+  }
+  return baseRate * (1 + direction * VIDEO_STEER);
+}
+
 // fallow-ignore-next-line complexity
 export function syncRuntimeMedia(params: {
  clips: RuntimeMediaClip[];
@@ -427,13 +452,8 @@ export function syncRuntimeMedia(params: {
      // (no-op when already "auto") and catches elements whose preload
      // was overridden after init.ts set it.
      if (el.preload !== "auto") el.preload = "auto";
-      try {
-        // Per-element rate × global transport rate
-        el.playbackRate = rateAt(clipRate, params.timeSeconds - clip.start) * params.playbackRate;
-      } catch (err) {
-        // ignore unsupported playbackRate
-        swallow("runtime.media.site1", err);
-      }
+      // Per-element rate × global transport rate
+      const baseRate = rateAt(clipRate, params.timeSeconds - clip.start) * params.playbackRate;
      // Drift correction — three tiers:
      //
      // 1. Hard sync (0.5s): first tick, timeline jumps (scrub), catastrophic
@@ -452,7 +472,6 @@ export function syncRuntimeMedia(params: {
      // The first tick a clip is active has no previous offset to compare —
      // treated as hard resync so sub-compositions with non-zero mediaStart
      // land on the right frame.
-      const STRICT_DRIFT_THRESHOLD = 0.04;
      const STRICT_REQUIRED_SAMPLES = 2;

      const currentElTime = el.currentTime || 0;
@@ -469,7 +488,7 @@ export function syncRuntimeMedia(params: {
      const staleAudioOnFirstTick =
        el.tagName === "AUDIO" &&
        firstTickOfClip &&
-        currentElTime - relTime > STRICT_DRIFT_THRESHOLD;
+        currentElTime - relTime > MEDIA_SYNC_TOLERANCE_SECONDS;
      const hardSync =
        (isHeldVideoTail && drift > 0.001) ||
        (el.ended && canSeekEndedMediaBackward && drift > 0.001) ||
@@ -492,7 +511,7 @@ export function syncRuntimeMedia(params: {
        !hardSync &&
        !firstTickOfClip &&
        offsetStabilized &&
-        drift > STRICT_DRIFT_THRESHOLD
+        drift > MEDIA_SYNC_TOLERANCE_SECONDS
      ) {
        const samples = (strictDriftSamples.get(el) ?? 0) + 1;
        strictDriftSamples.set(el, samples);
@@ -500,10 +519,21 @@ export function syncRuntimeMedia(params: {
          strictSync = true;
          strictDriftSamples.set(el, 0);
        }
-      } else if (drift <= STRICT_DRIFT_THRESHOLD) {
+      } else if (drift <= MEDIA_SYNC_TOLERANCE_SECONDS) {
        strictDriftSamples.set(el, 0);
      }
      const forceSync = !isPlayingVideo && params.forceSync && drift > 0.02;
+      try {
+        // A hard sync lands the video on the playhead, so its pre-seek offset says nothing.
+        if (!isPlayingVideo || hardSync) videoSteering.delete(el);
+        const rate =
+          isPlayingVideo && !hardSync ? steeredVideoRate(el, offset, baseRate) : baseRate;
+        // Some engines read a rate back at lower precision; an equal-enough rate is not rewritten.
+        if (Math.abs(el.playbackRate - rate) > 1e-6) el.playbackRate = rate;
+      } catch (err) {
+        // ignore unsupported playbackRate
+        swallow("runtime.media.site1", err);
+      }
      if (hardSync || strictSync || forceSync) {
        // Skip the per-tick seek (and the `el.load()` drift-recovery retry
        // below) for `<video>` elements that have a sibling