Merge pull request #714 from heygen-com/fix/713-webaudio-playback-rate

fix(core): thread playback rate into WebAudio audio sources

packages/core/src/runtime/init.ts

@@ -1589,6 +1589,7 @@ export function initSandboxRuntimeModular(): void {
     onSetPlaybackRate: (rate) => {
       applyPlaybackRate(rate);
       if (state.transportClock) state.transportClock.setRate(state.playbackRate);
+      webAudio.setRate(state.playbackRate);
     },
     onEnablePickMode: () => picker.enablePickMode(),
     onDisablePickMode: () => picker.disablePickMode(),
@@ -1858,6 +1859,7 @@ export function initSandboxRuntimeModular(): void {
             clock.now(),
             vol * state.bridgeVolume,
             gen,
+            state.playbackRate,
           );
         });
       }

packages/core/src/runtime/webAudioTransport.test.ts

@@ -1,6 +1,49 @@
 import { describe, it, expect, vi } from "vitest";
 import { WebAudioTransport } from "./webAudioTransport";
 
+function createMockAudioContext(currentTime = 100) {
+  const startFn = vi.fn();
+  const sourceNode = {
+    buffer: null as AudioBuffer | null,
+    playbackRate: { value: 1 },
+    start: startFn,
+    stop: vi.fn(),
+    disconnect: vi.fn(),
+    connect: vi.fn(),
+  };
+  const gainNode = {
+    gain: { value: 1 },
+    connect: vi.fn(),
+    disconnect: vi.fn(),
+  };
+  const masterGain = {
+    gain: { value: 1 },
+    connect: vi.fn(),
+  };
+  const ctx = {
+    currentTime,
+    state: "running",
+    resume: vi.fn(),
+    createBufferSource: vi.fn(() => sourceNode),
+    createGain: vi.fn(() => gainNode),
+    destination: {},
+    close: vi.fn(),
+  };
+  return { ctx, sourceNode, gainNode, masterGain, startFn };
+}
+
+function setupTransport(currentTime = 100) {
+  const transport = new WebAudioTransport();
+  const mock = createMockAudioContext(currentTime);
+  (transport as unknown as { _ctx: unknown })._ctx = mock.ctx;
+  (transport as unknown as { _masterGain: unknown })._masterGain = mock.masterGain;
+  const gen = transport.startGeneration();
+  return { transport, mock, gen };
+}
+
+const mockBuffer = {} as AudioBuffer;
+const mockEl = { muted: false } as HTMLMediaElement;
+
 describe("WebAudioTransport", () => {
   it("tracks play generation for async race prevention", () => {
     const transport = new WebAudioTransport();
@@ -82,45 +125,6 @@ describe("WebAudioTransport", () => {
   });
 
   describe("schedulePlayback timing", () => {
-    function createMockAudioContext(currentTime = 100) {
-      const startFn = vi.fn();
-      const sourceNode = {
-        buffer: null as AudioBuffer | null,
-        start: startFn,
-        connect: vi.fn(),
-      };
-      const gainNode = {
-        gain: { value: 1 },
-        connect: vi.fn(),
-      };
-      const masterGain = {
-        gain: { value: 1 },
-        connect: vi.fn(),
-      };
-      const ctx = {
-        currentTime,
-        state: "running",
-        resume: vi.fn(),
-        createBufferSource: vi.fn(() => sourceNode),
-        createGain: vi.fn(() => gainNode),
-        destination: {},
-        close: vi.fn(),
-      };
-      return { ctx, sourceNode, gainNode, masterGain, startFn };
-    }
-
-    function setupTransport(currentTime = 100) {
-      const transport = new WebAudioTransport();
-      const mock = createMockAudioContext(currentTime);
-      (transport as unknown as { _ctx: unknown })._ctx = mock.ctx;
-      (transport as unknown as { _masterGain: unknown })._masterGain = mock.masterGain;
-      const gen = transport.startGeneration();
-      return { transport, mock, gen };
-    }
-
-    const mockBuffer = {} as AudioBuffer;
-    const mockEl = { muted: false } as HTMLMediaElement;
-
     it("starts in-progress clips immediately with correct buffer offset", async () => {
       const { transport, mock, gen } = setupTransport(100);
 
@@ -161,4 +165,120 @@ describe("WebAudioTransport", () => {
       expect(mock.startFn).toHaveBeenCalledWith(0, 0);
     });
   });
+
+  describe("playback rate", () => {
+    it("sets sourceNode.playbackRate.value when rate is provided", async () => {
+      const { transport, mock, gen } = setupTransport(100);
+
+      await transport.schedulePlayback(mockEl, mockBuffer, 5, 0, 8, 1, gen, 2);
+
+      expect(mock.sourceNode.playbackRate.value).toBe(2);
+    });
+
+    it("defaults rate to 1 when not provided", async () => {
+      const { transport, mock, gen } = setupTransport(100);
+
+      await transport.schedulePlayback(mockEl, mockBuffer, 5, 0, 8, 1, gen);
+
+      expect(mock.sourceNode.playbackRate.value).toBe(1);
+    });
+
+    it("scales delay by rate for future clips so they fire at the right wallclock", async () => {
+      const { transport, mock, gen } = setupTransport(100);
+
+      // compStart=10, compositionTime=2, rate=2 → 8s of comp time = 4s wallclock
+      await transport.schedulePlayback(mockEl, mockBuffer, 10, 0, 2, 1, gen, 2);
+
+      expect(mock.startFn).toHaveBeenCalledWith(104, 0);
+    });
+
+    it("keeps in-progress buffer offset at elapsed + mediaStart regardless of rate", async () => {
+      const { transport, mock, gen } = setupTransport(100);
+
+      await transport.schedulePlayback(mockEl, mockBuffer, 5, 0, 8, 1, gen, 2);
+
+      expect(mock.startFn).toHaveBeenCalledWith(0, 3);
+    });
+
+    it("setRate updates active sources in place", async () => {
+      const { transport, mock, gen } = setupTransport(100);
+
+      await transport.schedulePlayback(mockEl, mockBuffer, 5, 0, 8, 1, gen, 1);
+      expect(mock.sourceNode.playbackRate.value).toBe(1);
+
+      transport.setRate(2);
+
+      expect(mock.sourceNode.playbackRate.value).toBe(2);
+    });
+
+    it("setRate before any sources are scheduled does not throw", () => {
+      const transport = new WebAudioTransport();
+      expect(() => transport.setRate(2)).not.toThrow();
+    });
+
+    it("setRate is a no-op when the rate is unchanged", async () => {
+      const { transport, mock, gen } = setupTransport(100);
+      await transport.schedulePlayback(mockEl, mockBuffer, 5, 0, 8, 1, gen, 2);
+
+      mock.ctx.currentTime = 100.5;
+      const timeBefore = transport.getTime();
+      transport.setRate(2);
+      const timeAfter = transport.getTime();
+
+      expect(timeAfter).toBe(timeBefore);
+      // No re-anchor, so the next 0.5s of wallclock still maps to 1s of comp time.
+      mock.ctx.currentTime = 101;
+      expect(transport.getTime()).toBeCloseTo(10, 10);
+    });
+
+    it("setRate clamps non-finite or non-positive values to 1", async () => {
+      const { transport, mock, gen } = setupTransport(100);
+      await transport.schedulePlayback(mockEl, mockBuffer, 5, 0, 8, 1, gen, 2);
+      expect(mock.sourceNode.playbackRate.value).toBe(2);
+
+      transport.setRate(Number.NaN);
+      expect(mock.sourceNode.playbackRate.value).toBe(1);
+
+      transport.setRate(2);
+      transport.setRate(0);
+      expect(mock.sourceNode.playbackRate.value).toBe(1);
+
+      transport.setRate(2);
+      transport.setRate(-1);
+      expect(mock.sourceNode.playbackRate.value).toBe(1);
+    });
+
+    it("getTime advances at the configured rate", async () => {
+      const { transport, mock, gen } = setupTransport(100);
+
+      await transport.schedulePlayback(mockEl, mockBuffer, 5, 0, 8, 1, gen, 2);
+
+      // At schedule time, ctx.currentTime=100, compositionTime=8.
+      expect(transport.getTime()).toBeCloseTo(8, 10);
+
+      // Advance the audio-context clock by 0.5 wallclock seconds; at rate=2,
+      // composition time should have advanced 1s.
+      mock.ctx.currentTime = 100.5;
+      expect(transport.getTime()).toBeCloseTo(9, 10);
+    });
+
+    it("getTime tracks composition time after a mid-playback setRate", async () => {
+      const { transport, mock, gen } = setupTransport(100);
+
+      await transport.schedulePlayback(mockEl, mockBuffer, 5, 0, 8, 1, gen, 1);
+      expect(transport.getTime()).toBeCloseTo(8, 10);
+
+      // 0.5s passes at rate=1 → composition time = 8.5
+      mock.ctx.currentTime = 100.5;
+      expect(transport.getTime()).toBeCloseTo(8.5, 10);
+
+      // Bump rate to 2 — composition time should NOT jump.
+      transport.setRate(2);
+      expect(transport.getTime()).toBeCloseTo(8.5, 10);
+
+      // Another 0.5s wallclock at rate=2 → composition time = 9.5
+      mock.ctx.currentTime = 101;
+      expect(transport.getTime()).toBeCloseTo(9.5, 10);
+    });
+  });
 });

packages/core/src/runtime/webAudioTransport.ts

@@ -1,5 +1,10 @@
 import { swallow } from "./diagnostics";
 
+function normalizeRate(rate: number): number {
+  if (!Number.isFinite(rate) || rate <= 0) return 1;
+  return rate;
+}
+
 export type ScheduledSource = {
   el: HTMLMediaElement;
   sourceNode: AudioBufferSourceNode;
@@ -15,7 +20,12 @@ export class WebAudioTransport {
   private _bufferCache = new Map<string, AudioBuffer>();
   private _activeSources: ScheduledSource[] = [];
   private _masterGain: GainNode | null = null;
-  private _scheduleOffset = 0;
+  // Composition-time reference frame: at AudioContext time `_rateAnchorCtx`,
+  // composition time was `_rateAnchorComp`, and time has been advancing at
+  // `_rate` composition-seconds per wallclock-second since.
+  private _rateAnchorCtx = 0;
+  private _rateAnchorComp = 0;
+  private _rate = 1;
   private _paused = true;
   private _playGeneration = 0;
 
@@ -36,7 +46,7 @@ export class WebAudioTransport {
 
   getTime(): number {
     if (!this._ctx || this._paused) return -1;
-    return this._ctx.currentTime - this._scheduleOffset;
+    return this._rateAnchorComp + (this._ctx.currentTime - this._rateAnchorCtx) * this._rate;
   }
 
   async decodeAudioElement(el: HTMLMediaElement): Promise<AudioBuffer | null> {
@@ -73,6 +83,7 @@ export class WebAudioTransport {
     compositionTime: number,
     volume: number,
     generation: number,
+    rate = 1,
   ): Promise<ScheduledSource | null> {
     if (!this._ctx || !this._masterGain) return null;
     if (generation !== this._playGeneration) return null;
@@ -83,8 +94,11 @@ export class WebAudioTransport {
       }
       if (generation !== this._playGeneration) return null;
 
+      const safeRate = normalizeRate(rate);
+
       const sourceNode = this._ctx.createBufferSource();
       sourceNode.buffer = buffer;
+      sourceNode.playbackRate.value = safeRate;
 
       const gainNode = this._ctx.createGain();
       gainNode.gain.value = volume;
@@ -93,12 +107,14 @@ export class WebAudioTransport {
 
       const elapsed = compositionTime - compositionStart;
       const scheduledAt = this._ctx.currentTime;
-      this._scheduleOffset = scheduledAt - compositionTime;
+      this._rate = safeRate;
+      this._rateAnchorCtx = scheduledAt;
+      this._rateAnchorComp = compositionTime;
 
       if (elapsed >= 0) {
         sourceNode.start(0, elapsed + mediaStart);
       } else {
-        const delay = -elapsed;
+        const delay = -elapsed / safeRate;
         sourceNode.start(scheduledAt + delay, mediaStart);
       }
 
@@ -123,6 +139,29 @@ export class WebAudioTransport {
     }
   }
 
+  /**
+   * Rebases the composition-time reference frame before swapping rate so
+   * `getTime()` stays continuous across the change. Sources scheduled to
+   * start in the future keep their original wallclock start time — callers
+   * that need rate-correct future starts should `stopAll()` and reschedule.
+   */
+  setRate(rate: number): void {
+    const safeRate = normalizeRate(rate);
+    if (safeRate === this._rate) return;
+    if (this._ctx && !this._paused) {
+      this._rateAnchorComp = this.getTime();
+      this._rateAnchorCtx = this._ctx.currentTime;
+    }
+    this._rate = safeRate;
+    for (const source of this._activeSources) {
+      try {
+        source.sourceNode.playbackRate.value = safeRate;
+      } catch (err) {
+        swallow("webAudioTransport.setRate", err);
+      }
+    }
+  }
+
   stopAll(): void {
     for (const source of this._activeSources) {
       try {