From ceb54811c65d8c5aa916f559d0d9e7aa8b06449a Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Miguel=20=C3=81ngel?= <miguel.sierra@heygen.com>
Date: Fri, 10 Apr 2026 00:05:32 +0200
Subject: [PATCH] fix(runtime): preload media on init to prevent broken
 first-play audio (#234)
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

## Summary
- Audio (and video) sounds broken/choppy on first play in the studio preview, but works fine on second play
- Root cause: `<audio>` elements default to `preload="metadata"`, which only fetches enough data to determine duration — not enough for smooth playback. When `el.play()` fires, the browser hasn't buffered the audio data yet
- The runtime now eagerly sets `preload="auto"` and calls `load()` during init, ensuring media is fully buffered before the user clicks play
- `syncRuntimeMedia` now defers `play()` on unbuffered media by registering a `canplay` listener, instead of silently swallowing the failure

## Testing
Verified with agent-browser against a 26s narration composition (soulscape-film):

```
# After fix — audio element state at init:
preload: "auto"
readyState: 4 (HAVE_ENOUGH_DATA)
buffered: 26.07s (entire file)
duration: 26.07s
```

Audio is fully buffered before any play attempt, so first-play works identically to subsequent plays.

## Files changed
- `packages/core/src/runtime/init.ts` — set `preload="auto"` + `load()` in `bindMediaMetadataListeners`
- `packages/core/src/runtime/media.ts` — defer `play()` on unbuffered media via `canplay` listener
- `packages/core/src/runtime/media.test.ts` — updated test + added unbuffered media test case
---
 packages/core/src/runtime/init.ts       | 10 ++++++++
 packages/core/src/runtime/media.test.ts | 34 ++++++++++++++++++++++++-
 packages/core/src/runtime/media.ts      | 24 +++++++++++++++--
 3 files changed, 65 insertions(+), 3 deletions(-)

diff --git a/packages/core/src/runtime/init.ts b/packages/core/src/runtime/init.ts
index 25ac6c16f..ff82b291e 100644
--- a/packages/core/src/runtime/init.ts
+++ b/packages/core/src/runtime/init.ts
@@ -1167,6 +1167,16 @@ export function initSandboxRuntimeModular(): void {
       metadataBoundMedia.add(mediaEl);
       mediaEl.addEventListener("loadedmetadata", scheduleMetadataDurationHydration);
       mediaEl.addEventListener("durationchange", scheduleMetadataDurationHydration);
+
+      // Eagerly preload media data so audio/video is buffered before the user
+      // clicks play. Without this, the first play() call fires on un-fetched
+      // media, producing silence or choppy audio until the browser caches it.
+      if (mediaEl.preload !== "auto") {
+        mediaEl.preload = "auto";
+      }
+      if (mediaEl.readyState < HTMLMediaElement.HAVE_FUTURE_DATA) {
+        mediaEl.load();
+      }
     }
   };
 
diff --git a/packages/core/src/runtime/media.test.ts b/packages/core/src/runtime/media.test.ts
index 460bdf142..0cc605e05 100644
--- a/packages/core/src/runtime/media.test.ts
+++ b/packages/core/src/runtime/media.test.ts
@@ -171,12 +171,44 @@ describe("syncRuntimeMedia", () => {
     document.body.innerHTML = "";
   });
 
-  it("plays active clip when playing", () => {
+  it("plays active clip when playing and buffered", () => {
     const clip = createMockClip({ start: 0, end: 10 });
+    Object.defineProperty(clip.el, "readyState", { value: 4, writable: true });
     syncRuntimeMedia({ clips: [clip], timeSeconds: 5, playing: true, playbackRate: 1 });
     expect(clip.el.play).toHaveBeenCalled();
   });
 
+  it("defers play on unbuffered media and calls load()", () => {
+    const clip = createMockClip({ start: 0, end: 10 });
+    Object.defineProperty(clip.el, "readyState", { value: 0, writable: true });
+    const loadSpy = vi.spyOn(clip.el, "load").mockImplementation(() => {});
+    const addEventSpy = vi.spyOn(clip.el, "addEventListener");
+    syncRuntimeMedia({ clips: [clip], timeSeconds: 5, playing: true, playbackRate: 1 });
+    expect(clip.el.play).not.toHaveBeenCalled();
+    expect(loadSpy).toHaveBeenCalledOnce();
+    expect(addEventSpy).toHaveBeenCalledWith("canplay", expect.any(Function), { once: true });
+  });
+
+  it("plays when canplay fires after deferred play", () => {
+    const clip = createMockClip({ start: 0, end: 10 });
+    Object.defineProperty(clip.el, "readyState", { value: 0, writable: true });
+    vi.spyOn(clip.el, "load").mockImplementation(() => {});
+    syncRuntimeMedia({ clips: [clip], timeSeconds: 5, playing: true, playbackRate: 1 });
+    expect(clip.el.play).not.toHaveBeenCalled();
+    clip.el.dispatchEvent(new Event("canplay"));
+    expect(clip.el.play).toHaveBeenCalled();
+  });
+
+  it("does not re-register listener on repeated ticks while unbuffered", () => {
+    const clip = createMockClip({ start: 0, end: 10 });
+    Object.defineProperty(clip.el, "readyState", { value: 0, writable: true });
+    const loadSpy = vi.spyOn(clip.el, "load").mockImplementation(() => {});
+    syncRuntimeMedia({ clips: [clip], timeSeconds: 5, playing: true, playbackRate: 1 });
+    syncRuntimeMedia({ clips: [clip], timeSeconds: 5.1, playing: true, playbackRate: 1 });
+    syncRuntimeMedia({ clips: [clip], timeSeconds: 5.2, playing: true, playbackRate: 1 });
+    expect(loadSpy).toHaveBeenCalledOnce();
+  });
+
   it("pauses active clip when not playing", () => {
     const clip = createMockClip({ start: 0, end: 10 });
     Object.defineProperty(clip.el, "paused", { value: false, writable: true });
diff --git a/packages/core/src/runtime/media.ts b/packages/core/src/runtime/media.ts
index 4df4a17bd..0630845dc 100644
--- a/packages/core/src/runtime/media.ts
+++ b/packages/core/src/runtime/media.ts
@@ -66,6 +66,10 @@ export function refreshRuntimeMediaCache(params?: {
   return { timedMediaEls: mediaEls, mediaClips, videoClips, maxMediaEnd };
 }
 
+// Elements with a pending deferred play — prevents re-calling load()/addEventListener
+// on every tick while the media is still buffering.
+const pendingPlay = new WeakSet<HTMLMediaElement>();
+
 export function syncRuntimeMedia(params: {
   clips: RuntimeMediaClip[];
   timeSeconds: number;
@@ -100,8 +104,24 @@ export function syncRuntimeMedia(params: {
           // ignore browser seek restrictions
         }
       }
-      if (params.playing && el.paused) {
-        void el.play().catch(() => {});
+      if (params.playing && el.paused && !pendingPlay.has(el)) {
+        if (el.readyState >= HTMLMediaElement.HAVE_FUTURE_DATA) {
+          void el.play().catch(() => {});
+        } else {
+          pendingPlay.add(el);
+          if (el.preload !== "auto") el.preload = "auto";
+          el.addEventListener(
+            "canplay",
+            () => {
+              pendingPlay.delete(el);
+              if (!el.paused) return;
+              void el.play().catch(() => {});
+            },
+            { once: true },
+          );
+          el.addEventListener("error", () => pendingPlay.delete(el), { once: true });
+          el.load();
+        }
       } else if (!params.playing && !el.paused) {
         el.pause();
       }