fix(runtime): preload media on init to prevent broken first-play audio (#234)

## Summary
- Audio (and video) sounds broken/choppy on first play in the studio preview, but works fine on second play
- Root cause: `<audio>` elements default to `preload="metadata"`, which only fetches enough data to determine duration — not enough for smooth playback. When `el.play()` fires, the browser hasn't buffered the audio data yet
- The runtime now eagerly sets `preload="auto"` and calls `load()` during init, ensuring media is fully buffered before the user clicks play
- `syncRuntimeMedia` now defers `play()` on unbuffered media by registering a `canplay` listener, instead of silently swallowing the failure

## Testing
Verified with agent-browser against a 26s narration composition (soulscape-film):

```
# After fix — audio element state at init:
preload: "auto"
readyState: 4 (HAVE_ENOUGH_DATA)
buffered: 26.07s (entire file)
duration: 26.07s
```

Audio is fully buffered before any play attempt, so first-play works identically to subsequent plays.

## Files changed
- `packages/core/src/runtime/init.ts` — set `preload="auto"` + `load()` in `bindMediaMetadataListeners`
- `packages/core/src/runtime/media.ts` — defer `play()` on unbuffered media via `canplay` listener
- `packages/core/src/runtime/media.test.ts` — updated test + added unbuffered media test case
This commit is contained in:
Miguel Ángel
2026-04-10 00:05:32 +02:00
committed by GitHub
parent 9115d7364c
commit ceb54811c6
3 changed files with 65 additions and 3 deletions
+10
View File
@@ -1167,6 +1167,16 @@ export function initSandboxRuntimeModular(): void {
metadataBoundMedia.add(mediaEl);
mediaEl.addEventListener("loadedmetadata", scheduleMetadataDurationHydration);
mediaEl.addEventListener("durationchange", scheduleMetadataDurationHydration);
// Eagerly preload media data so audio/video is buffered before the user
// clicks play. Without this, the first play() call fires on un-fetched
// media, producing silence or choppy audio until the browser caches it.
if (mediaEl.preload !== "auto") {
mediaEl.preload = "auto";
}
if (mediaEl.readyState < HTMLMediaElement.HAVE_FUTURE_DATA) {
mediaEl.load();
}
}
};
+33 -1
View File
@@ -171,12 +171,44 @@ describe("syncRuntimeMedia", () => {
document.body.innerHTML = "";
});
it("plays active clip when playing", () => {
it("plays active clip when playing and buffered", () => {
const clip = createMockClip({ start: 0, end: 10 });
Object.defineProperty(clip.el, "readyState", { value: 4, writable: true });
syncRuntimeMedia({ clips: [clip], timeSeconds: 5, playing: true, playbackRate: 1 });
expect(clip.el.play).toHaveBeenCalled();
});
it("defers play on unbuffered media and calls load()", () => {
const clip = createMockClip({ start: 0, end: 10 });
Object.defineProperty(clip.el, "readyState", { value: 0, writable: true });
const loadSpy = vi.spyOn(clip.el, "load").mockImplementation(() => {});
const addEventSpy = vi.spyOn(clip.el, "addEventListener");
syncRuntimeMedia({ clips: [clip], timeSeconds: 5, playing: true, playbackRate: 1 });
expect(clip.el.play).not.toHaveBeenCalled();
expect(loadSpy).toHaveBeenCalledOnce();
expect(addEventSpy).toHaveBeenCalledWith("canplay", expect.any(Function), { once: true });
});
it("plays when canplay fires after deferred play", () => {
const clip = createMockClip({ start: 0, end: 10 });
Object.defineProperty(clip.el, "readyState", { value: 0, writable: true });
vi.spyOn(clip.el, "load").mockImplementation(() => {});
syncRuntimeMedia({ clips: [clip], timeSeconds: 5, playing: true, playbackRate: 1 });
expect(clip.el.play).not.toHaveBeenCalled();
clip.el.dispatchEvent(new Event("canplay"));
expect(clip.el.play).toHaveBeenCalled();
});
it("does not re-register listener on repeated ticks while unbuffered", () => {
const clip = createMockClip({ start: 0, end: 10 });
Object.defineProperty(clip.el, "readyState", { value: 0, writable: true });
const loadSpy = vi.spyOn(clip.el, "load").mockImplementation(() => {});
syncRuntimeMedia({ clips: [clip], timeSeconds: 5, playing: true, playbackRate: 1 });
syncRuntimeMedia({ clips: [clip], timeSeconds: 5.1, playing: true, playbackRate: 1 });
syncRuntimeMedia({ clips: [clip], timeSeconds: 5.2, playing: true, playbackRate: 1 });
expect(loadSpy).toHaveBeenCalledOnce();
});
it("pauses active clip when not playing", () => {
const clip = createMockClip({ start: 0, end: 10 });
Object.defineProperty(clip.el, "paused", { value: false, writable: true });
+22 -2
View File
@@ -66,6 +66,10 @@ export function refreshRuntimeMediaCache(params?: {
return { timedMediaEls: mediaEls, mediaClips, videoClips, maxMediaEnd };
}
// Elements with a pending deferred play — prevents re-calling load()/addEventListener
// on every tick while the media is still buffering.
const pendingPlay = new WeakSet<HTMLMediaElement>();
export function syncRuntimeMedia(params: {
clips: RuntimeMediaClip[];
timeSeconds: number;
@@ -100,8 +104,24 @@ export function syncRuntimeMedia(params: {
// ignore browser seek restrictions
}
}
if (params.playing && el.paused) {
void el.play().catch(() => {});
if (params.playing && el.paused && !pendingPlay.has(el)) {
if (el.readyState >= HTMLMediaElement.HAVE_FUTURE_DATA) {
void el.play().catch(() => {});
} else {
pendingPlay.add(el);
if (el.preload !== "auto") el.preload = "auto";
el.addEventListener(
"canplay",
() => {
pendingPlay.delete(el);
if (!el.paused) return;
void el.play().catch(() => {});
},
{ once: true },
);
el.addEventListener("error", () => pendingPlay.delete(el), { once: true });
el.load();
}
} else if (!params.playing && !el.paused) {
el.pause();
}