mirror of
https://github.com/heygen-com/hyperframes.git
synced 2026-09-03 04:38:33 +00:00
fix(runtime): preload media on init to prevent broken first-play audio (#234)
## Summary - Audio (and video) sounds broken/choppy on first play in the studio preview, but works fine on second play - Root cause: `<audio>` elements default to `preload="metadata"`, which only fetches enough data to determine duration — not enough for smooth playback. When `el.play()` fires, the browser hasn't buffered the audio data yet - The runtime now eagerly sets `preload="auto"` and calls `load()` during init, ensuring media is fully buffered before the user clicks play - `syncRuntimeMedia` now defers `play()` on unbuffered media by registering a `canplay` listener, instead of silently swallowing the failure ## Testing Verified with agent-browser against a 26s narration composition (soulscape-film): ``` # After fix — audio element state at init: preload: "auto" readyState: 4 (HAVE_ENOUGH_DATA) buffered: 26.07s (entire file) duration: 26.07s ``` Audio is fully buffered before any play attempt, so first-play works identically to subsequent plays. ## Files changed - `packages/core/src/runtime/init.ts` — set `preload="auto"` + `load()` in `bindMediaMetadataListeners` - `packages/core/src/runtime/media.ts` — defer `play()` on unbuffered media via `canplay` listener - `packages/core/src/runtime/media.test.ts` — updated test + added unbuffered media test case
This commit is contained in:
@@ -1167,6 +1167,16 @@ export function initSandboxRuntimeModular(): void {
|
||||
metadataBoundMedia.add(mediaEl);
|
||||
mediaEl.addEventListener("loadedmetadata", scheduleMetadataDurationHydration);
|
||||
mediaEl.addEventListener("durationchange", scheduleMetadataDurationHydration);
|
||||
|
||||
// Eagerly preload media data so audio/video is buffered before the user
|
||||
// clicks play. Without this, the first play() call fires on un-fetched
|
||||
// media, producing silence or choppy audio until the browser caches it.
|
||||
if (mediaEl.preload !== "auto") {
|
||||
mediaEl.preload = "auto";
|
||||
}
|
||||
if (mediaEl.readyState < HTMLMediaElement.HAVE_FUTURE_DATA) {
|
||||
mediaEl.load();
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
|
||||
@@ -171,12 +171,44 @@ describe("syncRuntimeMedia", () => {
|
||||
document.body.innerHTML = "";
|
||||
});
|
||||
|
||||
it("plays active clip when playing", () => {
|
||||
it("plays active clip when playing and buffered", () => {
|
||||
const clip = createMockClip({ start: 0, end: 10 });
|
||||
Object.defineProperty(clip.el, "readyState", { value: 4, writable: true });
|
||||
syncRuntimeMedia({ clips: [clip], timeSeconds: 5, playing: true, playbackRate: 1 });
|
||||
expect(clip.el.play).toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it("defers play on unbuffered media and calls load()", () => {
|
||||
const clip = createMockClip({ start: 0, end: 10 });
|
||||
Object.defineProperty(clip.el, "readyState", { value: 0, writable: true });
|
||||
const loadSpy = vi.spyOn(clip.el, "load").mockImplementation(() => {});
|
||||
const addEventSpy = vi.spyOn(clip.el, "addEventListener");
|
||||
syncRuntimeMedia({ clips: [clip], timeSeconds: 5, playing: true, playbackRate: 1 });
|
||||
expect(clip.el.play).not.toHaveBeenCalled();
|
||||
expect(loadSpy).toHaveBeenCalledOnce();
|
||||
expect(addEventSpy).toHaveBeenCalledWith("canplay", expect.any(Function), { once: true });
|
||||
});
|
||||
|
||||
it("plays when canplay fires after deferred play", () => {
|
||||
const clip = createMockClip({ start: 0, end: 10 });
|
||||
Object.defineProperty(clip.el, "readyState", { value: 0, writable: true });
|
||||
vi.spyOn(clip.el, "load").mockImplementation(() => {});
|
||||
syncRuntimeMedia({ clips: [clip], timeSeconds: 5, playing: true, playbackRate: 1 });
|
||||
expect(clip.el.play).not.toHaveBeenCalled();
|
||||
clip.el.dispatchEvent(new Event("canplay"));
|
||||
expect(clip.el.play).toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it("does not re-register listener on repeated ticks while unbuffered", () => {
|
||||
const clip = createMockClip({ start: 0, end: 10 });
|
||||
Object.defineProperty(clip.el, "readyState", { value: 0, writable: true });
|
||||
const loadSpy = vi.spyOn(clip.el, "load").mockImplementation(() => {});
|
||||
syncRuntimeMedia({ clips: [clip], timeSeconds: 5, playing: true, playbackRate: 1 });
|
||||
syncRuntimeMedia({ clips: [clip], timeSeconds: 5.1, playing: true, playbackRate: 1 });
|
||||
syncRuntimeMedia({ clips: [clip], timeSeconds: 5.2, playing: true, playbackRate: 1 });
|
||||
expect(loadSpy).toHaveBeenCalledOnce();
|
||||
});
|
||||
|
||||
it("pauses active clip when not playing", () => {
|
||||
const clip = createMockClip({ start: 0, end: 10 });
|
||||
Object.defineProperty(clip.el, "paused", { value: false, writable: true });
|
||||
|
||||
@@ -66,6 +66,10 @@ export function refreshRuntimeMediaCache(params?: {
|
||||
return { timedMediaEls: mediaEls, mediaClips, videoClips, maxMediaEnd };
|
||||
}
|
||||
|
||||
// Elements with a pending deferred play — prevents re-calling load()/addEventListener
|
||||
// on every tick while the media is still buffering.
|
||||
const pendingPlay = new WeakSet<HTMLMediaElement>();
|
||||
|
||||
export function syncRuntimeMedia(params: {
|
||||
clips: RuntimeMediaClip[];
|
||||
timeSeconds: number;
|
||||
@@ -100,8 +104,24 @@ export function syncRuntimeMedia(params: {
|
||||
// ignore browser seek restrictions
|
||||
}
|
||||
}
|
||||
if (params.playing && el.paused) {
|
||||
if (params.playing && el.paused && !pendingPlay.has(el)) {
|
||||
if (el.readyState >= HTMLMediaElement.HAVE_FUTURE_DATA) {
|
||||
void el.play().catch(() => {});
|
||||
} else {
|
||||
pendingPlay.add(el);
|
||||
if (el.preload !== "auto") el.preload = "auto";
|
||||
el.addEventListener(
|
||||
"canplay",
|
||||
() => {
|
||||
pendingPlay.delete(el);
|
||||
if (!el.paused) return;
|
||||
void el.play().catch(() => {});
|
||||
},
|
||||
{ once: true },
|
||||
);
|
||||
el.addEventListener("error", () => pendingPlay.delete(el), { once: true });
|
||||
el.load();
|
||||
}
|
||||
} else if (!params.playing && !el.paused) {
|
||||
el.pause();
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user