mirror of
https://github.com/heygen-com/hyperframes.git
synced 2026-11-18 07:50:13 +00:00
fix(core): decode WebKit sources by memory budget, not by length (#7232)
This commit is contained in:
@@ -6076,6 +6076,11 @@ describe("initSandboxRuntimeModular", () => {
|
||||
await vi.waitFor(() => expect(release).toHaveBeenCalledWith(audio));
|
||||
});
|
||||
|
||||
it("hands a 650 s source to a decoded buffer, past the old 600 s limit", async () => {
|
||||
const { audio, release } = await startRouted(SAFARI, 650);
|
||||
await vi.waitFor(() => expect(release).toHaveBeenCalledWith(audio));
|
||||
});
|
||||
|
||||
it("leaves an hour-long source on its element", async () => {
|
||||
const { schedule, release } = await startRouted(SAFARI, 3600);
|
||||
await Promise.resolve();
|
||||
@@ -6084,7 +6089,7 @@ describe("initSandboxRuntimeModular", () => {
|
||||
});
|
||||
|
||||
it("gives a clip that has not loaded its source's verdict", async () => {
|
||||
vi.spyOn(WebAudioTransport.prototype, "isDecodedOrDecoding").mockReturnValue(true);
|
||||
vi.spyOn(WebAudioTransport.prototype, "fitsDecodeBudget").mockReturnValue(true);
|
||||
const { audio, release } = await startRouted(SAFARI, Number.NaN);
|
||||
await vi.waitFor(() => expect(release).toHaveBeenCalledWith(audio));
|
||||
});
|
||||
|
||||
@@ -499,18 +499,14 @@ const joinsWebAudio = (el: Element): el is HTMLMediaElement =>
|
||||
masterChain: readMasterFxChain(el.ownerDocument),
|
||||
}));
|
||||
const WEB_AUDIO_MEDIA = "audio[data-start], video[data-start]";
|
||||
const DECODE_FIRST_MAX_SECONDS = 600;
|
||||
|
||||
/** WebKit restarts a media element's audio 150 to 400 ms late; a decoded buffer on the context clock starts on time. */
|
||||
function decodesFirst(el: HTMLMediaElement, webAudio: WebAudioTransport): boolean {
|
||||
const shortEnough = Number.isFinite(el.duration)
|
||||
? el.duration <= DECODE_FIRST_MAX_SECONDS
|
||||
: webAudio.isDecodedOrDecoding(el);
|
||||
return (
|
||||
isWebKit() &&
|
||||
isAudioElement(el) &&
|
||||
!el.loop &&
|
||||
shortEnough &&
|
||||
webAudio.fitsDecodeBudget(el) &&
|
||||
typeof readElementRateSpec(el) === "number" &&
|
||||
readElementPlaybackRate(el) === 1
|
||||
);
|
||||
|
||||
@@ -1767,6 +1767,44 @@ describe("WebAudioTransport", () => {
|
||||
});
|
||||
});
|
||||
|
||||
describe("decode budget", () => {
|
||||
const audio = (src: string, duration: number) =>
|
||||
({
|
||||
tagName: "AUDIO",
|
||||
getAttribute: (name: string) => (name === "src" ? src : null),
|
||||
currentSrc: "",
|
||||
duration,
|
||||
}) as unknown as HTMLMediaElement;
|
||||
const transport = () => {
|
||||
const t = new WebAudioTransport();
|
||||
(t as unknown as { _ctx: unknown })._ctx = { sampleRate: 48000, decodeAudioData: vi.fn() };
|
||||
return t;
|
||||
};
|
||||
|
||||
it("takes a ten minute source and refuses a one hour one", () => {
|
||||
const t = transport();
|
||||
expect(t.fitsDecodeBudget(audio("a.mp4", 600))).toBe(true);
|
||||
expect(t.fitsDecodeBudget(audio("b.mp4", 3600))).toBe(false);
|
||||
});
|
||||
|
||||
it("counts a source that is decoding against the next one", async () => {
|
||||
const t = transport();
|
||||
vi.stubGlobal(
|
||||
"fetch",
|
||||
vi.fn(() => new Promise(() => {})),
|
||||
);
|
||||
void t.decodeAudioElement(audio("a.mp4", 600));
|
||||
expect(t.fitsDecodeBudget(audio("a.mp4", 600))).toBe(true);
|
||||
expect(t.fitsDecodeBudget(audio("b.mp4", 100))).toBe(false);
|
||||
expect(t.fitsDecodeBudget(audio("c.mp4", 10))).toBe(true);
|
||||
vi.unstubAllGlobals();
|
||||
});
|
||||
|
||||
it("leaves a clip that has not loaded to its source", () => {
|
||||
expect(transport().fitsDecodeBudget(audio("a.mp4", Number.NaN))).toBe(false);
|
||||
});
|
||||
});
|
||||
|
||||
describe("decodeAudioElement retry policy (late-asset self-heal)", () => {
|
||||
function transportWithDecode(decodeImpl: () => Promise<AudioBuffer>) {
|
||||
const transport = new WebAudioTransport();
|
||||
|
||||
@@ -229,11 +229,15 @@ const FOLLOWED_EVENTS = ["playing", "seeked", "ratechange"] as const;
|
||||
// A pause to scrub or a reschedule mid-play keeps the output awake; a real idle lets it sleep.
|
||||
const IDLE_SUSPEND_MS = 10000;
|
||||
const SHORT_DECODE_SECONDS = 0.25;
|
||||
const DECODE_BUDGET_BYTES = 256 * 1024 * 1024;
|
||||
const ASSUMED_CHANNELS = 2;
|
||||
const BYTES_PER_SAMPLE = 4;
|
||||
|
||||
export class WebAudioTransport {
|
||||
private _ctx: AudioContext | null = null;
|
||||
private _bufferCache = new Map<string, AudioBuffer>();
|
||||
private _decoding = new Map<string, Promise<AudioBuffer | null>>();
|
||||
private _pendingDecodeBytes = new Map<string, number>();
|
||||
private _failedSrcs = new Set<string>();
|
||||
private _mediaElementSources = new WeakMap<HTMLMediaElement, MediaElementAudioSourceNode>();
|
||||
private _handedOver = new WeakSet<HTMLMediaElement>();
|
||||
@@ -358,15 +362,35 @@ export class WebAudioTransport {
|
||||
if (!src) return Promise.resolve(null);
|
||||
const pending = this._decoding.get(src);
|
||||
if (pending) return pending;
|
||||
const decoding = this.fetchAndDecode(src, el).finally(() => this._decoding.delete(src));
|
||||
this._pendingDecodeBytes.set(src, this.estimateDecodedBytes(el.duration));
|
||||
const decoding = this.fetchAndDecode(src, el).finally(() => {
|
||||
this._decoding.delete(src);
|
||||
this._pendingDecodeBytes.delete(src);
|
||||
});
|
||||
this._decoding.set(src, decoding);
|
||||
return decoding;
|
||||
}
|
||||
|
||||
/** True once the source of `el` is decoded or decoding. */
|
||||
isDecodedOrDecoding(el: HTMLMediaElement): boolean {
|
||||
fitsDecodeBudget(el: HTMLMediaElement): boolean {
|
||||
const src = decodeKey(el);
|
||||
return !!src && (this._bufferCache.has(src) || this._decoding.has(src));
|
||||
if (!src) return false;
|
||||
if (this._bufferCache.has(src) || this._decoding.has(src)) return true;
|
||||
if (!Number.isFinite(el.duration)) return false;
|
||||
return this.decodedBytes() + this.estimateDecodedBytes(el.duration) <= DECODE_BUDGET_BYTES;
|
||||
}
|
||||
|
||||
private estimateDecodedBytes(seconds: number): number {
|
||||
const rate = this._ctx?.sampleRate ?? 48000;
|
||||
if (!Number.isFinite(seconds)) return 0;
|
||||
return seconds * rate * ASSUMED_CHANNELS * BYTES_PER_SAMPLE;
|
||||
}
|
||||
|
||||
private decodedBytes(): number {
|
||||
let bytes = 0;
|
||||
for (const buffer of this._bufferCache.values())
|
||||
bytes += buffer.length * buffer.numberOfChannels * BYTES_PER_SAMPLE;
|
||||
for (const pending of this._pendingDecodeBytes.values()) bytes += pending;
|
||||
return bytes;
|
||||
}
|
||||
|
||||
private async fetchAndDecode(src: string, el: HTMLMediaElement): Promise<AudioBuffer | null> {
|
||||
|
||||
Reference in New Issue
Block a user