import { attachElementFxChain, readElementAutomation, type ElementFxHandle } from "./audioFx.js"; import { scheduleParamLane, volumeLane, type AutomationTiming, } from "../audio/audioFxAutomation.js"; import { VOLUME_RANGE } from "../audioAutomation.js"; import { audioGroupOf, isAudibleUnderSolo } from "../audioGroups.js"; import { swallow } from "./diagnostics"; import { clampAudioGain } from "../audioGain.js"; import { getDebugSurface } from "./globals.js"; import { readElementPlaybackRate } from "./media.js"; function normalizeRate(rate: number): number { if (!Number.isFinite(rate) || rate <= 0) return 1; return rate; } /** * Breadcrumb for the per-element-mute handoff: the transport just claimed a track * that was audibly playing through the HTMLMedia fallback. Quiet unless * `__hfDebug` — a hook for diagnosing the race if it ever regresses. */ function logFallbackHandoff(el: HTMLMediaElement, priorMuted: boolean): void { if (priorMuted || el.paused || !getDebugSurface().__hfDebug) return; // eslint-disable-next-line no-console -- intentional debug surface console.debug( "[hyperframes] webAudioTransport claimed fallback-playing element:", el.currentSrc || el.getAttribute("src") || "", ); } /** * Start a buffer source, bounding it to the clip's authored window * (`data-duration`) so a trimmed clip stops at its edge instead of running the * buffer to the source file's natural end. `clipSourceLen` is the clip span in * buffer seconds; the third `start()` arg is the portion to play from the * offset. An infinite `clipDuration` plays unbounded (legacy behavior). * * Returns false when the playhead is already past the clip end (nothing to * play); the caller should discard the source. */ function startBoundedSource( node: AudioBufferSourceNode, opts: { elapsed: number; mediaStart: number; scheduledAt: number; globalRate: number; mediaRate: number; clipDuration: number; }, ): boolean { const { elapsed, mediaStart, scheduledAt, globalRate, mediaRate, clipDuration } = opts; const hasBound = Number.isFinite(clipDuration) && clipDuration > 0; const clipSourceLen = clipDuration * mediaRate; if (elapsed >= 0) { const sourceElapsed = elapsed * mediaRate; const remaining = clipSourceLen - sourceElapsed; if (hasBound && remaining <= 0) return false; if (hasBound) node.start(0, sourceElapsed + mediaStart, remaining); else node.start(0, sourceElapsed + mediaStart); return true; } const delay = -elapsed / globalRate; if (hasBound) node.start(scheduledAt + delay, mediaStart, clipSourceLen); else node.start(scheduledAt + delay, mediaStart); return true; } /** * The volume lane rides the fader, after the effects — where a DAW puts it, * and the order the render bakes it in. * * Typed against the attribute reader rather than `HTMLMediaElement` so a group * bus (an ``, not a media element) can ride the same path. */ function scheduleVolumeLane( el: { getAttribute?(name: string): string | null }, gainNode: GainNode, timing: AutomationTiming, ): void { const lane = volumeLane(readElementAutomation(el)); if (!lane) return; scheduleParamLane([{ param: gainNode.gain }], lane, VOLUME_RANGE.scale, timing); } type ScheduledSourceBase = { el: HTMLMediaElement; gainNode: GainNode; /** Solo ("Hear only this") attenuation — dedicated node, parallel to the * volume gain, so a solo toggle never fights `scheduleVolumeLane`'s ramps * on the same param (same hazard B5's group-mute gain was split out to * avoid). 0 while silenced by an active solo elsewhere, 1 otherwise. */ soloGain: GainNode; /** FX chain spliced between source and gain, when the element carries one. */ fx?: ElementFxHandle | null; compositionStart: number; mediaStart: number; scheduledAt: number; priorMuted: boolean; priorVolume: number; mediaPlaybackRate: number; // The clip had a finite window, so start() was given a fixed duration in // buffer-sample seconds. That bound can't be rescaled in place on a rate // change — callers must stopAll()+reschedule (see hasBoundedActiveSources). bounded: boolean; }; export type ScheduledSource = ScheduledSourceBase & ( | { sourceKind: "buffer"; sourceNode: AudioBufferSourceNode } | { sourceKind: "media-element"; sourceNode: MediaElementAudioSourceNode } ); function isBufferSource( source: ScheduledSource, ): source is ScheduledSourceBase & { sourceKind: "buffer"; sourceNode: AudioBufferSourceNode } { return source.sourceKind === "buffer"; } export class WebAudioTransport { private _ctx: AudioContext | null = null; private _bufferCache = new Map(); private _failedSrcs = new Set(); private _mediaElementSources = new WeakMap(); private _activeSources: ScheduledSource[] = []; private _masterGain: GainNode | null = null; private _masterVolume = 1; private _masterMuted = false; // One shared bus per group id, lazily built the first time a member of that // group is scheduled. Lives for the session (mirrors `_masterGain`'s own // lifecycle) rather than being torn down on every `stopAll()`, so replaying // a group does not rebuild its chain; only `destroy()` disposes these. private _groups = new Map< string, { input: GainNode; muteGain: GainNode; analyser: AnalyserNode; levelBuf: Float32Array; dispose(): void; } >(); // Composition-time reference frame: at AudioContext time `_rateAnchorCtx`, // composition time was `_rateAnchorComp`, and time has been advancing at // `_rate` composition-seconds per wallclock-second since. private _rateAnchorCtx = 0; private _rateAnchorComp = 0; private _rate = 1; private _paused = true; private _playGeneration = 0; // Session-only "Hear only this" set (clip ids and group ids). Never read // from or written to any attribute — studio pushes it in directly via // `setSolo`; see `isAudibleUnderSolo` for the exact predicate. private _soloed: ReadonlySet = new Set(); async init(): Promise { try { this._ctx = new AudioContext(); this._masterGain = this._ctx.createGain(); this._masterGain.connect(this._ctx.destination); this.applyMasterGain(); return true; } catch { return false; } } get context(): AudioContext | null { return this._ctx; } getTime(): number { if (!this._ctx || this._paused) return -1; return this._rateAnchorComp + (this._ctx.currentTime - this._rateAnchorCtx) * this._rate; } async decodeAudioElement(el: HTMLMediaElement): Promise { const src = el.currentSrc || el.getAttribute("src"); if (!src) return null; if (this._bufferCache.has(src)) return this._bufferCache.get(src)!; if (this._failedSrcs.has(src)) return null; if (!this._ctx) return null; // Fetch the bytes. A network error or non-OK status (e.g. a 404 for an // asset that simply has not been uploaded yet) is TRANSIENT — return null // WITHOUT blacklisting, so the next play/seek generation retries once the // asset becomes available. (Previously these were added to `_failedSrcs`, // which is never cleared, permanently silencing a merely-late track.) let arrayBuffer: ArrayBuffer; try { // `no-store`: a retry must actually re-request the asset — not replay a // cached 404/stale response from the failed attempt that we chose not to // blacklist. const response = await fetch(src, { cache: "no-store" }); if (!response.ok) { swallow("webAudioTransport.fetch", new Error(`${response.status} ${src}`)); return null; } arrayBuffer = await response.arrayBuffer(); } catch (err) { swallow("webAudioTransport.fetch", err); return null; } // A decode failure means the bytes themselves are unusable (corrupt or an // unsupported codec) — that IS permanent, so blacklist to avoid re-decoding // the same bad payload on every generation. try { const audioBuffer = await this._ctx.decodeAudioData(arrayBuffer); this._bufferCache.set(src, audioBuffer); return audioBuffer; } catch (err) { this._failedSrcs.add(src); swallow("webAudioTransport.decode", err); return null; } } startGeneration(): number { this._playGeneration += 1; return this._playGeneration; } currentGeneration(): number { return this._playGeneration; } /** * Route the browser's pitch-preserving HTMLMediaElement transport through the * same FX, automation, element-gain, and master graph used by final audio. * The media element remains the source-time/rate owner; Web Audio is strictly * downstream and therefore never resamples it for Studio global speed. */ async scheduleMediaElementPlayback( el: HTMLMediaElement, compositionStart: number, _mediaStart: number, compositionTime: number, volume: number, generation: number, rate = 1, ): Promise { if (!this._ctx || !this._masterGain) return null; if (generation !== this._playGeneration) return null; try { if (this._ctx.state === "suspended") await this._ctx.resume(); if (generation !== this._playGeneration) return null; let sourceNode = this._mediaElementSources.get(el); if (!sourceNode) { sourceNode = this._ctx.createMediaElementSource(el); this._mediaElementSources.set(el, sourceNode); } const safeRate = normalizeRate(rate); const gainNode = this._ctx.createGain(); gainNode.gain.value = volume; const scheduledAt = this._ctx.currentTime; const elapsed = compositionTime - compositionStart; const timing: AutomationTiming = { scheduledAt, elapsed, rate: safeRate }; const fx = attachElementFxChain(this._ctx, el, sourceNode, gainNode, timing); gainNode.connect(this._masterGain); scheduleVolumeLane(el, gainNode, timing); this._rate = safeRate; this._rateAnchorCtx = scheduledAt; this._rateAnchorComp = compositionTime; const scheduled: ScheduledSource = { fx, el, sourceNode, sourceKind: "media-element", gainNode, compositionStart, mediaStart: _mediaStart, scheduledAt, priorMuted: el.muted, priorVolume: el.volume, mediaPlaybackRate: readElementPlaybackRate(el), bounded: false, }; // Chrome applies HTMLMediaElement.volume before MediaElementAudioSource. // Keep that upstream stage at unity so the existing downstream gain is // the single owner of author × user volume and automation. el.volume = 1; this._activeSources.push(scheduled); this._paused = false; return scheduled; } catch (err) { swallow("webAudioTransport.mediaElementSource", err); return null; } } /** * The gain a grouped member's signal should land on, building it on first * use. A group's clock is COMPOSITION time (design doc §1.3) — it has no * `data-start`, and a missing start parses as 0, which is exactly * composition time — so its chain and volume lane are scheduled once here * against that zero-offset timing, not the member's own clip-local timing. * A group id with no matching `` element still gets a bus * (flat, no chain) so a hand-authored `data-audio-group` degrades to a * plain sum rather than losing the member's audio. */ private groupInput(groupId: string, doc: Document, timing: AutomationTiming): GainNode | null { const existing = this._groups.get(groupId); if (existing) return existing.input; if (!this._ctx || !this._masterGain) return null; const input = this._ctx.createGain(); // Stable point the FX chain (or, when there's none, the dry passthrough — // see `attachElementFxChain`'s `detach()`) always lands on before master, // regardless of whether a chain is attached/detached/rebuilt later. B7's // meter taps here. The mute gain splices in BEFORE `output` (between the // FX chain and here), never after — the meter is defined to read the // group's true, honestly-muted level (design doc §5), and this node is // that contract's anchor. const output = this._ctx.createGain(); output.connect(this._masterGain); const analyser = this._ctx.createAnalyser(); analyser.fftSize = 256; // level, not spectrum output.connect(analyser); const groupEl = doc.getElementById(groupId); const muteGain = this._ctx.createGain(); muteGain.gain.value = groupEl?.hasAttribute("data-hidden") ? 0 : 1; muteGain.connect(output); const fx = attachElementFxChain( this._ctx, groupEl ?? { getAttribute: () => null }, input, muteGain, timing, ); if (groupEl) scheduleVolumeLane(groupEl, input, timing); this._groups.set(groupId, { input, muteGain, analyser, levelBuf: new Float32Array(analyser.fftSize), dispose: () => { try { fx?.dispose(); input.disconnect(); muteGain.disconnect(); output.disconnect(); analyser.disconnect(); } catch { // Already torn down. } }, }); return input; } /** * Group mute, preview side — a separate gain from `input`'s volume fader * (B7) so a mute toggle never fights `scheduleVolumeLane`'s ramps on the * same param (the same hazard the design doc flags for §2.1). A no-op * until the group has an active member: at that point `groupInput` reads * the element's own `data-hidden` for its initial value, so there is * nothing to catch up on here. */ setGroupMuted(groupId: string, muted: boolean): void { const group = this._groups.get(groupId); if (!group) return; try { group.muteGain.gain.value = muted ? 0 : 1; } catch (err) { swallow("webAudioTransport.setGroupMuted", err); } } /** Every group id currently routing audio (built lazily by `groupInput` — * a group with no active member yet has no entry here). */ groupIds(): string[] { return [...this._groups.keys()]; } /** * RMS-ish level 0..1 and whether the last block clipped, for the group's * meter — or null when the group has no active member (idle/unknown). * Reuses a per-group buffer; no per-frame allocation. */ groupLevel(groupId: string): { level: number; clipped: boolean } | null { const g = this._groups.get(groupId); if (!g) return null; g.analyser.getFloatTimeDomainData(g.levelBuf); let sumSquares = 0; let clipped = false; for (const sample of g.levelBuf) { sumSquares += sample * sample; if (Math.abs(sample) >= 0.99) clipped = true; } return { level: Math.sqrt(sumSquares / g.levelBuf.length), clipped }; } /** Master, unless `el` belongs to a group — then that group's bus (built on * first use, per `groupInput`). */ private resolveDestination( el: HTMLMediaElement, scheduledAt: number, compositionTime: number, safeRate: number, ): GainNode | null { if (!this._masterGain) return null; const groupId = audioGroupOf(el); if (!groupId) return this._masterGain; const groupTiming: AutomationTiming = { scheduledAt, elapsed: compositionTime, rate: safeRate }; return this.groupInput(groupId, el.ownerDocument, groupTiming) ?? this._masterGain; } /** * The graph goes with it. Splicing alone left the FX handle alive and then * UNREACHABLE — `stopAll()` disposes by walking `_activeSources`, which the * splice just emptied of this entry. Every clip that finished naturally * leaked its MutationObserver for the session, and each one still answered * later `data-fx-chain` edits by rebuilding a whole graph (impulse response, * chorus/phaser oscillators started and never stopped) around a dead * source. Not disposed when the index is already -1: `stopAll()` has * already done it, and `stop()` is what fired this event. */ private handleSourceEnded( sourceNode: AudioBufferSourceNode, scheduled: ScheduledSource, el: HTMLMediaElement, priorMuted: boolean, ): void { const idx = this._activeSources.indexOf(scheduled); if (idx === -1) return; this._activeSources.splice(idx, 1); el.muted = priorMuted; try { sourceNode.disconnect(); scheduled.fx?.dispose(); scheduled.gainNode.disconnect(); scheduled.soloGain.disconnect(); } catch { // Already torn down. } if (this._activeSources.length === 0) this._paused = true; } // Pre-existing size (110 lines before this diff, which shrank it to under // 95 via two extractions — see `handleSourceEnded`/`resolveDestination`); // the remainder is inherently sequential graph-wiring, not a nested // decision tree, and further splitting would cost more readability than it // buys. Same call the B2 step took on `TimelineLogicalRow`. // fallow-ignore-next-line complexity async schedulePlayback( el: HTMLMediaElement, buffer: AudioBuffer, compositionStart: number, mediaStart: number, compositionTime: number, volume: number, generation: number, rate = 1, clipDuration = Number.POSITIVE_INFINITY, ): Promise { if (!this._ctx || !this._masterGain) return null; if (generation !== this._playGeneration) return null; try { if (this._ctx.state === "suspended") { await this._ctx.resume(); } if (generation !== this._playGeneration) return null; const safeRate = normalizeRate(rate); const mediaRate = readElementPlaybackRate(el); const sourceRate = safeRate * mediaRate; const sourceNode = this._ctx.createBufferSource(); sourceNode.buffer = buffer; sourceNode.playbackRate.value = sourceRate; const gainNode = this._ctx.createGain(); gainNode.gain.value = volume; const elapsed = compositionTime - compositionStart; const scheduledAt = this._ctx.currentTime; const timing: AutomationTiming = { scheduledAt, elapsed, rate: safeRate }; // Splice the element's FX chain between the decoded source and its gain, // so effects see the raw signal and volume automation rides on their // output — the same order the offline render uses. Preview and render run // the identical graph builders, so what is heard here is what is written. const fx = attachElementFxChain(this._ctx, el, sourceNode, gainNode, timing); const soloGain = this._ctx.createGain(); soloGain.gain.value = isAudibleUnderSolo(this._soloed, el.id, audioGroupOf(el)) ? 1 : 0; gainNode.connect(soloGain); soloGain.connect( this.resolveDestination(el, scheduledAt, compositionTime, safeRate) ?? this._masterGain, ); scheduleVolumeLane(el, gainNode, timing); this._rate = safeRate; this._rateAnchorCtx = scheduledAt; this._rateAnchorComp = compositionTime; if ( !startBoundedSource(sourceNode, { elapsed, mediaStart, scheduledAt, globalRate: safeRate, mediaRate, clipDuration, }) ) { // Playhead already past the clip end — discard the nodes we built. sourceNode.disconnect(); fx?.dispose(); gainNode.disconnect(); soloGain.disconnect(); return null; } const priorMuted = el.muted; el.muted = true; logFallbackHandoff(el, priorMuted); const scheduled: ScheduledSource = { fx, el, sourceNode, sourceKind: "buffer", gainNode, soloGain, compositionStart, mediaStart, scheduledAt, priorMuted, priorVolume: el.volume, mediaPlaybackRate: mediaRate, bounded: Number.isFinite(clipDuration) && clipDuration > 0, }; this._activeSources.push(scheduled); this._paused = false; sourceNode.addEventListener("ended", () => this.handleSourceEnded(sourceNode, scheduled, el, priorMuted), ); return scheduled; } catch (err) { swallow("webAudioTransport.schedule", err); return null; } } /** * Rebases the composition-time reference frame before swapping rate so * `getTime()` stays continuous across the change. Sources scheduled to * start in the future keep their original wallclock start time — callers * that need rate-correct future starts should `stopAll()` and reschedule. * * Each source's FX automation is re-aimed too. Lanes are committed to * absolute context times when the source is scheduled, so bumping only * `playbackRate` left every automated parameter running its original plan * over audio moving at a different speed. The `stopAll()`+reschedule recovery * in the runtime is no help here: it only fires for bounded sources, and a * project-level music bed with no `data-duration` is unbounded, so it never * recovered at all. */ setRate(rate: number): boolean { const safeRate = normalizeRate(rate); if (safeRate === this._rate) return false; if (this._ctx && !this._paused) { this._rateAnchorComp = this.getTime(); this._rateAnchorCtx = this._ctx.currentTime; } this._rate = safeRate; for (const source of this._activeSources) { try { if (isBufferSource(source)) { source.sourceNode.playbackRate.value = safeRate * source.mediaPlaybackRate; } source.fx?.setRate(safeRate); } catch (err) { swallow("webAudioTransport.setRate", err); } } return true; } // A bounded source's wall-clock duration was baked into start()'s duration // arg at its original rate; a later rate change can't rescale it in place, so // the caller must stopAll()+reschedule to keep trimmed clips ending on time. hasBoundedActiveSources(): boolean { return this._activeSources.some((s) => isBufferSource(s) && s.bounded); } stopAll(): void { for (const source of this._activeSources) { try { if (isBufferSource(source)) source.sourceNode.stop(); source.sourceNode.disconnect(); source.fx?.dispose(); source.gainNode.disconnect(); source.soloGain.disconnect(); } catch { // already stopped } if (isBufferSource(source)) source.el.muted = source.priorMuted; else source.el.volume = source.priorVolume; } this._activeSources = []; this._paused = true; } setVolume(volume: number): void { this._masterVolume = Math.max(0, Math.min(1, volume)); this.applyMasterGain(); } /** * The per-element gain carries the clip's AUTHOR gain, which reaches * MAX_AUDIO_GAIN — so it is clamped against that ceiling, not the spec's * [0,1]. `setVolume` above is the opposite case and stays spec-clamped: the * user's master volume is a fader, not a gain. * * Clamping this one at unity capped every static above-unity `data-volume` on * the preview path while the render honoured it — the exact preview/render * divergence this ceiling exists to close. Automation lanes hid it, because * they schedule ramps onto the param directly and never pass through here. */ setElementVolume(el: HTMLMediaElement, volume: number): void { const safeVolume = clampAudioGain(volume); for (const source of this._activeSources) { if (source.el !== el) continue; try { if (source.sourceKind === "media-element") source.el.volume = 1; source.gainNode.gain.value = safeVolume; } catch (err) { swallow("webAudioTransport.setElementVolume", err); } } } setMuted(muted: boolean): void { this._masterMuted = muted; this.applyMasterGain(); } private applyMasterGain(): void { if (this._masterGain) this._masterGain.gain.value = this._masterMuted ? 0 : this._masterVolume; } /** * Push the current "Hear only this" set and re-evaluate every active * source's solo gain against it — a gain-stage update, never a graph * rebuild (rule 3 of B5's step doc). Group buses are never touched here: * per `isAudibleUnderSolo`, a group is never attenuated by solo, so a * soloed member's path through its (unattenuated) group stays open by * construction. */ setSolo(soloed: ReadonlySet): void { this._soloed = soloed; for (const source of this._activeSources) { try { source.soloGain.gain.value = isAudibleUnderSolo( this._soloed, source.el.id, audioGroupOf(source.el), ) ? 1 : 0; } catch (err) { swallow("webAudioTransport.setSolo", err); } } } isActive(): boolean { return this._activeSources.length > 0 && !this._paused; } /** Whether the transport currently plays THIS element (the runtime mutes it to * avoid double audio; an unclaimed track stays audible). */ ownsElement(el: HTMLMediaElement): boolean { return !this._paused && this._activeSources.some((s) => s.el === el && isBufferSource(s)); } /** Whether this element's native signal currently flows through the WebAudio graph. */ routesElement(el: HTMLMediaElement): boolean { return !this._paused && this._activeSources.some((source) => source.el === el); } destroy(): void { this.stopAll(); for (const group of this._groups.values()) group.dispose(); this._groups.clear(); this._bufferCache.clear(); this._failedSrcs.clear(); this._mediaElementSources = new WeakMap(); if (this._ctx) { try { void this._ctx.close(); } catch { // ignore } } this._ctx = null; this._masterGain = null; this._masterVolume = 1; this._masterMuted = false; } }