mirror of
https://github.com/heygen-com/hyperframes.git
synced 2026-09-03 12:54:29 +00:00
feat(core): route grouped audio through a group bus in preview
An audio element carrying `data-audio-group` no longer lands its gain on the master bus directly — it feeds a per-group `GainNode` (built lazily on first use, one per group id) which itself feeds master, so members of the same group sum before the ear, ready for a group-level FX chain and volume/mute in later steps. An id with no matching `<hf-audio-group>` element still gets a plain, unprocessed bus rather than losing the track. The group's own chain and volume lane are wired through the same `attachElementFxChain`/`scheduleVolumeLane` every element already uses, against the group's clock — composition time (design doc §1.3), since a group has no `data-start` and a missing one parses as 0. The bus persists across `stopAll()` (mirroring `_masterGain`'s own lifecycle) so replaying a group does not rebuild its chain; only `destroy()` disposes it. Render is untouched — stays flat until B4; `audio-groups` is still a 0% canary so nothing ships this to a real composition without hand-authoring `data-audio-group`. Also: `audioGroupOf` (B1) crashed on any element lacking a real `tagName`/ `getAttribute` — exactly the shape of most `HTMLMediaElement` test doubles in this suite, including this file's own `mockEl`. Made it tolerant, same style as `readChain`'s existing guard in `runtime/audioFx.ts`. `schedulePlayback` was already 110 lines pre-existing before this diff; extracted `resolveDestination` and `handleSourceEnded` to shrink it to 92, then suppressed the remainder (inherently sequential graph wiring, not a decision tree) per the same precedent B2 used on `TimelineLogicalRow`. Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com>
This commit is contained in:
Vance Ingalls
co-authored by
Claude Sonnet 5
Claude Sonnet 5
parent
dd3212d11b
commit
5dc93a25ad
@@ -52,8 +52,14 @@ export function resolveAudioGroups(root: ParentNode): HfAudioGroup[] {
|
||||
}
|
||||
|
||||
/** The group a member belongs to, or null. Groups do not nest — this ignores
|
||||
* `data-audio-group` on an `<hf-audio-group>` element itself. */
|
||||
* `data-audio-group` on an `<hf-audio-group>` element itself.
|
||||
*
|
||||
* Tolerant of objects that only partially implement `Element` (test doubles
|
||||
* for `HTMLMediaElement` commonly do) — anything missing `tagName` or
|
||||
* `getAttribute` simply has no group, mirroring `readChain`'s style in
|
||||
* `runtime/audioFx.ts`. */
|
||||
export function audioGroupOf(el: Element): string | null {
|
||||
if (typeof el.tagName !== "string") return null;
|
||||
if (el.tagName.toLowerCase() === HF_AUDIO_GROUP_TAG) return null;
|
||||
return el.getAttribute(HF_AUDIO_GROUP_ATTR);
|
||||
return typeof el.getAttribute === "function" ? el.getAttribute(HF_AUDIO_GROUP_ATTR) : null;
|
||||
}
|
||||
|
||||
@@ -625,6 +625,167 @@ describe("WebAudioTransport", () => {
|
||||
});
|
||||
});
|
||||
|
||||
describe("group routing (preview)", () => {
|
||||
// Real jsdom elements — `groupInput` looks the group up via
|
||||
// `el.ownerDocument.getElementById`, and `audioGroupOf` reads `tagName` /
|
||||
// `getAttribute`, neither of which the plain-object mocks above implement.
|
||||
function createGroupMockAudioContext(currentTime = 100) {
|
||||
const gainNodes: {
|
||||
gain: { value: number };
|
||||
connect: ReturnType<typeof vi.fn>;
|
||||
disconnect: ReturnType<typeof vi.fn>;
|
||||
}[] = [];
|
||||
const masterGain = { gain: { value: 1 }, connect: vi.fn(), disconnect: vi.fn() };
|
||||
const ctx = {
|
||||
currentTime,
|
||||
state: "running",
|
||||
resume: vi.fn(),
|
||||
createBufferSource: vi.fn(() => ({
|
||||
buffer: null as AudioBuffer | null,
|
||||
playbackRate: { value: 1 },
|
||||
start: vi.fn(),
|
||||
stop: vi.fn(),
|
||||
disconnect: vi.fn(),
|
||||
connect: vi.fn(),
|
||||
addEventListener: vi.fn(),
|
||||
})),
|
||||
createGain: vi.fn(() => {
|
||||
const node = { gain: { value: 1 }, connect: vi.fn(), disconnect: vi.fn() };
|
||||
gainNodes.push(node);
|
||||
return node;
|
||||
}),
|
||||
destination: {},
|
||||
close: vi.fn(),
|
||||
};
|
||||
return { ctx, gainNodes, masterGain };
|
||||
}
|
||||
|
||||
function setupGroupTransport(currentTime = 100) {
|
||||
const transport = new WebAudioTransport();
|
||||
const mock = createGroupMockAudioContext(currentTime);
|
||||
(transport as unknown as { _ctx: unknown })._ctx = mock.ctx;
|
||||
(transport as unknown as { _masterGain: unknown })._masterGain = mock.masterGain;
|
||||
const gen = transport.startGeneration();
|
||||
return { transport, mock, gen };
|
||||
}
|
||||
|
||||
function groupedAudioEl(id: string, groupId?: string): HTMLMediaElement {
|
||||
const el = document.createElement("audio");
|
||||
el.id = id;
|
||||
if (groupId) el.setAttribute("data-audio-group", groupId);
|
||||
document.body.appendChild(el);
|
||||
return el as unknown as HTMLMediaElement;
|
||||
}
|
||||
|
||||
/** Create a grouped member and schedule it in one step — the shape every
|
||||
* test below needs, differing only in id/group/generation. */
|
||||
async function scheduleGrouped(
|
||||
transport: WebAudioTransport,
|
||||
gen: number,
|
||||
id: string,
|
||||
groupId?: string,
|
||||
): Promise<HTMLMediaElement> {
|
||||
const el = groupedAudioEl(id, groupId);
|
||||
await transport.schedulePlayback(el, mockBuffer, 0, 0, 0, 1, gen);
|
||||
return el;
|
||||
}
|
||||
|
||||
/** The group's own input gain is built lazily on the first member —
|
||||
* index 1 in creation order (that member's gain is index 0). */
|
||||
const firstGroupInput = (mock: ReturnType<typeof createGroupMockAudioContext>) =>
|
||||
mock.gainNodes[1]!;
|
||||
|
||||
beforeEach(() => {
|
||||
document.body.innerHTML = "";
|
||||
});
|
||||
|
||||
it("routes an ungrouped member straight to master, unchanged", async () => {
|
||||
const { transport, mock, gen } = setupGroupTransport();
|
||||
|
||||
await scheduleGrouped(transport, gen, "solo");
|
||||
|
||||
// One gain node — the member's own — connected directly to master.
|
||||
expect(mock.gainNodes).toHaveLength(1);
|
||||
expect(mock.gainNodes[0]!.connect).toHaveBeenCalledWith(mock.masterGain);
|
||||
});
|
||||
|
||||
it("two members of the same group land on ONE shared group gain, not master directly", async () => {
|
||||
const { transport, mock, gen } = setupGroupTransport();
|
||||
|
||||
await scheduleGrouped(transport, gen, "a", "vo");
|
||||
await scheduleGrouped(transport, gen, "b", "vo");
|
||||
|
||||
// Member gain nodes: index 0 (a) and index 2 (b) — index 1 is the
|
||||
// group's own input gain, built inside a's schedule call.
|
||||
expect(mock.gainNodes.length).toBeGreaterThanOrEqual(3);
|
||||
const groupInput = firstGroupInput(mock);
|
||||
const aGain = mock.gainNodes[0]!;
|
||||
const bGain = mock.gainNodes[2]!;
|
||||
|
||||
// Neither member connects straight to master — both feed the shared bus.
|
||||
expect(aGain.connect).toHaveBeenCalledWith(groupInput);
|
||||
expect(bGain.connect).toHaveBeenCalledWith(groupInput);
|
||||
expect(aGain.connect).not.toHaveBeenCalledWith(mock.masterGain);
|
||||
expect(bGain.connect).not.toHaveBeenCalledWith(mock.masterGain);
|
||||
|
||||
// The bus itself is what reaches master — a plain sum, no processing,
|
||||
// since neither member's group has a chain-bearing `<hf-audio-group>`.
|
||||
expect(groupInput.connect).toHaveBeenCalledWith(mock.masterGain);
|
||||
});
|
||||
|
||||
it("a second member of an already-open group does not rebuild the group bus", async () => {
|
||||
const { transport, mock, gen } = setupGroupTransport();
|
||||
|
||||
await scheduleGrouped(transport, gen, "a", "vo");
|
||||
const gainCountAfterFirst = mock.gainNodes.length; // a-gain + group-input
|
||||
await scheduleGrouped(transport, gen, "b", "vo");
|
||||
|
||||
// Only b's own gain is new — no second group-input gain minted.
|
||||
expect(mock.gainNodes.length).toBe(gainCountAfterFirst + 1);
|
||||
});
|
||||
|
||||
it("a group id with no matching <hf-audio-group> element still gets a flat bus", async () => {
|
||||
const { transport, mock, gen } = setupGroupTransport();
|
||||
|
||||
await scheduleGrouped(transport, gen, "a", "orphan-group"); // no matching element
|
||||
|
||||
expect(firstGroupInput(mock).connect).toHaveBeenCalledWith(mock.masterGain);
|
||||
});
|
||||
|
||||
it("group volume rides the group's own data-volume via its automation lane, not the member's", async () => {
|
||||
document.body.innerHTML = `<hf-audio-group id="vo" data-label="Voiceover"></hf-audio-group>`;
|
||||
const { transport, gen } = setupGroupTransport();
|
||||
|
||||
// No throw wiring the group's automation reader against a real
|
||||
// <hf-audio-group> element that carries no fx/automation attrs.
|
||||
await expect(scheduleGrouped(transport, gen, "a", "vo")).resolves.not.toBeNull();
|
||||
});
|
||||
|
||||
it("destroy() disposes every group bus", async () => {
|
||||
const { transport, mock, gen } = setupGroupTransport();
|
||||
await scheduleGrouped(transport, gen, "a", "vo");
|
||||
const groupInput = firstGroupInput(mock);
|
||||
|
||||
transport.destroy();
|
||||
|
||||
expect(groupInput.disconnect).toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it("stopAll() does NOT dispose group buses — replaying the group does not rebuild it", async () => {
|
||||
const { transport, mock, gen } = setupGroupTransport();
|
||||
await scheduleGrouped(transport, gen, "a", "vo");
|
||||
const groupInput = firstGroupInput(mock);
|
||||
|
||||
transport.stopAll();
|
||||
expect(groupInput.disconnect).not.toHaveBeenCalled();
|
||||
|
||||
const gen2 = transport.startGeneration();
|
||||
await scheduleGrouped(transport, gen2, "a", "vo");
|
||||
// Still only one group-input gain ever created for "vo".
|
||||
expect(mock.gainNodes.filter((n) => n === groupInput)).toHaveLength(1);
|
||||
});
|
||||
});
|
||||
|
||||
describe("decodeAudioElement retry policy (late-asset self-heal)", () => {
|
||||
function transportWithDecode(decodeImpl: () => Promise<AudioBuffer>) {
|
||||
const transport = new WebAudioTransport();
|
||||
|
||||
@@ -5,6 +5,7 @@ import {
|
||||
type AutomationTiming,
|
||||
} from "../audio/audioFxAutomation.js";
|
||||
import { VOLUME_RANGE } from "../audioAutomation.js";
|
||||
import { audioGroupOf } from "../audioGroups.js";
|
||||
import { swallow } from "./diagnostics";
|
||||
import { clampAudioGain } from "../audioGain.js";
|
||||
import { getDebugSurface } from "./globals.js";
|
||||
@@ -70,9 +71,12 @@ function startBoundedSource(
|
||||
/**
|
||||
* The volume lane rides the fader, after the effects — where a DAW puts it,
|
||||
* and the order the render bakes it in.
|
||||
*
|
||||
* Typed against the attribute reader rather than `HTMLMediaElement` so a group
|
||||
* bus (an `<hf-audio-group>`, not a media element) can ride the same path.
|
||||
*/
|
||||
function scheduleVolumeLane(
|
||||
el: HTMLMediaElement,
|
||||
el: { getAttribute?(name: string): string | null },
|
||||
gainNode: GainNode,
|
||||
timing: AutomationTiming,
|
||||
): void {
|
||||
@@ -119,6 +123,11 @@ export class WebAudioTransport {
|
||||
private _masterGain: GainNode | null = null;
|
||||
private _masterVolume = 1;
|
||||
private _masterMuted = false;
|
||||
// One shared bus per group id, lazily built the first time a member of that
|
||||
// group is scheduled. Lives for the session (mirrors `_masterGain`'s own
|
||||
// lifecycle) rather than being torn down on every `stopAll()`, so replaying
|
||||
// a group does not rebuild its chain; only `destroy()` disposes these.
|
||||
private _groups = new Map<string, { input: GainNode; dispose(): void }>();
|
||||
// Composition-time reference frame: at AudioContext time `_rateAnchorCtx`,
|
||||
// composition time was `_rateAnchorComp`, and time has been advancing at
|
||||
// `_rate` composition-seconds per wallclock-second since.
|
||||
@@ -269,6 +278,97 @@ export class WebAudioTransport {
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* The gain a grouped member's signal should land on, building it on first
|
||||
* use. A group's clock is COMPOSITION time (design doc §1.3) — it has no
|
||||
* `data-start`, and a missing start parses as 0, which is exactly
|
||||
* composition time — so its chain and volume lane are scheduled once here
|
||||
* against that zero-offset timing, not the member's own clip-local timing.
|
||||
* A group id with no matching `<hf-audio-group>` element still gets a bus
|
||||
* (flat, no chain) so a hand-authored `data-audio-group` degrades to a
|
||||
* plain sum rather than losing the member's audio.
|
||||
*/
|
||||
private groupInput(groupId: string, doc: Document, timing: AutomationTiming): GainNode | null {
|
||||
const existing = this._groups.get(groupId);
|
||||
if (existing) return existing.input;
|
||||
if (!this._ctx || !this._masterGain) return null;
|
||||
|
||||
const input = this._ctx.createGain();
|
||||
const groupEl = doc.getElementById(groupId);
|
||||
const fx = attachElementFxChain(
|
||||
this._ctx,
|
||||
groupEl ?? { getAttribute: () => null },
|
||||
input,
|
||||
this._masterGain,
|
||||
timing,
|
||||
);
|
||||
if (groupEl) scheduleVolumeLane(groupEl, input, timing);
|
||||
|
||||
this._groups.set(groupId, {
|
||||
input,
|
||||
dispose: () => {
|
||||
try {
|
||||
fx?.dispose();
|
||||
input.disconnect();
|
||||
} catch {
|
||||
// Already torn down.
|
||||
}
|
||||
},
|
||||
});
|
||||
return input;
|
||||
}
|
||||
|
||||
/** Master, unless `el` belongs to a group — then that group's bus (built on
|
||||
* first use, per `groupInput`). */
|
||||
private resolveDestination(
|
||||
el: HTMLMediaElement,
|
||||
scheduledAt: number,
|
||||
compositionTime: number,
|
||||
safeRate: number,
|
||||
): GainNode | null {
|
||||
if (!this._masterGain) return null;
|
||||
const groupId = audioGroupOf(el);
|
||||
if (!groupId) return this._masterGain;
|
||||
const groupTiming: AutomationTiming = { scheduledAt, elapsed: compositionTime, rate: safeRate };
|
||||
return this.groupInput(groupId, el.ownerDocument, groupTiming) ?? this._masterGain;
|
||||
}
|
||||
|
||||
/**
|
||||
* The graph goes with it. Splicing alone left the FX handle alive and then
|
||||
* UNREACHABLE — `stopAll()` disposes by walking `_activeSources`, which the
|
||||
* splice just emptied of this entry. Every clip that finished naturally
|
||||
* leaked its MutationObserver for the session, and each one still answered
|
||||
* later `data-fx-chain` edits by rebuilding a whole graph (impulse response,
|
||||
* chorus/phaser oscillators started and never stopped) around a dead
|
||||
* source. Not disposed when the index is already -1: `stopAll()` has
|
||||
* already done it, and `stop()` is what fired this event.
|
||||
*/
|
||||
private handleSourceEnded(
|
||||
sourceNode: AudioBufferSourceNode,
|
||||
scheduled: ScheduledSource,
|
||||
el: HTMLMediaElement,
|
||||
priorMuted: boolean,
|
||||
): void {
|
||||
const idx = this._activeSources.indexOf(scheduled);
|
||||
if (idx === -1) return;
|
||||
this._activeSources.splice(idx, 1);
|
||||
el.muted = priorMuted;
|
||||
try {
|
||||
sourceNode.disconnect();
|
||||
scheduled.fx?.dispose();
|
||||
scheduled.gainNode.disconnect();
|
||||
} catch {
|
||||
// Already torn down.
|
||||
}
|
||||
if (this._activeSources.length === 0) this._paused = true;
|
||||
}
|
||||
|
||||
// Pre-existing size (110 lines before this diff, which shrank it to under
|
||||
// 95 via two extractions — see `handleSourceEnded`/`resolveDestination`);
|
||||
// the remainder is inherently sequential graph-wiring, not a nested
|
||||
// decision tree, and further splitting would cost more readability than it
|
||||
// buys. Same call the B2 step took on `TimelineLogicalRow`.
|
||||
// fallow-ignore-next-line complexity
|
||||
async schedulePlayback(
|
||||
el: HTMLMediaElement,
|
||||
buffer: AudioBuffer,
|
||||
@@ -309,7 +409,9 @@ export class WebAudioTransport {
|
||||
// output — the same order the offline render uses. Preview and render run
|
||||
// the identical graph builders, so what is heard here is what is written.
|
||||
const fx = attachElementFxChain(this._ctx, el, sourceNode, gainNode, timing);
|
||||
gainNode.connect(this._masterGain);
|
||||
gainNode.connect(
|
||||
this.resolveDestination(el, scheduledAt, compositionTime, safeRate) ?? this._masterGain,
|
||||
);
|
||||
|
||||
scheduleVolumeLane(el, gainNode, timing);
|
||||
|
||||
@@ -355,29 +457,9 @@ export class WebAudioTransport {
|
||||
this._activeSources.push(scheduled);
|
||||
this._paused = false;
|
||||
|
||||
sourceNode.addEventListener("ended", () => {
|
||||
const idx = this._activeSources.indexOf(scheduled);
|
||||
if (idx !== -1) {
|
||||
this._activeSources.splice(idx, 1);
|
||||
el.muted = priorMuted;
|
||||
// The graph goes with it. Splicing alone left the FX handle alive and
|
||||
// then UNREACHABLE — stopAll() disposes by walking this array, which
|
||||
// the splice just emptied of this entry. Every clip that finished
|
||||
// naturally leaked its MutationObserver for the session, and each one
|
||||
// still answered later `data-fx-chain` edits by rebuilding a whole
|
||||
// graph (impulse response, chorus/phaser oscillators started and never
|
||||
// stopped) around a dead source. Not disposed when idx is -1: stopAll()
|
||||
// has already done it, and `stop()` is what fired this event.
|
||||
try {
|
||||
sourceNode.disconnect();
|
||||
fx?.dispose();
|
||||
gainNode.disconnect();
|
||||
} catch {
|
||||
// Already torn down.
|
||||
}
|
||||
if (this._activeSources.length === 0) this._paused = true;
|
||||
}
|
||||
});
|
||||
sourceNode.addEventListener("ended", () =>
|
||||
this.handleSourceEnded(sourceNode, scheduled, el, priorMuted),
|
||||
);
|
||||
|
||||
return scheduled;
|
||||
} catch (err) {
|
||||
@@ -500,6 +582,8 @@ export class WebAudioTransport {
|
||||
|
||||
destroy(): void {
|
||||
this.stopAll();
|
||||
for (const group of this._groups.values()) group.dispose();
|
||||
this._groups.clear();
|
||||
this._bufferCache.clear();
|
||||
this._failedSrcs.clear();
|
||||
this._mediaElementSources = new WeakMap();
|
||||
|
||||
Reference in New Issue
Block a user