From 93921f0fff1e8e67ae4fbd18a895d79a1653e645 Mon Sep 17 00:00:00 2001 From: Vance Ingalls Date: Thu, 13 Aug 2026 01:37:38 -0700 Subject: [PATCH 1/2] feat(core): carve against every voice over a bed, always (#3212) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * feat(core): carve against every voice over a bed, always dynamically A bed usually runs under a whole sequence — a narrator, an interview answer, a second presenter — and carving against one of them left the others fighting it. `source` becomes `sources`, and `mixCarveSources` sums every voice onto the BED's clock before anything is measured. That is what keeps one analysis sufficient: the chain is fixed, so there is no per-voice filter to switch between, and bands drawn from all the speech there is with envelopes that rise wherever any of it happens answer the actual question — where and when is speech masking this bed. Summed rather than averaged: two people talking at once mask more than either alone. Audio before the bed starts is dropped rather than folded in at zero, since it plays over nothing and shifting it would put a cut where there is no voice. `dynamic` is gone. A fixed depth thins the bed through every pause, and once both have been heard there is no reason to want it, so every carve follows the speech. Two helpers the panel and the headless script now share instead of each carrying a copy — two definitions of "what does this name suggest" drift, and then the two disagree about which track is the voice: - `classifyAudioName` reads a track's kind from its id and filename together. `unknown` is deliberately common: treating an unrecognised name as "not a voice" would hide the one track somebody needs to pick. - `clipsOverlap` keeps out a voice that never plays while the bed does. An unwritten duration counts as unbounded, not zero — refusing a clip whose length the composition leaves to the media would drop the commonest case there is. Files written before this still load: a single `source` reads as a one-voice list, a stored `dynamic` is ignored, and an absent attribute means the defaults whole. Co-Authored-By: Claude Opus 5 (1M context) * fix(core): stop \b from missing underscore-separated names, guard clipsOverlap's negative duration \b treats `_` as a word character, so \bbed\b never matched bed_01, music_bed_loop, or theme_song, and \bvo\b/\bvox\b/\btts\b had the same gap — an underscore-separated bed classified as "unknown" and could end up offered as its own carve source. Replaced the short hints with a boundary that actually excludes letters and digits on both sides. clipsOverlap computed end = start + duration without guarding sign, so a negative duration put end before start — an interval that does not describe anything, and one specific case showed it silently dropping a real overlap (a shorter, earlier broken end rejected a clip that genuinely contained the point). Duration clamps to zero instead: a clip cannot un-play time, and a zero-length clip at its start is the sane reading of "duration nobody wrote down as positive." Review by Miga (PR #3212). --------- Co-authored-by: Claude Opus 5 (1M context) --- packages/core/src/audioCarve.test.ts | 153 ++++++++++++++++++++++ packages/core/src/audioCarve.ts | 182 +++++++++++++++++++++++++-- 2 files changed, 323 insertions(+), 12 deletions(-) diff --git a/packages/core/src/audioCarve.test.ts b/packages/core/src/audioCarve.test.ts index cbd337520c..c2cb260198 100644 --- a/packages/core/src/audioCarve.test.ts +++ b/packages/core/src/audioCarve.test.ts @@ -6,6 +6,10 @@ import { analyseCarveDynamics, carveBandsToChain, carveProfile, + classifyAudioName, + clipsOverlap, + mixCarveSources, + couldBeCarveSource, DEFAULT_CARVE, normalizeCarveSettings, } from "./audioCarve.js"; @@ -458,3 +462,152 @@ describe("analyseCarveDynamics", () => { expect(analyseCarveDynamics(new Float32Array(0), SR, [BAND])).toEqual([]); }); }); + +describe("classifyAudioName", () => { + it("reads a track's kind from its id and its filename together", () => { + // Either can be the informative one: elements named a1/a2 may still have + // narration.mp3 and bgm.mp3 behind them. + expect(classifyAudioName("narration")).toBe("voice"); + expect(classifyAudioName("a1", "voiceover-take3.wav")).toBe("voice"); + expect(classifyAudioName("music-bed")).toBe("music"); + expect(classifyAudioName("a2", "bgm_loop.m4a")).toBe("music"); + expect(classifyAudioName("sfx-explosion")).toBe("sfx"); + expect(classifyAudioName("whoosh-01")).toBe("sfx"); + }); + + it("says nothing about a name that says nothing", () => { + // The common case, and the reason nothing downstream may treat "unknown" as + // "not a voice": it would hide the one track somebody needs to pick. + expect(classifyAudioName("a1")).toBe("unknown"); + expect(classifyAudioName("clip-2", "0f9c1a.mp3")).toBe("unknown"); + expect(classifyAudioName(undefined, null)).toBe("unknown"); + }); + + it("prefers voice when a name carries both hints", () => { + // A file called voiceover-over-music-bed.wav is the voiceover, and a track + // matching both is better offered than hidden. + expect(classifyAudioName("voiceover-over-music-bed.wav")).toBe("voice"); + }); + + it("offers speech and unnamed tracks as carve sources, never music or effects", () => { + expect(couldBeCarveSource("recap-audio")).toBe(true); + expect(couldBeCarveSource("a1")).toBe(true); + expect(couldBeCarveSource("music-bed")).toBe(false); + expect(couldBeCarveSource("sfx-explosion")).toBe(false); + }); + + it("treats underscores as separators, not word characters, for short hints", () => { + // `\b` treats `_` as a word character, so `\bbed\b` used to miss `bed_01` — + // an underscore-separated bed classified as "unknown" and could end up + // offered as its own carve source. + expect(classifyAudioName("bed_01")).toBe("music"); + expect(classifyAudioName("my_bed")).toBe("music"); + expect(classifyAudioName("music_bed_loop")).toBe("music"); + expect(classifyAudioName("theme_song")).toBe("music"); + expect(classifyAudioName("vo_take3")).toBe("voice"); + expect(classifyAudioName("main_vox")).toBe("voice"); + expect(couldBeCarveSource("bed_01")).toBe(false); + }); +}); + +describe("clipsOverlap", () => { + it("overlaps when the spans genuinely share time", () => { + expect(clipsOverlap({ start: 0, duration: 5 }, { start: 3, duration: 5 })).toBe(true); + }); + + it("does not overlap when one span ends before the other starts", () => { + expect(clipsOverlap({ start: 0, duration: 5 }, { start: 5, duration: 5 })).toBe(false); + expect(clipsOverlap({ start: 10, duration: 5 }, { start: 0, duration: 5 })).toBe(false); + }); + + it("does not overlap two clips that only touch at an edge", () => { + // Half-open: an end exactly at the other's start shares no time to carve. + expect(clipsOverlap({ start: 0, duration: 5 }, { start: 5, duration: 5 })).toBe(false); + }); + + it("treats a null or undefined duration as unbounded", () => { + expect(clipsOverlap({ start: 0, duration: null }, { start: 100, duration: 1 })).toBe(true); + expect(clipsOverlap({ start: 0 }, { start: 100, duration: 1 })).toBe(true); + // Symmetric: the unbounded span can be on either side. + expect(clipsOverlap({ start: 100, duration: 1 }, { start: 0, duration: undefined })).toBe(true); + }); + + it("gives a zero-duration clip a single instant, not a span", () => { + expect(clipsOverlap({ start: 5, duration: 0 }, { start: 5, duration: 5 })).toBe(false); + expect(clipsOverlap({ start: 5, duration: 0 }, { start: 4, duration: 5 })).toBe(true); + }); + + it("clamps a negative duration to zero rather than inverting the interval", () => { + // The regression: end = start + duration puts a negative-duration clip's + // end BEFORE its start, and end(a) is what the other clip's start gets + // compared against — so a smaller (earlier) broken end silently rejects + // real overlaps too. {start:10, duration:-5} clamped is a zero-length + // clip AT t=10, which genuinely sits inside {start:6, duration:20}'s + // [6, 26) span; the unclamped math missed it (end(a) came out to 5). + expect(clipsOverlap({ start: 10, duration: -5 }, { start: 6, duration: 20 })).toBe(true); + // And it stays correct where it isn't inside anything. + expect(clipsOverlap({ start: 10, duration: -5 }, { start: 20, duration: 5 })).toBe(false); + }); +}); + +describe("mixCarveSources", () => { + const tone = (seconds: number, level: number, sampleRate = 48000) => + new Float32Array(Math.round(seconds * sampleRate)).fill(level); + + it("places every voice where it starts on the bed's clock", () => { + // Three people talking at different times is still one question — where and + // when is speech masking this bed — so they become one signal. + const mixed = mixCarveSources( + [ + { samples: tone(1, 0.5), offsetSeconds: 1 }, + { samples: tone(1, 0.25), offsetSeconds: 3 }, + ], + 48000, + ); + expect(mixed.length).toBe(4 * 48000); + const at = (t: number) => mixed[Math.round(t * 48000)]; + expect(at(0.5)).toBe(0); // before anyone speaks + expect(at(1.5)).toBeCloseTo(0.5, 5); + expect(at(2.5)).toBe(0); // the gap between them + expect(at(3.5)).toBeCloseTo(0.25, 5); + }); + + it("sums voices that overlap, because two at once mask more than one", () => { + const mixed = mixCarveSources( + [ + { samples: tone(1, 0.3), offsetSeconds: 0 }, + { samples: tone(1, 0.3), offsetSeconds: 0 }, + ], + 48000, + ); + expect(mixed[0]).toBeCloseTo(0.6, 5); + }); + + it("drops the part of a voice that plays before the bed starts", () => { + // It masks nothing there, and folding it in at zero would put a cut where + // there is no voice. + const mixed = mixCarveSources([{ samples: tone(1, 0.5), offsetSeconds: -0.5 }], 48000); + expect(mixed.length).toBe(0.5 * 48000); + expect(mixed[0]).toBeCloseTo(0.5, 5); + }); + + it("has nothing to mix when there are no voices", () => { + expect(mixCarveSources([], 48000)).toHaveLength(0); + }); +}); + +describe("carve settings written before this took a list of voices", () => { + it("reads a single `source` as a one-voice list, and forgets `dynamic`", () => { + // Every carve is dynamic now: a static one thinned the bed through every pause, + // and nobody wanted that once they had heard both. + const read = normalizeCarveSettings({ source: "vo", strength: 0.4, dynamic: false } as never); + expect(read.sources).toEqual(["vo"]); + expect(read.strength).toBe(0.4); + expect("dynamic" in read).toBe(false); + }); + + it("drops empty ids rather than carrying a source that names nothing", () => { + expect(normalizeCarveSettings({ sources: ["", "vo", ""] } as never).sources).toEqual(["vo"]); + expect(normalizeCarveSettings({ source: "" } as never).sources).toEqual([]); + }); +}); diff --git a/packages/core/src/audioCarve.ts b/packages/core/src/audioCarve.ts index 1844010518..080638c4cd 100644 --- a/packages/core/src/audioCarve.ts +++ b/packages/core/src/audioCarve.ts @@ -46,19 +46,28 @@ export interface HfCarveBand { * once. */ export interface HfCarveSettings { - /** Element id of the voice track to analyse. */ - source: string; + /** + * Element ids of every voice track this bed makes room for. + * + * More than one because a bed usually runs under a whole sequence: a narrator, an + * interview answer, a second presenter. Each occupies its own stretch of the bed, + * and carving against only one of them leaves the others fighting it. They are + * analysed together — see `mixCarveSources` — so the cuts follow whoever is + * speaking rather than averaging strangers. + */ + sources: string[]; /** How hard to carve, 0..1. */ strength: number; /** - * Follow the voice rather than sitting at a fixed depth. + * Whether the carve is applied at all. * - * A static carve holds its cuts for the whole clip, including every pause — the - * bed is thinned where there is nothing to make room for. Dynamic turns every - * value into an envelope of the voice's own level, so silence leaves the bed - * alone and a loud passage pushes the carve to full depth. + * A bed under a voice wants carving, so a track that has never been configured + * is treated as on and carved without being asked. That default needs an off + * switch that survives: with "off" represented by having no settings at all, + * selecting the clip again would read it as never-configured and re-apply. So + * switching it off writes `enabled: false` and the default stops applying. */ - dynamic: boolean; + enabled: boolean; } /** The numbers the analysis actually works in, all derived from `strength`. */ @@ -82,13 +91,110 @@ export interface HfCarveProfile { headroomDb: number; } +/** + * What a track's name suggests it holds. + * + * Only ever a hint — a name is what the author called something, not what is in the + * file — so this is used to order and to filter a list of candidates, never to + * decide alone. `unknown` is deliberately common: a track called `a1` could be + * anything, and treating an unrecognised name as "not a voice" would hide the one + * track somebody needs to pick. + */ +export type HfAudioNameKind = "voice" | "music" | "sfx" | "unknown"; + +/** Short, deliberately dull effects. Nothing here is ever a voiceover. */ +const SFX_NAME = + /sfx|foley|whoosh|impact|riser|stinger|swoosh|thud|boom|click|ding|beep|ambien|room[-_ ]?tone/i; +// `\b` treats `_` as a word character, so `\bbed\b` does not match `bed_01` +// or `music_bed_loop` — exactly the separator an asset name is likely to use. +// These short words need a boundary that actually excludes letters and +// digits on both sides; everything else here is long enough that a +// substring match is already the intent (`music` inside `bgmusic` is fine). +const NOT_WORD = "(? `${NOT_WORD}${term}${NOT_WORD_END}`; + +/** A bed, which is the thing being carved rather than the thing carving it. */ +const MUSIC_NAME = new RegExp( + `music|bgm|${wordish("bed")}|soundtrack|score|${wordish("song")}|theme|instrumental|track\\d`, + "i", +); +/** Speech. */ +const VOICE_NAME = new RegExp( + `voice|${wordish("vo")}|${wordish("vox")}|narrat|speech|dialog|monolog|announce|` + + `${wordish("tts")}|talk|interview|podcast|recap|script`, + "i", +); + +/** + * Classify a track from its id and filename together. + * + * Both, because either can be the informative one: an author naming elements `a1` + * and `a2` may still have `narration.mp3` and `bgm.mp3` as their sources, and one + * naming them `voice` and `music` may have opaque hashes for filenames. + * + * Voice is tested first: a file called `voiceover-music-bed.wav` is more likely the + * voiceover than the bed, and a track matching both hints is better offered than + * hidden. + */ +export function classifyAudioName( + ...parts: readonly (string | null | undefined)[] +): HfAudioNameKind { + const text = parts.filter(Boolean).join(" "); + if (VOICE_NAME.test(text)) return "voice"; + if (SFX_NAME.test(text)) return "sfx"; + if (MUSIC_NAME.test(text)) return "music"; + return "unknown"; +} + +/** A clip's place on the timeline. A duration that is not a number is unbounded. */ +export interface HfClipSpan { + start: number; + duration?: number | null; +} + +/** + * Do these two clips share any time at all? + * + * A voice that never plays while the bed does cannot mask it, so it has no business + * in the carve: it would contribute silence to the analysis and, worse, invite the + * author to wonder why including it changed nothing. + * + * An unknown duration counts as unbounded rather than as zero. Refusing a track + * because its length is not written down would drop the commonest case there is — a + * clip whose duration the composition leaves to the media itself. + */ +export function clipsOverlap(a: HfClipSpan, b: HfClipSpan): boolean { + const end = (clip: HfClipSpan): number => + typeof clip.duration === "number" && Number.isFinite(clip.duration) + ? // Negative is clamped to zero-length rather than passed through: a + // clip cannot un-play time, and letting it through inverts the + // interval (end before start), which reads as overlapping everything + // it is nowhere near. + clip.start + Math.max(0, clip.duration) + : Number.POSITIVE_INFINITY; + return a.start < end(b) && b.start < end(a); +} + +/** + * Could this track be the voice a carve listens to? + * + * Music and SFX are out: a bed is the thing being carved, and a 200 ms whoosh has + * no speech to make room for. Everything else stays in, including names that say + * nothing — see `HfAudioNameKind`. + */ +export function couldBeCarveSource(...parts: readonly (string | null | undefined)[]): boolean { + const kind = classifyAudioName(...parts); + return kind === "voice" || kind === "unknown"; +} + export const DEFAULT_CARVE: HfCarveSettings = { - source: "", + enabled: true, + sources: [], // A quarter, because the knob's range was doubled and this is the point on the // new scale that produces what the panel has always defaulted to. Switching // carve on sounds the same as it did; the extra range is above, not under. strength: 0.25, - dynamic: false, }; /** @@ -133,10 +239,16 @@ export function carveProfile(strength: number): HfCarveProfile { export function normalizeCarveSettings( raw: Partial | undefined, ): HfCarveSettings { + // `source` and `dynamic` are gone from the type but still out there in files. + const legacy = raw as (Partial & { source?: unknown }) | undefined; const num = (v: unknown): number | null => { const n = typeof v === "number" ? v : Number(v); return Number.isFinite(n) ? n : null; }; + // No attribute at all is not a carve to read, it is the absence of one — so the + // defaults apply whole, dynamic included. Only a stored object gets the reading + // below, where a missing `dynamic` means the static carve it was written as. + if (raw === undefined || raw === null) return { ...DEFAULT_CARVE }; const strength = num(raw?.strength); const legacyDepth = num(raw?.maxCutDb); const resolved = @@ -147,13 +259,59 @@ export function normalizeCarveSettings( // reads back as the strength that produces 6 dB. (legacyDepth - 2) / 16 : DEFAULT_CARVE.strength; + // A carve written before this took a list names its one voice in `source`. + const stored = Array.isArray(raw?.sources) + ? raw.sources + : typeof legacy?.source === "string" + ? [legacy.source] + : []; return { - source: typeof raw?.source === "string" ? raw.source : "", + // Absent means on: every carve written before the flag existed was applied. + enabled: raw?.enabled !== false, + sources: stored.filter((id): id is string => typeof id === "string" && id !== ""), strength: Math.min(1, Math.max(0, resolved)), - dynamic: raw?.dynamic === true, }; } +/** + * Every voice as one signal on the BED's clock. + * + * The analysis asks one question — where and when is speech masking this bed — and + * that question has one answer even when three people are talking at different + * times. Summing them onto the bed's timeline first means the existing analysis + * needs no notion of "which voice": bands come out of all the speech there is, and + * the envelopes rise wherever any of it is happening. + * + * `offsetSeconds` is where each voice starts relative to the bed. Audio before the + * bed begins is dropped rather than folded in at zero: it plays over nothing and + * cannot mask anything, and shifting it would put a cut where there is no voice. + * + * Summed, not averaged. Two people speaking at once mask more than either alone, + * which is exactly what the carve should answer to. + */ +export function mixCarveSources( + parts: readonly { samples: Float32Array; offsetSeconds: number }[], + sampleRate: number, +): Float32Array { + const placed = parts.map((part) => ({ + samples: part.samples, + at: Math.round(part.offsetSeconds * sampleRate), + })); + const length = placed.reduce((max, p) => Math.max(max, p.at + p.samples.length), 0); + if (length <= 0) return new Float32Array(0); + const mixed = new Float32Array(length); + for (const { samples, at } of placed) { + // A voice starting before the bed contributes only the part that overlaps it. + const from = at < 0 ? -at : 0; + for (let i = from; i < samples.length; i += 1) { + const target = at + i; + if (target < 0 || target >= length) continue; + mixed[target] = (mixed[target] ?? 0) + (samples[i] ?? 0); + } + } + return mixed; +} + /** Averaged power spectrum, Welch-style. */ function powerSpectrum( mono: Float32Array, From 03c1634911fb99ff49052363a5e5ad58b0f14130 Mon Sep 17 00:00:00 2001 From: Vance Ingalls Date: Thu, 13 Aug 2026 01:59:41 -0700 Subject: [PATCH 2/2] fix(studio): port the carve UI off the removed source/dynamic fields MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit #3212 (accidentally squash-merged into this branch instead of main) changed HfCarveSettings from a single `source` + `dynamic` toggle to a `sources` list with dynamic mode removed outright — the multi-voice UI consumer that goes with that shape lands in the very next PR, so this branch was left with a type that no longer matched its own code. Minimal port, not the multi-voice redesign that PR does properly: the "Listen to" picker and analyse() treat sources[0] as the one voice this UI still understands, and every dynamic-mode branch (the automated envelope lanes, the toggle, the checkbox) is gone along with the field — a carve is now always the static value the analysis computes, matching what the type change made permanent. Test suite trimmed the same way: the automation-lane and toggle tests covered behavior that no longer exists. --- .../editor/propertyPanelAudioFxGroup.test.tsx | 194 ++---------------- .../editor/propertyPanelAudioFxGroup.tsx | 65 ++---- .../editor/propertyPanelFxSection.test.tsx | 6 +- .../editor/propertyPanelFxSection.tsx | 24 +-- 4 files changed, 34 insertions(+), 255 deletions(-) diff --git a/packages/studio/src/components/editor/propertyPanelAudioFxGroup.test.tsx b/packages/studio/src/components/editor/propertyPanelAudioFxGroup.test.tsx index f9cad3b146..40d693dfee 100644 --- a/packages/studio/src/components/editor/propertyPanelAudioFxGroup.test.tsx +++ b/packages/studio/src/components/editor/propertyPanelAudioFxGroup.test.tsx @@ -246,15 +246,15 @@ describe("AudioFxGroup carve", () => { }); }); -describe("AudioFxGroup dynamic carve", () => { +describe("AudioFxGroup carve analysis", () => { const carvedChain = JSON.stringify({ version: 1, nodes: [{ type: "lowpass", id: "n1", params: { frequency: 400, q: 0.9, poles: "2" } }], }); // Strength 0 carves frequencies only — no level ducking — so the spectral // cases measure just the spectral half. A case that wants the duck raises it. - const settings = (dynamic: boolean, over: Record = {}) => - JSON.stringify({ source: "vo", strength: 0, dynamic, ...over }); + const settings = (over: Record = {}) => + JSON.stringify({ sources: ["vo"], strength: 0, ...over }); /** The value written for one attribute, whatever order the writes landed in. */ const writeFor = (calls: unknown[][], attr: string) => @@ -267,9 +267,6 @@ describe("AudioFxGroup dynamic carve", () => { select.dispatchEvent(new Event("change", { bubbles: true })); }; - const dynamicBox = (host: HTMLElement) => - host.querySelector(".hf-fx-carve-dynamic")!; - /** A voice with a pause in it, decoded through a stubbed offline context. */ function stubDecode(): void { const sampleRate = 48000; @@ -292,114 +289,11 @@ describe("AudioFxGroup dynamic carve", () => { afterEach(() => vi.unstubAllGlobals()); - it("records the choice in the carve settings", () => { - const { host, onSetAttributeQuiet } = mount({ - "fx-chain": carvedChain, - "fx-carve": settings(false), - }); - act(() => dynamicBox(host).click()); - const write = onSetAttributeQuiet.mock.calls.find((c) => c[0] === "data-fx-carve"); - expect(JSON.parse(String(write![1])).dynamic).toBe(true); - }); - - it("automates the carve filters' gain from the voice, in the bed's own time", async () => { - stubDecode(); - // Voice starts 10s into the composition, bed at 0: the envelope is measured - // against the voice but read from the start of the bed, so it has to shift. - const { host, onSetAttributeQuiet } = mount({ - "fx-chain": carvedChain, - // No source yet: picking one is what applies the carve. - "fx-carve": settings(true, { source: "" }), - start: "0", - }); - const vo = document.getElementById("vo")!; - vo.setAttribute("data-start", "10"); - vo.setAttribute("src", "voice.wav"); - await act(async () => { - pickSource(host, "vo"); - }); - - // Chain first, then automation: a lane naming a node the chain does not - // carry yet is dropped when it is read back. - const order = onSetAttributeQuiet.mock.calls.map((c) => c[0]); - // The settings land first, then the filters they imply, then the envelopes. - expect(order.indexOf("data-fx-chain")).toBeLessThan(order.indexOf("data-automation")); - - const carved = writeFor(onSetAttributeQuiet.mock.calls, "data-fx-chain").nodes; - const carveNode = carved.find((n: { fromCarve?: boolean }) => n.fromCarve); - expect(carveNode.id).toBeTruthy(); - - const lanes = writeFor(onSetAttributeQuiet.mock.calls, "data-automation").lanes; - const lane = lanes.find((l: { target: string }) => l.target === `fx.${carveNode.id}.gain`) as { - points: { t: number; v: number }[]; - }; - expect(lane).toBeTruthy(); - // Flat at the bed's own start, before the voice exists at all. - expect(lane.points[0]).toMatchObject({ t: 0, v: 0 }); - // The voice's pause is at 0-1s of its own clip, so 10-11s of the bed's. - expect(lane.points.find((p) => p.t > 10.5 && p.t < 11)?.v ?? 0).toBe(0); - // And it cuts once the voice speaks, a second later. Depth is per band and - // relative to that band's own peak in the voice, so the invariant is that the - // envelope gets most of the way to what the analysis put on the node — not a - // fixed number of dB, which changes with the band the analysis chose. - const bandGain = Number(carveNode.params?.gain ?? 0); - // At least half the depth the analysis put on the node; the exact floor - // depends on which band it chose and how the envelope was thinned. - expect(Math.min(...lane.points.map((p) => p.v))).toBeLessThanOrEqual(bandGain * 0.5); - // Ends back at no cut, so the bed is not left dipped for the rest of the clip. - expect(lane.points.at(-1)!.v).toBe(0); - }); - - it("adds a gain stage that ducks the bed under the voice, automated when dynamic", async () => { - // Carving frequencies cannot beat a bed that is simply louder than the - // voice. The level half rides a gain node the carve owns, so the track's own - // volume lane is left alone. + it("holds one measured value from the voice and bed", async () => { stubDecode(); const { host, onSetAttributeQuiet } = mount({ "fx-chain": carvedChain, - "fx-carve": settings(true, { strength: 1, source: "" }), - start: "0", - }); - const vo = document.getElementById("vo")!; - vo.setAttribute("data-start", "0"); - vo.setAttribute("src", "voice.wav"); - // The bed is measured too — "how far over the voice is it" needs both. - document.getElementById("bed")!.setAttribute("src", "bed.m4a"); - await act(async () => { - pickSource(host, "vo"); - }); - - const nodes = writeFor(onSetAttributeQuiet.mock.calls, "data-fx-chain").nodes; - const gain = nodes.find((n: { type: string }) => n.type === "gain"); - expect(gain).toBeTruthy(); - expect(gain.fromCarve).toBe(true); - // Dynamic hands the value to the envelope, so the static one stays at unity. - expect(gain.params.gain).toBe(0); - - const lanes = writeFor(onSetAttributeQuiet.mock.calls, "data-automation").lanes; - const duckLane = lanes.find((l: { target: string }) => l.target === `fx.${gain.id}.gain`); - expect(duckLane).toBeTruthy(); - expect(Math.min(...duckLane.points.map((p: { v: number }) => p.v))).toBeLessThan(0); - // Every carved band gets an envelope reaching that band's own analysed depth. - for (const node of nodes.filter((n: { type: string }) => n.type === "peaking")) { - const lane = lanes.find((l: { target: string }) => l.target === `fx.${node.id}.gain`) as - | { points: { v: number }[] } - | undefined; - expect(lane, `band ${node.id} has no envelope`).toBeTruthy(); - const deepest = Math.min(...lane!.points.map((p) => p.v)); - expect(deepest).toBeLessThanOrEqual(0); - expect(deepest).toBeGreaterThanOrEqual(node.params.gain - 0.2); - expect(deepest).toBeLessThanOrEqual(node.params.gain * 0.5); - } - // The author's own volume lane is not something a carve gets to touch. - expect(lanes.some((l: { target: string }) => l.target === "volume")).toBe(false); - }); - - it("holds one measured value when the carve is not dynamic", async () => { - stubDecode(); - const { host, onSetAttributeQuiet } = mount({ - "fx-chain": carvedChain, - "fx-carve": settings(false, { strength: 1, source: "" }), + "fx-carve": settings({ strength: 1, sources: [] }), start: "0", }); const vo = document.getElementById("vo")!; @@ -413,7 +307,7 @@ describe("AudioFxGroup dynamic carve", () => { const nodes = writeFor(onSetAttributeQuiet.mock.calls, "data-fx-chain").nodes; const gain = nodes.find((n: { type: string }) => n.type === "gain"); expect(gain.params.gain).toBeLessThan(0); - // Nothing to schedule: a static carve is a value, not an envelope. + // Nothing to schedule: a carve is a value, not an envelope. expect(onSetAttributeQuiet.mock.calls.some((c) => c[0] === "data-automation")).toBe(false); }); @@ -421,7 +315,7 @@ describe("AudioFxGroup dynamic carve", () => { stubDecode(); const { host, onSetAttributeQuiet } = mount({ "fx-chain": carvedChain, - "fx-carve": settings(true, { strength: 0, source: "" }), + "fx-carve": settings({ strength: 0, sources: [] }), start: "0", }); document.getElementById("vo")!.setAttribute("src", "voice.wav"); @@ -439,7 +333,7 @@ describe("AudioFxGroup dynamic carve", () => { stubDecode(); const { host, onSetAttributeQuiet } = mount({ "fx-chain": JSON.stringify({ version: 1, nodes: [] }), - "fx-carve": JSON.stringify({ source: "", strength: 0.25, dynamic: true }), + "fx-carve": JSON.stringify({ sources: [], strength: 0.25 }), start: "0", }); document.getElementById("vo")!.setAttribute("src", "voice.wav"); @@ -448,44 +342,17 @@ describe("AudioFxGroup dynamic carve", () => { pickSource(host, "vo"); }); const written = onSetAttributeQuiet.mock.calls.map((c) => c[0]); - expect(written).toEqual(["data-fx-carve", "data-fx-chain", "data-automation"]); + expect(written).toEqual(["data-fx-carve", "data-fx-chain"]); const nodes = JSON.parse( String(onSetAttributeQuiet.mock.calls.find((c) => c[0] === "data-fx-chain")![1]), ).nodes; expect(nodes.every((n: { fromCarve?: boolean }) => n.fromCarve)).toBe(true); }); - it("re-applies when dynamic is switched on, not just when strength moves", async () => { - stubDecode(); - const carvedAlready = JSON.stringify({ - version: 1, - nodes: [ - { - type: "peaking", - id: "n1", - fromCarve: true, - params: { frequency: 1000, gain: -6, q: 1.4 }, - }, - ], - }); - const { host, onSetAttributeQuiet } = mount({ - "fx-chain": carvedAlready, - "fx-carve": settings(false, { strength: 0.25 }), - start: "0", - }); - document.getElementById("vo")!.setAttribute("src", "voice.wav"); - document.getElementById("bed")!.setAttribute("src", "bed.m4a"); - await act(async () => { - host.querySelector(".hf-fx-carve-dynamic")!.click(); - }); - // Static and dynamic are different chains, so the switch has to rebuild them. - expect(onSetAttributeQuiet.mock.calls.some((c) => c[0] === "data-fx-chain")).toBe(true); - }); - it("re-applies an existing carve when strength moves", async () => { // Strength is the whole control surface, so it has to act on what is already - // applied. Left to the button alone, a carve kept the filters and envelopes - // its old strength produced and the knob silently described nothing. + // applied. Left to the button alone, a carve kept the filters its old + // strength produced and the knob silently described nothing. stubDecode(); const carvedAlready = JSON.stringify({ version: 1, @@ -501,7 +368,7 @@ describe("AudioFxGroup dynamic carve", () => { }); const { host, onSetAttributeQuiet } = mount({ "fx-chain": carvedAlready, - "fx-carve": settings(true, { strength: 0.25 }), + "fx-carve": settings({ strength: 0.25 }), start: "0", }); document.getElementById("vo")!.setAttribute("src", "voice.wav"); @@ -516,9 +383,8 @@ describe("AudioFxGroup dynamic carve", () => { const written = onSetAttributeQuiet.mock.calls.map((c) => c[0]); expect(written).toContain("data-fx-carve"); - // The settings land first, then the filters they imply, then the envelopes. + // The settings land first, then the filters they imply. expect(written.indexOf("data-fx-carve")).toBeLessThan(written.indexOf("data-fx-chain")); - expect(written.indexOf("data-fx-chain")).toBeLessThan(written.indexOf("data-automation")); const chainWrite = onSetAttributeQuiet.mock.calls.find((c) => c[0] === "data-fx-chain"); const nodes = JSON.parse(String(chainWrite![1])).nodes; @@ -537,7 +403,7 @@ describe("AudioFxGroup dynamic carve", () => { stubDecode(); const { host, onSetAttributeQuiet } = mount({ "fx-chain": JSON.stringify({ version: 1, nodes: [] }), - "fx-carve": settings(true, { strength: 0.25, source: "" }), + "fx-carve": settings({ strength: 0.25, sources: [] }), start: "0", }); const dial = host.querySelector(".hf-fx-carve input[type=range]")!; @@ -566,7 +432,7 @@ describe("AudioFxGroup dynamic carve", () => { }); const { host, onSetAttributeQuiet, onSetAttributeLive } = mount({ "fx-chain": carvedAlready, - "fx-carve": settings(true, { strength: 0.25 }), + "fx-carve": settings({ strength: 0.25 }), start: "0", }); document.getElementById("vo")!.setAttribute("src", "voice.wav"); @@ -580,36 +446,6 @@ describe("AudioFxGroup dynamic carve", () => { expect(onSetAttributeLive.mock.calls.every((c) => c[0] === "data-fx-carve")).toBe(true); expect(onSetAttributeQuiet.mock.calls.some((c) => c[0] === "data-fx-chain")).toBe(false); }); - - it("drops the envelopes when dynamic is switched back off", async () => { - // An automated gain ignores the panel's depth, so leaving the lanes behind - // would keep the filters following a voice with nothing saying they do. - const automation = JSON.stringify({ - version: 1, - lanes: [ - { target: "fx.n2.gain", points: [{ t: 0, v: 0 }] }, - { target: "volume", points: [{ t: 0, v: 1 }] }, - ], - }); - const withCarveNode = JSON.stringify({ - version: 1, - nodes: [ - { type: "peaking", id: "n2", fromCarve: true, params: { frequency: 1000, gain: -6 } }, - ], - }); - const { host, onSetAttributeQuiet } = mount({ - "fx-chain": withCarveNode, - "fx-carve": settings(true), - automation, - }); - await act(async () => { - dynamicBox(host).click(); - }); - const write = onSetAttributeQuiet.mock.calls.find((c) => c[0] === "data-automation"); - expect(write).toBeTruthy(); - const lanes = JSON.parse(String(write![1])).lanes; - expect(lanes.map((l: { target: string }) => l.target)).toEqual(["volume"]); - }); }); describe("AudioFxGroup successive edits", () => { diff --git a/packages/studio/src/components/editor/propertyPanelAudioFxGroup.tsx b/packages/studio/src/components/editor/propertyPanelAudioFxGroup.tsx index 9401d6aae7..4261533c5c 100644 --- a/packages/studio/src/components/editor/propertyPanelAudioFxGroup.tsx +++ b/packages/studio/src/components/editor/propertyPanelAudioFxGroup.tsx @@ -20,7 +20,6 @@ import { import { analyseCarveBands, analyseCarveDuck, - analyseCarveDynamics, carveBandsToChain, carveProfile, HF_AUDIO_CARVE_ATTR, @@ -31,7 +30,6 @@ import { fxAutomationTarget, sampleAutomationLane, type HfAutomation, - type HfAutomationLane, } from "@hyperframes/core/audio-automation"; import { automatedTargetsOf, @@ -185,10 +183,7 @@ export function AudioFxGroup({ * commit, which does not exist yet. */ const setCarve = async (next: HfCarveSettings | null): Promise => { - // Envelopes the carve wrote outlive it otherwise, and an automated gain - // ignores the panel's own depth — so switching dynamic off would leave the - // filters still following the voice with nothing saying they do. - if (!next || (carve?.dynamic && !next.dynamic)) { + if (!next) { const carriedOver = withoutCarveLanes(automation, chain); if (carriedOver.lanes.length !== automation.lanes.length) { await onSetAttributeQuiet( @@ -215,11 +210,10 @@ export function AudioFxGroup({ // is already there. A carve with no source yet has nothing to analyse. const changed = next && - next.source && + next.sources.length > 0 && (!carve || - next.source !== carve.source || - next.strength !== carve.strength || - next.dynamic !== carve.dynamic); + next.sources.join("") !== carve.sources.join("") || + next.strength !== carve.strength); if (next && changed) await analyse(next); }; @@ -258,7 +252,7 @@ export function AudioFxGroup({ if (other.id === element.id) continue; try { const raw = other.getAttribute(HF_AUDIO_CARVE_ATTR); - if (raw && normalizeCarveSettings(JSON.parse(raw)).source === element.id) { + if (raw && normalizeCarveSettings(JSON.parse(raw)).sources.includes(element.id)) { return other.id || "another track"; } } catch { @@ -284,9 +278,10 @@ export function AudioFxGroup({ * hand-added effects alone, so re-analysing does not discard other work. */ const analyse = async (active: HfCarveSettings | null = carve): Promise => { - if (!active?.source) return; + const activeSource = active?.sources[0]; + if (!activeSource) return; const doc = element.element?.ownerDocument; - const voice = doc?.getElementById(active.source) as HTMLAudioElement | null; + const voice = doc?.getElementById(activeSource) as HTMLAudioElement | null; const src = voice?.getAttribute("src"); if (!src) return; setAnalysing(true); @@ -348,14 +343,13 @@ export function AudioFxGroup({ }; const carvedNodes: HfAudioFxNode[] = carved.nodes.map(mint); // The gain stage sits after the filters, and only exists when the carve was - // asked to make level room. Dynamic drives it from the envelope; static - // holds the one value above. + // asked to make level room, holding the one value computed above. const duckNode = duck.length > 0 ? mint({ type: "gain", enabled: true, - params: { ...defaultAudioFxParams("gain"), gain: active.dynamic ? 0 : staticDuckDb }, + params: { ...defaultAudioFxParams("gain"), gain: staticDuckDb }, }) : null; const next = { @@ -372,42 +366,11 @@ export function AudioFxGroup({ // pruned when it is read back. await onSetAttributeQuiet(HF_AUDIO_FX_ATTR, serializeAudioFxChain(next)); - /** - * One carve envelope as a lane on this bed's clock. - * - * Everything the analysis returns is timed from the start of the voice, so - * it shifts by the gap between the two clips; and a lane holds its first - * value backwards to the start of its own clip, so a bed that begins before - * the voice needs an explicit "no cut" at zero or it starts out ducked. - */ - const laneFor = (id: string, points: { t: number; v: number }[]): HfAutomationLane[] => { - const shifted = points - .map((p) => ({ t: Number((p.t + offset).toFixed(3)), v: p.v })) - .filter((p) => p.t >= 0); - if ((shifted[0]?.t ?? 0) > 0) shifted.unshift({ t: 0, v: 0 }); - return shifted.length > 1 - ? [{ target: fxAutomationTarget(id, "gain"), points: shifted }] - : []; - }; - - // Dynamic carve: each filter's depth becomes an envelope of the voice's - // level in that band, so pauses leave the bed alone. - const lanes: HfAutomationLane[] = active.dynamic - ? analyseCarveDynamics(buffer.getChannelData(0), buffer.sampleRate, bands).flatMap( - (dyn, i) => { - const id = carvedNodes[i]?.id; - if (!id) return []; - return laneFor(id, dyn.points); - }, - ) - : []; - // The level envelope rides the gain stage, on the same clock as the bands. - if (active.dynamic && duckNode?.id && duck.length > 0) { - lanes.push(...laneFor(duckNode.id, duck)); - } + // A carve written before dynamic mode was removed may still carry the + // envelope lanes it automated; a re-run is static now, so they are stale. const carriedOver = withoutCarveLanes(automation, chain); - if (lanes.length > 0 || carriedOver.lanes.length !== automation.lanes.length) { - writeAutomation({ version: 1, lanes: [...carriedOver.lanes, ...lanes] }); + if (carriedOver.lanes.length !== automation.lanes.length) { + writeAutomation(carriedOver); } } catch { // Leave the chain as it was; the button simply re-enables. diff --git a/packages/studio/src/components/editor/propertyPanelFxSection.test.tsx b/packages/studio/src/components/editor/propertyPanelFxSection.test.tsx index cd6e5aeea6..f129763252 100644 --- a/packages/studio/src/components/editor/propertyPanelFxSection.test.tsx +++ b/packages/studio/src/components/editor/propertyPanelFxSection.test.tsx @@ -346,13 +346,13 @@ describe("FxSection carve", () => { it("offers no analyse button — picking a voice is the whole gesture", () => { // A carve with a source and no filters is a setting nobody applied; the // button was a second step for something the panel already knew to do. - const { host } = mount({ carve: { ...DEFAULT_CARVE, source: "vo" } }); + const { host } = mount({ carve: { ...DEFAULT_CARVE, sources: ["vo"] } }); expect(host.querySelector(".hf-fx-analyse")).toBeNull(); expect(host.textContent).not.toMatch(/Analyse/i); }); it("says when it is working, since there is no button to grey out", () => { - const { host } = mount({ carve: { ...DEFAULT_CARVE, source: "vo" }, analysing: true }); + const { host } = mount({ carve: { ...DEFAULT_CARVE, sources: ["vo"] }, analysing: true }); expect(host.querySelector(".hf-fx-carve-working")?.textContent).toMatch(/Analysing/i); }); @@ -516,7 +516,7 @@ describe("voiceover carve visibility", () => { const { host } = mount({ chain: chainOf("lowpass"), sourceOptions: [], - carve: { ...DEFAULT_CARVE, source: "vo" }, + carve: { ...DEFAULT_CARVE, sources: ["vo"] }, }); expect(carveBlock(host)).toBeTruthy(); }); diff --git a/packages/studio/src/components/editor/propertyPanelFxSection.tsx b/packages/studio/src/components/editor/propertyPanelFxSection.tsx index 3a99c77862..dd8ea9a866 100644 --- a/packages/studio/src/components/editor/propertyPanelFxSection.tsx +++ b/packages/studio/src/components/editor/propertyPanelFxSection.tsx @@ -748,9 +748,9 @@ export function FxSection({ onCarveChange({ ...carve, dynamic: e.target.checked })} - /> - - follows the voice, flat where it is silent - - {analysing ? (

Analysing…