mirror of
https://github.com/heygen-com/hyperframes.git
synced 2026-09-03 04:38:33 +00:00
fix(studio): stop the carve measuring the bed, reciprocating, and firing twice
Review findings 9, 10, 11 plus the prune-sentinel tail item. All four are the group-first carve shape catching up with code written when `sources` held clip ids. **9 — the far-end guard was defeated by the shape lint asks for.** `carvesAgainst` matched a carve's `sources` against raw clip ids and never expanded a group. A plural carve now names a GROUP (that is what `audio_carve_ungrouped_sources` exists to push authors toward), so `.includes(memberId)` stopped matching: `carverAgainst` returned null, the carve module was offered on a voice clip a bed is already ducking against, and switching it on wrote a reciprocal carve — each side measuring audio the other is already attenuating. Sources are expanded through `resolveCarveSourceIds` first now, which `useFxCarve` already imported for exactly this. **10 — the bed was measured as one of its own voices.** A group expands to its CURRENT members, so once the bed joins the group its own carve names — one timeline drag — `resolveCarveVoices` accepted it: peaking notches at the bed's own spectral peaks and a duck envelope that dips whenever the bed is loud, written to `data-fx-chain` / `data-automation` and baked into the export. The bed's id is excluded at the analysis entry, not inside core's generic resolver, because the exclusion is a fact about this analysis and not about resolution. `excludedFor` only guards the picker while it is offering options. **11 — the await opened a double-fire window the `<= 1` / `!== 1` split cannot see across.** `setCarve` awaits group creation, which live-patches `data-audio-group` and calls `updateElement` per member — a store notification. React re-renders mid-flight and before the carve attribute is written, so the candidate count collapses 2 → 1 while `carve` is still null and the sibling single-candidate effect fires a SECOND concurrent `setCarve`: two read-modify-write saves of `data-fx-carve` against one file (lost update) and two analyse() runs — two decodes, two FFT passes, two competing chain/automation writes. One in-flight latch now guards both effects, applied only to AUTO decisions: a manual change from the panel stays interruptible. **Tail — the prune never ran for a group bed.** Its "is the timeline loaded" sentinel was `present.has(element.id)`, and a group is not a timeline element, so a bus carrying its own carve always bailed. It now accepts either proof. `useFxCarve.ts` would have gone to 619 lines, so the compile half — `mintCarveNodes`, `measureCarve`, `carveLanes`, `carveLaneFor` — moved to `useFxCarveNodes.ts` first. 580 -> 442 + 153, both under the cap, and the split is the natural one: that half is pure, the hook keeps effects and persistence. Three tests for 9, each verified against a revert. studio: 390 files.
This commit is contained in:
Vance Ingalls
@@ -7,24 +7,15 @@
|
||||
* the file grew past a size where "the carve" was still one thing to read.
|
||||
*/
|
||||
|
||||
import { useEffect } from "react";
|
||||
import { useEffect, useRef } from "react";
|
||||
import {
|
||||
defaultAudioFxParams,
|
||||
HF_AUDIO_FX_ATTR,
|
||||
mintAudioFxNodeId,
|
||||
serializeAudioFxChain,
|
||||
type HfAudioFxChain,
|
||||
type HfAudioFxNode,
|
||||
} from "@hyperframes/core/audio-fx";
|
||||
import {
|
||||
analyseCarveBands,
|
||||
analyseCarveDuck,
|
||||
analyseCarveDynamics,
|
||||
carveBandsToChain,
|
||||
carveProfile,
|
||||
clipsOverlap,
|
||||
DEFAULT_CARVE,
|
||||
mixCarveSources,
|
||||
HF_AUDIO_CARVE_ATTR,
|
||||
type HfCarveSettings,
|
||||
} from "@hyperframes/core/audio-carve";
|
||||
@@ -37,23 +28,19 @@ import {
|
||||
isPromiseLike,
|
||||
resolveNextCarveSettings,
|
||||
} from "./useFxCarveGrouping.js";
|
||||
import {
|
||||
fxAutomationTarget,
|
||||
type HfAutomation,
|
||||
type HfAutomationLane,
|
||||
} from "@hyperframes/core/audio-automation";
|
||||
import { type HfAutomation } from "@hyperframes/core/audio-automation";
|
||||
import { automationAttrValue, HF_AUDIO_AUTOMATION_ATTR } from "./propertyPanelAutomation";
|
||||
import { trackCarveChanged } from "./audioFxTelemetry.js";
|
||||
import type { DomEditSelection } from "./domEditingTypes";
|
||||
import { usePlayerStore } from "../../player";
|
||||
import { clipStart, spanOf } from "./propertyPanelAudioFxGroupUtils.js";
|
||||
import { carveLanes, measureCarve, mintCarveNodes } from "./useFxCarveNodes.js";
|
||||
import { spanOf } from "./propertyPanelAudioFxGroupUtils.js";
|
||||
import type { AudioTrackOption } from "./propertyPanelFxCarveModule.js";
|
||||
|
||||
/**
|
||||
* Rate the carve source is decoded at. Analysis is self-consistent because it
|
||||
* reads the decoded buffer's own rate, so this only has to be a sane audio rate.
|
||||
*/
|
||||
const DECODE_SAMPLE_RATE = 48000;
|
||||
|
||||
/**
|
||||
* Which carve setting actually moved, by comparing the two snapshots.
|
||||
@@ -129,131 +116,6 @@ function resolveCarveVoices(
|
||||
return voices;
|
||||
}
|
||||
|
||||
/**
|
||||
* Turns the analysed bands (and, if the carve is asked to match levels, a
|
||||
* ducking envelope) into chain nodes — tagged so a re-run replaces them
|
||||
* instead of stacking, and minted against the nodes already claiming an id
|
||||
* because a dynamic carve automates these filters and a lane addresses its
|
||||
* node by id.
|
||||
*/
|
||||
function mintCarveNodes(
|
||||
chain: HfAudioFxChain,
|
||||
carved: HfAudioFxChain,
|
||||
duck: { t: number; v: number }[],
|
||||
): { next: HfAudioFxChain; carvedNodes: HfAudioFxNode[]; duckNode: HfAudioFxNode | null } {
|
||||
const kept = chain.nodes.filter((n) => !n.fromCarve);
|
||||
let claimed: HfAudioFxChain = { version: 1, nodes: kept };
|
||||
const mint = (node: HfAudioFxNode): HfAudioFxNode => {
|
||||
const withId = { ...node, id: mintAudioFxNodeId(claimed), fromCarve: true };
|
||||
claimed = { version: 1, nodes: [...claimed.nodes, withId] };
|
||||
return withId;
|
||||
};
|
||||
const carvedNodes: HfAudioFxNode[] = carved.nodes.map(mint);
|
||||
// The gain stage sits after the filters, and only exists when the carve was
|
||||
// asked to make level room. It sits at 0 and is driven by the envelope below.
|
||||
const duckNode =
|
||||
duck.length > 0
|
||||
? mint({ type: "gain", enabled: true, params: { ...defaultAudioFxParams("gain"), gain: 0 } })
|
||||
: null;
|
||||
return {
|
||||
next: { version: 1, nodes: [...carvedNodes, ...(duckNode ? [duckNode] : []), ...kept] },
|
||||
carvedNodes,
|
||||
duckNode,
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Decode every voice, mix them onto the bed's own clock, and measure the
|
||||
* bands (and, if the profile calls for it, the ducking envelope) from that
|
||||
* mix. Null on anything that leaves nothing to build a carve from — the
|
||||
* platform lacking an offline context, or a mix that decoded to silence.
|
||||
*/
|
||||
async function measureCarve(
|
||||
doc: Document,
|
||||
voices: { src: string; start: string | null }[],
|
||||
strength: number,
|
||||
bedStartAttr: string | null | undefined,
|
||||
bedSrc: string | null | undefined,
|
||||
): Promise<{
|
||||
bands: ReturnType<typeof analyseCarveBands>;
|
||||
carved: HfAudioFxChain;
|
||||
duck: { t: number; v: number }[];
|
||||
voiceMix: Float32Array;
|
||||
} | null> {
|
||||
// Decoded in an OfflineAudioContext, not a live one. Opening a second output
|
||||
// device mid-playback makes the running track glitch while the hardware is
|
||||
// reconfigured; an offline context touches no device.
|
||||
const Ctor =
|
||||
window.OfflineAudioContext ??
|
||||
(window as unknown as { webkitOfflineAudioContext?: typeof OfflineAudioContext })
|
||||
.webkitOfflineAudioContext;
|
||||
if (!Ctor) return null;
|
||||
const decode = async (relative: string): Promise<AudioBuffer> => {
|
||||
const res = await fetch(new URL(relative, doc.baseURI).href);
|
||||
return new Ctor(1, 1, DECODE_SAMPLE_RATE).decodeAudioData(await res.arrayBuffer());
|
||||
};
|
||||
const bedStart = clipStart(bedStartAttr);
|
||||
// Every voice, summed onto the bed's own clock. One question — where and
|
||||
// when is speech masking this bed — with one answer, even when the answer
|
||||
// comes from three people talking at different times. Doing this before the
|
||||
// analysis is also what lets the bands and the envelopes stay a single set:
|
||||
// the chain is fixed, so there is no per-voice filter to switch between.
|
||||
const decoded = await Promise.all(
|
||||
voices.map(async (voice) => ({
|
||||
samples: (await decode(voice.src)).getChannelData(0),
|
||||
offsetSeconds: clipStart(voice.start) - bedStart,
|
||||
})),
|
||||
);
|
||||
const voiceMix = mixCarveSources(decoded, DECODE_SAMPLE_RATE);
|
||||
if (voiceMix.length === 0) return null;
|
||||
// Strength is what the author set; these are the numbers it means.
|
||||
const profile = carveProfile(strength);
|
||||
// The bed as well as the voice, when the carve is asked to match levels:
|
||||
// "how far over the voice is this bed" cannot be answered by listening to
|
||||
// one of them.
|
||||
const bedBuffer = profile.duckDb > 0 && bedSrc ? await decode(bedSrc).catch(() => null) : null;
|
||||
const bands = analyseCarveBands(voiceMix, DECODE_SAMPLE_RATE, profile);
|
||||
// The level half of the carve, measured against the speech it has to sit
|
||||
// under. No offset to apply: the mix is already on the bed's clock.
|
||||
const duck = bedBuffer
|
||||
? analyseCarveDuck(voiceMix, bedBuffer.getChannelData(0), DECODE_SAMPLE_RATE, profile, 0)
|
||||
: [];
|
||||
return { bands, carved: carveBandsToChain(bands), duck, voiceMix };
|
||||
}
|
||||
|
||||
/** Each filter's depth as an envelope, plus the level envelope if there is one. */
|
||||
function carveLanes(
|
||||
carvedNodes: HfAudioFxNode[],
|
||||
duckNode: HfAudioFxNode | null,
|
||||
duck: { t: number; v: number }[],
|
||||
voiceMix: Float32Array,
|
||||
bands: ReturnType<typeof analyseCarveBands>,
|
||||
): HfAutomationLane[] {
|
||||
// Each filter's depth becomes an envelope of the speech's level in that
|
||||
// band, so pauses leave the bed alone and whoever is talking sets the depth.
|
||||
const lanes = analyseCarveDynamics(voiceMix, DECODE_SAMPLE_RATE, bands).flatMap((dyn, i) => {
|
||||
const id = carvedNodes[i]?.id;
|
||||
return id ? carveLaneFor(id, dyn.points) : [];
|
||||
});
|
||||
// The level envelope rides the gain stage, on the same clock as the bands.
|
||||
if (duckNode?.id && duck.length > 0) lanes.push(...carveLaneFor(duckNode.id, duck));
|
||||
return lanes;
|
||||
}
|
||||
|
||||
/**
|
||||
* One carve envelope as a lane on this bed's clock.
|
||||
*
|
||||
* No shifting: the voices were summed onto the bed's clock before the analysis
|
||||
* ran, so what comes back is already in the bed's own time. A lane does hold
|
||||
* its first value backwards to the start of its clip, so an envelope that
|
||||
* begins later needs an explicit "no cut" at zero or the bed starts out ducked.
|
||||
*/
|
||||
function carveLaneFor(id: string, points: { t: number; v: number }[]): HfAutomationLane[] {
|
||||
const timed = points.map((p) => ({ t: Number(p.t.toFixed(3)), v: p.v })).filter((p) => p.t >= 0);
|
||||
if ((timed[0]?.t ?? 0) > 0) timed.unshift({ t: 0, v: 0 });
|
||||
return timed.length > 1 ? [{ target: fxAutomationTarget(id, "gain"), points: timed }] : [];
|
||||
}
|
||||
|
||||
/**
|
||||
* What the carve generated is only justified by the voices it was measured
|
||||
* from: switched off, or left naming none — every source deleted, say — there
|
||||
@@ -374,13 +236,18 @@ export function useFxCarve(
|
||||
// player — reads as "every voice was deleted" and throws away a carve that
|
||||
// is perfectly fine. Unchanged sources are what the prune treats as nothing
|
||||
// to do.
|
||||
if (!element.id || !present.has(element.id)) return carve.sources;
|
||||
// A group id is never in `present` — it's not a timeline element — so it
|
||||
// needs its own existence check: a group survives as long as it still has
|
||||
// at least one member, checked against the live document the same way the
|
||||
// picker resolves groups above.
|
||||
const doc = element.element?.ownerDocument;
|
||||
const groupIds = doc ? new Set(resolveAudioGroups(doc).map((g) => g.id)) : new Set<string>();
|
||||
// The BED can be a group too — a bus carrying its own carve — and a group id
|
||||
// is never in `present`, so the sentinel rejected every group bed and the
|
||||
// prune simply never ran for one. Accept either proof that the timeline
|
||||
// describes this composition.
|
||||
if (!element.id || !(present.has(element.id) || groupIds.has(element.id))) {
|
||||
return carve.sources;
|
||||
}
|
||||
// Same reason a group source needs its own check: it is not a timeline
|
||||
// element. A group survives as long as it still has at least one member,
|
||||
// read off the live document the way the picker resolves groups above.
|
||||
return carve.sources.filter((id) => present.has(id) || groupIds.has(id));
|
||||
})();
|
||||
|
||||
@@ -427,6 +294,59 @@ export function useFxCarve(
|
||||
// carveLanes); what is left is the orchestration between them, including the
|
||||
// two-attribute write order the comments below explain the reason for.
|
||||
// fallow-ignore-next-line complexity
|
||||
/**
|
||||
* One auto-carve decision at a time.
|
||||
*
|
||||
* `setCarve` awaits `resolveNextCarveSettings`, which awaits group creation —
|
||||
* and that live-patches `data-audio-group` onto each member and calls
|
||||
* `updateElement` per member, a store notification. So React re-renders while
|
||||
* the first `setCarve` is still in flight and BEFORE the carve attribute has
|
||||
* been written: the candidate count legitimately collapses 2 → 1 (two clips
|
||||
* became one group) while `carve` is still null, and the sibling
|
||||
* single-candidate effect fires a second concurrent `setCarve`. That is two
|
||||
* read-modify-write saves of `data-fx-carve` against the same file (a lost
|
||||
* update) and two `analyse()` runs — two decodes, two FFT passes, two
|
||||
* competing `data-fx-chain` / `data-automation` writes.
|
||||
*
|
||||
* The `<= 1` / `!== 1` split between the two effects prevents the same
|
||||
* double-fire within one render pass; it cannot see across an await.
|
||||
*/
|
||||
const autoCarveInFlight = useRef(false);
|
||||
|
||||
/** `setCarve` from an auto-decision: latched, so the sibling effect cannot
|
||||
* start a second one across the await inside it. A manual change from the
|
||||
* panel is deliberately NOT latched — the author is allowed to interrupt. */
|
||||
const setCarveAuto = (next: HfCarveSettings): void => {
|
||||
autoCarveInFlight.current = true;
|
||||
void setCarve(next).finally(() => {
|
||||
autoCarveInFlight.current = false;
|
||||
});
|
||||
};
|
||||
|
||||
/**
|
||||
* Write what a measurement compiled to: the chain first, then the lanes.
|
||||
*
|
||||
* The chain write is live, like every other one — the runtime swaps the graph
|
||||
* in place, so a reload would only interrupt the audio to reach the same
|
||||
* filters. It is AWAITED because the automation write is a second
|
||||
* read-modify-write against the same file (fired together the later one drops
|
||||
* the earlier), and because a lane naming a node the chain does not carry yet
|
||||
* is pruned when it is read back.
|
||||
*/
|
||||
const persistMeasuredCarve = async (
|
||||
measured: NonNullable<Awaited<ReturnType<typeof measureCarve>>>,
|
||||
): Promise<void> => {
|
||||
const { bands, carved, duck, voiceMix } = measured;
|
||||
const { next, carvedNodes, duckNode } = mintCarveNodes(chain, carved, duck);
|
||||
await onSetAttributeQuiet(HF_AUDIO_FX_ATTR, serializeAudioFxChain(next));
|
||||
|
||||
const lanes = carveLanes(carvedNodes, duckNode, duck, voiceMix, bands);
|
||||
const carriedOver = withoutCarveLanes(automation, chain);
|
||||
if (lanes.length > 0 || carriedOver.lanes.length !== automation.lanes.length) {
|
||||
writeAutomation({ version: 1, lanes: [...carriedOver.lanes, ...lanes] });
|
||||
}
|
||||
};
|
||||
|
||||
const analyse = async (active: HfCarveSettings | null = carve): Promise<void> => {
|
||||
if (!active?.sources.length) return;
|
||||
const doc = element.element?.ownerDocument;
|
||||
@@ -435,7 +355,16 @@ export function useFxCarve(
|
||||
// a group named here expands to whoever is in it right now, so a fourth
|
||||
// voice added to an already-carved group is heard on the next analysis
|
||||
// without anyone editing the carve itself.
|
||||
const voices = resolveCarveVoices(doc, resolveCarveSourceIds(doc, active.sources));
|
||||
// The bed is excluded from its own voice list. A group expands to its
|
||||
// CURRENT members, so once the bed joins the group its own carve names —
|
||||
// one timeline drag — it was measured as one of its own voices: peaking
|
||||
// notches at the bed's own spectral peaks and a duck envelope that dips
|
||||
// whenever the bed is loud, written into `data-fx-chain` and
|
||||
// `data-automation` and baked into the export. The candidate-list guard
|
||||
// (`excludedFor`) only runs while the picker is offering options, never
|
||||
// against sources already persisted.
|
||||
const expanded = resolveCarveSourceIds(doc, active.sources).filter((id) => id !== element.id);
|
||||
const voices = resolveCarveVoices(doc, expanded);
|
||||
if (voices.length === 0) return;
|
||||
setAnalysing(true);
|
||||
try {
|
||||
@@ -448,24 +377,7 @@ export function useFxCarve(
|
||||
bedSrc,
|
||||
);
|
||||
if (!measured) return;
|
||||
const { bands, carved, duck, voiceMix } = measured;
|
||||
|
||||
const { next, carvedNodes, duckNode } = mintCarveNodes(chain, carved, duck);
|
||||
// Live, like every other chain write: the runtime swaps the graph in
|
||||
// place, so a reload would only interrupt the audio to reach the same
|
||||
// filters.
|
||||
//
|
||||
// Awaited, because the automation write below is a second read-modify-write
|
||||
// against the same file — fired together the later one would drop the
|
||||
// earlier — and because a lane naming a node the chain does not have yet is
|
||||
// pruned when it is read back.
|
||||
await onSetAttributeQuiet(HF_AUDIO_FX_ATTR, serializeAudioFxChain(next));
|
||||
|
||||
const lanes = carveLanes(carvedNodes, duckNode, duck, voiceMix, bands);
|
||||
const carriedOver = withoutCarveLanes(automation, chain);
|
||||
if (lanes.length > 0 || carriedOver.lanes.length !== automation.lanes.length) {
|
||||
writeAutomation({ version: 1, lanes: [...carriedOver.lanes, ...lanes] });
|
||||
}
|
||||
await persistMeasuredCarve(measured);
|
||||
} catch {
|
||||
// Leave the chain as it was; the button simply re-enables.
|
||||
} finally {
|
||||
@@ -524,16 +436,17 @@ export function useFxCarve(
|
||||
// the same result — two decodes, two FFT runs, two concurrent attribute
|
||||
// writes.
|
||||
if (carvedAgainstBy || !autoBed || autoSourceIds.length <= 1) return;
|
||||
if (autoCarveInFlight.current) return;
|
||||
const all = autoSourceIds;
|
||||
// Nothing configured: the default carve, pointed at everything it could hear.
|
||||
if (carve === null) {
|
||||
void setCarve({ ...DEFAULT_CARVE, sources: all });
|
||||
setCarveAuto({ ...DEFAULT_CARVE, sources: all });
|
||||
return;
|
||||
}
|
||||
// Configured but naming no voice — switched on before there was anything to
|
||||
// listen to, or a source list emptied. The card reads the candidates out, so
|
||||
// they have to be the stored ones too.
|
||||
if (carve.enabled && carve.sources.length === 0) void setCarve({ ...carve, sources: all });
|
||||
if (carve.enabled && carve.sources.length === 0) setCarveAuto({ ...carve, sources: all });
|
||||
// Keyed on the identity of the decision, not on setCarve — which is rebuilt
|
||||
// every render and would re-fire this.
|
||||
// eslint-disable-next-line react-hooks/exhaustive-deps
|
||||
@@ -559,18 +472,19 @@ export function useFxCarve(
|
||||
*/
|
||||
useEffect(() => {
|
||||
if (carvedAgainstBy || !autoBed || autoSourceIds.length !== 1) return;
|
||||
if (autoCarveInFlight.current) return;
|
||||
const only = autoSourceIds[0];
|
||||
if (!only) return;
|
||||
// Nothing configured: the default carve, pointed at the one candidate.
|
||||
if (carve === null) {
|
||||
void setCarve({ ...DEFAULT_CARVE, sources: [only] });
|
||||
setCarveAuto({ ...DEFAULT_CARVE, sources: [only] });
|
||||
return;
|
||||
}
|
||||
// Configured but with no voice yet — a carve switched on before there was
|
||||
// anything to listen to, or one whose source was cleared. The panel reads the
|
||||
// sole candidate out as the source, so it has to be the stored one too;
|
||||
// otherwise the card claims a relationship the attribute does not record.
|
||||
if (carve.enabled && carve.sources.length === 0) void setCarve({ ...carve, sources: [only] });
|
||||
if (carve.enabled && carve.sources.length === 0) setCarveAuto({ ...carve, sources: [only] });
|
||||
// Deliberately keyed on the identity of the decision, not on setCarve — which
|
||||
// is rebuilt every render and would re-fire this.
|
||||
// eslint-disable-next-line react-hooks/exhaustive-deps
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
// @vitest-environment happy-dom
|
||||
import { describe, expect, it } from "vitest";
|
||||
import { collectCarveCandidates } from "./useFxCarveGrouping";
|
||||
import { carverAgainst, collectCarveCandidates } from "./useFxCarveGrouping";
|
||||
|
||||
function previewDoc(html: string): Document {
|
||||
const doc = document.implementation.createHTMLDocument("preview");
|
||||
@@ -53,3 +53,32 @@ describe("collectCarveCandidates", () => {
|
||||
expect(candidatesFor(doc, "music-bed")).toEqual(["vo-1"]);
|
||||
});
|
||||
});
|
||||
|
||||
describe("carverAgainst", () => {
|
||||
// The far-end guard has to see through a GROUP source. A plural carve names a
|
||||
// group — that is what the lint rule pushes authors toward — so matching raw
|
||||
// ids never found the member, the carve module was offered on a voice already
|
||||
// being ducked against, and switching it on wrote a reciprocal carve.
|
||||
it("finds the bed carving a voice through its group", () => {
|
||||
const doc = previewDoc(`
|
||||
${GROUPED_VOICES}
|
||||
<audio id="bed" data-fx-carve='{"enabled":true,"sources":["voiceover"],"strength":0.3}'></audio>
|
||||
`);
|
||||
expect(carverAgainst(doc, "vo-1")).toBe("bed");
|
||||
expect(carverAgainst(doc, "vo-2")).toBe("bed");
|
||||
});
|
||||
|
||||
it("still finds a carve that names the clip directly", () => {
|
||||
const doc = previewDoc(`
|
||||
${GROUPED_VOICES}
|
||||
<audio id="bed" data-fx-carve='{"enabled":true,"sources":["vo-1"],"strength":0.3}'></audio>
|
||||
`);
|
||||
expect(carverAgainst(doc, "vo-1")).toBe("bed");
|
||||
expect(carverAgainst(doc, "vo-2")).toBeNull();
|
||||
});
|
||||
|
||||
it("is null for a track nobody carves against", () => {
|
||||
const doc = previewDoc(GROUPED_VOICES);
|
||||
expect(carverAgainst(doc, "vo-1")).toBeNull();
|
||||
});
|
||||
});
|
||||
|
||||
@@ -15,7 +15,7 @@ import {
|
||||
isNamedCarveBed,
|
||||
type HfCarveSettings,
|
||||
} from "@hyperframes/core/audio-carve";
|
||||
import { resolveAudioGroups } from "@hyperframes/core/audio-groups";
|
||||
import { resolveAudioGroups, resolveCarveSourceIds } from "@hyperframes/core/audio-groups";
|
||||
|
||||
/**
|
||||
* An id for a new voiceover group, de-duped against every id already in the
|
||||
@@ -200,11 +200,23 @@ export function carveBedRoles(
|
||||
return { couldBeBed: couldBeCarveBed(...parts), autoBed: isNamedCarveBed(...parts) };
|
||||
}
|
||||
|
||||
/** Whether some element's own carve attribute names `targetId` as a source. */
|
||||
function carvesAgainst(other: HTMLElement, targetId: string): boolean {
|
||||
/**
|
||||
* Whether some element's own carve attribute names `targetId` as a source.
|
||||
*
|
||||
* Sources are EXPANDED first. A plural carve now names a group rather than a
|
||||
* clip list — that is what `audio_carve_ungrouped_sources` exists to push
|
||||
* authors toward — so a raw `.includes(targetId)` never matched a member again,
|
||||
* and the far-end guard this feeds was silently defeated by the very shape the
|
||||
* lint rule asks for. The result: the carve module was offered on a voice clip
|
||||
* a bed is already ducking against, and switching it on wrote a reciprocal
|
||||
* carve — each side measuring audio the other is already attenuating.
|
||||
*/
|
||||
function carvesAgainst(doc: Document, other: HTMLElement, targetId: string): boolean {
|
||||
try {
|
||||
const raw = other.getAttribute(HF_AUDIO_CARVE_ATTR);
|
||||
return Boolean(raw && normalizeCarveSettings(JSON.parse(raw)).sources.includes(targetId));
|
||||
if (!raw) return false;
|
||||
const sources = normalizeCarveSettings(JSON.parse(raw)).sources;
|
||||
return resolveCarveSourceIds(doc, sources).includes(targetId);
|
||||
} catch {
|
||||
// An unreadable carve on some other element says nothing about this one.
|
||||
return false;
|
||||
@@ -226,6 +238,6 @@ export function carverAgainst(
|
||||
): string | null {
|
||||
if (!doc || !id) return null;
|
||||
const others = Array.from(doc.querySelectorAll<HTMLElement>(`[${HF_AUDIO_CARVE_ATTR}]`));
|
||||
const carver = others.find((other) => other.id !== id && carvesAgainst(other, id));
|
||||
const carver = others.find((other) => other.id !== id && carvesAgainst(doc, other, id));
|
||||
return carver ? carver.id || "another track" : null;
|
||||
}
|
||||
|
||||
@@ -0,0 +1,153 @@
|
||||
/**
|
||||
* What a carve analysis COMPILES TO: the filter nodes it mints and the
|
||||
* automation lanes it writes for them.
|
||||
*
|
||||
* Split out of `useFxCarve.ts` to keep it under the studio's 600-line cap. This
|
||||
* half is pure — it takes measurements and returns nodes and lanes — while the
|
||||
* hook keeps the effects, the persistence and the auto-carve decisions.
|
||||
*/
|
||||
|
||||
import {
|
||||
defaultAudioFxParams,
|
||||
mintAudioFxNodeId,
|
||||
type HfAudioFxChain,
|
||||
type HfAudioFxNode,
|
||||
} from "@hyperframes/core/audio-fx";
|
||||
import {
|
||||
analyseCarveBands,
|
||||
analyseCarveDuck,
|
||||
analyseCarveDynamics,
|
||||
carveBandsToChain,
|
||||
carveProfile,
|
||||
mixCarveSources,
|
||||
} from "@hyperframes/core/audio-carve";
|
||||
import { fxAutomationTarget, type HfAutomationLane } from "@hyperframes/core/audio-automation";
|
||||
import { clipStart } from "./propertyPanelAudioFxGroupUtils.js";
|
||||
|
||||
/** Decode rate every carve measurement shares — see `measureCarve`. */
|
||||
const DECODE_SAMPLE_RATE = 48000;
|
||||
|
||||
/**
|
||||
* Turns the analysed bands (and, if the carve is asked to match levels, a
|
||||
* ducking envelope) into chain nodes — tagged so a re-run replaces them
|
||||
* instead of stacking, and minted against the nodes already claiming an id
|
||||
* because a dynamic carve automates these filters and a lane addresses its
|
||||
* node by id.
|
||||
*/
|
||||
export function mintCarveNodes(
|
||||
chain: HfAudioFxChain,
|
||||
carved: HfAudioFxChain,
|
||||
duck: { t: number; v: number }[],
|
||||
): { next: HfAudioFxChain; carvedNodes: HfAudioFxNode[]; duckNode: HfAudioFxNode | null } {
|
||||
const kept = chain.nodes.filter((n) => !n.fromCarve);
|
||||
let claimed: HfAudioFxChain = { version: 1, nodes: kept };
|
||||
const mint = (node: HfAudioFxNode): HfAudioFxNode => {
|
||||
const withId = { ...node, id: mintAudioFxNodeId(claimed), fromCarve: true };
|
||||
claimed = { version: 1, nodes: [...claimed.nodes, withId] };
|
||||
return withId;
|
||||
};
|
||||
const carvedNodes: HfAudioFxNode[] = carved.nodes.map(mint);
|
||||
// The gain stage sits after the filters, and only exists when the carve was
|
||||
// asked to make level room. It sits at 0 and is driven by the envelope below.
|
||||
const duckNode =
|
||||
duck.length > 0
|
||||
? mint({ type: "gain", enabled: true, params: { ...defaultAudioFxParams("gain"), gain: 0 } })
|
||||
: null;
|
||||
return {
|
||||
next: { version: 1, nodes: [...carvedNodes, ...(duckNode ? [duckNode] : []), ...kept] },
|
||||
carvedNodes,
|
||||
duckNode,
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Decode every voice, mix them onto the bed's own clock, and measure the
|
||||
* bands (and, if the profile calls for it, the ducking envelope) from that
|
||||
* mix. Null on anything that leaves nothing to build a carve from — the
|
||||
* platform lacking an offline context, or a mix that decoded to silence.
|
||||
*/
|
||||
export async function measureCarve(
|
||||
doc: Document,
|
||||
voices: { src: string; start: string | null }[],
|
||||
strength: number,
|
||||
bedStartAttr: string | null | undefined,
|
||||
bedSrc: string | null | undefined,
|
||||
): Promise<{
|
||||
bands: ReturnType<typeof analyseCarveBands>;
|
||||
carved: HfAudioFxChain;
|
||||
duck: { t: number; v: number }[];
|
||||
voiceMix: Float32Array;
|
||||
} | null> {
|
||||
// Decoded in an OfflineAudioContext, not a live one. Opening a second output
|
||||
// device mid-playback makes the running track glitch while the hardware is
|
||||
// reconfigured; an offline context touches no device.
|
||||
const Ctor =
|
||||
window.OfflineAudioContext ??
|
||||
(window as unknown as { webkitOfflineAudioContext?: typeof OfflineAudioContext })
|
||||
.webkitOfflineAudioContext;
|
||||
if (!Ctor) return null;
|
||||
const decode = async (relative: string): Promise<AudioBuffer> => {
|
||||
const res = await fetch(new URL(relative, doc.baseURI).href);
|
||||
return new Ctor(1, 1, DECODE_SAMPLE_RATE).decodeAudioData(await res.arrayBuffer());
|
||||
};
|
||||
const bedStart = clipStart(bedStartAttr);
|
||||
// Every voice, summed onto the bed's own clock. One question — where and
|
||||
// when is speech masking this bed — with one answer, even when the answer
|
||||
// comes from three people talking at different times. Doing this before the
|
||||
// analysis is also what lets the bands and the envelopes stay a single set:
|
||||
// the chain is fixed, so there is no per-voice filter to switch between.
|
||||
const decoded = await Promise.all(
|
||||
voices.map(async (voice) => ({
|
||||
samples: (await decode(voice.src)).getChannelData(0),
|
||||
offsetSeconds: clipStart(voice.start) - bedStart,
|
||||
})),
|
||||
);
|
||||
const voiceMix = mixCarveSources(decoded, DECODE_SAMPLE_RATE);
|
||||
if (voiceMix.length === 0) return null;
|
||||
// Strength is what the author set; these are the numbers it means.
|
||||
const profile = carveProfile(strength);
|
||||
// The bed as well as the voice, when the carve is asked to match levels:
|
||||
// "how far over the voice is this bed" cannot be answered by listening to
|
||||
// one of them.
|
||||
const bedBuffer = profile.duckDb > 0 && bedSrc ? await decode(bedSrc).catch(() => null) : null;
|
||||
const bands = analyseCarveBands(voiceMix, DECODE_SAMPLE_RATE, profile);
|
||||
// The level half of the carve, measured against the speech it has to sit
|
||||
// under. No offset to apply: the mix is already on the bed's clock.
|
||||
const duck = bedBuffer
|
||||
? analyseCarveDuck(voiceMix, bedBuffer.getChannelData(0), DECODE_SAMPLE_RATE, profile, 0)
|
||||
: [];
|
||||
return { bands, carved: carveBandsToChain(bands), duck, voiceMix };
|
||||
}
|
||||
|
||||
/** Each filter's depth as an envelope, plus the level envelope if there is one. */
|
||||
export function carveLanes(
|
||||
carvedNodes: HfAudioFxNode[],
|
||||
duckNode: HfAudioFxNode | null,
|
||||
duck: { t: number; v: number }[],
|
||||
voiceMix: Float32Array,
|
||||
bands: ReturnType<typeof analyseCarveBands>,
|
||||
): HfAutomationLane[] {
|
||||
// Each filter's depth becomes an envelope of the speech's level in that
|
||||
// band, so pauses leave the bed alone and whoever is talking sets the depth.
|
||||
const lanes = analyseCarveDynamics(voiceMix, DECODE_SAMPLE_RATE, bands).flatMap((dyn, i) => {
|
||||
const id = carvedNodes[i]?.id;
|
||||
return id ? carveLaneFor(id, dyn.points) : [];
|
||||
});
|
||||
// The level envelope rides the gain stage, on the same clock as the bands.
|
||||
if (duckNode?.id && duck.length > 0) lanes.push(...carveLaneFor(duckNode.id, duck));
|
||||
return lanes;
|
||||
}
|
||||
|
||||
/**
|
||||
* One carve envelope as a lane on this bed's clock.
|
||||
*
|
||||
* No shifting: the voices were summed onto the bed's clock before the analysis
|
||||
* ran, so what comes back is already in the bed's own time. A lane does hold
|
||||
* its first value backwards to the start of its clip, so an envelope that
|
||||
* begins later needs an explicit "no cut" at zero or the bed starts out ducked.
|
||||
*/
|
||||
function carveLaneFor(id: string, points: { t: number; v: number }[]): HfAutomationLane[] {
|
||||
const timed = points.map((p) => ({ t: Number(p.t.toFixed(3)), v: p.v })).filter((p) => p.t >= 0);
|
||||
if ((timed[0]?.t ?? 0) > 0) timed.unshift({ t: 0, v: 0 });
|
||||
return timed.length > 1 ? [{ target: fxAutomationTarget(id, "gain"), points: timed }] : [];
|
||||
}
|
||||
Reference in New Issue
Block a user