mirror of
https://github.com/heygen-com/hyperframes.git
synced 2026-09-04 07:19:52 +00:00
fix(core,engine): make a group's preview match what the render bakes
Findings 8-11 and 16 — the render/preview parity set. All group audio, all invisible to preview or to export alone, which is why each had a passing test beside it. 9 — The preview bus wired only the automation lane, and readElementAutomation reads data-automation, so a group's own `data-volume` never reached the graph at all: the bus stayed at unity while the render applied it. The export was ~8 dB quieter than what had been auditioned. The test named for this asserted only `resolves.not.toBeNull()`. 10 — The volume lane was scheduled on the FX chain's SOURCE, ahead of the effects, breaking the contract scheduleVolumeLane's own docstring states: "the volume lane rides the fader, after the effects — where a DAW puts it, and the order the render bakes it in". The element path honoured it; the group path did not, so any nonlinear group effect previewed differently than it rendered. Both now land on a dedicated post-FX fader node: input → chain → fader → mute → output. 11 — The bus deliberately outlives stopAll(), and groupInput early -returned on an existing entry, so after a replay or a seek no new ramps were booked and the gain held the previous pass's last value — 0 after a fade-out, i.e. silent for the rest of the session. It re-anchors once per play generation now (ElementFxHandle gains `reanchor`), and the record keeps its `fx` handle so setRate can re-aim a group's automation, which its docblock already claimed it did. 8 — The sub-mix summed members at unity (normalize=0) into a pcm_s16le intermediate, so an over-unity sum hard-clipped at ±1 BEFORE the group's FX and fader ran: pulling the fader down, or the Giant preset's compressor, then operated on distortion. The intermediate is float now. Both downstream readers already took float; applyVolumeEnvelopeToWav did not, and does now, so a group's envelope is still baked sample-accurately rather than silently degrading to the expression path. The new level test is the one that matters here: EVERY tone in that file is built on ffmpeg's `sine` source, which peaks at ~0.125 full scale, so nothing in the suite could reach a clip whatever gain it asked for. `writePeakTone` states the amplitude outright, and the test fails against the 16-bit intermediate. 16 — mixGroupMembers forked mixAudioTracks' filter build and dropped its automation-degradation retry, so a member envelope past this ffmpeg build's expression limits failed the whole composition's audio where an ungrouped one degrades to base volume with a warning. The retry is back, reported on a successful result the same way. The group track is also pushed with its volumeKeyframes when the envelope could not be baked, instead of losing the group's automation silently. Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
This commit is contained in:
co-authored by
Claude Opus 5
parent
86ef1ed33e
commit
0103c2459c
@@ -35,7 +35,13 @@ export interface HfAudioGroup {
|
||||
hidden: boolean;
|
||||
}
|
||||
|
||||
function parseGroupVolume(el: Element | undefined): number {
|
||||
/**
|
||||
* A group element's `data-volume`, defaulting to 1 for a missing, unparseable
|
||||
* or absent element. Shared so the preview bus and `resolveAudioGroups` (which
|
||||
* the render reads) cannot drift: the export was ~8 dB quieter than what was
|
||||
* auditioned for exactly as long as the preview ignored this.
|
||||
*/
|
||||
export function readAudioGroupVolume(el: Element | null | undefined): number {
|
||||
const raw = el?.getAttribute("data-volume");
|
||||
const parsed = raw ? parseFloat(raw) : 1;
|
||||
return Number.isFinite(parsed) ? parsed : 1;
|
||||
@@ -50,7 +56,7 @@ function buildGroup(id: string, memberIds: string[], el: Element | undefined): H
|
||||
memberIds,
|
||||
...(fxChain ? { fxChain } : {}),
|
||||
...(automation ? { automation } : {}),
|
||||
volume: parseGroupVolume(el),
|
||||
volume: readAudioGroupVolume(el),
|
||||
hidden: el?.hasAttribute("data-hidden") ?? false,
|
||||
};
|
||||
}
|
||||
|
||||
@@ -98,6 +98,16 @@ export function readElementAutomation(el: {
|
||||
*/
|
||||
export interface ElementFxHandle {
|
||||
dispose(): void;
|
||||
/**
|
||||
* Re-book every envelope against a fresh reference frame.
|
||||
*
|
||||
* For a graph that OUTLIVES the source feeding it — a group bus, which is
|
||||
* built once and kept for the session — a replay or a seek starts a new pass
|
||||
* over the same chain. Without re-anchoring, the envelopes stay committed to
|
||||
* the first pass's absolute context times: past their last point that is a
|
||||
* stuck value, which for a fade-out is silence for the rest of the session.
|
||||
*/
|
||||
reanchor(timing: AutomationTiming): void;
|
||||
/**
|
||||
* Re-aim every booked envelope at a new playback rate.
|
||||
*
|
||||
@@ -277,6 +287,13 @@ export function attachElementFxChain(
|
||||
}
|
||||
|
||||
return {
|
||||
reanchor: (next: AutomationTiming) => {
|
||||
if (disposed) return;
|
||||
const at = timingNow();
|
||||
if (at) cancelParamLane(automated, at.scheduledAt);
|
||||
frame = { ...next };
|
||||
scheduleFor(readChain(el).chain, frame);
|
||||
},
|
||||
setRate: (rate: number) => {
|
||||
const at = timingNow();
|
||||
if (disposed || !at || !Number.isFinite(rate) || rate <= 0 || rate === at.rate) return;
|
||||
|
||||
@@ -726,16 +726,17 @@ describe("WebAudioTransport", () => {
|
||||
await scheduleGrouped(transport, gen, "b", "vo");
|
||||
|
||||
// Creation order for a: a-gain(0), a-solo(1), groupInput(2), groupOutput(3),
|
||||
// muteGain(4) — the group bus is built lazily inside a's schedule call.
|
||||
// Then b: b-gain(5), b-solo(6).
|
||||
expect(mock.gainNodes.length).toBeGreaterThanOrEqual(7);
|
||||
// muteGain(4), fader(5) — the group bus is built lazily inside a's
|
||||
// schedule call. Then b: b-gain(6), b-solo(7).
|
||||
expect(mock.gainNodes.length).toBeGreaterThanOrEqual(8);
|
||||
const aGain = mock.gainNodes[0]!;
|
||||
const aSolo = mock.gainNodes[1]!;
|
||||
const groupInput = firstGroupInput(mock);
|
||||
const groupOutput = mock.gainNodes[3]!;
|
||||
const muteGain = mock.gainNodes[4]!;
|
||||
const bGain = mock.gainNodes[5]!;
|
||||
const bSolo = mock.gainNodes[6]!;
|
||||
const fader = mock.gainNodes[5]!;
|
||||
const bGain = mock.gainNodes[6]!;
|
||||
const bSolo = mock.gainNodes[7]!;
|
||||
|
||||
// Each member feeds its own solo gain, and both solo gains feed the
|
||||
// shared bus — neither connects straight to master.
|
||||
@@ -746,11 +747,14 @@ describe("WebAudioTransport", () => {
|
||||
expect(aSolo.connect).not.toHaveBeenCalledWith(mock.masterGain);
|
||||
expect(bSolo.connect).not.toHaveBeenCalledWith(mock.masterGain);
|
||||
|
||||
// The bus's input never reaches master directly — it lands on the mute
|
||||
// gain (B5) first (the dry passthrough, since neither member's group has
|
||||
// a chain-bearing `<hf-audio-group>`), then the output gain, then master.
|
||||
// The bus's input never reaches master directly. It runs through the
|
||||
// chain (dry here — neither member's group has a chain-bearing
|
||||
// `<hf-audio-group>`) onto the FADER, then the mute gain (B5), then the
|
||||
// output gain, then master. The fader sits POST-FX because that is where
|
||||
// the render bakes group volume in.
|
||||
expect(groupInput.connect).not.toHaveBeenCalledWith(mock.masterGain);
|
||||
expect(groupInput.connect).toHaveBeenCalledWith(muteGain);
|
||||
expect(groupInput.connect).toHaveBeenCalledWith(fader);
|
||||
expect(fader.connect).toHaveBeenCalledWith(muteGain);
|
||||
expect(muteGain.connect).toHaveBeenCalledWith(groupOutput);
|
||||
expect(groupOutput.connect).toHaveBeenCalledWith(mock.masterGain);
|
||||
});
|
||||
@@ -773,18 +777,36 @@ describe("WebAudioTransport", () => {
|
||||
|
||||
const muteGain = mock.gainNodes[4]!;
|
||||
const groupOutput = mock.gainNodes[3]!;
|
||||
expect(firstGroupInput(mock).connect).toHaveBeenCalledWith(muteGain);
|
||||
const fader = mock.gainNodes[5]!;
|
||||
expect(firstGroupInput(mock).connect).toHaveBeenCalledWith(fader);
|
||||
expect(fader.connect).toHaveBeenCalledWith(muteGain);
|
||||
expect(muteGain.connect).toHaveBeenCalledWith(groupOutput);
|
||||
expect(groupOutput.connect).toHaveBeenCalledWith(mock.masterGain);
|
||||
// No element to read, so the fader sits at unity.
|
||||
expect(fader.gain.value).toBe(1);
|
||||
});
|
||||
|
||||
it("group volume rides the group's own data-volume via its automation lane, not the member's", async () => {
|
||||
// The old assertion here was `resolves.not.toBeNull()` — it was named for
|
||||
// group volume and checked only that scheduling did not throw, so the bus
|
||||
// sitting at unity while the render applied data-volume went unseen. The
|
||||
// export was ~8 dB quieter than what had been auditioned.
|
||||
it("puts the group's own data-volume on the bus fader", async () => {
|
||||
document.body.innerHTML = `<hf-audio-group id="vo" data-label="Voiceover" data-volume="0.4"></hf-audio-group>`;
|
||||
const { transport, mock, gen } = setupGroupTransport();
|
||||
|
||||
await expect(scheduleGrouped(transport, gen, "a", "vo")).resolves.not.toBeNull();
|
||||
|
||||
expect(mock.gainNodes[5]!.gain.value).toBeCloseTo(0.4, 6);
|
||||
});
|
||||
|
||||
it("leaves the fader at unity when the group carries no data-volume", async () => {
|
||||
document.body.innerHTML = `<hf-audio-group id="vo" data-label="Voiceover"></hf-audio-group>`;
|
||||
const { transport, gen } = setupGroupTransport();
|
||||
const { transport, mock, gen } = setupGroupTransport();
|
||||
|
||||
// No throw wiring the group's automation reader against a real
|
||||
// <hf-audio-group> element that carries no fx/automation attrs.
|
||||
await expect(scheduleGrouped(transport, gen, "a", "vo")).resolves.not.toBeNull();
|
||||
expect(mock.gainNodes[5]!.gain.value).toBe(1);
|
||||
});
|
||||
|
||||
it("destroy() disposes every group bus", async () => {
|
||||
@@ -811,13 +833,37 @@ describe("WebAudioTransport", () => {
|
||||
expect(mock.gainNodes.filter((n) => n === groupInput)).toHaveLength(1);
|
||||
});
|
||||
|
||||
// Surviving stopAll() is the point of the bus — and the trap. Its envelopes
|
||||
// were booked against the FIRST pass's absolute context times, so a replay
|
||||
// or a seek left the fader holding that pass's last value: 0 after a
|
||||
// fade-out, i.e. silent for the rest of the session.
|
||||
it("re-anchors the reused bus once per play generation, and only once", async () => {
|
||||
document.body.innerHTML = `<hf-audio-group id="vo" data-volume="0.5"></hf-audio-group>`;
|
||||
const { transport, mock, gen } = setupGroupTransport();
|
||||
await scheduleGrouped(transport, gen, "a", "vo");
|
||||
const fader = mock.gainNodes[5]!;
|
||||
|
||||
// Something moved the fader mid-pass (a ramp reaching its last point).
|
||||
fader.gain.value = 0;
|
||||
transport.stopAll();
|
||||
|
||||
const gen2 = transport.startGeneration();
|
||||
await scheduleGrouped(transport, gen2, "a", "vo");
|
||||
expect(fader.gain.value).toBeCloseTo(0.5, 6);
|
||||
|
||||
// A second member in the SAME pass must not re-book on top of the first.
|
||||
fader.gain.value = 0;
|
||||
await scheduleGrouped(transport, gen2, "b", "vo");
|
||||
expect(fader.gain.value).toBe(0);
|
||||
});
|
||||
|
||||
describe('solo — "Hear only this" (B5)', () => {
|
||||
it("silences a non-soloed member via its own solo gain, without touching the group bus", async () => {
|
||||
const { transport, mock, gen } = setupGroupTransport();
|
||||
await scheduleGrouped(transport, gen, "a", "vo");
|
||||
await scheduleGrouped(transport, gen, "b", "vo");
|
||||
const aSolo = mock.gainNodes[1]!;
|
||||
const bSolo = mock.gainNodes[6]!;
|
||||
const bSolo = mock.gainNodes[7]!;
|
||||
const groupInput = firstGroupInput(mock);
|
||||
|
||||
transport.setSolo(new Set(["other-clip"]));
|
||||
@@ -834,7 +880,7 @@ describe("WebAudioTransport", () => {
|
||||
await scheduleGrouped(transport, gen, "a", "vo");
|
||||
await scheduleGrouped(transport, gen, "b", "vo");
|
||||
const aSolo = mock.gainNodes[1]!;
|
||||
const bSolo = mock.gainNodes[6]!;
|
||||
const bSolo = mock.gainNodes[7]!;
|
||||
const groupInput = firstGroupInput(mock);
|
||||
|
||||
transport.setSolo(new Set(["a"]));
|
||||
@@ -849,7 +895,7 @@ describe("WebAudioTransport", () => {
|
||||
await scheduleGrouped(transport, gen, "a", "vo");
|
||||
await scheduleGrouped(transport, gen, "b", "vo");
|
||||
const aSolo = mock.gainNodes[1]!;
|
||||
const bSolo = mock.gainNodes[6]!;
|
||||
const bSolo = mock.gainNodes[7]!;
|
||||
|
||||
transport.setSolo(new Set(["vo"]));
|
||||
|
||||
|
||||
@@ -5,7 +5,7 @@ import {
|
||||
type AutomationTiming,
|
||||
} from "../audio/audioFxAutomation.js";
|
||||
import { VOLUME_RANGE } from "../audioAutomation.js";
|
||||
import { audioGroupOf, isAudibleUnderSolo } from "../audioGroups.js";
|
||||
import { audioGroupOf, isAudibleUnderSolo, readAudioGroupVolume } from "../audioGroups.js";
|
||||
import { swallow } from "./diagnostics";
|
||||
import { clampAudioGain } from "../audioGain.js";
|
||||
import { getDebugSurface } from "./globals.js";
|
||||
@@ -136,9 +136,17 @@ export class WebAudioTransport {
|
||||
string,
|
||||
{
|
||||
input: GainNode;
|
||||
/** Post-FX fader: `data-volume` plus the volume lane. */
|
||||
fader: GainNode;
|
||||
muteGain: GainNode;
|
||||
analyser: AnalyserNode;
|
||||
levelBuf: Float32Array;
|
||||
/** Kept so `setRate` can re-aim this bus's FX automation, the way it does
|
||||
* every source's — its docblock claims it already did. */
|
||||
fx: ElementFxHandle | null;
|
||||
/** Play generation the current envelopes were booked against. */
|
||||
generation: number;
|
||||
reanchor(timing: AutomationTiming): void;
|
||||
dispose(): void;
|
||||
}
|
||||
>();
|
||||
@@ -308,7 +316,18 @@ export class WebAudioTransport {
|
||||
*/
|
||||
private groupInput(groupId: string, doc: Document, timing: AutomationTiming): GainNode | null {
|
||||
const existing = this._groups.get(groupId);
|
||||
if (existing) return existing.input;
|
||||
if (existing) {
|
||||
// The bus outlives `stopAll()` on purpose, so a replay or a seek reuses
|
||||
// this graph — but its envelopes were committed to the FIRST pass's
|
||||
// absolute context times. Left alone they hold their last value forever,
|
||||
// which for a fade-out is silence for the rest of the session. Re-anchor
|
||||
// once per play generation, not once per member scheduled.
|
||||
if (existing.generation !== this._playGeneration) {
|
||||
existing.generation = this._playGeneration;
|
||||
existing.reanchor(timing);
|
||||
}
|
||||
return existing.input;
|
||||
}
|
||||
if (!this._ctx || !this._masterGain) return null;
|
||||
|
||||
const input = this._ctx.createGain();
|
||||
@@ -329,24 +348,42 @@ export class WebAudioTransport {
|
||||
const muteGain = this._ctx.createGain();
|
||||
muteGain.gain.value = groupEl?.hasAttribute("data-hidden") ? 0 : 1;
|
||||
muteGain.connect(output);
|
||||
// The group's fader, POST-FX: `data-volume` is the static position and the
|
||||
// volume lane rides it, which is where a DAW puts it and the order the
|
||||
// render bakes it in (`scheduleVolumeLane`'s own contract). Scheduling it
|
||||
// on `input` instead put the fader ahead of the effects, so any nonlinear
|
||||
// group effect — a compressor, the Giant preset — previewed differently
|
||||
// than it rendered.
|
||||
const fader = this._ctx.createGain();
|
||||
fader.gain.value = readAudioGroupVolume(groupEl);
|
||||
fader.connect(muteGain);
|
||||
const fx = attachElementFxChain(
|
||||
this._ctx,
|
||||
groupEl ?? { getAttribute: () => null },
|
||||
input,
|
||||
muteGain,
|
||||
fader,
|
||||
timing,
|
||||
);
|
||||
if (groupEl) scheduleVolumeLane(groupEl, input, timing);
|
||||
if (groupEl) scheduleVolumeLane(groupEl, fader, timing);
|
||||
|
||||
this._groups.set(groupId, {
|
||||
input,
|
||||
fader,
|
||||
muteGain,
|
||||
analyser,
|
||||
levelBuf: new Float32Array(analyser.fftSize),
|
||||
fx,
|
||||
generation: this._playGeneration,
|
||||
reanchor: (at: AutomationTiming) => {
|
||||
fader.gain.value = readAudioGroupVolume(groupEl);
|
||||
fx?.reanchor(at);
|
||||
if (groupEl) scheduleVolumeLane(groupEl, fader, at);
|
||||
},
|
||||
dispose: () => {
|
||||
try {
|
||||
fx?.dispose();
|
||||
input.disconnect();
|
||||
fader.disconnect();
|
||||
muteGain.disconnect();
|
||||
output.disconnect();
|
||||
analyser.disconnect();
|
||||
@@ -588,6 +625,16 @@ export class WebAudioTransport {
|
||||
swallow("webAudioTransport.setRate", err);
|
||||
}
|
||||
}
|
||||
// Group buses are not in `_activeSources` — they outlive it — so their FX
|
||||
// automation needs re-aiming here too, or a rate change leaves a group's
|
||||
// envelopes running the old plan over audio at the new speed.
|
||||
for (const group of this._groups.values()) {
|
||||
try {
|
||||
group.fx?.setRate(safeRate);
|
||||
} catch (err) {
|
||||
swallow("webAudioTransport.setRate.group", err);
|
||||
}
|
||||
}
|
||||
return true;
|
||||
}
|
||||
|
||||
|
||||
@@ -63,6 +63,35 @@ function writeTone(path: string, freq: number, seconds: number, gain: number): v
|
||||
if (result.status !== 0) throw new Error(`Could not write tone: ${result.stderr}`);
|
||||
}
|
||||
|
||||
/**
|
||||
* A sine of an EXACT peak amplitude.
|
||||
*
|
||||
* `writeTone` builds on ffmpeg's `sine` source, whose output is ~0.125 full
|
||||
* scale, so its `gain` argument is a relative knob rather than a level: a tone
|
||||
* asked for at 0.7 lands at about -21 dBFS. Fine for the relative comparisons
|
||||
* above, useless for anything about headroom — `aevalsrc` states the amplitude
|
||||
* outright.
|
||||
*/
|
||||
function writePeakTone(path: string, freq: number, seconds: number, peak: number): void {
|
||||
const result = spawnSync(
|
||||
getFfmpegBinary(),
|
||||
[
|
||||
"-nostdin",
|
||||
"-v",
|
||||
"error",
|
||||
"-f",
|
||||
"lavfi",
|
||||
"-i",
|
||||
`aevalsrc=${peak}*sin(2*PI*${freq}*t):d=${seconds}:s=48000`,
|
||||
"-c:a",
|
||||
"pcm_s16le",
|
||||
path,
|
||||
],
|
||||
{ encoding: "utf-8" },
|
||||
);
|
||||
if (result.status !== 0) throw new Error(`Could not write tone: ${result.stderr}`);
|
||||
}
|
||||
|
||||
const track = (id: string, end: number, volume = 1) => ({
|
||||
id,
|
||||
src: `${id}.wav`,
|
||||
@@ -268,4 +297,54 @@ describe.skipIf(!HAS_FFMPEG)("mix level arithmetic", () => {
|
||||
const flatTail = meanVolumeDb(flatOut, 3, 4);
|
||||
expect(Math.abs(groupedTail - flatTail)).toBeLessThan(0.5);
|
||||
});
|
||||
|
||||
// Members sum at unity (normalize=0), so an over-unity sum used to hard-clip
|
||||
// at ±1 in the 16-bit intermediate BEFORE the group's fader and FX chain ran
|
||||
// — pulling the group down then operated on distortion. Every existing probe
|
||||
// here sums to ≤ 0.8, which is exactly why nothing caught it; preview cannot
|
||||
// reproduce it either, because its bus is float.
|
||||
it("does not clip an over-unity member sum before the group fader", async () => {
|
||||
const projectDir = mkdtempSync(join(tmpdir(), "hf-grp-clip-"));
|
||||
const workDir = mkdtempSync(join(tmpdir(), "hf-grp-clip-work-"));
|
||||
tempDirs.push(projectDir, workDir);
|
||||
|
||||
// Two coherent copies of the same tone: 0.7 + 0.7 = 1.4, comfortably over.
|
||||
// `writePeakTone`, not `writeTone` — ffmpeg's `sine` source is nowhere near
|
||||
// full scale (its output peaks at ~0.125), so every tone in this file sits
|
||||
// around -21 dBFS and NOTHING here can reach a clip no matter what gain is
|
||||
// asked for. That is a large part of why this class of bug survived.
|
||||
writePeakTone(join(projectDir, "a.wav"), 440, 2, 0.7);
|
||||
writePeakTone(join(projectDir, "b.wav"), 440, 2, 0.7);
|
||||
// The reference: the level that sum SHOULD reach once the group's 0.5
|
||||
// fader has been applied — 1.4 × 0.5 = 0.7, one tone's worth.
|
||||
writePeakTone(join(projectDir, "ref.wav"), 440, 2, 0.7);
|
||||
|
||||
const groupedOut = join(projectDir, `clip-grouped-${MIXED_AUDIO_FILENAME}`);
|
||||
const refOut = join(projectDir, `clip-ref-${MIXED_AUDIO_FILENAME}`);
|
||||
|
||||
const grouped = await processCompositionAudio(
|
||||
[
|
||||
{ ...track("a", 2), groupId: "vo", groupVolume: 0.5 },
|
||||
{ ...track("b", 2), groupId: "vo", groupVolume: 0.5 },
|
||||
],
|
||||
projectDir,
|
||||
workDir,
|
||||
groupedOut,
|
||||
2,
|
||||
);
|
||||
const reference = await processCompositionAudio(
|
||||
[track("ref", 2)],
|
||||
projectDir,
|
||||
workDir,
|
||||
refOut,
|
||||
2,
|
||||
);
|
||||
expect(grouped.success).toBe(true);
|
||||
expect(reference.success).toBe(true);
|
||||
|
||||
// Both are ONE track into the outer mix, so the outer graph is identical
|
||||
// and the levels are directly comparable. Clipped, the flat-topped sum
|
||||
// reads well over a dB hot even after the fader halves it.
|
||||
expect(Math.abs(meanVolumeDb(groupedOut) - meanVolumeDb(refOut))).toBeLessThan(0.5);
|
||||
});
|
||||
});
|
||||
|
||||
@@ -885,20 +885,25 @@ async function mixGroupMembers(
|
||||
totalDuration: number,
|
||||
signal?: AbortSignal,
|
||||
config?: Partial<Pick<EngineConfig, "ffmpegProcessTimeout">>,
|
||||
): Promise<{ success: boolean; error?: string }> {
|
||||
): Promise<{ success: boolean; error?: string; degradedAutomation?: boolean }> {
|
||||
const ffmpegProcessTimeout = config?.ffmpegProcessTimeout ?? DEFAULT_CONFIG.ffmpegProcessTimeout;
|
||||
const outputDir = dirname(outputPath);
|
||||
if (!existsSync(outputDir)) mkdirSync(outputDir, { recursive: true });
|
||||
|
||||
const inputFilters = memberTracks.map((track, i) => {
|
||||
const delayMs = Math.round(track.start * 1000);
|
||||
const trimDuration = track.end - track.start + (track.tailSeconds ?? 0);
|
||||
const volumeFilter = buildVolumeExpression(track);
|
||||
return `[${i}:a]atrim=0:${formatFilterNumber(trimDuration)},${volumeFilter},adelay=${delayMs}|${delayMs},apad,atrim=0:${formatFilterNumber(totalDuration)}[a${i}]`;
|
||||
});
|
||||
const buildInputFilters = (ignoreKeyframes: boolean) =>
|
||||
memberTracks.map((track, i) => {
|
||||
const delayMs = Math.round(track.start * 1000);
|
||||
const trimDuration = track.end - track.start + (track.tailSeconds ?? 0);
|
||||
const volumeFilter = buildVolumeExpression(track, ignoreKeyframes);
|
||||
return `[${i}:a]atrim=0:${formatFilterNumber(trimDuration)},${volumeFilter},adelay=${delayMs}|${delayMs},apad,atrim=0:${formatFilterNumber(totalDuration)}[a${i}]`;
|
||||
});
|
||||
const mixInputs = memberTracks.map((_, i) => `[a${i}]`).join("");
|
||||
|
||||
const runOnce = async (useNormalize: boolean): Promise<RunFfmpegResult> => {
|
||||
const runOnce = async (
|
||||
useNormalize: boolean,
|
||||
ignoreKeyframes = false,
|
||||
): Promise<RunFfmpegResult> => {
|
||||
const inputFilters = buildInputFilters(ignoreKeyframes);
|
||||
const mixFilter = useNormalize
|
||||
? `${mixInputs}amix=inputs=${memberTracks.length}:duration=longest:dropout_transition=0:normalize=0[out]`
|
||||
: // amix's default normalize divides by input count; compensate by THIS
|
||||
@@ -922,8 +927,15 @@ async function mixGroupMembers(
|
||||
scriptPath,
|
||||
"-map",
|
||||
"[out]",
|
||||
// Float, not pcm_s16le: `normalize=0` sums the members at unity, so any
|
||||
// over-unity sum hard-clipped at ±1 in the intermediate — BEFORE the
|
||||
// group's FX chain and its fader ran. Pulling the group down, or the
|
||||
// Giant preset's compressor, then operated on distortion. Preview
|
||||
// cannot reproduce it (its bus is float), and it only shows up in the
|
||||
// export. Both readers downstream take float: `readWav` (format 3) and
|
||||
// `applyVolumeEnvelopeToWav`.
|
||||
"-acodec",
|
||||
"pcm_s16le",
|
||||
"pcm_f32le",
|
||||
"-ar",
|
||||
"48000",
|
||||
"-t",
|
||||
@@ -943,14 +955,32 @@ async function mixGroupMembers(
|
||||
}
|
||||
};
|
||||
|
||||
let result = await runOnce(true);
|
||||
let useNormalize = true;
|
||||
let result = await runOnce(useNormalize);
|
||||
if (!result.success && groupNormalizeOptionUnsupported(result.stderr)) {
|
||||
result = await runOnce(false);
|
||||
useNormalize = false;
|
||||
result = await runOnce(useNormalize);
|
||||
}
|
||||
|
||||
// The same defence `mixAudioTracks` has, which this forked without: a
|
||||
// member's volume automation becomes an ffmpeg `volume` expression whose
|
||||
// evaluator limits are build-dependent, so a dense envelope can fail the
|
||||
// whole run. Ungrouped, that track degrades to base volume with a warning;
|
||||
// grouped, it took the entire composition's audio down with it.
|
||||
let degradedAutomation = false;
|
||||
const hasAutomation = memberTracks.some((track) => (track.volumeKeyframes?.length ?? 0) > 0);
|
||||
if (!result.success && !signal?.aborted && hasAutomation) {
|
||||
const retry = await runOnce(useNormalize, true);
|
||||
if (retry.success) {
|
||||
result = retry;
|
||||
degradedAutomation = true;
|
||||
}
|
||||
}
|
||||
|
||||
if (signal?.aborted) return { success: false, error: "Group sub-mix cancelled" };
|
||||
if (!result.success)
|
||||
return { success: false, error: formatFfmpegError(result.exitCode, result.stderr) };
|
||||
return { success: true };
|
||||
return { success: true, degradedAutomation };
|
||||
}
|
||||
|
||||
export async function processCompositionAudio(
|
||||
@@ -1305,6 +1335,7 @@ export async function processCompositionAudio(
|
||||
// clock is composition time (offset 0): a group has no `data-start` of its
|
||||
// own, and members are already delayed to their composition positions
|
||||
// inside the sub-mix, so the group WAV's t=0 IS composition time.
|
||||
const groupsDegradedAutomation: string[] = [];
|
||||
for (const [groupId, memberTracks] of groupTracks) {
|
||||
const meta = groupMeta.get(groupId);
|
||||
if (!meta) continue;
|
||||
@@ -1329,6 +1360,7 @@ export async function processCompositionAudio(
|
||||
});
|
||||
continue;
|
||||
}
|
||||
if (subMix.degradedAutomation) groupsDegradedAutomation.push(groupId);
|
||||
|
||||
// Composition-time automation (offset 0, duration totalDuration) — same
|
||||
// resolve/lane/bake path a member uses, just anchored at the group clock
|
||||
@@ -1385,6 +1417,10 @@ export async function processCompositionAudio(
|
||||
mediaStart: 0,
|
||||
duration: totalDuration,
|
||||
volume: bakedEnvelope ? 1.0 : meta.volume,
|
||||
// Same fallback an ungrouped track gets: when the envelope could not be
|
||||
// baked into the samples, hand the keyframes to the outer mix's volume
|
||||
// expression instead of dropping the group's automation on the floor.
|
||||
...(bakedEnvelope || !laneKeyframes?.length ? {} : { volumeKeyframes: laneKeyframes }),
|
||||
});
|
||||
}
|
||||
if (failures.length > 0) return bail();
|
||||
@@ -1397,9 +1433,18 @@ export async function processCompositionAudio(
|
||||
/* ignore */
|
||||
}
|
||||
|
||||
// A group whose sub-mix had to drop member automation reports it the same
|
||||
// way mixAudioTracks reports its own degradation: on a SUCCESSFUL result, so
|
||||
// the render ships and the caller can still say what was lost.
|
||||
const degradedGroups = [...groupsDegradedAutomation];
|
||||
const degradedNote =
|
||||
degradedGroups.length > 0
|
||||
? `Volume automation exceeded this ffmpeg build's expression limits in group(s) ${degradedGroups.join(", ")}; rendered at base volume`
|
||||
: undefined;
|
||||
|
||||
return {
|
||||
...mixResult,
|
||||
durationMs: Date.now() - startMs,
|
||||
error: mixResult.error,
|
||||
error: mixResult.error ?? degradedNote,
|
||||
};
|
||||
}
|
||||
|
||||
@@ -173,4 +173,76 @@ describe("applyVolumeEnvelopeToWav", () => {
|
||||
),
|
||||
).toBe(false);
|
||||
});
|
||||
|
||||
// The group sub-mix writes float precisely so an over-unity member sum keeps
|
||||
// its headroom until the group's own fader and FX act on it. If the baker
|
||||
// could not read that file it returned false, and the group's automation was
|
||||
// dropped on the floor by the caller.
|
||||
describe("32-bit float input", () => {
|
||||
/** Stereo float32 WAV, every sample `value` — over 1.0 on purpose. */
|
||||
function writeConstantFloatWav(path: string, frames: number, value: number): void {
|
||||
const dataSize = frames * CHANNELS * 4;
|
||||
const buffer = Buffer.alloc(44 + dataSize);
|
||||
buffer.write("RIFF", 0, "ascii");
|
||||
buffer.writeUInt32LE(36 + dataSize, 4);
|
||||
buffer.write("WAVE", 8, "ascii");
|
||||
buffer.write("fmt ", 12, "ascii");
|
||||
buffer.writeUInt32LE(16, 16);
|
||||
buffer.writeUInt16LE(3, 20); // WAVE_FORMAT_IEEE_FLOAT
|
||||
buffer.writeUInt16LE(CHANNELS, 22);
|
||||
buffer.writeUInt32LE(SAMPLE_RATE, 24);
|
||||
buffer.writeUInt32LE(SAMPLE_RATE * CHANNELS * 4, 28);
|
||||
buffer.writeUInt16LE(CHANNELS * 4, 32);
|
||||
buffer.writeUInt16LE(32, 34);
|
||||
buffer.write("data", 36, "ascii");
|
||||
buffer.writeUInt32LE(dataSize, 40);
|
||||
for (let i = 0; i < frames * CHANNELS; i += 1) buffer.writeFloatLE(value, 44 + i * 4);
|
||||
writeFileSync(path, buffer);
|
||||
}
|
||||
|
||||
const floatSampleAt = (path: string, frame: number, channel = 0): number =>
|
||||
readFileSync(path).readFloatLE(44 + (frame * CHANNELS + channel) * 4);
|
||||
|
||||
it("scales float samples and keeps them above 1.0 unclamped", () => {
|
||||
const path = join(tmp(), "float.wav");
|
||||
writeConstantFloatWav(path, SAMPLE_RATE, 1.4);
|
||||
|
||||
expect(
|
||||
applyVolumeEnvelopeToWav(
|
||||
path,
|
||||
[
|
||||
{ time: 0, volume: 1 },
|
||||
{ time: 1, volume: 1 },
|
||||
],
|
||||
0,
|
||||
1,
|
||||
),
|
||||
).toBe(true);
|
||||
|
||||
// Unity envelope: unchanged, and NOT clamped down to 1.0.
|
||||
expect(floatSampleAt(path, 0)).toBeCloseTo(1.4, 5);
|
||||
expect(floatSampleAt(path, SAMPLE_RATE - 1)).toBeCloseTo(1.4, 5);
|
||||
});
|
||||
|
||||
it("applies the envelope across the file", () => {
|
||||
const path = join(tmp(), "float-fade.wav");
|
||||
writeConstantFloatWav(path, SAMPLE_RATE, 1.4);
|
||||
|
||||
expect(
|
||||
applyVolumeEnvelopeToWav(
|
||||
path,
|
||||
[
|
||||
{ time: 0, volume: 1 },
|
||||
{ time: 1, volume: 0 },
|
||||
],
|
||||
0,
|
||||
1,
|
||||
),
|
||||
).toBe(true);
|
||||
|
||||
expect(floatSampleAt(path, 0)).toBeCloseTo(1.4, 5);
|
||||
expect(floatSampleAt(path, Math.floor(SAMPLE_RATE / 2))).toBeCloseTo(0.7, 2);
|
||||
expect(floatSampleAt(path, SAMPLE_RATE - 1)).toBeCloseTo(0, 3);
|
||||
});
|
||||
});
|
||||
});
|
||||
|
||||
@@ -22,13 +22,17 @@ import { normaliseEnvelope } from "@hyperframes/core/media-volume-envelope";
|
||||
import { riffChunks } from "./wavChunks.js";
|
||||
|
||||
const PCM_FORMAT = 1; // WAVE_FORMAT_PCM
|
||||
const SUPPORTED_BITS = 16;
|
||||
const FLOAT_FORMAT = 3; // WAVE_FORMAT_IEEE_FLOAT
|
||||
|
||||
interface WavLayout {
|
||||
numChannels: number;
|
||||
sampleRate: number;
|
||||
dataOffset: number;
|
||||
dataSize: number;
|
||||
/** 16-bit integer, or 32-bit float — the group sub-mix writes float so an
|
||||
* over-unity member sum is not hard-clipped before the group's own FX and
|
||||
* fader get to act on it. */
|
||||
float: boolean;
|
||||
}
|
||||
|
||||
/**
|
||||
@@ -39,34 +43,49 @@ interface WavLayout {
|
||||
* and trailing chunks (LIST/fact/etc.) are skipped harmlessly. Returns null on
|
||||
* anything unexpected so the caller falls back to the expression path.
|
||||
*/
|
||||
function parseWavLayout(buffer: Buffer): WavLayout | null {
|
||||
if (buffer.length < 12 || buffer.toString("ascii", 0, 4) !== "RIFF") return null;
|
||||
if (buffer.toString("ascii", 8, 12) !== "WAVE") return null;
|
||||
interface WavFmt {
|
||||
numChannels: number;
|
||||
sampleRate: number;
|
||||
/** 16-bit integer PCM, or 32-bit IEEE float. Anything else is unreadable. */
|
||||
float: boolean;
|
||||
}
|
||||
|
||||
let fmt: { numChannels: number; sampleRate: number; bitsPerSample: number } | null = null;
|
||||
/** The `fmt ` chunk, or null for a format this cannot safely edit in place. */
|
||||
function readFmtChunk(buffer: Buffer, body: number): WavFmt | null {
|
||||
const format = buffer.readUInt16LE(body);
|
||||
const bits = buffer.readUInt16LE(body + 14);
|
||||
const float = format === FLOAT_FORMAT;
|
||||
if (!float && format !== PCM_FORMAT) return null;
|
||||
if (bits !== (float ? 32 : 16)) return null;
|
||||
const numChannels = buffer.readUInt16LE(body + 2);
|
||||
if (numChannels < 1) return null;
|
||||
return { numChannels, sampleRate: buffer.readUInt32LE(body + 4), float };
|
||||
}
|
||||
|
||||
function isRiffWave(buffer: Buffer): boolean {
|
||||
return (
|
||||
buffer.length >= 12 &&
|
||||
buffer.toString("ascii", 0, 4) === "RIFF" &&
|
||||
buffer.toString("ascii", 8, 12) === "WAVE"
|
||||
);
|
||||
}
|
||||
|
||||
function parseWavLayout(buffer: Buffer): WavLayout | null {
|
||||
if (!isRiffWave(buffer)) return null;
|
||||
|
||||
let fmt: WavFmt | null = null;
|
||||
let data: { offset: number; size: number } | null = null;
|
||||
|
||||
for (const { id, body, size } of riffChunks(buffer)) {
|
||||
if (id === "fmt " && body + 16 <= buffer.length) {
|
||||
if (buffer.readUInt16LE(body) !== PCM_FORMAT) return null;
|
||||
fmt = {
|
||||
numChannels: buffer.readUInt16LE(body + 2),
|
||||
sampleRate: buffer.readUInt32LE(body + 4),
|
||||
bitsPerSample: buffer.readUInt16LE(body + 14),
|
||||
};
|
||||
fmt = readFmtChunk(buffer, body);
|
||||
} else if (id === "data") {
|
||||
data = { offset: body, size: Math.min(size, buffer.length - body) };
|
||||
}
|
||||
}
|
||||
|
||||
if (!fmt || !data) return null;
|
||||
if (fmt.bitsPerSample !== SUPPORTED_BITS || fmt.numChannels < 1) return null;
|
||||
return {
|
||||
numChannels: fmt.numChannels,
|
||||
sampleRate: fmt.sampleRate,
|
||||
dataOffset: data.offset,
|
||||
dataSize: data.size,
|
||||
};
|
||||
return { ...fmt, dataOffset: data.offset, dataSize: data.size };
|
||||
}
|
||||
|
||||
/**
|
||||
@@ -103,11 +122,41 @@ export function createEnvelopeWalker(
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
/** Every sample scaled by the envelope, in place, in whichever of the two
|
||||
* formats the layout reports. Float is NOT clamped: it is the format the group
|
||||
* sub-mix writes precisely so an over-unity sum keeps its headroom until
|
||||
* something downstream chooses to reduce it. */
|
||||
function scaleSamples(
|
||||
buffer: Buffer,
|
||||
layout: WavLayout,
|
||||
gainAt: (seconds: number) => number,
|
||||
): void {
|
||||
const { numChannels, sampleRate, dataOffset, dataSize, float } = layout;
|
||||
const bytesPerSample = float ? 4 : 2;
|
||||
const frameBytes = numChannels * bytesPerSample;
|
||||
const frameCount = Math.floor(dataSize / frameBytes);
|
||||
const scaleOne = float
|
||||
? (at: number, gain: number) => buffer.writeFloatLE(buffer.readFloatLE(at) * gain, at)
|
||||
: (at: number, gain: number) => {
|
||||
const scaled = Math.round(buffer.readInt16LE(at) * gain);
|
||||
buffer.writeInt16LE(scaled < -32768 ? -32768 : scaled > 32767 ? 32767 : scaled, at);
|
||||
};
|
||||
|
||||
for (let frame = 0; frame < frameCount; frame += 1) {
|
||||
const gain = gainAt(frame / sampleRate);
|
||||
const base = dataOffset + frame * frameBytes;
|
||||
for (let channel = 0; channel < numChannels; channel += 1) {
|
||||
scaleOne(base + channel * bytesPerSample, gain);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Multiply a prepared WAV's samples by a time-varying gain envelope in place.
|
||||
*
|
||||
* @returns `true` if the envelope was applied; `false` if the file isn't the
|
||||
* expected 16-bit PCM (caller should fall back to the expression path).
|
||||
* @returns `true` if the envelope was applied; `false` if the file is neither
|
||||
* 16-bit PCM nor 32-bit float (caller should fall back to the expression path).
|
||||
*/
|
||||
export function applyVolumeEnvelopeToWav(
|
||||
wavPath: string,
|
||||
@@ -123,20 +172,7 @@ export function applyVolumeEnvelopeToWav(
|
||||
const layout = parseWavLayout(buffer);
|
||||
if (!layout) return false;
|
||||
|
||||
const { numChannels, sampleRate, dataOffset, dataSize } = layout;
|
||||
const bytesPerSample = SUPPORTED_BITS / 8;
|
||||
const frameBytes = numChannels * bytesPerSample;
|
||||
const frameCount = Math.floor(dataSize / frameBytes);
|
||||
|
||||
for (let frame = 0; frame < frameCount; frame += 1) {
|
||||
const gain = gainAt(frame / sampleRate);
|
||||
const base = dataOffset + frame * frameBytes;
|
||||
for (let channel = 0; channel < numChannels; channel += 1) {
|
||||
const at = base + channel * bytesPerSample;
|
||||
const scaled = Math.round(buffer.readInt16LE(at) * gain);
|
||||
buffer.writeInt16LE(scaled < -32768 ? -32768 : scaled > 32767 ? 32767 : scaled, at);
|
||||
}
|
||||
}
|
||||
scaleSamples(buffer, layout, gainAt);
|
||||
|
||||
// Write to a uniquely-named sibling then atomically rename over the
|
||||
// original. The random name avoids following a pre-planted symlink at a
|
||||
|
||||
Reference in New Issue
Block a user