feat(cli): audio --read --layer reads a one-layer file whole, no separation (#7345)

* feat(cli): audio --read --layer music|sfx|voice reads a one-layer file whole

A caller that knows a file holds a single layer can declare it: the file is
read whole as that layer with no stem separation and nothing uploaded.
Without the flag the read is unchanged. An unknown layer is rejected.

Consumer: heygen-com/experiment-framework#66322. Proposal OK'd by Rory
(C0BEH487Q7R, 2026-11-07).

Co-Authored-By: Jerrai

* chore(cli): drop added comments from audio read layer

Co-Authored-By: Jerrai
This commit is contained in:
Jerrai
2026-11-07 03:25:46 +00:00
committed by GitHub
co-authored by Jerrai
parent 1ddd425587
commit b37db11225
6 changed files with 179 additions and 2 deletions
+6
View File
@@ -885,6 +885,12 @@ Sentences come from a transcript: `--transcript <file>`, or
`<recording>.transcript.json` beside the recording. Without one, voice lines
carry no words.
A file you know holds one layer is read whole with `--layer music|sfx|voice`:
no stems are fetched and nothing leaves the machine, whatever `--quality` says.
`music` reads the file as the music bed (sections, tempo, levels), `voice` reads
words and pauses from it, `sfx` finds its effects. The JSON result records the
declared `layer`; without the flag the read is unchanged.
Effects and music are read from stems. At `--quality best` the
recording is separated into vocals, drums, bass, guitar, piano and other by
HeyGen's `/v3/audio/stems`, with your HeyGen credential (see
+57
View File
@@ -495,3 +495,60 @@ describe.skipIf(!ffmpeg)("readAudio", () => {
expect(analysis.warnings[1]).toContain("no transcript and no stems");
});
});
describe.skipIf(!ffmpeg)("readAudio with a declared layer", () => {
const read = (path: string, name: string, cache: string, layer?: "music" | "sfx" | "voice") => {
vi.stubEnv("HEYGEN_API_KEY", "test-key");
return readAudio(path, name, { cacheDir: join(dir, cache), quality: "best", layer });
};
it("reads a music file whole, without separating it, and keeps its cache apart from a mix read", async () => {
const path = wav("layer-music.wav", 8, bed);
const callsBefore = stemCalls;
const first = await read(path, "layer-music.wav", "cache-layer-music", "music");
expect(stemCalls).toBe(callsBefore);
expect(first.analysis).toMatchObject({ separation: "mix", layer: "music", voice: null });
expect(first.analysis.music!.segments.length).toBeGreaterThan(0);
expect(first.analysis.bpm).toBe(first.analysis.music!.segments[0]!.bpm);
expect((await read(path, "layer-music.wav", "cache-layer-music", "music")).cached).toBe(true);
const asMix = await read(path, "layer-music.wav", "cache-layer-music");
expect(asMix.cached).toBe(false);
expect(asMix.analysis.layer).toBeUndefined();
});
it("reads a voice file whole: words and pauses from the file itself", async () => {
const path = wav("layer-voice.wav", 6, voiced);
writeFileSync(
`${path}.transcript.json`,
JSON.stringify({
words: [
{ text: "one", start: 0.5, end: 2 },
{ text: "two", start: 3, end: 4.5 },
],
}),
);
const callsBefore = stemCalls;
const { analysis } = await read(path, "layer-voice.wav", "cache-layer-voice", "voice");
expect(stemCalls).toBe(callsBefore);
expect(analysis).toMatchObject({ separation: "mix", layer: "voice", effects: null });
expect(analysis.voice!.sentences.map((s) => s.text)).toEqual(["one", "two"]);
expect(analysis.voice!.pauses.length).toBeGreaterThan(0);
});
it("reads a sound-effect file whole, with no music read", async () => {
const path = wav("layer-sfx.wav", 5, boom);
const callsBefore = stemCalls;
const { analysis } = await read(path, "layer-sfx.wav", "cache-layer-sfx", "sfx");
expect(stemCalls).toBe(callsBefore);
expect(analysis).toMatchObject({ separation: "mix", layer: "sfx", voice: null, music: null });
expect(analysis.effects!.events.length).toBeGreaterThan(0);
});
it("records no layer when none is declared", async () => {
const path = wav("layer-none.wav", 3, voiced);
const { analysis } = await readAudio(path, "layer-none.wav", {
cacheDir: join(dir, "cache-layer-none"),
});
expect("layer" in analysis).toBe(false);
});
});
+55 -2
View File
@@ -28,7 +28,14 @@ import {
spanDb,
} from "./signal.js";
import { DRUM_STEM, VOICE_STEM, cachedStems, fetchStems, stemsAccess } from "./stems.js";
import { READ_VERSION, type AudibleStem, type Music, type ReadAudio, type Voice } from "./types.js";
import {
READ_VERSION,
type AudibleStem,
type Layer,
type Music,
type ReadAudio,
type Voice,
} from "./types.js";
import { voiceFromActivity, voiceFromWords, type TimedWord } from "./voice.js";
export interface ReadOptions {
@@ -37,6 +44,7 @@ export interface ReadOptions {
/** A transcript of the recording; `<file>.transcript.json` next to it is used when absent. */
transcript?: string;
quality?: AnalysisQuality;
layer?: Layer;
}
export interface ReadResult {
@@ -379,6 +387,46 @@ async function analyze(
return { ...about, voice, effects, music: null, ambience: null, warnings };
}
async function analyzeLayer(
path: string,
file: string,
sha: string,
transcript: string | null,
layer: Layer,
): Promise<ReadAudio> {
const samples = await decodeMono(path);
const about = {
...base(file, sha, round(samples.length / READ_RATE, 3), null, transcript),
layer,
};
const none = { voice: null, effects: null, music: null, ambience: null, warnings: [] };
if (layer === "music") {
const { music, ambience } = musicOf({ music: samples }, null, samples);
return { ...about, ...none, music, ambience, bpm: tempoOf(music) };
}
if (layer === "sfx") {
const effects = findEffects({
sources: [{ samples }],
sampleRate: READ_RATE,
mix: samples,
music: null,
beats: [],
speech: [],
});
return { ...about, ...none, effects };
}
const words = transcript ? wordsOf(transcript) : [];
const pitch = pitchTrack(await decodeMono(path, PITCH_RATE));
const levelOf = (span: Span) => spanDb(samples, READ_RATE, span.start, span.end);
const voice = words.length
? voiceFromWords(words, pitch, levelOf)
: voiceFromActivity(samples, READ_RATE, pitch, levelOf);
const warnings = words.length
? []
: [`no transcript: voice lines carry no words; ${transcribeHint(file)}`];
return { ...about, ...none, voice, warnings };
}
function base(
file: string,
sha: string,
@@ -408,6 +456,7 @@ export async function readAudio(
options: ReadOptions,
): Promise<ReadResult> {
const quality = options.quality ?? "local";
const { layer } = options;
const sha = await sha256(path);
const transcript = transcriptFor(path, options.transcript);
const transcriptKey = fingerprint(transcript);
@@ -419,12 +468,14 @@ export async function readAudio(
if (existsSync(jsonPath) && existsSync(textPath)) {
const entry = JSON.parse(readFileSync(jsonPath, "utf8")) as CacheEntry;
const stemsNowPossible =
!layer &&
entry.analysis.separation === "mix" &&
quality === "best" &&
(cachedStems(stemDir) !== null || typeof (await stemsAccess()) !== "string");
if (
(entry.analysis.separation !== "stems" || cachedStems(stemDir) !== null) &&
entry.analysis.version === READ_VERSION &&
entry.analysis.layer === layer &&
entry.revision === CACHE_REVISION &&
entry.transcriptKey === transcriptKey &&
!stemsNowPossible &&
@@ -437,7 +488,9 @@ export async function readAudio(
)
return { analysis: entry.analysis, textPath, jsonPath, cached: true };
}
const analysis = await analyze(path, file, sha, transcript, stemDir, quality, options.cacheDir);
const analysis = layer
? await analyzeLayer(path, file, sha, transcript, layer)
: await analyze(path, file, sha, transcript, stemDir, quality, options.cacheDir);
writeAtomic(
jsonPath,
JSON.stringify({ revision: CACHE_REVISION, analysis, transcriptKey } satisfies CacheEntry),
+4
View File
@@ -4,6 +4,9 @@ export const READ_VERSION = 7;
export type Separation = "stems" | "mix";
export const LAYERS = ["music", "sfx", "voice"] as const;
export type Layer = (typeof LAYERS)[number];
export type RateUnit = "chars" | "words";
export type Ending = "rising" | "falling" | "level";
@@ -121,6 +124,7 @@ export interface ReadAudio {
sha256: string;
duration: number;
separation: Separation;
layer?: Layer;
stems: AudibleStem[];
stemsDir: string | null;
voiceFreeMix: string | null;
@@ -0,0 +1,39 @@
import { runCommand } from "citty";
import { beforeEach, describe, expect, it, vi } from "vitest";
const { readAudio } = vi.hoisted(() => ({ readAudio: vi.fn() }));
vi.mock("../audio/read/read.js", () => ({ readAudio }));
import audioCommand from "./audio.js";
describe("audio --read --layer", () => {
beforeEach(() => {
readAudio.mockReset();
readAudio.mockResolvedValue({ analysis: {}, textPath: "x.txt", jsonPath: "x.json" });
vi.spyOn(console, "log").mockImplementation(() => {});
});
it.each(["music", "sfx", "voice"])("passes --layer %s to the read", async (layer) => {
await runCommand(audioCommand, { rawArgs: ["a.wav", "--read", "--layer", layer, "--json"] });
expect(readAudio.mock.calls[0]![2]).toMatchObject({ layer });
});
it("passes no layer when the flag is absent", async () => {
await runCommand(audioCommand, { rawArgs: ["a.wav", "--read", "--json"] });
expect(readAudio.mock.calls[0]![2].layer).toBeUndefined();
});
it("rejects an unknown layer and names the valid ones", async () => {
await expect(
runCommand(audioCommand, { rawArgs: ["a.wav", "--read", "--layer", "drums"] }),
).rejects.toThrow("--layer is music, sfx, voice; got drums");
expect(readAudio).not.toHaveBeenCalled();
});
it("rejects --layer without --read", async () => {
await expect(
runCommand(audioCommand, { rawArgs: ["a.wav", "--layer", "music"] }),
).rejects.toThrow("--layer only works with --read");
});
});
+18
View File
@@ -12,6 +12,7 @@ import { analyzeTargets, previewTarget } from "../audio/analysis/run.js";
import { filmTargets, type AudioTarget, type FilmQuery } from "../audio/analysis/targets.js";
import { formatRead } from "../audio/read/format.js";
import { readAudio } from "../audio/read/read.js";
import { LAYERS, type Layer } from "../audio/read/types.js";
import { describeProject } from "../timeline/describeProject.js";
import { CliUsageError } from "../utils/commandResult.js";
import { ensureDOMParser } from "../utils/dom.js";
@@ -122,6 +123,7 @@ interface AudioFlags {
"silence-db"?: string;
read?: boolean;
transcript?: string;
layer?: string;
json?: boolean;
}
@@ -181,13 +183,23 @@ function readQuality(files: string[], flags: AudioFlags): AnalysisQuality {
return parseQuality(flags);
}
function parseLayer(flags: AudioFlags): Layer | undefined {
if (flags.layer === undefined) return undefined;
if (!flags.read) throw new CliUsageError("--layer only works with --read");
if (!LAYERS.includes(flags.layer as Layer))
throw new CliUsageError(`--layer is ${LAYERS.join(", ")}; got ${flags.layer}`);
return flags.layer as Layer;
}
async function read(files: string[], flags: AudioFlags) {
const quality = readQuality(files, flags);
const range = pair("range", flags.range);
const layer = parseLayer(flags);
const result = await readAudio(resolve(process.cwd(), files[0]!), files[0]!, {
cacheDir: join(resolve(flags.dir ?? process.cwd()), ".hyperframes", "read-audio"),
transcript: flags.transcript ? resolve(process.cwd(), flags.transcript) : undefined,
quality,
layer,
});
if (flags.json) {
console.log(JSON.stringify(withMeta(result)));
@@ -259,6 +271,11 @@ export default defineCommand({
type: "string",
description: "With --read: the recording's transcript (default <file>.transcript.json)",
},
layer: {
type: "string",
description:
"With --read: the file holds only this layer (music, sfx or voice); it is read whole, with no separation",
},
json: { type: "boolean", description: "Output as JSON", default: false },
},
setup({ rawArgs, cmd }) {
@@ -268,6 +285,7 @@ export default defineCommand({
const flags = args as unknown as AudioFlags;
const files = args._.map(String);
if (flags.read) return read(files, flags);
if (flags.layer !== undefined) throw new CliUsageError("--layer only works with --read");
if (flags.transcript) throw new CliUsageError("--transcript only works with --read");
const query = filmQuery(flags);
if (files.length && query) throw new CliUsageError("give files or a film query, not both");