mirror of
https://github.com/heygen-com/hyperframes.git
synced 2026-11-18 16:08:56 +00:00
feat(cli): audio --read --layer reads a one-layer file whole, no separation (#7345)
* feat(cli): audio --read --layer music|sfx|voice reads a one-layer file whole A caller that knows a file holds a single layer can declare it: the file is read whole as that layer with no stem separation and nothing uploaded. Without the flag the read is unchanged. An unknown layer is rejected. Consumer: heygen-com/experiment-framework#66322. Proposal OK'd by Rory (C0BEH487Q7R, 2026-11-07). Co-Authored-By: Jerrai * chore(cli): drop added comments from audio read layer Co-Authored-By: Jerrai
This commit is contained in:
@@ -885,6 +885,12 @@ Sentences come from a transcript: `--transcript <file>`, or
|
||||
`<recording>.transcript.json` beside the recording. Without one, voice lines
|
||||
carry no words.
|
||||
|
||||
A file you know holds one layer is read whole with `--layer music|sfx|voice`:
|
||||
no stems are fetched and nothing leaves the machine, whatever `--quality` says.
|
||||
`music` reads the file as the music bed (sections, tempo, levels), `voice` reads
|
||||
words and pauses from it, `sfx` finds its effects. The JSON result records the
|
||||
declared `layer`; without the flag the read is unchanged.
|
||||
|
||||
Effects and music are read from stems. At `--quality best` the
|
||||
recording is separated into vocals, drums, bass, guitar, piano and other by
|
||||
HeyGen's `/v3/audio/stems`, with your HeyGen credential (see
|
||||
|
||||
@@ -495,3 +495,60 @@ describe.skipIf(!ffmpeg)("readAudio", () => {
|
||||
expect(analysis.warnings[1]).toContain("no transcript and no stems");
|
||||
});
|
||||
});
|
||||
|
||||
describe.skipIf(!ffmpeg)("readAudio with a declared layer", () => {
|
||||
const read = (path: string, name: string, cache: string, layer?: "music" | "sfx" | "voice") => {
|
||||
vi.stubEnv("HEYGEN_API_KEY", "test-key");
|
||||
return readAudio(path, name, { cacheDir: join(dir, cache), quality: "best", layer });
|
||||
};
|
||||
|
||||
it("reads a music file whole, without separating it, and keeps its cache apart from a mix read", async () => {
|
||||
const path = wav("layer-music.wav", 8, bed);
|
||||
const callsBefore = stemCalls;
|
||||
const first = await read(path, "layer-music.wav", "cache-layer-music", "music");
|
||||
expect(stemCalls).toBe(callsBefore);
|
||||
expect(first.analysis).toMatchObject({ separation: "mix", layer: "music", voice: null });
|
||||
expect(first.analysis.music!.segments.length).toBeGreaterThan(0);
|
||||
expect(first.analysis.bpm).toBe(first.analysis.music!.segments[0]!.bpm);
|
||||
expect((await read(path, "layer-music.wav", "cache-layer-music", "music")).cached).toBe(true);
|
||||
const asMix = await read(path, "layer-music.wav", "cache-layer-music");
|
||||
expect(asMix.cached).toBe(false);
|
||||
expect(asMix.analysis.layer).toBeUndefined();
|
||||
});
|
||||
|
||||
it("reads a voice file whole: words and pauses from the file itself", async () => {
|
||||
const path = wav("layer-voice.wav", 6, voiced);
|
||||
writeFileSync(
|
||||
`${path}.transcript.json`,
|
||||
JSON.stringify({
|
||||
words: [
|
||||
{ text: "one", start: 0.5, end: 2 },
|
||||
{ text: "two", start: 3, end: 4.5 },
|
||||
],
|
||||
}),
|
||||
);
|
||||
const callsBefore = stemCalls;
|
||||
const { analysis } = await read(path, "layer-voice.wav", "cache-layer-voice", "voice");
|
||||
expect(stemCalls).toBe(callsBefore);
|
||||
expect(analysis).toMatchObject({ separation: "mix", layer: "voice", effects: null });
|
||||
expect(analysis.voice!.sentences.map((s) => s.text)).toEqual(["one", "two"]);
|
||||
expect(analysis.voice!.pauses.length).toBeGreaterThan(0);
|
||||
});
|
||||
|
||||
it("reads a sound-effect file whole, with no music read", async () => {
|
||||
const path = wav("layer-sfx.wav", 5, boom);
|
||||
const callsBefore = stemCalls;
|
||||
const { analysis } = await read(path, "layer-sfx.wav", "cache-layer-sfx", "sfx");
|
||||
expect(stemCalls).toBe(callsBefore);
|
||||
expect(analysis).toMatchObject({ separation: "mix", layer: "sfx", voice: null, music: null });
|
||||
expect(analysis.effects!.events.length).toBeGreaterThan(0);
|
||||
});
|
||||
|
||||
it("records no layer when none is declared", async () => {
|
||||
const path = wav("layer-none.wav", 3, voiced);
|
||||
const { analysis } = await readAudio(path, "layer-none.wav", {
|
||||
cacheDir: join(dir, "cache-layer-none"),
|
||||
});
|
||||
expect("layer" in analysis).toBe(false);
|
||||
});
|
||||
});
|
||||
|
||||
@@ -28,7 +28,14 @@ import {
|
||||
spanDb,
|
||||
} from "./signal.js";
|
||||
import { DRUM_STEM, VOICE_STEM, cachedStems, fetchStems, stemsAccess } from "./stems.js";
|
||||
import { READ_VERSION, type AudibleStem, type Music, type ReadAudio, type Voice } from "./types.js";
|
||||
import {
|
||||
READ_VERSION,
|
||||
type AudibleStem,
|
||||
type Layer,
|
||||
type Music,
|
||||
type ReadAudio,
|
||||
type Voice,
|
||||
} from "./types.js";
|
||||
import { voiceFromActivity, voiceFromWords, type TimedWord } from "./voice.js";
|
||||
|
||||
export interface ReadOptions {
|
||||
@@ -37,6 +44,7 @@ export interface ReadOptions {
|
||||
/** A transcript of the recording; `<file>.transcript.json` next to it is used when absent. */
|
||||
transcript?: string;
|
||||
quality?: AnalysisQuality;
|
||||
layer?: Layer;
|
||||
}
|
||||
|
||||
export interface ReadResult {
|
||||
@@ -379,6 +387,46 @@ async function analyze(
|
||||
return { ...about, voice, effects, music: null, ambience: null, warnings };
|
||||
}
|
||||
|
||||
async function analyzeLayer(
|
||||
path: string,
|
||||
file: string,
|
||||
sha: string,
|
||||
transcript: string | null,
|
||||
layer: Layer,
|
||||
): Promise<ReadAudio> {
|
||||
const samples = await decodeMono(path);
|
||||
const about = {
|
||||
...base(file, sha, round(samples.length / READ_RATE, 3), null, transcript),
|
||||
layer,
|
||||
};
|
||||
const none = { voice: null, effects: null, music: null, ambience: null, warnings: [] };
|
||||
if (layer === "music") {
|
||||
const { music, ambience } = musicOf({ music: samples }, null, samples);
|
||||
return { ...about, ...none, music, ambience, bpm: tempoOf(music) };
|
||||
}
|
||||
if (layer === "sfx") {
|
||||
const effects = findEffects({
|
||||
sources: [{ samples }],
|
||||
sampleRate: READ_RATE,
|
||||
mix: samples,
|
||||
music: null,
|
||||
beats: [],
|
||||
speech: [],
|
||||
});
|
||||
return { ...about, ...none, effects };
|
||||
}
|
||||
const words = transcript ? wordsOf(transcript) : [];
|
||||
const pitch = pitchTrack(await decodeMono(path, PITCH_RATE));
|
||||
const levelOf = (span: Span) => spanDb(samples, READ_RATE, span.start, span.end);
|
||||
const voice = words.length
|
||||
? voiceFromWords(words, pitch, levelOf)
|
||||
: voiceFromActivity(samples, READ_RATE, pitch, levelOf);
|
||||
const warnings = words.length
|
||||
? []
|
||||
: [`no transcript: voice lines carry no words; ${transcribeHint(file)}`];
|
||||
return { ...about, ...none, voice, warnings };
|
||||
}
|
||||
|
||||
function base(
|
||||
file: string,
|
||||
sha: string,
|
||||
@@ -408,6 +456,7 @@ export async function readAudio(
|
||||
options: ReadOptions,
|
||||
): Promise<ReadResult> {
|
||||
const quality = options.quality ?? "local";
|
||||
const { layer } = options;
|
||||
const sha = await sha256(path);
|
||||
const transcript = transcriptFor(path, options.transcript);
|
||||
const transcriptKey = fingerprint(transcript);
|
||||
@@ -419,12 +468,14 @@ export async function readAudio(
|
||||
if (existsSync(jsonPath) && existsSync(textPath)) {
|
||||
const entry = JSON.parse(readFileSync(jsonPath, "utf8")) as CacheEntry;
|
||||
const stemsNowPossible =
|
||||
!layer &&
|
||||
entry.analysis.separation === "mix" &&
|
||||
quality === "best" &&
|
||||
(cachedStems(stemDir) !== null || typeof (await stemsAccess()) !== "string");
|
||||
if (
|
||||
(entry.analysis.separation !== "stems" || cachedStems(stemDir) !== null) &&
|
||||
entry.analysis.version === READ_VERSION &&
|
||||
entry.analysis.layer === layer &&
|
||||
entry.revision === CACHE_REVISION &&
|
||||
entry.transcriptKey === transcriptKey &&
|
||||
!stemsNowPossible &&
|
||||
@@ -437,7 +488,9 @@ export async function readAudio(
|
||||
)
|
||||
return { analysis: entry.analysis, textPath, jsonPath, cached: true };
|
||||
}
|
||||
const analysis = await analyze(path, file, sha, transcript, stemDir, quality, options.cacheDir);
|
||||
const analysis = layer
|
||||
? await analyzeLayer(path, file, sha, transcript, layer)
|
||||
: await analyze(path, file, sha, transcript, stemDir, quality, options.cacheDir);
|
||||
writeAtomic(
|
||||
jsonPath,
|
||||
JSON.stringify({ revision: CACHE_REVISION, analysis, transcriptKey } satisfies CacheEntry),
|
||||
|
||||
@@ -4,6 +4,9 @@ export const READ_VERSION = 7;
|
||||
|
||||
export type Separation = "stems" | "mix";
|
||||
|
||||
export const LAYERS = ["music", "sfx", "voice"] as const;
|
||||
export type Layer = (typeof LAYERS)[number];
|
||||
|
||||
export type RateUnit = "chars" | "words";
|
||||
|
||||
export type Ending = "rising" | "falling" | "level";
|
||||
@@ -121,6 +124,7 @@ export interface ReadAudio {
|
||||
sha256: string;
|
||||
duration: number;
|
||||
separation: Separation;
|
||||
layer?: Layer;
|
||||
stems: AudibleStem[];
|
||||
stemsDir: string | null;
|
||||
voiceFreeMix: string | null;
|
||||
|
||||
@@ -0,0 +1,39 @@
|
||||
import { runCommand } from "citty";
|
||||
import { beforeEach, describe, expect, it, vi } from "vitest";
|
||||
|
||||
const { readAudio } = vi.hoisted(() => ({ readAudio: vi.fn() }));
|
||||
|
||||
vi.mock("../audio/read/read.js", () => ({ readAudio }));
|
||||
|
||||
import audioCommand from "./audio.js";
|
||||
|
||||
describe("audio --read --layer", () => {
|
||||
beforeEach(() => {
|
||||
readAudio.mockReset();
|
||||
readAudio.mockResolvedValue({ analysis: {}, textPath: "x.txt", jsonPath: "x.json" });
|
||||
vi.spyOn(console, "log").mockImplementation(() => {});
|
||||
});
|
||||
|
||||
it.each(["music", "sfx", "voice"])("passes --layer %s to the read", async (layer) => {
|
||||
await runCommand(audioCommand, { rawArgs: ["a.wav", "--read", "--layer", layer, "--json"] });
|
||||
expect(readAudio.mock.calls[0]![2]).toMatchObject({ layer });
|
||||
});
|
||||
|
||||
it("passes no layer when the flag is absent", async () => {
|
||||
await runCommand(audioCommand, { rawArgs: ["a.wav", "--read", "--json"] });
|
||||
expect(readAudio.mock.calls[0]![2].layer).toBeUndefined();
|
||||
});
|
||||
|
||||
it("rejects an unknown layer and names the valid ones", async () => {
|
||||
await expect(
|
||||
runCommand(audioCommand, { rawArgs: ["a.wav", "--read", "--layer", "drums"] }),
|
||||
).rejects.toThrow("--layer is music, sfx, voice; got drums");
|
||||
expect(readAudio).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it("rejects --layer without --read", async () => {
|
||||
await expect(
|
||||
runCommand(audioCommand, { rawArgs: ["a.wav", "--layer", "music"] }),
|
||||
).rejects.toThrow("--layer only works with --read");
|
||||
});
|
||||
});
|
||||
@@ -12,6 +12,7 @@ import { analyzeTargets, previewTarget } from "../audio/analysis/run.js";
|
||||
import { filmTargets, type AudioTarget, type FilmQuery } from "../audio/analysis/targets.js";
|
||||
import { formatRead } from "../audio/read/format.js";
|
||||
import { readAudio } from "../audio/read/read.js";
|
||||
import { LAYERS, type Layer } from "../audio/read/types.js";
|
||||
import { describeProject } from "../timeline/describeProject.js";
|
||||
import { CliUsageError } from "../utils/commandResult.js";
|
||||
import { ensureDOMParser } from "../utils/dom.js";
|
||||
@@ -122,6 +123,7 @@ interface AudioFlags {
|
||||
"silence-db"?: string;
|
||||
read?: boolean;
|
||||
transcript?: string;
|
||||
layer?: string;
|
||||
json?: boolean;
|
||||
}
|
||||
|
||||
@@ -181,13 +183,23 @@ function readQuality(files: string[], flags: AudioFlags): AnalysisQuality {
|
||||
return parseQuality(flags);
|
||||
}
|
||||
|
||||
function parseLayer(flags: AudioFlags): Layer | undefined {
|
||||
if (flags.layer === undefined) return undefined;
|
||||
if (!flags.read) throw new CliUsageError("--layer only works with --read");
|
||||
if (!LAYERS.includes(flags.layer as Layer))
|
||||
throw new CliUsageError(`--layer is ${LAYERS.join(", ")}; got ${flags.layer}`);
|
||||
return flags.layer as Layer;
|
||||
}
|
||||
|
||||
async function read(files: string[], flags: AudioFlags) {
|
||||
const quality = readQuality(files, flags);
|
||||
const range = pair("range", flags.range);
|
||||
const layer = parseLayer(flags);
|
||||
const result = await readAudio(resolve(process.cwd(), files[0]!), files[0]!, {
|
||||
cacheDir: join(resolve(flags.dir ?? process.cwd()), ".hyperframes", "read-audio"),
|
||||
transcript: flags.transcript ? resolve(process.cwd(), flags.transcript) : undefined,
|
||||
quality,
|
||||
layer,
|
||||
});
|
||||
if (flags.json) {
|
||||
console.log(JSON.stringify(withMeta(result)));
|
||||
@@ -259,6 +271,11 @@ export default defineCommand({
|
||||
type: "string",
|
||||
description: "With --read: the recording's transcript (default <file>.transcript.json)",
|
||||
},
|
||||
layer: {
|
||||
type: "string",
|
||||
description:
|
||||
"With --read: the file holds only this layer (music, sfx or voice); it is read whole, with no separation",
|
||||
},
|
||||
json: { type: "boolean", description: "Output as JSON", default: false },
|
||||
},
|
||||
setup({ rawArgs, cmd }) {
|
||||
@@ -268,6 +285,7 @@ export default defineCommand({
|
||||
const flags = args as unknown as AudioFlags;
|
||||
const files = args._.map(String);
|
||||
if (flags.read) return read(files, flags);
|
||||
if (flags.layer !== undefined) throw new CliUsageError("--layer only works with --read");
|
||||
if (flags.transcript) throw new CliUsageError("--transcript only works with --read");
|
||||
const query = filmQuery(flags);
|
||||
if (files.length && query) throw new CliUsageError("give files or a film query, not both");
|
||||
|
||||
Reference in New Issue
Block a user