diff --git a/docs/packages/cli.mdx b/docs/packages/cli.mdx index 6e2ba82d63..9e64e74786 100644 --- a/docs/packages/cli.mdx +++ b/docs/packages/cli.mdx @@ -885,6 +885,12 @@ Sentences come from a transcript: `--transcript `, or `.transcript.json` beside the recording. Without one, voice lines carry no words. +A file you know holds one layer is read whole with `--layer music|sfx|voice`: +no stems are fetched and nothing leaves the machine, whatever `--quality` says. +`music` reads the file as the music bed (sections, tempo, levels), `voice` reads +words and pauses from it, `sfx` finds its effects. The JSON result records the +declared `layer`; without the flag the read is unchanged. + Effects and music are read from stems. At `--quality best` the recording is separated into vocals, drums, bass, guitar, piano and other by HeyGen's `/v3/audio/stems`, with your HeyGen credential (see diff --git a/packages/cli/src/audio/read/read.test.ts b/packages/cli/src/audio/read/read.test.ts index 69f23568ec..346760e1e4 100644 --- a/packages/cli/src/audio/read/read.test.ts +++ b/packages/cli/src/audio/read/read.test.ts @@ -495,3 +495,60 @@ describe.skipIf(!ffmpeg)("readAudio", () => { expect(analysis.warnings[1]).toContain("no transcript and no stems"); }); }); + +describe.skipIf(!ffmpeg)("readAudio with a declared layer", () => { + const read = (path: string, name: string, cache: string, layer?: "music" | "sfx" | "voice") => { + vi.stubEnv("HEYGEN_API_KEY", "test-key"); + return readAudio(path, name, { cacheDir: join(dir, cache), quality: "best", layer }); + }; + + it("reads a music file whole, without separating it, and keeps its cache apart from a mix read", async () => { + const path = wav("layer-music.wav", 8, bed); + const callsBefore = stemCalls; + const first = await read(path, "layer-music.wav", "cache-layer-music", "music"); + expect(stemCalls).toBe(callsBefore); + expect(first.analysis).toMatchObject({ separation: "mix", layer: "music", voice: null }); + expect(first.analysis.music!.segments.length).toBeGreaterThan(0); + expect(first.analysis.bpm).toBe(first.analysis.music!.segments[0]!.bpm); + expect((await read(path, "layer-music.wav", "cache-layer-music", "music")).cached).toBe(true); + const asMix = await read(path, "layer-music.wav", "cache-layer-music"); + expect(asMix.cached).toBe(false); + expect(asMix.analysis.layer).toBeUndefined(); + }); + + it("reads a voice file whole: words and pauses from the file itself", async () => { + const path = wav("layer-voice.wav", 6, voiced); + writeFileSync( + `${path}.transcript.json`, + JSON.stringify({ + words: [ + { text: "one", start: 0.5, end: 2 }, + { text: "two", start: 3, end: 4.5 }, + ], + }), + ); + const callsBefore = stemCalls; + const { analysis } = await read(path, "layer-voice.wav", "cache-layer-voice", "voice"); + expect(stemCalls).toBe(callsBefore); + expect(analysis).toMatchObject({ separation: "mix", layer: "voice", effects: null }); + expect(analysis.voice!.sentences.map((s) => s.text)).toEqual(["one", "two"]); + expect(analysis.voice!.pauses.length).toBeGreaterThan(0); + }); + + it("reads a sound-effect file whole, with no music read", async () => { + const path = wav("layer-sfx.wav", 5, boom); + const callsBefore = stemCalls; + const { analysis } = await read(path, "layer-sfx.wav", "cache-layer-sfx", "sfx"); + expect(stemCalls).toBe(callsBefore); + expect(analysis).toMatchObject({ separation: "mix", layer: "sfx", voice: null, music: null }); + expect(analysis.effects!.events.length).toBeGreaterThan(0); + }); + + it("records no layer when none is declared", async () => { + const path = wav("layer-none.wav", 3, voiced); + const { analysis } = await readAudio(path, "layer-none.wav", { + cacheDir: join(dir, "cache-layer-none"), + }); + expect("layer" in analysis).toBe(false); + }); +}); diff --git a/packages/cli/src/audio/read/read.ts b/packages/cli/src/audio/read/read.ts index f0a77c7c27..ee37907810 100644 --- a/packages/cli/src/audio/read/read.ts +++ b/packages/cli/src/audio/read/read.ts @@ -28,7 +28,14 @@ import { spanDb, } from "./signal.js"; import { DRUM_STEM, VOICE_STEM, cachedStems, fetchStems, stemsAccess } from "./stems.js"; -import { READ_VERSION, type AudibleStem, type Music, type ReadAudio, type Voice } from "./types.js"; +import { + READ_VERSION, + type AudibleStem, + type Layer, + type Music, + type ReadAudio, + type Voice, +} from "./types.js"; import { voiceFromActivity, voiceFromWords, type TimedWord } from "./voice.js"; export interface ReadOptions { @@ -37,6 +44,7 @@ export interface ReadOptions { /** A transcript of the recording; `.transcript.json` next to it is used when absent. */ transcript?: string; quality?: AnalysisQuality; + layer?: Layer; } export interface ReadResult { @@ -379,6 +387,46 @@ async function analyze( return { ...about, voice, effects, music: null, ambience: null, warnings }; } +async function analyzeLayer( + path: string, + file: string, + sha: string, + transcript: string | null, + layer: Layer, +): Promise { + const samples = await decodeMono(path); + const about = { + ...base(file, sha, round(samples.length / READ_RATE, 3), null, transcript), + layer, + }; + const none = { voice: null, effects: null, music: null, ambience: null, warnings: [] }; + if (layer === "music") { + const { music, ambience } = musicOf({ music: samples }, null, samples); + return { ...about, ...none, music, ambience, bpm: tempoOf(music) }; + } + if (layer === "sfx") { + const effects = findEffects({ + sources: [{ samples }], + sampleRate: READ_RATE, + mix: samples, + music: null, + beats: [], + speech: [], + }); + return { ...about, ...none, effects }; + } + const words = transcript ? wordsOf(transcript) : []; + const pitch = pitchTrack(await decodeMono(path, PITCH_RATE)); + const levelOf = (span: Span) => spanDb(samples, READ_RATE, span.start, span.end); + const voice = words.length + ? voiceFromWords(words, pitch, levelOf) + : voiceFromActivity(samples, READ_RATE, pitch, levelOf); + const warnings = words.length + ? [] + : [`no transcript: voice lines carry no words; ${transcribeHint(file)}`]; + return { ...about, ...none, voice, warnings }; +} + function base( file: string, sha: string, @@ -408,6 +456,7 @@ export async function readAudio( options: ReadOptions, ): Promise { const quality = options.quality ?? "local"; + const { layer } = options; const sha = await sha256(path); const transcript = transcriptFor(path, options.transcript); const transcriptKey = fingerprint(transcript); @@ -419,12 +468,14 @@ export async function readAudio( if (existsSync(jsonPath) && existsSync(textPath)) { const entry = JSON.parse(readFileSync(jsonPath, "utf8")) as CacheEntry; const stemsNowPossible = + !layer && entry.analysis.separation === "mix" && quality === "best" && (cachedStems(stemDir) !== null || typeof (await stemsAccess()) !== "string"); if ( (entry.analysis.separation !== "stems" || cachedStems(stemDir) !== null) && entry.analysis.version === READ_VERSION && + entry.analysis.layer === layer && entry.revision === CACHE_REVISION && entry.transcriptKey === transcriptKey && !stemsNowPossible && @@ -437,7 +488,9 @@ export async function readAudio( ) return { analysis: entry.analysis, textPath, jsonPath, cached: true }; } - const analysis = await analyze(path, file, sha, transcript, stemDir, quality, options.cacheDir); + const analysis = layer + ? await analyzeLayer(path, file, sha, transcript, layer) + : await analyze(path, file, sha, transcript, stemDir, quality, options.cacheDir); writeAtomic( jsonPath, JSON.stringify({ revision: CACHE_REVISION, analysis, transcriptKey } satisfies CacheEntry), diff --git a/packages/cli/src/audio/read/types.ts b/packages/cli/src/audio/read/types.ts index 66ff639bd0..01367d74c5 100644 --- a/packages/cli/src/audio/read/types.ts +++ b/packages/cli/src/audio/read/types.ts @@ -4,6 +4,9 @@ export const READ_VERSION = 7; export type Separation = "stems" | "mix"; +export const LAYERS = ["music", "sfx", "voice"] as const; +export type Layer = (typeof LAYERS)[number]; + export type RateUnit = "chars" | "words"; export type Ending = "rising" | "falling" | "level"; @@ -121,6 +124,7 @@ export interface ReadAudio { sha256: string; duration: number; separation: Separation; + layer?: Layer; stems: AudibleStem[]; stemsDir: string | null; voiceFreeMix: string | null; diff --git a/packages/cli/src/commands/audio.command.test.ts b/packages/cli/src/commands/audio.command.test.ts new file mode 100644 index 0000000000..332337f70a --- /dev/null +++ b/packages/cli/src/commands/audio.command.test.ts @@ -0,0 +1,39 @@ +import { runCommand } from "citty"; +import { beforeEach, describe, expect, it, vi } from "vitest"; + +const { readAudio } = vi.hoisted(() => ({ readAudio: vi.fn() })); + +vi.mock("../audio/read/read.js", () => ({ readAudio })); + +import audioCommand from "./audio.js"; + +describe("audio --read --layer", () => { + beforeEach(() => { + readAudio.mockReset(); + readAudio.mockResolvedValue({ analysis: {}, textPath: "x.txt", jsonPath: "x.json" }); + vi.spyOn(console, "log").mockImplementation(() => {}); + }); + + it.each(["music", "sfx", "voice"])("passes --layer %s to the read", async (layer) => { + await runCommand(audioCommand, { rawArgs: ["a.wav", "--read", "--layer", layer, "--json"] }); + expect(readAudio.mock.calls[0]![2]).toMatchObject({ layer }); + }); + + it("passes no layer when the flag is absent", async () => { + await runCommand(audioCommand, { rawArgs: ["a.wav", "--read", "--json"] }); + expect(readAudio.mock.calls[0]![2].layer).toBeUndefined(); + }); + + it("rejects an unknown layer and names the valid ones", async () => { + await expect( + runCommand(audioCommand, { rawArgs: ["a.wav", "--read", "--layer", "drums"] }), + ).rejects.toThrow("--layer is music, sfx, voice; got drums"); + expect(readAudio).not.toHaveBeenCalled(); + }); + + it("rejects --layer without --read", async () => { + await expect( + runCommand(audioCommand, { rawArgs: ["a.wav", "--layer", "music"] }), + ).rejects.toThrow("--layer only works with --read"); + }); +}); diff --git a/packages/cli/src/commands/audio.ts b/packages/cli/src/commands/audio.ts index a3760a53c1..c9406b77dd 100644 --- a/packages/cli/src/commands/audio.ts +++ b/packages/cli/src/commands/audio.ts @@ -12,6 +12,7 @@ import { analyzeTargets, previewTarget } from "../audio/analysis/run.js"; import { filmTargets, type AudioTarget, type FilmQuery } from "../audio/analysis/targets.js"; import { formatRead } from "../audio/read/format.js"; import { readAudio } from "../audio/read/read.js"; +import { LAYERS, type Layer } from "../audio/read/types.js"; import { describeProject } from "../timeline/describeProject.js"; import { CliUsageError } from "../utils/commandResult.js"; import { ensureDOMParser } from "../utils/dom.js"; @@ -122,6 +123,7 @@ interface AudioFlags { "silence-db"?: string; read?: boolean; transcript?: string; + layer?: string; json?: boolean; } @@ -181,13 +183,23 @@ function readQuality(files: string[], flags: AudioFlags): AnalysisQuality { return parseQuality(flags); } +function parseLayer(flags: AudioFlags): Layer | undefined { + if (flags.layer === undefined) return undefined; + if (!flags.read) throw new CliUsageError("--layer only works with --read"); + if (!LAYERS.includes(flags.layer as Layer)) + throw new CliUsageError(`--layer is ${LAYERS.join(", ")}; got ${flags.layer}`); + return flags.layer as Layer; +} + async function read(files: string[], flags: AudioFlags) { const quality = readQuality(files, flags); const range = pair("range", flags.range); + const layer = parseLayer(flags); const result = await readAudio(resolve(process.cwd(), files[0]!), files[0]!, { cacheDir: join(resolve(flags.dir ?? process.cwd()), ".hyperframes", "read-audio"), transcript: flags.transcript ? resolve(process.cwd(), flags.transcript) : undefined, quality, + layer, }); if (flags.json) { console.log(JSON.stringify(withMeta(result))); @@ -259,6 +271,11 @@ export default defineCommand({ type: "string", description: "With --read: the recording's transcript (default .transcript.json)", }, + layer: { + type: "string", + description: + "With --read: the file holds only this layer (music, sfx or voice); it is read whole, with no separation", + }, json: { type: "boolean", description: "Output as JSON", default: false }, }, setup({ rawArgs, cmd }) { @@ -268,6 +285,7 @@ export default defineCommand({ const flags = args as unknown as AudioFlags; const files = args._.map(String); if (flags.read) return read(files, flags); + if (flags.layer !== undefined) throw new CliUsageError("--layer only works with --read"); if (flags.transcript) throw new CliUsageError("--transcript only works with --read"); const query = filmQuery(flags); if (files.length && query) throw new CliUsageError("give files or a film query, not both");