From 1df31d8717578e31d4c80c194410bbc65783ff6f Mon Sep 17 00:00:00 2001 From: Xuanru Li <157947275+xuanruli@users.noreply.github.com> Date: Wed, 28 Oct 2026 12:48:08 +0000 Subject: [PATCH] fix(audio): a sound is voice only with a voice-range pitch, judged on the whole clip (#6653) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * fix(audio): a sound is voice only with a voice-range pitch, judged on the whole clip Two ways the kind guess called a non-voice sound voice: - A short hit, click or whoosh can have a speech-shaped envelope. classify() now also measures the share of 40 ms frames with a 70-500 Hz fundamental (YIN, moved from the CLI into core) and calls a sound voice only when that share reaches 0.15. Speech measured 0.22-0.43; impacts, clicks and whooshes 0. - A range query judged a clip's kind on the slice it covered, so 1.7 s of a music bed read as voice. A film clip's kind is now judged on all of the clip that plays. Even-paced speech (a 64 s TTS take) moves less at syllable rate and was called music; with no beat and a voiced share of 0.25 or more it is now voice. * chore(audio): keep each touched file's comment share at or under its base The Comments check failed on run.ts, targets.ts and audioAnalysis.ts because the PR added comment lines to files whose share may only go down. Drops the added doc comments; no code change. — Jerrai * fix(audio): preserve clip evidence and stereo pitch during analysis * fix(audio): bound and cache whole-clip kind analysis * fix(audio): retain native-rate kind evidence when available --------- Co-authored-by: Jerrai --- docs/guides/voice-and-audio.mdx | 11 ++ .../cli/src/audio/analysis/kindCache.test.ts | 44 ++++++ packages/cli/src/audio/analysis/kindCache.ts | 112 ++++++++++++++ packages/cli/src/audio/analysis/run.test.ts | 65 ++++++++ packages/cli/src/audio/analysis/run.ts | 109 +++++++++++++- .../cli/src/audio/analysis/runKind.test.ts | 140 ++++++++++++++++++ .../cli/src/audio/analysis/targets.test.ts | 1 + packages/cli/src/audio/analysis/targets.ts | 5 + packages/cli/src/audio/read/pitch.ts | 61 +------- packages/cli/src/audio/read/signal.ts | 4 +- packages/cli/src/commands/audio.ts | 7 +- packages/core/src/audioAnalysis.test.ts | 48 +++++- packages/core/src/audioAnalysis.ts | 16 +- packages/core/src/audioPitch.test.ts | 39 +++++ packages/core/src/audioPitch.ts | 95 ++++++++++++ 15 files changed, 685 insertions(+), 72 deletions(-) create mode 100644 packages/cli/src/audio/analysis/kindCache.test.ts create mode 100644 packages/cli/src/audio/analysis/kindCache.ts create mode 100644 packages/cli/src/audio/analysis/runKind.test.ts create mode 100644 packages/core/src/audioPitch.test.ts create mode 100644 packages/core/src/audioPitch.ts diff --git a/docs/guides/voice-and-audio.mdx b/docs/guides/voice-and-audio.mdx index 9b79364bb7..594b686720 100644 --- a/docs/guides/voice-and-audio.mdx +++ b/docs/guides/voice-and-audio.mdx @@ -85,6 +85,17 @@ Watch once with sound and once muted, then listen once without watching. - Nothing cuts off a word or a breath at a scene change. - The music ends on purpose, not at the end of the file. +The audio analysis gives each sound a first guess of `voice`, `music` or `sfx`. +Voice needs both a speech-like level pattern and voice-range pitch, so a hit +with a speech-like tail is less likely to be mistaken for narration. These are +guesses, not transcript labels. When you inspect a time range, kind uses the +whole source span played by the clip; loudness, hits and the other measurements +still describe the selected range. A short slice of a music bed therefore keeps +the bed’s kind. Repeated range queries reuse that guess until the source or clip +changes. If the whole clip cannot be analyzed, the result warns that kind uses +only the selected range; the range’s other measurements are still returned. +See [audio analysis](/packages/cli#audio) for the command and output. + To dip the music only where the voice sits instead of turning it all down, use a [voiceover carve](/studio/audio-effects#make-music-sit-under-narration). diff --git a/packages/cli/src/audio/analysis/kindCache.test.ts b/packages/cli/src/audio/analysis/kindCache.test.ts new file mode 100644 index 0000000000..e46d71fc0f --- /dev/null +++ b/packages/cli/src/audio/analysis/kindCache.test.ts @@ -0,0 +1,44 @@ +import { mkdtempSync, readdirSync, rmSync, writeFileSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { afterAll, describe, expect, it } from "vitest"; +import { readKindCache, writeKindCache } from "./kindCache.js"; + +const dir = mkdtempSync(join(tmpdir(), "hf-kind-cache-")); +afterAll(() => rmSync(dir, { recursive: true, force: true })); +const kind = { + value: "music" as const, + confidence: 0.8, + features: { duration: 20, regularity: 0.3, syllables: 0.1, sustain: 1 }, +}; + +describe("kind cache", () => { + it("reconstructs successful signal evidence without retaining temporary files", () => { + const file = join(dir, "good.json"); + expect(writeKindCache(file, kind)).toBeNull(); + expect(readKindCache(file)).toEqual({ + analyzed: 20, + values: { kind }, + provenance: { kind: "signal" }, + ladder: { kind: [{ provider: "signal", outcome: "answered" }] }, + warnings: [], + }); + expect(readdirSync(dir).some((name) => name.endsWith(".tmp"))).toBe(false); + }); + + it.each(["{", "null", "{}", '{"value": "voice", "confidence": 1, "features": {}}'])( + "ignores incomplete or corrupt cached JSON %s", + (json) => { + const file = join(dir, "bad.json"); + writeFileSync(file, json); + expect(readKindCache(file)).toBeNull(); + }, + ); + + it("treats absent entries as misses and reports unwritable destinations without throwing", () => { + expect(readKindCache(join(dir, "absent.json"))).toBeNull(); + const blocked = join(dir, "not-a-directory"); + writeFileSync(blocked, "file"); + expect(writeKindCache(join(blocked, "entry.json"), kind)).toMatch(/could not be cached/); + }); +}); diff --git a/packages/cli/src/audio/analysis/kindCache.ts b/packages/cli/src/audio/analysis/kindCache.ts new file mode 100644 index 0000000000..434992a0d6 --- /dev/null +++ b/packages/cli/src/audio/analysis/kindCache.ts @@ -0,0 +1,112 @@ +import { createHash, randomUUID } from "node:crypto"; +import { mkdirSync, readFileSync, renameSync, rmSync, statSync, writeFileSync } from "node:fs"; +import { dirname, join } from "node:path"; +import type { KindFeatures, KindGuess } from "@hyperframes/core/audio-analysis"; +import type { AnalyzeOptions, Analysis } from "./analyze.js"; +import type { AudioTarget } from "./targets.js"; + +const CACHE_VERSION = 1; + +const record = (value: unknown): value is Record => + value !== null && typeof value === "object" && !Array.isArray(value); +const finite = (value: unknown): value is number => + typeof value === "number" && Number.isFinite(value); + +const probability = (value: unknown): value is number => finite(value) && value >= 0 && value <= 1; + +function isKindFeatures(value: unknown): value is KindFeatures { + return ( + record(value) && + finite(value.duration) && + value.duration >= 0 && + [value.regularity, value.syllables, value.sustain].every(probability) && + (value.voiced === undefined || probability(value.voiced)) + ); +} + +function isKindGuess(value: unknown): value is KindGuess { + return ( + record(value) && + typeof value.value === "string" && + ["sfx", "music", "voice", "mixed"].includes(value.value) && + probability(value.confidence) && + isKindFeatures(value.features) + ); +} + +export interface KindCacheSource { + dir: string; + identity: string; +} + +export function kindSourceIdentity(path: string): string { + const { size, mtimeMs, ctimeMs, ino } = statSync(path); + return JSON.stringify({ size, mtimeMs, ctimeMs, ino }); +} + +export function kindCacheFile( + dir: string, + target: AudioTarget, + request: AnalyzeOptions, + sampleRate: number, + source = kindSourceIdentity(target.path), +): string { + const key = JSON.stringify({ + version: CACHE_VERSION, + path: target.path, + source, + clip: target.clip, + sampleRate, + quality: request.quality, + provider: request.provider?.kind, + options: request.options?.kind, + }); + return join(dir, `${createHash("sha256").update(key).digest("hex")}.json`); +} + +export function readKindCache(file: string | null): Analysis | null { + if (file === null) return null; + let value: unknown; + try { + value = JSON.parse(readFileSync(file, "utf8")); + } catch { + return null; + } + if (!isKindGuess(value)) return null; + return { + analyzed: value.features.duration, + values: { kind: value }, + provenance: { kind: "signal" }, + ladder: { kind: [{ provider: "signal", outcome: "answered" }] }, + warnings: [], + }; +} + +export function writeKindCache(file: string, value: KindGuess): string | null { + const temporary = `${file}.${randomUUID()}.tmp`; + try { + mkdirSync(dirname(file), { recursive: true }); + writeFileSync(temporary, JSON.stringify(value)); + renameSync(temporary, file); + return null; + } catch { + return "kind could not be cached; another query will analyze the clip again"; + } finally { + try { + rmSync(temporary, { force: true }); + } catch { + // A failed cache cleanup must not discard measured audio. + } + } +} + +export function cacheKindAnalysis(file: string, analysis: Analysis): void { + if ( + analysis.provenance.kind !== "signal" || + analysis.warnings.length || + !isKindGuess(analysis.values.kind) + ) + return; + const warning = writeKindCache(file, analysis.values.kind); + if (warning) analysis.warnings.push(warning); +} diff --git a/packages/cli/src/audio/analysis/run.test.ts b/packages/cli/src/audio/analysis/run.test.ts index a5efe8c05e..91a72210f9 100644 --- a/packages/cli/src/audio/analysis/run.test.ts +++ b/packages/cli/src/audio/analysis/run.test.ts @@ -157,6 +157,71 @@ describe.skipIf(!ffmpeg)("analyzeTargets", () => { expect(hits.filter((t) => Math.abs(t - 2) >= 0.01)).toEqual([]); }); + it("judges a clip's kind on all it plays, not on the slice a range query covers", async () => { + const bed = (t: number) => + 0.3 * Math.sin(2 * Math.PI * 220 * t) + 0.6 * hit(0)(t % 0.5) + 0.05 * noise(); + const path = wav("music-bed.wav", 12, bed); + const target = (clip?: { from: number; seconds: number }) => ({ + path, + file: "music-bed.wav", + ref: "#music", + kind: "audio" as const, + placement: null, + window: { from: 3, seconds: 1.7 }, + ...(clip && { clip }), + }); + const slice = await analyzeTargets([target()], { capabilities: ["kind"] }); + expect((slice.items[0]!.kind as { value: string }).value).not.toBe("music"); + const report = await analyzeTargets( + [target({ from: 0, seconds: 12 })], + { + capabilities: ["kind", "moments"], + }, + true, + ); + expect(report.items[0]!.kind).toMatchObject({ features: { duration: 12 } }); + expect(report.items[0]!.ladder?.kind).toEqual([{ provider: "signal", outcome: "answered" }]); + const unavailable = await analyzeTargets( + [target({ from: 0, seconds: 12 })], + { + capabilities: ["kind"], + provider: { kind: "missing" }, + }, + true, + ); + expect(unavailable.items[0]!.warnings.length).toBeGreaterThan(0); + expect(unavailable.items[0]!.ladder?.kind?.length).toBeGreaterThan(0); + expect((report.items[0]!.kind as { value: string }).value).toBe("music"); + expect(report.items[0]!.window).toEqual({ from: 3, to: 4.7 }); + const windowOnly = await analyzeTargets([target()], { capabilities: ["moments"] }); + expect(report.items[0]!.moments).toEqual(windowOnly.items[0]!.moments); + }); + + it.each([ + { from: 1, seconds: 8, duration: 8 }, + { from: 11, seconds: 3, duration: 1 }, + ])( + "keeps kind inside the played source span $from + $seconds, bounded by EOF", + async ({ from, seconds, duration }) => { + const path = wav("trimmed.wav", 12, (t) => 0.3 * Math.sin(2 * Math.PI * 200 * t)); + const target = { + path, + file: "trimmed.wav", + ref: "#trimmed", + kind: "audio" as const, + placement: null, + clip: { from, seconds }, + window: { from: from + 0.2, seconds: 0.3 }, + }; + const [whole, slice] = await Promise.all([ + analyzeTargets([{ ...target, window: target.clip }], { capabilities: ["kind"] }), + analyzeTargets([target], { capabilities: ["kind"] }), + ]); + expect(slice.items[0]!.kind).toMatchObject({ features: { duration } }); + expect(slice.items[0]!.kind).toEqual(whole.items[0]!.kind); + }, + ); + it("reports a file it cannot read on that item and goes on to the next", async () => { const broken = join(dir, "notes.txt"); writeFileSync(broken, "not audio"); diff --git a/packages/cli/src/audio/analysis/run.ts b/packages/cli/src/audio/analysis/run.ts index 2cadc018b6..176c00dd68 100644 --- a/packages/cli/src/audio/analysis/run.ts +++ b/packages/cli/src/audio/analysis/run.ts @@ -8,6 +8,13 @@ import { import { analyzeSignal, type AnalyzeOptions } from "./analyze.js"; import { decodeAudio, probeAudio, type AudioStreamInfo } from "./decode.js"; import type { AudioTarget, Placement } from "./targets.js"; +import { + cacheKindAnalysis, + kindCacheFile, + kindSourceIdentity, + readKindCache, + type KindCacheSource, +} from "./kindCache.js"; export interface AudioItem { file: string; @@ -37,6 +44,8 @@ const round = (t: number) => Math.round(t * 1000) / 1000; /** Onsets at a window's edges need the audio just past them to tell a hit from the cut. */ const EDGE_CONTEXT_SECONDS = 0.1; +const KIND_SAMPLE_RATE = 16_000; +const KIND_DECODE_BYTES = 256 * 1024 * 1024; /** A file the film does not play yet, placed where the caller would put it. */ export async function previewTarget( @@ -128,20 +137,115 @@ async function skippedPeak(target: AudioTarget, kept: AudioSignal, stream: Audio ]; } +function coversPart({ window, clip }: AudioTarget): boolean { + return ( + !!window && + !!clip && + (window.from > clip.from || window.from + window.seconds < clip.from + clip.seconds) + ); +} + +function clipKindRate(target: AudioTarget, stream: AudioStreamInfo): number { + const seconds = Math.min(target.clip!.seconds, Math.max(0, stream.duration - target.clip!.from)); + const bytes = seconds * stream.sampleRate * stream.channels * 4; + return bytes <= KIND_DECODE_BYTES + ? stream.sampleRate + : Math.min(stream.sampleRate, KIND_SAMPLE_RATE); +} + +async function clipKindSignal( + target: AudioTarget, + stream: AudioStreamInfo, + sampleRate: number, + windowSignal?: AudioSignal, +): Promise { + if (windowSignal) return windowSignal; + const seconds = Math.min(target.clip!.seconds, Math.max(0, stream.duration - target.clip!.from)); + if (seconds * sampleRate * stream.channels * 4 > KIND_DECODE_BYTES) + throw new Error("whole clip exceeds the kind analysis decode limit"); + return (await decodeAudio(target.path, target.clip!, stream, { sampleRate }))?.signal; +} + +async function clipKind( + target: AudioTarget, + request: AnalyzeOptions, + stream: AudioStreamInfo, + cache?: KindCacheSource, + windowSignal?: AudioSignal, +) { + const sampleRate = windowSignal ? stream.sampleRate : clipKindRate(target, stream); + const file = cache ? kindCacheFile(cache.dir, target, request, sampleRate, cache.identity) : null; + const cached = readKindCache(file); + if (cached) return cached; + const signal = await clipKindSignal(target, stream, sampleRate, windowSignal); + if (!signal) return null; + const analysis = await analyzeSignal(signal, { + ...request, + capabilities: ["kind"], + fileOffset: target.clip!.from, + context: windowSignal ? request.context : undefined, + }); + if (file && cache && file === kindCacheFile(cache.dir, target, request, sampleRate)) + cacheKindAnalysis(file, analysis); + return analysis; +} + +/** Analyzes the window, except a kind the window would misjudge, which comes from the whole clip. */ +async function analyzeWindow( + target: AudioTarget, + signal: AudioSignal, + request: AnalyzeOptions, + stream: AudioStreamInfo, + cache?: KindCacheSource, +) { + const kindApart = request.capabilities.includes("kind") && target.clip !== undefined; + if (!kindApart) return analyzeSignal(signal, request); + const analysis = await analyzeSignal(signal, { + ...request, + capabilities: request.capabilities.filter((capability) => capability !== "kind"), + }); + let kind; + try { + kind = await clipKind(target, request, stream, cache, coversPart(target) ? undefined : signal); + if (!kind) throw new Error("no audio stream"); + } catch (error) { + kind = await analyzeSignal(signal, { ...request, capabilities: ["kind"] }); + const reason = error instanceof Error ? error.message.split("\n")[0] : String(error); + kind.warnings.push( + `kind uses only the selected range: could not analyze the whole clip (${reason})`, + ); + } + return { + ...analysis, + values: { ...analysis.values, kind: kind.values.kind }, + provenance: { ...analysis.provenance, ...kind.provenance }, + ladder: { ...analysis.ladder, ...kind.ladder }, + warnings: [...new Set([...analysis.warnings, ...kind.warnings])], + }; +} + async function analyzeTarget( target: AudioTarget, request: AnalyzeOptions, full: boolean, + cacheDir?: string, ): Promise { const item = emptyItem(target); const problem = unreadable(target); if (problem) return { ...item, error: problem }; const stream = await probeAudio(target.path); if (!stream) return { ...item, error: "no audio stream" }; + const cache = cacheDir ? { dir: cacheDir, identity: kindSourceIdentity(target.path) } : undefined; const { from, read, context } = decodeSpan(target, stream); const decoded = await decodeAudio(target.path, read, stream); if (!decoded) return { ...item, error: "no audio stream" }; - const analysis = await analyzeSignal(decoded.signal, { ...request, fileOffset: from, context }); + const analysis = await analyzeWindow( + target, + decoded.signal, + { ...request, fileOffset: from, context }, + stream, + cache, + ); return { ...item, window: { from: round(from), to: round(from + analysis.analyzed) }, @@ -166,11 +270,12 @@ export async function analyzeTargets( targets: readonly AudioTarget[], request: AnalyzeOptions, full = false, + cacheDir?: string, ): Promise { const items: AudioItem[] = []; for (const target of targets) { try { - items.push(await analyzeTarget(target, request, full)); + items.push(await analyzeTarget(target, request, full, cacheDir)); } catch (error) { const reason = error instanceof Error ? error.message.split("\n")[0]! : String(error); items.push({ ...emptyItem(target), error: `could not read: ${reason}` }); diff --git a/packages/cli/src/audio/analysis/runKind.test.ts b/packages/cli/src/audio/analysis/runKind.test.ts new file mode 100644 index 0000000000..a4e6f7d751 --- /dev/null +++ b/packages/cli/src/audio/analysis/runKind.test.ts @@ -0,0 +1,140 @@ +import { existsSync, mkdtempSync, rmSync, writeFileSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { afterAll, beforeEach, describe, expect, it, vi } from "vitest"; +import { decodeAudio, probeAudio } from "./decode.js"; +import { analyzeTargets } from "./run.js"; + +vi.mock("./decode.js", () => ({ decodeAudio: vi.fn(), probeAudio: vi.fn() })); +const dir = mkdtempSync(join(tmpdir(), "hf-kind-run-")); +const path = join(dir, "source.wav"); +const stream = { sampleRate: 48000, channels: 2, duration: 1200, audioStream: 0 }; +const signal = { sampleRate: 8000, channels: [new Float32Array(8000)] }; +const target = { + path, + file: "source.wav", + kind: "audio" as const, + ref: "#music", + placement: null, + window: { from: 30, seconds: 1 }, + clip: { from: 0, seconds: 1200 }, +}; +const request = { capabilities: ["kind", "moments"] as const, quality: "fast" as const }; +afterAll(() => rmSync(dir, { recursive: true, force: true })); +beforeEach(() => { + vi.resetAllMocks(); + writeFileSync(path, "source"); + vi.mocked(probeAudio).mockResolvedValue(stream); + vi.mocked(decodeAudio).mockResolvedValue({ signal, stream }); +}); + +describe("clip kind isolation", () => { + it("decodes long clip kind at a reduced rate without changing the window decode", async () => { + await analyzeTargets([target], request); + expect(decodeAudio).toHaveBeenNthCalledWith( + 1, + path, + { from: 29.9, seconds: expect.closeTo(1.2) }, + stream, + ); + expect(decodeAudio).toHaveBeenNthCalledWith(2, path, target.clip, stream, { + sampleRate: 16000, + }); + }); + + it("reuses the native-rate window signal when it covers the whole clip", async () => { + const whole = { ...target, window: target.clip }; + await analyzeTargets([whole], request); + expect(decodeAudio).toHaveBeenCalledTimes(1); + }); + + it("keeps the native rate for clip evidence that fits the budget", async () => { + const short = { ...target, clip: { from: 0, seconds: 60 } }; + await analyzeTargets([short], request); + expect(decodeAudio).toHaveBeenNthCalledWith(2, path, short.clip, stream, { sampleRate: 48000 }); + }); + + it("does not attempt an oversized whole-clip decode", async () => { + vi.mocked(probeAudio).mockResolvedValue({ ...stream, duration: 86400 }); + const oversized = { ...target, clip: { from: 0, seconds: 86400 } }; + const [item] = (await analyzeTargets([oversized], request)).items; + expect(decodeAudio).toHaveBeenCalledTimes(1); + expect(item?.error).toBeUndefined(); + expect(item?.moments).toBeNull(); + expect(item?.warnings[0]).toMatch(/kind analysis decode limit/); + }); + + it("keeps the window measurements and warns when full-clip decoding exceeds its limit", async () => { + vi.mocked(decodeAudio) + .mockResolvedValueOnce({ signal, stream }) + .mockRejectedValueOnce(new Error("stdout maxBuffer exceeded")); + const [item] = (await analyzeTargets([target], request, true)).items; + expect(item?.error).toBeUndefined(); + expect(item?.moments).toBeNull(); + expect(item?.kind).toMatchObject({ value: "sfx" }); + expect(item?.ladder?.kind).toEqual([{ provider: "signal", outcome: "answered" }]); + expect(item?.warnings).toEqual([ + "kind uses only the selected range: could not analyze the whole clip (stdout maxBuffer exceeded)", + ]); + }); + + it("reuses kind across calls and ranges, but invalidates media, trim and provider changes", async () => { + const cache = join(dir, "cache"); + const first = await analyzeTargets([target], request, true, cache); + const next = await analyzeTargets( + [{ ...target, window: { from: 40, seconds: 1 } }], + request, + true, + cache, + ); + expect(decodeAudio).toHaveBeenCalledTimes(3); + expect(next.items[0]?.kind).toEqual(first.items[0]?.kind); + expect(next.items[0]?.ladder).toEqual(first.items[0]?.ladder); + writeFileSync(path, "changed source"); + await analyzeTargets([target], request, false, cache); + expect(decodeAudio).toHaveBeenCalledTimes(5); + await analyzeTargets([{ ...target, clip: { from: 1, seconds: 1199 } }], request, false, cache); + expect(decodeAudio).toHaveBeenCalledTimes(7); + const pinned = await analyzeTargets( + [target], + { ...request, provider: { kind: "missing" } }, + true, + cache, + ); + expect(pinned.items[0]?.kind).toBeNull(); + expect(pinned.items[0]?.warnings.length).toBeGreaterThan(0); + }); + + it("does not cache a reused full-window signal under a source changed during decode", async () => { + const cache = join(dir, "changed-full-decode"); + vi.mocked(decodeAudio).mockImplementationOnce(async () => { + writeFileSync(path, "source replaced while decoding full window"); + return { signal, stream }; + }); + await analyzeTargets([{ ...target, window: target.clip }], request, false, cache); + expect(existsSync(cache)).toBe(false); + }); + + it("does not cache evidence if the source changes during its decode", async () => { + const cache = join(dir, "changed-during-decode"); + vi.mocked(decodeAudio) + .mockResolvedValueOnce({ signal, stream }) + .mockImplementationOnce(async () => { + writeFileSync(path, "new source while decoding"); + return { signal, stream }; + }); + await analyzeTargets([target], request, false, cache); + await analyzeTargets([target], request, false, cache); + expect(decodeAudio).toHaveBeenCalledTimes(4); + }); + + it("does not cache a failed full-clip attempt as the whole clip's kind", async () => { + const cache = join(dir, "retry-cache"); + vi.mocked(decodeAudio) + .mockResolvedValueOnce({ signal, stream }) + .mockRejectedValueOnce(new Error("decode failed")); + await analyzeTargets([target], request, false, cache); + await analyzeTargets([target], request, false, cache); + expect(decodeAudio).toHaveBeenCalledTimes(4); + }); +}); diff --git a/packages/cli/src/audio/analysis/targets.test.ts b/packages/cli/src/audio/analysis/targets.test.ts index 4884480067..3363de566f 100644 --- a/packages/cli/src/audio/analysis/targets.test.ts +++ b/packages/cli/src/audio/analysis/targets.test.ts @@ -114,6 +114,7 @@ describe("filmTargets", () => { (t) => t.ref === "#fast", ); expect(fast!.window).toEqual({ from: 3, seconds: 2 }); + expect(fast!.clip).toEqual({ from: 1, seconds: 4 }); }); it("resolves a source against the file that declares the clip, and keeps its local data-start", () => { diff --git a/packages/cli/src/audio/analysis/targets.ts b/packages/cli/src/audio/analysis/targets.ts index 3287fd2a35..11730c0687 100644 --- a/packages/cli/src/audio/analysis/targets.ts +++ b/packages/cli/src/audio/analysis/targets.ts @@ -23,6 +23,7 @@ export interface AudioTarget { placement: Placement | null; /** The part of the file to analyze, in file seconds; the whole file when absent. */ window?: { from: number; seconds: number }; + clip?: { from: number; seconds: number }; } export type FilmQuery = @@ -87,6 +88,10 @@ function targetOf(projectDir: string, row: TimelineRow, query: FilmQuery): Audio from: fileTime(placement, from), seconds: round((to - from) * placement.playbackRate), }, + clip: { + from: placement.mediaStart, + seconds: round((placement.end - placement.start) * placement.playbackRate), + }, }; } diff --git a/packages/cli/src/audio/read/pitch.ts b/packages/cli/src/audio/read/pitch.ts index ba4facec45..818485490f 100644 --- a/packages/cli/src/audio/read/pitch.ts +++ b/packages/cli/src/audio/read/pitch.ts @@ -1,62 +1,5 @@ -import { PITCH_RATE } from "./signal.js"; - -const FRAME = 320; -const HOP = 320; -const LOW_HZ = 70; -const HIGH_HZ = 500; -const VOICED_DB = -35; -const PERIODIC = 0.15; - -export interface PitchPoint { - t: number; - hz: number; -} - -const MIN_LAG = Math.floor(PITCH_RATE / HIGH_HZ); -const MAX_LAG = Math.ceil(PITCH_RATE / LOW_HZ); - -function voiced(pcm: Float32Array, start: number): boolean { - let energy = 0; - for (let i = 0; i < FRAME; i++) energy += pcm[start + i]! ** 2; - return 10 * Math.log10(energy / FRAME + 1e-12) >= VOICED_DB; -} - -function difference(pcm: Float32Array, start: number, tau: number): number { - let sum = 0; - for (let i = 0; i < FRAME; i++) { - const delta = pcm[start + i]! - pcm[start + i + tau]!; - sum += delta * delta; - } - return sum; -} - -/** The first lag whose normalized difference dips under the threshold, followed down to its minimum; -1 if none. */ -function periodOf(pcm: Float32Array, start: number, d: Float64Array): number { - let running = 0; - let lag = -1; - for (let tau = 1; tau <= MAX_LAG; tau++) { - const sum = difference(pcm, start, tau); - running += sum; - d[tau] = running > 0 ? (sum * tau) / running : 1; - if (lag === -1) { - if (tau >= MIN_LAG && d[tau]! < PERIODIC) lag = tau; - } else if (d[tau]! > d[tau - 1]!) break; - else if (d[tau]! < d[lag]!) lag = tau; - } - return lag; -} - -/** The fundamental of each voiced 40 ms frame of 8 kHz mono, by YIN's cumulative mean normalized difference. */ -export function pitchTrack(pcm: Float32Array): PitchPoint[] { - const out: PitchPoint[] = []; - const d = new Float64Array(MAX_LAG + 1); - for (let start = 0; start + FRAME + MAX_LAG <= pcm.length; start += HOP) { - if (!voiced(pcm, start)) continue; - const lag = periodOf(pcm, start, d); - if (lag > 0) out.push({ t: (start + FRAME / 2) / PITCH_RATE, hz: PITCH_RATE / lag }); - } - return out; -} +export { pitchTrack, type PitchPoint } from "@hyperframes/core/audio-analysis"; +import type { PitchPoint } from "@hyperframes/core/audio-analysis"; export function within(track: readonly PitchPoint[], start: number, end: number): number[] { let lo = 0; diff --git a/packages/cli/src/audio/read/signal.ts b/packages/cli/src/audio/read/signal.ts index c65bee988e..19d79be78f 100644 --- a/packages/cli/src/audio/read/signal.ts +++ b/packages/cli/src/audio/read/signal.ts @@ -1,9 +1,9 @@ -import type { AudioSignal } from "@hyperframes/core/audio-analysis"; +import { PITCH_RATE, type AudioSignal } from "@hyperframes/core/audio-analysis"; import { decodeAudio } from "../analysis/decode.js"; /** Every layer is read from mono at this rate: enough for onsets and band shapes, small enough for a long episode. */ export const READ_RATE = 22050; -export const PITCH_RATE = 8000; +export { PITCH_RATE }; const SILENT_DB = -120; export const ABSENT_DB = -55; diff --git a/packages/cli/src/commands/audio.ts b/packages/cli/src/commands/audio.ts index 11cd124adf..a3760a53c1 100644 --- a/packages/cli/src/commands/audio.ts +++ b/packages/cli/src/commands/audio.ts @@ -274,7 +274,12 @@ export default defineCommand({ const targets = files.length ? await fileTargets(files, flags) : await clipTargets(flags, query); - const report = await analyzeTargets(targets, analyzeRequest(flags), Boolean(args.full)); + const report = await analyzeTargets( + targets, + analyzeRequest(flags), + Boolean(args.full), + join(resolve(flags.dir ?? process.cwd()), ".hyperframes", "audio-kind"), + ); console.log(args.json ? JSON.stringify(withMeta(report)) : formatReport(report)); }, }); diff --git a/packages/core/src/audioAnalysis.test.ts b/packages/core/src/audioAnalysis.test.ts index c366e91725..c6e58970b5 100644 --- a/packages/core/src/audioAnalysis.test.ts +++ b/packages/core/src/audioAnalysis.test.ts @@ -172,12 +172,19 @@ describe("classify", () => { const gaps = noise(7); const syllables: number[] = []; for (let at = 0.1; at < 10; at += 0.12 + 0.25 * Math.abs(gaps())) syllables.push(at); - /** Noise in irregular syllable-length bursts over a quiet room tone. */ - const speechLike = (t: number, random: () => number) => { - const into = syllables.map((at) => t - at).find((d) => d >= 0 && d < 0.15); - const voiced = into === undefined ? 0 : 0.5 * Math.sin((Math.PI * into) / 0.15); - return (voiced + 0.02) * random(); + const syllableAt = (t: number) => syllables.findIndex((at) => t - at >= 0 && t - at < 0.15); + const burst = (t: number) => { + const n = syllableAt(t); + return n < 0 ? 0 : 0.5 * Math.sin((Math.PI * (t - syllables[n]!)) / 0.15); }; + /** A voice whose pitch moves from syllable to syllable, with breath noise, in irregular bursts over room tone. */ + const speechLike = (t: number, random: () => number) => { + const hz = 110 + 13 * (syllableAt(t) % 7); + const fold = Math.sin(2 * Math.PI * hz * t) + 0.5 * Math.sin(4 * Math.PI * hz * t); + return burst(t) * (fold + 0.3 * random()) + 0.02 * random(); + }; + /** The same envelope as noise: what a hit with a reverb tail looks like to the envelope alone. */ + const unpitched = (t: number, random: () => number) => (burst(t) + 0.02) * random(); it("calls a short sound an effect", () => { expect(classify(signal(1, impact(0.1, 0.01))).value).toBe("sfx"); @@ -198,12 +205,43 @@ describe("classify", () => { expect(classify(signal(3, speechLike)).value).toBe("voice"); }); + it("still calls speech voice when the stereo channels have opposite polarity", () => { + const speech = signal(3, speechLike); + const expected = classify(speech); + speech.channels.push(Float32Array.from(speech.channels[0]!, (sample) => -sample)); + expect(classify(speech)).toEqual(expected); + }); + + it("does not call speech-shaped noise voice: without a voice-range pitch it is an effect or a bed", () => { + const short = classify(signal(3, unpitched)); + expect(short.value).toBe("sfx"); + expect(short.features.voiced).toBe(0); + expect(classify(signal(10, unpitched)).value).not.toBe("voice"); + }); + + it("calls even-paced speech without a beat voice, and a held pitched pad still music", () => { + /** Voice that never stops: the syllables only swell a carried pitch, as a presenter or TTS reads. */ + const steady = (t: number, random: () => number) => { + const hz = 110 + 13 * (Math.max(0, syllableAt(t)) % 7); + const phrase = (t + 0.4 * Math.sin(t)) % 2.9 < 2.3 ? 1 : 0.03; + return phrase * (0.25 + 0.6 * burst(t)) * (Math.sin(2 * Math.PI * hz * t) + 0.1 * random()); + }; + const pad = (t: number, random: () => number) => + 0.3 * (Math.sin(2 * Math.PI * 220 * t) + 0.5 * Math.sin(2 * Math.PI * 330 * t)) + + 0.01 * random(); + const speech = classify(signal(10, steady)); + expect(speech.features.syllables).toBeLessThan(0.35); + expect(speech.value).toBe("voice"); + expect(classify(signal(10, pad)).value).toBe("music"); + }); + it("reports what the guess was made from", () => { expect(Object.keys(classify(signal(3, speechLike)).features)).toEqual([ "duration", "regularity", "syllables", "sustain", + "voiced", ]); }); }); diff --git a/packages/core/src/audioAnalysis.ts b/packages/core/src/audioAnalysis.ts index fcbf6095de..6506d6769a 100644 --- a/packages/core/src/audioAnalysis.ts +++ b/packages/core/src/audioAnalysis.ts @@ -6,6 +6,7 @@ import type { AudioAnalysisProvider, AudioCapability } from "./audioAnalysisLadder"; import { finish, type Steps } from "./audioSteps"; import { fft } from "./fft"; +import { voicedShare } from "./audioPitch"; import { rhythm } from "./audioRhythm"; import { sections, type SectionOptions } from "./audioSections"; @@ -116,6 +117,7 @@ export interface KindFeatures { syllables: number; /** Share of 50 ms frames within 20 dB of the loud end (95th percentile), 0–1: high for beds and speech, low for hits. */ sustain: number; + voiced?: number; } const DB_FLOOR = -120; @@ -588,26 +590,34 @@ export function kindFeatures(signal: AudioSignal): KindFeatures { }; } +const VOICED_SPEECH = 0.15; +const VOICED_STEADY_SPEECH = 0.25; + /** * A first guess at what the audio is. Length decides most of it: beds run long, effects short. Long audio is voice - * when its envelope moves at syllable rate without a beat; short audio is voice when it also holds its level. + * when its envelope moves at syllable rate without a beat; short audio also holds its level. Both need a voice pitch. */ export function classify(signal: AudioSignal): KindGuess { const features = kindFeatures(signal); const { duration, regularity, syllables, sustain } = features; const speechLike = syllables >= 0.35 && regularity < 0.35; + const voiced = () => (features.voiced ??= round(voicedShare(signal.channels, signal.sampleRate))); + const pitched = () => voiced() >= VOICED_SPEECH; if (duration >= 8) { if (sustain < 0.4) return { value: "sfx", confidence: 0.6, features }; - if (speechLike) + if (speechLike && pitched()) return { value: "voice", confidence: round(Math.min(0.9, 0.4 + syllables)), features }; + if (regularity < 0.2 && syllables >= 0.17 && voiced() >= VOICED_STEADY_SPEECH) + return { value: "voice", confidence: 0.6, features }; return { value: "music", confidence: round(Math.min(0.9, 0.5 + regularity)), features }; } - if (duration >= 1.5 && syllables >= 0.17 && regularity < 0.3 && sustain >= 0.7) + if (duration >= 1.5 && syllables >= 0.17 && regularity < 0.3 && sustain >= 0.7 && pitched()) return { value: "voice", confidence: speechLike ? 0.8 : 0.6, features }; return { value: "sfx", confidence: duration <= 2 ? 0.85 : 0.65, features }; } export * from "./audioAnalysisLadder"; +export * from "./audioPitch"; export * from "./audioRhythm"; export * from "./audioSections"; export * from "./audioSnap"; diff --git a/packages/core/src/audioPitch.test.ts b/packages/core/src/audioPitch.test.ts new file mode 100644 index 0000000000..b9e181c827 --- /dev/null +++ b/packages/core/src/audioPitch.test.ts @@ -0,0 +1,39 @@ +import { describe, expect, it } from "vitest"; +import { atPitchRate, pitchTrack, voicedShare, PITCH_RATE } from "./audioPitch"; + +const tone = (rate: number, seconds = 1, gain = 0.5) => + Float32Array.from( + { length: Math.floor(rate * seconds) }, + (_, i) => gain * Math.sin((2 * Math.PI * 200 * i) / rate), + ); + +describe("voice pitch evidence", () => { + it.each([8000, 16000, 22050, 44100, 48000, 96000])( + "keeps pitch and voiced share at %s Hz, in mono and duplicated stereo", + (rate) => { + const channel = tone(rate); + const mono = atPitchRate([channel], rate); + expect(mono).toHaveLength(PITCH_RATE); + expect(atPitchRate([channel, channel], rate)).toEqual(mono); + const track = pitchTrack(mono); + expect(track).toHaveLength(24); + expect(track[0]).toEqual({ t: 0.02, hz: 200 }); + expect(track.every(({ hz }) => hz === 200)).toBe(true); + expect(voicedShare([channel], rate)).toBe(24 / 25); + }, + ); + + it("keeps voiced evidence when stereo channels have opposite polarity", () => { + const channel = tone(48000); + const reversed = Float32Array.from(channel, (value) => -value); + expect(voicedShare([channel, reversed], 48000)).toBe(voicedShare([channel], 48000)); + }); + + it("has no evidence for empty, silent, too-short or below-gate signals", () => { + expect(voicedShare([], 48000)).toBe(0); + expect(voicedShare([new Float32Array(48000)], 48000)).toBe(0); + expect(voicedShare([tone(8000, 0.04)], 8000)).toBe(0); + expect(voicedShare([tone(8000, 1, 0.001)], 8000)).toBe(0); + expect(pitchTrack(tone(8000, 435 / 8000))).toHaveLength(1); + }); +}); diff --git a/packages/core/src/audioPitch.ts b/packages/core/src/audioPitch.ts new file mode 100644 index 0000000000..df26af6765 --- /dev/null +++ b/packages/core/src/audioPitch.ts @@ -0,0 +1,95 @@ +/** The rate pitch is tracked at: 8 kHz keeps the 70–500 Hz search cheap. */ +export const PITCH_RATE = 8000; + +const FRAME = 320; +const HOP = 320; +const LOW_HZ = 70; +const HIGH_HZ = 500; +const VOICED_DB = -35; +const PERIODIC = 0.15; + +export interface PitchPoint { + t: number; + hz: number; +} + +const MIN_LAG = Math.floor(PITCH_RATE / HIGH_HZ); +const MAX_LAG = Math.ceil(PITCH_RATE / LOW_HZ); + +function voiced(channels: readonly Float32Array[], start: number): boolean { + let energy = 0; + for (const pcm of channels) for (let i = 0; i < FRAME; i++) energy += pcm[start + i]! ** 2; + return 10 * Math.log10(energy / (FRAME * channels.length) + 1e-12) >= VOICED_DB; +} + +function difference(channels: readonly Float32Array[], start: number, tau: number): number { + let sum = 0; + for (const pcm of channels) { + for (let i = 0; i < FRAME; i++) { + const delta = pcm[start + i]! - pcm[start + i + tau]!; + sum += delta * delta; + } + } + return sum; +} + +/** The first lag whose normalized difference dips under the threshold, followed down to its minimum; -1 if none. */ +function periodOf(channels: readonly Float32Array[], start: number, d: Float64Array): number { + let running = 0; + let lag = -1; + for (let tau = 1; tau <= MAX_LAG; tau++) { + const sum = difference(channels, start, tau); + running += sum; + d[tau] = running > 0 ? (sum * tau) / running : 1; + if (lag === -1) { + if (tau >= MIN_LAG && d[tau]! < PERIODIC) lag = tau; + } else if (d[tau]! > d[tau - 1]!) break; + else if (d[tau]! < d[lag]!) lag = tau; + } + return lag; +} + +/** The fundamental of each voiced 40 ms frame of 8 kHz mono, by YIN's cumulative mean normalized difference. */ +export function pitchTrack(pcm: Float32Array): PitchPoint[] { + return channelPitchTrack([pcm]); +} + +function channelPitchTrack(channels: readonly Float32Array[]): PitchPoint[] { + const length = channels[0]?.length ?? 0; + const out: PitchPoint[] = []; + const d = new Float64Array(MAX_LAG + 1); + for (let start = 0; start + FRAME + MAX_LAG <= length; start += HOP) { + if (!voiced(channels, start)) continue; + const lag = periodOf(channels, start, d); + if (lag > 0) out.push({ t: (start + FRAME / 2) / PITCH_RATE, hz: PITCH_RATE / lag }); + } + return out; +} + +/** Mono at {@link PITCH_RATE}: channels averaged, a box filter as wide as the step against aliasing, then sampled. */ +export function atPitchRate(channels: readonly Float32Array[], sampleRate: number): Float32Array { + const length = channels[0]?.length ?? 0; + if (!length || !channels.length) return new Float32Array(0); + const step = sampleRate / PITCH_RATE; + const width = Math.max(1, Math.round(step)); + const sums = new Float64Array(length + 1); + for (let i = 0; i < length; i++) { + let mixed = 0; + for (const channel of channels) mixed += channel[i]!; + sums[i + 1] = sums[i]! + mixed / channels.length; + } + const out = new Float32Array(Math.floor(length / step)); + for (let n = 0; n < out.length; n++) { + const from = Math.floor(n * step); + const to = Math.min(length, from + width); + out[n] = (sums[to]! - sums[from]!) / (to - from); + } + return out; +} + +/** The share of 40 ms frames with a fundamental in the voice range; hits, whooshes and noise have none. */ +export function voicedShare(channels: readonly Float32Array[], sampleRate: number): number { + const pitched = channels.map((channel) => atPitchRate([channel], sampleRate)); + const frames = Math.floor((pitched[0]?.length ?? 0) / HOP); + return frames ? channelPitchTrack(pitched).length / frames : 0; +}