diff --git a/docs/guides/voice-and-audio.mdx b/docs/guides/voice-and-audio.mdx index 9b79364bb7..594b686720 100644 --- a/docs/guides/voice-and-audio.mdx +++ b/docs/guides/voice-and-audio.mdx @@ -85,6 +85,17 @@ Watch once with sound and once muted, then listen once without watching. - Nothing cuts off a word or a breath at a scene change. - The music ends on purpose, not at the end of the file. +The audio analysis gives each sound a first guess of `voice`, `music` or `sfx`. +Voice needs both a speech-like level pattern and voice-range pitch, so a hit +with a speech-like tail is less likely to be mistaken for narration. These are +guesses, not transcript labels. When you inspect a time range, kind uses the +whole source span played by the clip; loudness, hits and the other measurements +still describe the selected range. A short slice of a music bed therefore keeps +the bed’s kind. Repeated range queries reuse that guess until the source or clip +changes. If the whole clip cannot be analyzed, the result warns that kind uses +only the selected range; the range’s other measurements are still returned. +See [audio analysis](/packages/cli#audio) for the command and output. + To dip the music only where the voice sits instead of turning it all down, use a [voiceover carve](/studio/audio-effects#make-music-sit-under-narration). diff --git a/packages/cli/src/audio/analysis/kindCache.test.ts b/packages/cli/src/audio/analysis/kindCache.test.ts new file mode 100644 index 0000000000..e46d71fc0f --- /dev/null +++ b/packages/cli/src/audio/analysis/kindCache.test.ts @@ -0,0 +1,44 @@ +import { mkdtempSync, readdirSync, rmSync, writeFileSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { afterAll, describe, expect, it } from "vitest"; +import { readKindCache, writeKindCache } from "./kindCache.js"; + +const dir = mkdtempSync(join(tmpdir(), "hf-kind-cache-")); +afterAll(() => rmSync(dir, { recursive: true, force: true })); +const kind = { + value: "music" as const, + confidence: 0.8, + features: { duration: 20, regularity: 0.3, syllables: 0.1, sustain: 1 }, +}; + +describe("kind cache", () => { + it("reconstructs successful signal evidence without retaining temporary files", () => { + const file = join(dir, "good.json"); + expect(writeKindCache(file, kind)).toBeNull(); + expect(readKindCache(file)).toEqual({ + analyzed: 20, + values: { kind }, + provenance: { kind: "signal" }, + ladder: { kind: [{ provider: "signal", outcome: "answered" }] }, + warnings: [], + }); + expect(readdirSync(dir).some((name) => name.endsWith(".tmp"))).toBe(false); + }); + + it.each(["{", "null", "{}", '{"value": "voice", "confidence": 1, "features": {}}'])( + "ignores incomplete or corrupt cached JSON %s", + (json) => { + const file = join(dir, "bad.json"); + writeFileSync(file, json); + expect(readKindCache(file)).toBeNull(); + }, + ); + + it("treats absent entries as misses and reports unwritable destinations without throwing", () => { + expect(readKindCache(join(dir, "absent.json"))).toBeNull(); + const blocked = join(dir, "not-a-directory"); + writeFileSync(blocked, "file"); + expect(writeKindCache(join(blocked, "entry.json"), kind)).toMatch(/could not be cached/); + }); +}); diff --git a/packages/cli/src/audio/analysis/kindCache.ts b/packages/cli/src/audio/analysis/kindCache.ts new file mode 100644 index 0000000000..434992a0d6 --- /dev/null +++ b/packages/cli/src/audio/analysis/kindCache.ts @@ -0,0 +1,112 @@ +import { createHash, randomUUID } from "node:crypto"; +import { mkdirSync, readFileSync, renameSync, rmSync, statSync, writeFileSync } from "node:fs"; +import { dirname, join } from "node:path"; +import type { KindFeatures, KindGuess } from "@hyperframes/core/audio-analysis"; +import type { AnalyzeOptions, Analysis } from "./analyze.js"; +import type { AudioTarget } from "./targets.js"; + +const CACHE_VERSION = 1; + +const record = (value: unknown): value is Record => + value !== null && typeof value === "object" && !Array.isArray(value); +const finite = (value: unknown): value is number => + typeof value === "number" && Number.isFinite(value); + +const probability = (value: unknown): value is number => finite(value) && value >= 0 && value <= 1; + +function isKindFeatures(value: unknown): value is KindFeatures { + return ( + record(value) && + finite(value.duration) && + value.duration >= 0 && + [value.regularity, value.syllables, value.sustain].every(probability) && + (value.voiced === undefined || probability(value.voiced)) + ); +} + +function isKindGuess(value: unknown): value is KindGuess { + return ( + record(value) && + typeof value.value === "string" && + ["sfx", "music", "voice", "mixed"].includes(value.value) && + probability(value.confidence) && + isKindFeatures(value.features) + ); +} + +export interface KindCacheSource { + dir: string; + identity: string; +} + +export function kindSourceIdentity(path: string): string { + const { size, mtimeMs, ctimeMs, ino } = statSync(path); + return JSON.stringify({ size, mtimeMs, ctimeMs, ino }); +} + +export function kindCacheFile( + dir: string, + target: AudioTarget, + request: AnalyzeOptions, + sampleRate: number, + source = kindSourceIdentity(target.path), +): string { + const key = JSON.stringify({ + version: CACHE_VERSION, + path: target.path, + source, + clip: target.clip, + sampleRate, + quality: request.quality, + provider: request.provider?.kind, + options: request.options?.kind, + }); + return join(dir, `${createHash("sha256").update(key).digest("hex")}.json`); +} + +export function readKindCache(file: string | null): Analysis | null { + if (file === null) return null; + let value: unknown; + try { + value = JSON.parse(readFileSync(file, "utf8")); + } catch { + return null; + } + if (!isKindGuess(value)) return null; + return { + analyzed: value.features.duration, + values: { kind: value }, + provenance: { kind: "signal" }, + ladder: { kind: [{ provider: "signal", outcome: "answered" }] }, + warnings: [], + }; +} + +export function writeKindCache(file: string, value: KindGuess): string | null { + const temporary = `${file}.${randomUUID()}.tmp`; + try { + mkdirSync(dirname(file), { recursive: true }); + writeFileSync(temporary, JSON.stringify(value)); + renameSync(temporary, file); + return null; + } catch { + return "kind could not be cached; another query will analyze the clip again"; + } finally { + try { + rmSync(temporary, { force: true }); + } catch { + // A failed cache cleanup must not discard measured audio. + } + } +} + +export function cacheKindAnalysis(file: string, analysis: Analysis): void { + if ( + analysis.provenance.kind !== "signal" || + analysis.warnings.length || + !isKindGuess(analysis.values.kind) + ) + return; + const warning = writeKindCache(file, analysis.values.kind); + if (warning) analysis.warnings.push(warning); +} diff --git a/packages/cli/src/audio/analysis/run.test.ts b/packages/cli/src/audio/analysis/run.test.ts index a5efe8c05e..91a72210f9 100644 --- a/packages/cli/src/audio/analysis/run.test.ts +++ b/packages/cli/src/audio/analysis/run.test.ts @@ -157,6 +157,71 @@ describe.skipIf(!ffmpeg)("analyzeTargets", () => { expect(hits.filter((t) => Math.abs(t - 2) >= 0.01)).toEqual([]); }); + it("judges a clip's kind on all it plays, not on the slice a range query covers", async () => { + const bed = (t: number) => + 0.3 * Math.sin(2 * Math.PI * 220 * t) + 0.6 * hit(0)(t % 0.5) + 0.05 * noise(); + const path = wav("music-bed.wav", 12, bed); + const target = (clip?: { from: number; seconds: number }) => ({ + path, + file: "music-bed.wav", + ref: "#music", + kind: "audio" as const, + placement: null, + window: { from: 3, seconds: 1.7 }, + ...(clip && { clip }), + }); + const slice = await analyzeTargets([target()], { capabilities: ["kind"] }); + expect((slice.items[0]!.kind as { value: string }).value).not.toBe("music"); + const report = await analyzeTargets( + [target({ from: 0, seconds: 12 })], + { + capabilities: ["kind", "moments"], + }, + true, + ); + expect(report.items[0]!.kind).toMatchObject({ features: { duration: 12 } }); + expect(report.items[0]!.ladder?.kind).toEqual([{ provider: "signal", outcome: "answered" }]); + const unavailable = await analyzeTargets( + [target({ from: 0, seconds: 12 })], + { + capabilities: ["kind"], + provider: { kind: "missing" }, + }, + true, + ); + expect(unavailable.items[0]!.warnings.length).toBeGreaterThan(0); + expect(unavailable.items[0]!.ladder?.kind?.length).toBeGreaterThan(0); + expect((report.items[0]!.kind as { value: string }).value).toBe("music"); + expect(report.items[0]!.window).toEqual({ from: 3, to: 4.7 }); + const windowOnly = await analyzeTargets([target()], { capabilities: ["moments"] }); + expect(report.items[0]!.moments).toEqual(windowOnly.items[0]!.moments); + }); + + it.each([ + { from: 1, seconds: 8, duration: 8 }, + { from: 11, seconds: 3, duration: 1 }, + ])( + "keeps kind inside the played source span $from + $seconds, bounded by EOF", + async ({ from, seconds, duration }) => { + const path = wav("trimmed.wav", 12, (t) => 0.3 * Math.sin(2 * Math.PI * 200 * t)); + const target = { + path, + file: "trimmed.wav", + ref: "#trimmed", + kind: "audio" as const, + placement: null, + clip: { from, seconds }, + window: { from: from + 0.2, seconds: 0.3 }, + }; + const [whole, slice] = await Promise.all([ + analyzeTargets([{ ...target, window: target.clip }], { capabilities: ["kind"] }), + analyzeTargets([target], { capabilities: ["kind"] }), + ]); + expect(slice.items[0]!.kind).toMatchObject({ features: { duration } }); + expect(slice.items[0]!.kind).toEqual(whole.items[0]!.kind); + }, + ); + it("reports a file it cannot read on that item and goes on to the next", async () => { const broken = join(dir, "notes.txt"); writeFileSync(broken, "not audio"); diff --git a/packages/cli/src/audio/analysis/run.ts b/packages/cli/src/audio/analysis/run.ts index 2cadc018b6..176c00dd68 100644 --- a/packages/cli/src/audio/analysis/run.ts +++ b/packages/cli/src/audio/analysis/run.ts @@ -8,6 +8,13 @@ import { import { analyzeSignal, type AnalyzeOptions } from "./analyze.js"; import { decodeAudio, probeAudio, type AudioStreamInfo } from "./decode.js"; import type { AudioTarget, Placement } from "./targets.js"; +import { + cacheKindAnalysis, + kindCacheFile, + kindSourceIdentity, + readKindCache, + type KindCacheSource, +} from "./kindCache.js"; export interface AudioItem { file: string; @@ -37,6 +44,8 @@ const round = (t: number) => Math.round(t * 1000) / 1000; /** Onsets at a window's edges need the audio just past them to tell a hit from the cut. */ const EDGE_CONTEXT_SECONDS = 0.1; +const KIND_SAMPLE_RATE = 16_000; +const KIND_DECODE_BYTES = 256 * 1024 * 1024; /** A file the film does not play yet, placed where the caller would put it. */ export async function previewTarget( @@ -128,20 +137,115 @@ async function skippedPeak(target: AudioTarget, kept: AudioSignal, stream: Audio ]; } +function coversPart({ window, clip }: AudioTarget): boolean { + return ( + !!window && + !!clip && + (window.from > clip.from || window.from + window.seconds < clip.from + clip.seconds) + ); +} + +function clipKindRate(target: AudioTarget, stream: AudioStreamInfo): number { + const seconds = Math.min(target.clip!.seconds, Math.max(0, stream.duration - target.clip!.from)); + const bytes = seconds * stream.sampleRate * stream.channels * 4; + return bytes <= KIND_DECODE_BYTES + ? stream.sampleRate + : Math.min(stream.sampleRate, KIND_SAMPLE_RATE); +} + +async function clipKindSignal( + target: AudioTarget, + stream: AudioStreamInfo, + sampleRate: number, + windowSignal?: AudioSignal, +): Promise { + if (windowSignal) return windowSignal; + const seconds = Math.min(target.clip!.seconds, Math.max(0, stream.duration - target.clip!.from)); + if (seconds * sampleRate * stream.channels * 4 > KIND_DECODE_BYTES) + throw new Error("whole clip exceeds the kind analysis decode limit"); + return (await decodeAudio(target.path, target.clip!, stream, { sampleRate }))?.signal; +} + +async function clipKind( + target: AudioTarget, + request: AnalyzeOptions, + stream: AudioStreamInfo, + cache?: KindCacheSource, + windowSignal?: AudioSignal, +) { + const sampleRate = windowSignal ? stream.sampleRate : clipKindRate(target, stream); + const file = cache ? kindCacheFile(cache.dir, target, request, sampleRate, cache.identity) : null; + const cached = readKindCache(file); + if (cached) return cached; + const signal = await clipKindSignal(target, stream, sampleRate, windowSignal); + if (!signal) return null; + const analysis = await analyzeSignal(signal, { + ...request, + capabilities: ["kind"], + fileOffset: target.clip!.from, + context: windowSignal ? request.context : undefined, + }); + if (file && cache && file === kindCacheFile(cache.dir, target, request, sampleRate)) + cacheKindAnalysis(file, analysis); + return analysis; +} + +/** Analyzes the window, except a kind the window would misjudge, which comes from the whole clip. */ +async function analyzeWindow( + target: AudioTarget, + signal: AudioSignal, + request: AnalyzeOptions, + stream: AudioStreamInfo, + cache?: KindCacheSource, +) { + const kindApart = request.capabilities.includes("kind") && target.clip !== undefined; + if (!kindApart) return analyzeSignal(signal, request); + const analysis = await analyzeSignal(signal, { + ...request, + capabilities: request.capabilities.filter((capability) => capability !== "kind"), + }); + let kind; + try { + kind = await clipKind(target, request, stream, cache, coversPart(target) ? undefined : signal); + if (!kind) throw new Error("no audio stream"); + } catch (error) { + kind = await analyzeSignal(signal, { ...request, capabilities: ["kind"] }); + const reason = error instanceof Error ? error.message.split("\n")[0] : String(error); + kind.warnings.push( + `kind uses only the selected range: could not analyze the whole clip (${reason})`, + ); + } + return { + ...analysis, + values: { ...analysis.values, kind: kind.values.kind }, + provenance: { ...analysis.provenance, ...kind.provenance }, + ladder: { ...analysis.ladder, ...kind.ladder }, + warnings: [...new Set([...analysis.warnings, ...kind.warnings])], + }; +} + async function analyzeTarget( target: AudioTarget, request: AnalyzeOptions, full: boolean, + cacheDir?: string, ): Promise { const item = emptyItem(target); const problem = unreadable(target); if (problem) return { ...item, error: problem }; const stream = await probeAudio(target.path); if (!stream) return { ...item, error: "no audio stream" }; + const cache = cacheDir ? { dir: cacheDir, identity: kindSourceIdentity(target.path) } : undefined; const { from, read, context } = decodeSpan(target, stream); const decoded = await decodeAudio(target.path, read, stream); if (!decoded) return { ...item, error: "no audio stream" }; - const analysis = await analyzeSignal(decoded.signal, { ...request, fileOffset: from, context }); + const analysis = await analyzeWindow( + target, + decoded.signal, + { ...request, fileOffset: from, context }, + stream, + cache, + ); return { ...item, window: { from: round(from), to: round(from + analysis.analyzed) }, @@ -166,11 +270,12 @@ export async function analyzeTargets( targets: readonly AudioTarget[], request: AnalyzeOptions, full = false, + cacheDir?: string, ): Promise { const items: AudioItem[] = []; for (const target of targets) { try { - items.push(await analyzeTarget(target, request, full)); + items.push(await analyzeTarget(target, request, full, cacheDir)); } catch (error) { const reason = error instanceof Error ? error.message.split("\n")[0]! : String(error); items.push({ ...emptyItem(target), error: `could not read: ${reason}` }); diff --git a/packages/cli/src/audio/analysis/runKind.test.ts b/packages/cli/src/audio/analysis/runKind.test.ts new file mode 100644 index 0000000000..a4e6f7d751 --- /dev/null +++ b/packages/cli/src/audio/analysis/runKind.test.ts @@ -0,0 +1,140 @@ +import { existsSync, mkdtempSync, rmSync, writeFileSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { afterAll, beforeEach, describe, expect, it, vi } from "vitest"; +import { decodeAudio, probeAudio } from "./decode.js"; +import { analyzeTargets } from "./run.js"; + +vi.mock("./decode.js", () => ({ decodeAudio: vi.fn(), probeAudio: vi.fn() })); +const dir = mkdtempSync(join(tmpdir(), "hf-kind-run-")); +const path = join(dir, "source.wav"); +const stream = { sampleRate: 48000, channels: 2, duration: 1200, audioStream: 0 }; +const signal = { sampleRate: 8000, channels: [new Float32Array(8000)] }; +const target = { + path, + file: "source.wav", + kind: "audio" as const, + ref: "#music", + placement: null, + window: { from: 30, seconds: 1 }, + clip: { from: 0, seconds: 1200 }, +}; +const request = { capabilities: ["kind", "moments"] as const, quality: "fast" as const }; +afterAll(() => rmSync(dir, { recursive: true, force: true })); +beforeEach(() => { + vi.resetAllMocks(); + writeFileSync(path, "source"); + vi.mocked(probeAudio).mockResolvedValue(stream); + vi.mocked(decodeAudio).mockResolvedValue({ signal, stream }); +}); + +describe("clip kind isolation", () => { + it("decodes long clip kind at a reduced rate without changing the window decode", async () => { + await analyzeTargets([target], request); + expect(decodeAudio).toHaveBeenNthCalledWith( + 1, + path, + { from: 29.9, seconds: expect.closeTo(1.2) }, + stream, + ); + expect(decodeAudio).toHaveBeenNthCalledWith(2, path, target.clip, stream, { + sampleRate: 16000, + }); + }); + + it("reuses the native-rate window signal when it covers the whole clip", async () => { + const whole = { ...target, window: target.clip }; + await analyzeTargets([whole], request); + expect(decodeAudio).toHaveBeenCalledTimes(1); + }); + + it("keeps the native rate for clip evidence that fits the budget", async () => { + const short = { ...target, clip: { from: 0, seconds: 60 } }; + await analyzeTargets([short], request); + expect(decodeAudio).toHaveBeenNthCalledWith(2, path, short.clip, stream, { sampleRate: 48000 }); + }); + + it("does not attempt an oversized whole-clip decode", async () => { + vi.mocked(probeAudio).mockResolvedValue({ ...stream, duration: 86400 }); + const oversized = { ...target, clip: { from: 0, seconds: 86400 } }; + const [item] = (await analyzeTargets([oversized], request)).items; + expect(decodeAudio).toHaveBeenCalledTimes(1); + expect(item?.error).toBeUndefined(); + expect(item?.moments).toBeNull(); + expect(item?.warnings[0]).toMatch(/kind analysis decode limit/); + }); + + it("keeps the window measurements and warns when full-clip decoding exceeds its limit", async () => { + vi.mocked(decodeAudio) + .mockResolvedValueOnce({ signal, stream }) + .mockRejectedValueOnce(new Error("stdout maxBuffer exceeded")); + const [item] = (await analyzeTargets([target], request, true)).items; + expect(item?.error).toBeUndefined(); + expect(item?.moments).toBeNull(); + expect(item?.kind).toMatchObject({ value: "sfx" }); + expect(item?.ladder?.kind).toEqual([{ provider: "signal", outcome: "answered" }]); + expect(item?.warnings).toEqual([ + "kind uses only the selected range: could not analyze the whole clip (stdout maxBuffer exceeded)", + ]); + }); + + it("reuses kind across calls and ranges, but invalidates media, trim and provider changes", async () => { + const cache = join(dir, "cache"); + const first = await analyzeTargets([target], request, true, cache); + const next = await analyzeTargets( + [{ ...target, window: { from: 40, seconds: 1 } }], + request, + true, + cache, + ); + expect(decodeAudio).toHaveBeenCalledTimes(3); + expect(next.items[0]?.kind).toEqual(first.items[0]?.kind); + expect(next.items[0]?.ladder).toEqual(first.items[0]?.ladder); + writeFileSync(path, "changed source"); + await analyzeTargets([target], request, false, cache); + expect(decodeAudio).toHaveBeenCalledTimes(5); + await analyzeTargets([{ ...target, clip: { from: 1, seconds: 1199 } }], request, false, cache); + expect(decodeAudio).toHaveBeenCalledTimes(7); + const pinned = await analyzeTargets( + [target], + { ...request, provider: { kind: "missing" } }, + true, + cache, + ); + expect(pinned.items[0]?.kind).toBeNull(); + expect(pinned.items[0]?.warnings.length).toBeGreaterThan(0); + }); + + it("does not cache a reused full-window signal under a source changed during decode", async () => { + const cache = join(dir, "changed-full-decode"); + vi.mocked(decodeAudio).mockImplementationOnce(async () => { + writeFileSync(path, "source replaced while decoding full window"); + return { signal, stream }; + }); + await analyzeTargets([{ ...target, window: target.clip }], request, false, cache); + expect(existsSync(cache)).toBe(false); + }); + + it("does not cache evidence if the source changes during its decode", async () => { + const cache = join(dir, "changed-during-decode"); + vi.mocked(decodeAudio) + .mockResolvedValueOnce({ signal, stream }) + .mockImplementationOnce(async () => { + writeFileSync(path, "new source while decoding"); + return { signal, stream }; + }); + await analyzeTargets([target], request, false, cache); + await analyzeTargets([target], request, false, cache); + expect(decodeAudio).toHaveBeenCalledTimes(4); + }); + + it("does not cache a failed full-clip attempt as the whole clip's kind", async () => { + const cache = join(dir, "retry-cache"); + vi.mocked(decodeAudio) + .mockResolvedValueOnce({ signal, stream }) + .mockRejectedValueOnce(new Error("decode failed")); + await analyzeTargets([target], request, false, cache); + await analyzeTargets([target], request, false, cache); + expect(decodeAudio).toHaveBeenCalledTimes(4); + }); +}); diff --git a/packages/cli/src/audio/analysis/targets.test.ts b/packages/cli/src/audio/analysis/targets.test.ts index 4884480067..3363de566f 100644 --- a/packages/cli/src/audio/analysis/targets.test.ts +++ b/packages/cli/src/audio/analysis/targets.test.ts @@ -114,6 +114,7 @@ describe("filmTargets", () => { (t) => t.ref === "#fast", ); expect(fast!.window).toEqual({ from: 3, seconds: 2 }); + expect(fast!.clip).toEqual({ from: 1, seconds: 4 }); }); it("resolves a source against the file that declares the clip, and keeps its local data-start", () => { diff --git a/packages/cli/src/audio/analysis/targets.ts b/packages/cli/src/audio/analysis/targets.ts index 3287fd2a35..11730c0687 100644 --- a/packages/cli/src/audio/analysis/targets.ts +++ b/packages/cli/src/audio/analysis/targets.ts @@ -23,6 +23,7 @@ export interface AudioTarget { placement: Placement | null; /** The part of the file to analyze, in file seconds; the whole file when absent. */ window?: { from: number; seconds: number }; + clip?: { from: number; seconds: number }; } export type FilmQuery = @@ -87,6 +88,10 @@ function targetOf(projectDir: string, row: TimelineRow, query: FilmQuery): Audio from: fileTime(placement, from), seconds: round((to - from) * placement.playbackRate), }, + clip: { + from: placement.mediaStart, + seconds: round((placement.end - placement.start) * placement.playbackRate), + }, }; } diff --git a/packages/cli/src/audio/read/pitch.ts b/packages/cli/src/audio/read/pitch.ts index ba4facec45..818485490f 100644 --- a/packages/cli/src/audio/read/pitch.ts +++ b/packages/cli/src/audio/read/pitch.ts @@ -1,62 +1,5 @@ -import { PITCH_RATE } from "./signal.js"; - -const FRAME = 320; -const HOP = 320; -const LOW_HZ = 70; -const HIGH_HZ = 500; -const VOICED_DB = -35; -const PERIODIC = 0.15; - -export interface PitchPoint { - t: number; - hz: number; -} - -const MIN_LAG = Math.floor(PITCH_RATE / HIGH_HZ); -const MAX_LAG = Math.ceil(PITCH_RATE / LOW_HZ); - -function voiced(pcm: Float32Array, start: number): boolean { - let energy = 0; - for (let i = 0; i < FRAME; i++) energy += pcm[start + i]! ** 2; - return 10 * Math.log10(energy / FRAME + 1e-12) >= VOICED_DB; -} - -function difference(pcm: Float32Array, start: number, tau: number): number { - let sum = 0; - for (let i = 0; i < FRAME; i++) { - const delta = pcm[start + i]! - pcm[start + i + tau]!; - sum += delta * delta; - } - return sum; -} - -/** The first lag whose normalized difference dips under the threshold, followed down to its minimum; -1 if none. */ -function periodOf(pcm: Float32Array, start: number, d: Float64Array): number { - let running = 0; - let lag = -1; - for (let tau = 1; tau <= MAX_LAG; tau++) { - const sum = difference(pcm, start, tau); - running += sum; - d[tau] = running > 0 ? (sum * tau) / running : 1; - if (lag === -1) { - if (tau >= MIN_LAG && d[tau]! < PERIODIC) lag = tau; - } else if (d[tau]! > d[tau - 1]!) break; - else if (d[tau]! < d[lag]!) lag = tau; - } - return lag; -} - -/** The fundamental of each voiced 40 ms frame of 8 kHz mono, by YIN's cumulative mean normalized difference. */ -export function pitchTrack(pcm: Float32Array): PitchPoint[] { - const out: PitchPoint[] = []; - const d = new Float64Array(MAX_LAG + 1); - for (let start = 0; start + FRAME + MAX_LAG <= pcm.length; start += HOP) { - if (!voiced(pcm, start)) continue; - const lag = periodOf(pcm, start, d); - if (lag > 0) out.push({ t: (start + FRAME / 2) / PITCH_RATE, hz: PITCH_RATE / lag }); - } - return out; -} +export { pitchTrack, type PitchPoint } from "@hyperframes/core/audio-analysis"; +import type { PitchPoint } from "@hyperframes/core/audio-analysis"; export function within(track: readonly PitchPoint[], start: number, end: number): number[] { let lo = 0; diff --git a/packages/cli/src/audio/read/signal.ts b/packages/cli/src/audio/read/signal.ts index c65bee988e..19d79be78f 100644 --- a/packages/cli/src/audio/read/signal.ts +++ b/packages/cli/src/audio/read/signal.ts @@ -1,9 +1,9 @@ -import type { AudioSignal } from "@hyperframes/core/audio-analysis"; +import { PITCH_RATE, type AudioSignal } from "@hyperframes/core/audio-analysis"; import { decodeAudio } from "../analysis/decode.js"; /** Every layer is read from mono at this rate: enough for onsets and band shapes, small enough for a long episode. */ export const READ_RATE = 22050; -export const PITCH_RATE = 8000; +export { PITCH_RATE }; const SILENT_DB = -120; export const ABSENT_DB = -55; diff --git a/packages/cli/src/commands/audio.ts b/packages/cli/src/commands/audio.ts index 11cd124adf..a3760a53c1 100644 --- a/packages/cli/src/commands/audio.ts +++ b/packages/cli/src/commands/audio.ts @@ -274,7 +274,12 @@ export default defineCommand({ const targets = files.length ? await fileTargets(files, flags) : await clipTargets(flags, query); - const report = await analyzeTargets(targets, analyzeRequest(flags), Boolean(args.full)); + const report = await analyzeTargets( + targets, + analyzeRequest(flags), + Boolean(args.full), + join(resolve(flags.dir ?? process.cwd()), ".hyperframes", "audio-kind"), + ); console.log(args.json ? JSON.stringify(withMeta(report)) : formatReport(report)); }, }); diff --git a/packages/core/src/audioAnalysis.test.ts b/packages/core/src/audioAnalysis.test.ts index c366e91725..c6e58970b5 100644 --- a/packages/core/src/audioAnalysis.test.ts +++ b/packages/core/src/audioAnalysis.test.ts @@ -172,12 +172,19 @@ describe("classify", () => { const gaps = noise(7); const syllables: number[] = []; for (let at = 0.1; at < 10; at += 0.12 + 0.25 * Math.abs(gaps())) syllables.push(at); - /** Noise in irregular syllable-length bursts over a quiet room tone. */ - const speechLike = (t: number, random: () => number) => { - const into = syllables.map((at) => t - at).find((d) => d >= 0 && d < 0.15); - const voiced = into === undefined ? 0 : 0.5 * Math.sin((Math.PI * into) / 0.15); - return (voiced + 0.02) * random(); + const syllableAt = (t: number) => syllables.findIndex((at) => t - at >= 0 && t - at < 0.15); + const burst = (t: number) => { + const n = syllableAt(t); + return n < 0 ? 0 : 0.5 * Math.sin((Math.PI * (t - syllables[n]!)) / 0.15); }; + /** A voice whose pitch moves from syllable to syllable, with breath noise, in irregular bursts over room tone. */ + const speechLike = (t: number, random: () => number) => { + const hz = 110 + 13 * (syllableAt(t) % 7); + const fold = Math.sin(2 * Math.PI * hz * t) + 0.5 * Math.sin(4 * Math.PI * hz * t); + return burst(t) * (fold + 0.3 * random()) + 0.02 * random(); + }; + /** The same envelope as noise: what a hit with a reverb tail looks like to the envelope alone. */ + const unpitched = (t: number, random: () => number) => (burst(t) + 0.02) * random(); it("calls a short sound an effect", () => { expect(classify(signal(1, impact(0.1, 0.01))).value).toBe("sfx"); @@ -198,12 +205,43 @@ describe("classify", () => { expect(classify(signal(3, speechLike)).value).toBe("voice"); }); + it("still calls speech voice when the stereo channels have opposite polarity", () => { + const speech = signal(3, speechLike); + const expected = classify(speech); + speech.channels.push(Float32Array.from(speech.channels[0]!, (sample) => -sample)); + expect(classify(speech)).toEqual(expected); + }); + + it("does not call speech-shaped noise voice: without a voice-range pitch it is an effect or a bed", () => { + const short = classify(signal(3, unpitched)); + expect(short.value).toBe("sfx"); + expect(short.features.voiced).toBe(0); + expect(classify(signal(10, unpitched)).value).not.toBe("voice"); + }); + + it("calls even-paced speech without a beat voice, and a held pitched pad still music", () => { + /** Voice that never stops: the syllables only swell a carried pitch, as a presenter or TTS reads. */ + const steady = (t: number, random: () => number) => { + const hz = 110 + 13 * (Math.max(0, syllableAt(t)) % 7); + const phrase = (t + 0.4 * Math.sin(t)) % 2.9 < 2.3 ? 1 : 0.03; + return phrase * (0.25 + 0.6 * burst(t)) * (Math.sin(2 * Math.PI * hz * t) + 0.1 * random()); + }; + const pad = (t: number, random: () => number) => + 0.3 * (Math.sin(2 * Math.PI * 220 * t) + 0.5 * Math.sin(2 * Math.PI * 330 * t)) + + 0.01 * random(); + const speech = classify(signal(10, steady)); + expect(speech.features.syllables).toBeLessThan(0.35); + expect(speech.value).toBe("voice"); + expect(classify(signal(10, pad)).value).toBe("music"); + }); + it("reports what the guess was made from", () => { expect(Object.keys(classify(signal(3, speechLike)).features)).toEqual([ "duration", "regularity", "syllables", "sustain", + "voiced", ]); }); }); diff --git a/packages/core/src/audioAnalysis.ts b/packages/core/src/audioAnalysis.ts index fcbf6095de..6506d6769a 100644 --- a/packages/core/src/audioAnalysis.ts +++ b/packages/core/src/audioAnalysis.ts @@ -6,6 +6,7 @@ import type { AudioAnalysisProvider, AudioCapability } from "./audioAnalysisLadder"; import { finish, type Steps } from "./audioSteps"; import { fft } from "./fft"; +import { voicedShare } from "./audioPitch"; import { rhythm } from "./audioRhythm"; import { sections, type SectionOptions } from "./audioSections"; @@ -116,6 +117,7 @@ export interface KindFeatures { syllables: number; /** Share of 50 ms frames within 20 dB of the loud end (95th percentile), 0–1: high for beds and speech, low for hits. */ sustain: number; + voiced?: number; } const DB_FLOOR = -120; @@ -588,26 +590,34 @@ export function kindFeatures(signal: AudioSignal): KindFeatures { }; } +const VOICED_SPEECH = 0.15; +const VOICED_STEADY_SPEECH = 0.25; + /** * A first guess at what the audio is. Length decides most of it: beds run long, effects short. Long audio is voice - * when its envelope moves at syllable rate without a beat; short audio is voice when it also holds its level. + * when its envelope moves at syllable rate without a beat; short audio also holds its level. Both need a voice pitch. */ export function classify(signal: AudioSignal): KindGuess { const features = kindFeatures(signal); const { duration, regularity, syllables, sustain } = features; const speechLike = syllables >= 0.35 && regularity < 0.35; + const voiced = () => (features.voiced ??= round(voicedShare(signal.channels, signal.sampleRate))); + const pitched = () => voiced() >= VOICED_SPEECH; if (duration >= 8) { if (sustain < 0.4) return { value: "sfx", confidence: 0.6, features }; - if (speechLike) + if (speechLike && pitched()) return { value: "voice", confidence: round(Math.min(0.9, 0.4 + syllables)), features }; + if (regularity < 0.2 && syllables >= 0.17 && voiced() >= VOICED_STEADY_SPEECH) + return { value: "voice", confidence: 0.6, features }; return { value: "music", confidence: round(Math.min(0.9, 0.5 + regularity)), features }; } - if (duration >= 1.5 && syllables >= 0.17 && regularity < 0.3 && sustain >= 0.7) + if (duration >= 1.5 && syllables >= 0.17 && regularity < 0.3 && sustain >= 0.7 && pitched()) return { value: "voice", confidence: speechLike ? 0.8 : 0.6, features }; return { value: "sfx", confidence: duration <= 2 ? 0.85 : 0.65, features }; } export * from "./audioAnalysisLadder"; +export * from "./audioPitch"; export * from "./audioRhythm"; export * from "./audioSections"; export * from "./audioSnap"; diff --git a/packages/core/src/audioPitch.test.ts b/packages/core/src/audioPitch.test.ts new file mode 100644 index 0000000000..b9e181c827 --- /dev/null +++ b/packages/core/src/audioPitch.test.ts @@ -0,0 +1,39 @@ +import { describe, expect, it } from "vitest"; +import { atPitchRate, pitchTrack, voicedShare, PITCH_RATE } from "./audioPitch"; + +const tone = (rate: number, seconds = 1, gain = 0.5) => + Float32Array.from( + { length: Math.floor(rate * seconds) }, + (_, i) => gain * Math.sin((2 * Math.PI * 200 * i) / rate), + ); + +describe("voice pitch evidence", () => { + it.each([8000, 16000, 22050, 44100, 48000, 96000])( + "keeps pitch and voiced share at %s Hz, in mono and duplicated stereo", + (rate) => { + const channel = tone(rate); + const mono = atPitchRate([channel], rate); + expect(mono).toHaveLength(PITCH_RATE); + expect(atPitchRate([channel, channel], rate)).toEqual(mono); + const track = pitchTrack(mono); + expect(track).toHaveLength(24); + expect(track[0]).toEqual({ t: 0.02, hz: 200 }); + expect(track.every(({ hz }) => hz === 200)).toBe(true); + expect(voicedShare([channel], rate)).toBe(24 / 25); + }, + ); + + it("keeps voiced evidence when stereo channels have opposite polarity", () => { + const channel = tone(48000); + const reversed = Float32Array.from(channel, (value) => -value); + expect(voicedShare([channel, reversed], 48000)).toBe(voicedShare([channel], 48000)); + }); + + it("has no evidence for empty, silent, too-short or below-gate signals", () => { + expect(voicedShare([], 48000)).toBe(0); + expect(voicedShare([new Float32Array(48000)], 48000)).toBe(0); + expect(voicedShare([tone(8000, 0.04)], 8000)).toBe(0); + expect(voicedShare([tone(8000, 1, 0.001)], 8000)).toBe(0); + expect(pitchTrack(tone(8000, 435 / 8000))).toHaveLength(1); + }); +}); diff --git a/packages/core/src/audioPitch.ts b/packages/core/src/audioPitch.ts new file mode 100644 index 0000000000..df26af6765 --- /dev/null +++ b/packages/core/src/audioPitch.ts @@ -0,0 +1,95 @@ +/** The rate pitch is tracked at: 8 kHz keeps the 70–500 Hz search cheap. */ +export const PITCH_RATE = 8000; + +const FRAME = 320; +const HOP = 320; +const LOW_HZ = 70; +const HIGH_HZ = 500; +const VOICED_DB = -35; +const PERIODIC = 0.15; + +export interface PitchPoint { + t: number; + hz: number; +} + +const MIN_LAG = Math.floor(PITCH_RATE / HIGH_HZ); +const MAX_LAG = Math.ceil(PITCH_RATE / LOW_HZ); + +function voiced(channels: readonly Float32Array[], start: number): boolean { + let energy = 0; + for (const pcm of channels) for (let i = 0; i < FRAME; i++) energy += pcm[start + i]! ** 2; + return 10 * Math.log10(energy / (FRAME * channels.length) + 1e-12) >= VOICED_DB; +} + +function difference(channels: readonly Float32Array[], start: number, tau: number): number { + let sum = 0; + for (const pcm of channels) { + for (let i = 0; i < FRAME; i++) { + const delta = pcm[start + i]! - pcm[start + i + tau]!; + sum += delta * delta; + } + } + return sum; +} + +/** The first lag whose normalized difference dips under the threshold, followed down to its minimum; -1 if none. */ +function periodOf(channels: readonly Float32Array[], start: number, d: Float64Array): number { + let running = 0; + let lag = -1; + for (let tau = 1; tau <= MAX_LAG; tau++) { + const sum = difference(channels, start, tau); + running += sum; + d[tau] = running > 0 ? (sum * tau) / running : 1; + if (lag === -1) { + if (tau >= MIN_LAG && d[tau]! < PERIODIC) lag = tau; + } else if (d[tau]! > d[tau - 1]!) break; + else if (d[tau]! < d[lag]!) lag = tau; + } + return lag; +} + +/** The fundamental of each voiced 40 ms frame of 8 kHz mono, by YIN's cumulative mean normalized difference. */ +export function pitchTrack(pcm: Float32Array): PitchPoint[] { + return channelPitchTrack([pcm]); +} + +function channelPitchTrack(channels: readonly Float32Array[]): PitchPoint[] { + const length = channels[0]?.length ?? 0; + const out: PitchPoint[] = []; + const d = new Float64Array(MAX_LAG + 1); + for (let start = 0; start + FRAME + MAX_LAG <= length; start += HOP) { + if (!voiced(channels, start)) continue; + const lag = periodOf(channels, start, d); + if (lag > 0) out.push({ t: (start + FRAME / 2) / PITCH_RATE, hz: PITCH_RATE / lag }); + } + return out; +} + +/** Mono at {@link PITCH_RATE}: channels averaged, a box filter as wide as the step against aliasing, then sampled. */ +export function atPitchRate(channels: readonly Float32Array[], sampleRate: number): Float32Array { + const length = channels[0]?.length ?? 0; + if (!length || !channels.length) return new Float32Array(0); + const step = sampleRate / PITCH_RATE; + const width = Math.max(1, Math.round(step)); + const sums = new Float64Array(length + 1); + for (let i = 0; i < length; i++) { + let mixed = 0; + for (const channel of channels) mixed += channel[i]!; + sums[i + 1] = sums[i]! + mixed / channels.length; + } + const out = new Float32Array(Math.floor(length / step)); + for (let n = 0; n < out.length; n++) { + const from = Math.floor(n * step); + const to = Math.min(length, from + width); + out[n] = (sums[to]! - sums[from]!) / (to - from); + } + return out; +} + +/** The share of 40 ms frames with a fundamental in the voice range; hits, whooshes and noise have none. */ +export function voicedShare(channels: readonly Float32Array[], sampleRate: number): number { + const pitched = channels.map((channel) => atPitchRate([channel], sampleRate)); + const frames = Math.floor((pitched[0]?.length ?? 0) / HOP); + return frames ? channelPitchTrack(pitched).length / frames : 0; +}