diff --git a/packages/core/src/audioResample.test.ts b/packages/core/src/audioResample.test.ts new file mode 100644 index 0000000000..6d6daca630 --- /dev/null +++ b/packages/core/src/audioResample.test.ts @@ -0,0 +1,35 @@ +import { describe, expect, it } from "vitest"; +import { resample } from "./audioResample"; + +const tone = (hz: number, rate: number, seconds: number) => + Float32Array.from({ length: seconds * rate }, (_, i) => Math.sin((2 * Math.PI * hz * i) / rate)); + +const rms = (x: Float32Array) => Math.sqrt(x.reduce((a, v) => a + v * v, 0) / x.length); + +/** Upward zero crossings per second, over the middle of the signal. */ +function frequency(x: Float32Array, rate: number): number { + const from = Math.floor(x.length / 4); + const to = Math.floor((3 * x.length) / 4); + let crossings = 0; + for (let i = from + 1; i < to; i++) if (x[i - 1]! < 0 && x[i]! >= 0) crossings++; + return (crossings * rate) / (to - from); +} + +describe("resample", () => { + it("keeps a tone's pitch and level from 48 kHz to 22.05 kHz", () => { + const out = resample(tone(1000, 48_000, 2), 48_000, 22_050); + expect(out.length).toBe(44_100); + expect(frequency(out, 22_050)).toBeCloseTo(1000, -1); + expect(rms(out.subarray(1000, -1000))).toBeCloseTo(Math.SQRT1_2, 2); + }); + + it("removes what the lower rate cannot hold instead of folding it down", () => { + const out = resample(tone(15_000, 48_000, 1), 48_000, 22_050); + expect(rms(out.subarray(1000, -1000))).toBeLessThan(0.01); + }); + + it("returns the samples untouched at the same rate", () => { + const x = tone(440, 22_050, 1); + expect(resample(x, 22_050, 22_050)).toBe(x); + }); +}); diff --git a/packages/core/src/audioResample.ts b/packages/core/src/audioResample.ts new file mode 100644 index 0000000000..60add57aec --- /dev/null +++ b/packages/core/src/audioResample.ts @@ -0,0 +1,38 @@ +/** Band-limited resampling by windowed-sinc interpolation, so analysis reads the same at any decode rate. */ + +const ZERO_CROSSINGS = 8; +const STEPS_PER_CROSSING = 256; +/** Cutoff as a share of the lower Nyquist, leaving the window's transition band below it. */ +const CUTOFF = 0.95; + +const KERNEL = Float64Array.from({ length: ZERO_CROSSINGS * STEPS_PER_CROSSING + 2 }, (_, i) => { + const x = i / STEPS_PER_CROSSING; + if (x >= ZERO_CROSSINGS) return 0; + const sinc = x === 0 ? 1 : Math.sin(Math.PI * x) / (Math.PI * x); + return sinc * (0.5 + 0.5 * Math.cos((Math.PI * x) / ZERO_CROSSINGS)); +}); + +function kernelAt(x: number): number { + const at = Math.abs(x) * STEPS_PER_CROSSING; + const i = Math.floor(at); + if (i + 1 >= KERNEL.length) return 0; + const frac = at - i; + return KERNEL[i]! * (1 - frac) + KERNEL[i + 1]! * frac; +} + +export function resample(samples: Float32Array, from: number, to: number): Float32Array { + if (from === to) return samples; + const ratio = to / from; + const scale = Math.min(1, ratio) * CUTOFF; + const reach = ZERO_CROSSINGS / scale; + const out = new Float32Array(Math.floor(samples.length * ratio)); + for (let n = 0; n < out.length; n++) { + const t = n / ratio; + const last = Math.min(samples.length - 1, Math.floor(t + reach)); + let sum = 0; + for (let k = Math.max(0, Math.ceil(t - reach)); k <= last; k++) + sum += samples[k]! * kernelAt((t - k) * scale); + out[n] = sum * scale; + } + return out; +} diff --git a/packages/core/src/audioRhythm.test.ts b/packages/core/src/audioRhythm.test.ts index 2667d5d40c..760a85c1d5 100644 --- a/packages/core/src/audioRhythm.test.ts +++ b/packages/core/src/audioRhythm.test.ts @@ -15,14 +15,14 @@ function random(seed = 3) { } /** A drum pattern on a known grid: a kick with a low thump on each bar's first beat, a click on the others. */ -function drums(beatTimes: number[], meter: number, seconds: number): AudioSignal { - const samples = new Float32Array(Math.round(seconds * RATE)); +function drums(beatTimes: number[], meter: number, seconds: number, rate = RATE): AudioSignal { + const samples = new Float32Array(Math.round(seconds * rate)); const noise = random(); beatTimes.forEach((t, k) => { const downbeat = k % meter === 0; - const start = Math.round(t * RATE); - for (let i = 0; i < 0.12 * RATE && start + i < samples.length; i++) { - const since = i / RATE; + const start = Math.round(t * rate); + for (let i = 0; i < 0.12 * rate && start + i < samples.length; i++) { + const since = i / rate; const click = (downbeat ? 0.5 : 0.35) * noise() * Math.exp(-since / 0.02); const thump = downbeat ? 0.8 * Math.sin(2 * Math.PI * 55 * since) * Math.exp(-since / 0.06) @@ -30,6 +30,31 @@ function drums(beatTimes: number[], meter: number, seconds: number): AudioSignal samples[start + i]! += click + thump; } }); + return { sampleRate: rate, channels: [samples] }; +} + +/** The same hit on every beat, over chords that change every four beats from beat `firstChange`. */ +function chordsOverEvenHits( + beatTimes: number[], + firstChange: number, + seconds: number, +): AudioSignal { + const samples = new Float32Array(Math.round(seconds * RATE)); + const noise = random(7); + const roots = [220, 175, 262, 196]; + beatTimes.forEach((t) => { + const start = Math.round(t * RATE); + for (let i = 0; i < 0.08 * RATE && start + i < samples.length; i++) + samples[start + i]! += 0.4 * noise() * Math.exp(-i / RATE / 0.02); + }); + const period = beatTimes[1]! - beatTimes[0]!; + for (let i = 0; i < samples.length; i++) { + const t = i / RATE; + const bar = Math.floor((t - beatTimes[firstChange]!) / (4 * period)); + const root = roots[((bar % 4) + 4) % 4]!; + for (const ratio of [1, 1.26, 1.5]) + samples[i]! += 0.08 * Math.sin(2 * Math.PI * root * ratio * t); + } return { sampleRate: RATE, channels: [samples] }; } @@ -66,6 +91,23 @@ describe("rhythm", () => { expect(curve[curve.length - 1]!.bpm).toBeCloseTo(133, -1); }); + it("gives the same beats at any sample rate and for mono or the same sound on two channels", () => { + const truth = grid(0.25, 0.5, 19.5); + const reference = rhythm(drums(truth, 4, 20)).beats.times; + const wide = drums(truth, 4, 20, 44_100).channels[0]!; + const stereo = rhythm({ sampleRate: 44_100, channels: [wide, wide.slice()] }).beats.times; + expect(stereo).toHaveLength(reference.length); + stereo.forEach((t, k) => expect(Math.abs(t - reference[k]!)).toBeLessThanOrEqual(0.011)); + }); + + it("puts the bar where the chords change when every beat hits the same", () => { + const truth = grid(0.25, 0.5, 23.5); + const { downbeats } = rhythm(chordsOverEvenHits(truth, 1, 24)); + expect(downbeats.meter).toBe(4); + const changes = truth.filter((_, k) => k % 4 === 1); + expect(recall(downbeats.times, changes)).toBeGreaterThan(0.9); + }); + it("finds no steady beat in silence or in noise, and none in a clip too short to tell", () => { const silent = rhythm({ sampleRate: RATE, channels: [new Float32Array(10 * RATE)] }); expect(silent.beats).toMatchObject({ times: [], reason: "no steady beat" }); @@ -75,6 +117,7 @@ describe("rhythm", () => { channels: [Float32Array.from({ length: 10 * RATE }, () => 0.3 * noise())], }); expect(hiss.beats.times).toEqual([]); + expect(hiss.downbeats.times).toEqual([]); const short = rhythm(drums(grid(0.25, 0.5, 3), 4, 3)); expect(short.beats).toMatchObject({ times: [], reason: "too short for a steady beat" }); }); diff --git a/packages/core/src/audioRhythm.ts b/packages/core/src/audioRhythm.ts index 74fa42d36e..a0608630f3 100644 --- a/packages/core/src/audioRhythm.ts +++ b/packages/core/src/audioRhythm.ts @@ -4,6 +4,7 @@ */ import type { AudioSignal } from "./audioAnalysis"; +import { resample } from "./audioResample"; import { fft } from "./fft"; export interface Beats { @@ -18,7 +19,7 @@ export interface Downbeats { times: number[]; /** Beats per bar, 3 or 4; null when there are too few beats to tell. */ meter: number | null; - /** "low" for a guess from where the low end hits hardest. */ + /** "low": a guess from where the harmony changes and the low end hits. */ confidence?: "low"; } @@ -39,28 +40,39 @@ export interface Rhythm { downbeats: Downbeats; } -/** Log energy per frame in log-spaced bands, the frames `fps` per second. */ +/** Log energy per frame in log-spaced bands, `fps` frames per second, and a slower pitch-class profile. */ export interface BandFrames { fps: number; bands: Float64Array[]; + chroma: Float64Array[]; + chromaFps: number; } +/** Every signal is analyzed at this rate and as the mean of its channels' power, so decode rate and layout cannot change a result. */ +const ANALYSIS_RATE = 22050; +const FRAME_SIZE = 1024; +const CHROMA_SIZE = 4096; +const CHROMA_FPS = 10; const FRAMES_PER_SECOND = 100; const LOW_BAND_HZ = 150; const SLOWEST_BPM = 40; const FASTEST_BPM = 240; const TIGHTNESS = 100; +/** The onset curve peaks this much ahead of where listeners place the beat, so beats move this much later. */ +const ONSET_LAG_SECONDS = 0.015; const round = (t: number) => Math.round(t * 1000) / 1000; -function bandEdges(sampleRate: number, size: number): number[] { - const binHz = sampleRate / size; +function bandEdges(): number[] { + const binHz = ANALYSIS_RATE / FRAME_SIZE; const edges: number[] = []; - for (let hz = 30; hz < Math.min(9000, sampleRate / 2); hz *= 2 ** (1 / 5)) - edges.push(Math.max(1, Math.round(hz / binHz))); + for (let hz = 30; hz < 9000; hz *= 2 ** (1 / 5)) edges.push(Math.max(1, Math.round(hz / binHz))); return edges; } +const hann = (size: number) => + Float64Array.from({ length: size }, (_, i) => 0.5 - 0.5 * Math.cos((2 * Math.PI * i) / size)); + /** Add each band's power in the Hann-windowed frame centred on `center` into `into`. */ function addFramePower( samples: Float32Array, @@ -86,24 +98,61 @@ function addFramePower( } } -/** Per-band log power, summed over channels so a polarity-flipped channel cannot cancel. */ -export function bandFrames(signal: AudioSignal): BandFrames { - const { sampleRate, channels } = signal; - const hop = Math.max(1, Math.round(sampleRate / FRAMES_PER_SECOND)); - const size = 2 ** Math.round(Math.log2(sampleRate * 0.046)); - const window = Float64Array.from( - { length: size }, - (_, i) => 0.5 - 0.5 * Math.cos((2 * Math.PI * i) / size), - ); - const edges = bandEdges(sampleRate, size); - const length = channels[0]?.length ?? 0; +/** Per-band log power of every frame, averaged over channels so a polarity-flipped channel cannot cancel. */ +function bandPower(channels: Float32Array[]): Float64Array[] { + const window = hann(FRAME_SIZE); + const edges = bandEdges(); + const hop = ANALYSIS_RATE / FRAMES_PER_SECOND; const bands: Float64Array[] = []; - for (let center = 0; center < length; center += hop) { - const power = new Float64Array(Math.max(0, edges.length - 1)); - for (const channel of channels) addFramePower(channel, center, window, edges, power); - bands.push(power.map((p) => Math.log1p(1e9 * p))); + for (let f = 0; Math.round(f * hop) < (channels[0]?.length ?? 0); f++) { + const power = new Float64Array(edges.length - 1); + for (const channel of channels) + addFramePower(channel, Math.round(f * hop), window, edges, power); + bands.push(power.map((p) => Math.log1p((1e9 * p) / channels.length))); } - return { fps: sampleRate / hop, bands }; + return bands; +} + +/** Index of the pitch class, A = 9, of each FFT bin from 55 Hz to 2 kHz; -1 outside that range. */ +const PITCH_CLASS = Array.from({ length: CHROMA_SIZE / 2 }, (_, bin) => { + const hz = (bin * ANALYSIS_RATE) / CHROMA_SIZE; + if (hz < 55 || hz > 2000) return -1; + return (((Math.round(12 * Math.log2(hz / 440)) + 9) % 12) + 12) % 12; +}); + +function chromaFrames(channels: Float32Array[]): Float64Array[] { + const window = hann(CHROMA_SIZE); + const hop = ANALYSIS_RATE / CHROMA_FPS; + const re = new Float64Array(CHROMA_SIZE); + const im = new Float64Array(CHROMA_SIZE); + const frames: Float64Array[] = []; + for (let f = 0; Math.round(f * hop) < (channels[0]?.length ?? 0); f++) { + const chroma = new Float64Array(12); + const first = Math.round(f * hop) - CHROMA_SIZE / 2; + for (const channel of channels) { + for (let i = 0; i < CHROMA_SIZE; i++) { + const at = first + i; + re[i] = at >= 0 && at < channel.length ? channel[at]! * window[i]! : 0; + im[i] = 0; + } + fft(re, im); + PITCH_CLASS.forEach((pc, bin) => { + if (pc >= 0) chroma[pc]! += Math.hypot(re[bin]!, im[bin]!); + }); + } + frames.push(chroma); + } + return frames; +} + +function bandFrames(signal: AudioSignal): BandFrames { + const channels = signal.channels.map((c) => resample(c, signal.sampleRate, ANALYSIS_RATE)); + return { + fps: FRAMES_PER_SECOND, + bands: bandPower(channels), + chroma: chromaFrames(channels), + chromaFps: CHROMA_FPS, + }; } /** Subtract the mean of the surrounding ±`radius` frames and keep what rises above it. */ @@ -131,10 +180,9 @@ interface OnsetCurves { low: Float64Array; } -function onsetCurves({ fps, bands }: BandFrames, sampleRate: number): OnsetCurves { - const size = 2 ** Math.round(Math.log2(sampleRate * 0.046)); - const lowBands = bandEdges(sampleRate, size).filter( - (bin) => (bin * sampleRate) / size < LOW_BAND_HZ, +function onsetCurves({ fps, bands }: BandFrames): OnsetCurves { + const lowBands = bandEdges().filter( + (bin) => (bin * ANALYSIS_RATE) / FRAME_SIZE < LOW_BAND_HZ, ).length; const broad = new Float64Array(bands.length); const low = new Float64Array(bands.length); @@ -170,7 +218,7 @@ function autocorrelation(curve: Float64Array, maxLag: number): Float64Array { } const tempoPrior = (lag: number, fps: number) => - Math.exp(-0.5 * Math.log2(60 / (lag / fps) / 120) ** 2); + Math.exp(-0.5 * (Math.log2(60 / (lag / fps) / 120) / 0.8) ** 2); interface LagRange { shortest: number; @@ -256,13 +304,13 @@ function trimmed(beats: number[], local: Float64Array): number[] { const LOCAL_WINDOW_SECONDS = 8; -/** The period through the piece: eight-second windows a second apart, held near the global period, median-smoothed. */ +/** The period through the piece: eight-second windows a second apart, held within 10% of the global period, median-smoothed. */ function localPeriods(curve: Float64Array, fps: number, period: number): (frame: number) => number { const window = Math.round(LOCAL_WINDOW_SECONDS * fps); const hop = Math.round(fps); const range = { - shortest: Math.max(fullRange(fps).shortest, Math.floor(period * 0.7)), - longest: Math.min(fullRange(fps).longest, Math.ceil(period * 1.4)), + shortest: Math.max(fullRange(fps).shortest, Math.floor(period * 0.9)), + longest: Math.min(fullRange(fps).longest, Math.ceil(period * 1.1)), }; const centres: number[] = []; const periods: number[] = []; @@ -303,25 +351,81 @@ function onsetLift(frames: number[], local: Float64Array): number { return overall > 0 ? onBeats / overall - 1 : 0; } -/** The meter and bar phase whose beats carry the most low-band onset energy against the others. */ -function barPhase(beats: number[], curves: OnsetCurves): { meter: number; phase: number } { - const strength = (f: number) => - Math.max( - ...[-1, 0, 1, 2].map((d) => (curves.low[f + d] ?? 0) + 0.3 * (curves.broad[f + d] ?? 0)), - ); - const levels = beats.map(strength); +const mean = (values: readonly number[]) => + values.reduce((a, b) => a + b, 0) / (values.length || 1); + +function zScored(values: number[]): number[] { + const centre = mean(values); + const spread = Math.sqrt(mean(values.map((v) => (v - centre) ** 2))) || 1; + return values.map((v) => (v - centre) / spread); +} + +const CHROMA_REACH_SECONDS = CHROMA_SIZE / 2 / ANALYSIS_RATE; + +/** The unit-length mean chroma from one time to another, in seconds, leaving out frames that already hear the next onset. */ +function chromaBetween({ chroma, chromaFps }: BandFrames, from: number, to: number): number[] { + const middle = Math.round(((from + to) / 2) * chromaFps); + let first = Math.ceil(from * chromaFps); + let end = Math.floor((to - CHROMA_REACH_SECONDS) * chromaFps) + 1; + if (end <= first) [first, end] = [middle, middle + 1]; + const sum = new Array(12).fill(0); + for (let f = first; f < end; f++) for (let c = 0; c < 12; c++) sum[c]! += chroma[f]?.[c] ?? 0; + const length = Math.hypot(...sum) || 1; + return sum.map((v) => v / length); +} + +const cosine = (a: number[], b: number[]) => a.reduce((sum, v, i) => sum + v * b[i]!, 0); + +/** How much the harmony changes at each beat: the chroma `span` beats before it against the `span` after. */ +function harmonicChange(beats: number[], fps: number, frames: BandFrames, span: number): number[] { + return beats.map((f, k) => { + if (k < span || k + span >= beats.length) return 0; + const before = chromaBetween(frames, beats[k - span]! / fps, f / fps); + return 1 - cosine(before, chromaBetween(frames, f / fps, beats[k + span]! / fps)); + }); +} + +/** Each beat's evidence of starting a bar: a low-end hit, a chord change, and a chord change over half a bar. */ +function barStartEvidence(beats: number[], curves: OnsetCurves, frames: BandFrames): number[] { + const lowHit = (f: number) => Math.max(...[-1, 0, 1, 2].map((d) => curves.low[f + d] ?? 0)); + const cues = [ + zScored(beats.map(lowHit)), + zScored(harmonicChange(beats, curves.fps, frames, 1)), + zScored(harmonicChange(beats, curves.fps, frames, 2)), + ]; + return beats.map((_, k) => cues.reduce((sum, cue) => sum + cue[k]!, 0)); +} + +/** Most music is in four, so three must win by a clear margin. */ +const TRIPLE_METER_HANDICAP = 0.7; + +function barPhase(beats: number[], curves: OnsetCurves, frames: BandFrames) { + const evidence = barStartEvidence(beats, curves, frames); let best = { meter: 4, phase: 0, contrast: -Infinity }; for (const meter of [4, 3]) for (let phase = 0; phase < meter; phase++) { - const on = levels.filter((_, k) => (k - phase) % meter === 0); - const off = levels.filter((_, k) => (k - phase) % meter !== 0); - const mean = (v: number[]) => v.reduce((a, b) => a + b, 0) / (v.length || 1); - const contrast = (mean(on) - mean(off)) * (meter === 3 ? 0.85 : 1); + const on = evidence.filter((_, k) => (k - phase) % meter === 0); + const off = evidence.filter((_, k) => (k - phase) % meter !== 0); + const contrast = (mean(on) - mean(off)) * (meter === 3 ? TRIPLE_METER_HANDICAP : 1); if (contrast > best.contrast) best = { meter, phase, contrast }; } return best; } +function downbeatsOf( + beatList: number[], + times: number[], + curves: OnsetCurves, + frames: BandFrames, +): Downbeats { + const bar = beatList.length >= 6 ? barPhase(beatList, curves, frames) : null; + return { + times: bar ? times.filter((_, k) => (k - bar.phase) % bar.meter === 0) : [], + meter: bar?.meter ?? null, + confidence: "low", + }; +} + export function medianInterval(times: readonly number[]): number | null { const gaps = times .slice(1) @@ -339,15 +443,15 @@ const noRhythm = (confidence: number, reason = "no steady beat"): Rhythm => ({ const SHORTEST_SECONDS = 6; /** Below this the beats are more likely the tracker's own grid than the music's. */ -export const STEADY_BEAT_CONFIDENCE = 0.05; +export const STEADY_BEAT_CONFIDENCE = 0.15; /** Clarity of the period, scaled down when the beats are under three times as strong as the onsets around them. */ const beatConfidence = (clarity: number, lift: number) => round(Math.max(0, Math.min(1, clarity)) * Math.max(0, Math.min(1, lift / 3))); -/** Beats and downbeats of `frames`, the band energies of a signal at `sampleRate`. */ -export function rhythmOf(frames: BandFrames, sampleRate: number): Rhythm { - const curves = onsetCurves(frames, sampleRate); +/** Beats and downbeats of `frames`, a signal's band energies and chroma. */ +function rhythmOf(frames: BandFrames): Rhythm { + const curves = onsetCurves(frames); const { period, clarity } = beatPeriod(curves.broad, curves.fps); if (curves.broad.length < SHORTEST_SECONDS * curves.fps) return noRhythm(0, "too short for a steady beat"); @@ -355,16 +459,8 @@ export function rhythmOf(frames: BandFrames, sampleRate: number): Rhythm { const { frames: beatList, local } = beatFrames(curves.broad, curves.fps, period); const confidence = beatConfidence(clarity, onsetLift(beatList, local)); if (beatList.length < 4 || confidence < STEADY_BEAT_CONFIDENCE) return noRhythm(confidence); - const times = beatList.map((f) => round(f / curves.fps)); - const bar = beatList.length >= 6 ? barPhase(beatList, curves) : null; - return { - beats: { times, confidence }, - downbeats: { - times: bar ? times.filter((_, k) => (k - bar.phase) % bar.meter === 0) : [], - meter: bar?.meter ?? null, - confidence: "low", - }, - }; + const times = beatList.map((f) => round(f / curves.fps + ONSET_LAG_SECONDS)); + return { beats: { times, confidence }, downbeats: downbeatsOf(beatList, times, curves, frames) }; } const TEMPO_STEP_SECONDS = 4; @@ -405,5 +501,5 @@ export function bandFramesOf(signal: AudioSignal): BandFrames { export function rhythm(signal: AudioSignal): Rhythm { const frames = bandFramesOf(signal); const entry = measured.get(signal)!; - return (entry.rhythm ??= rhythmOf(frames, signal.sampleRate)); + return (entry.rhythm ??= rhythmOf(frames)); } diff --git a/packages/core/src/audioSections.ts b/packages/core/src/audioSections.ts index 334adb8d6e..efd4cd0907 100644 --- a/packages/core/src/audioSections.ts +++ b/packages/core/src/audioSections.ts @@ -1,5 +1,5 @@ /** - * Sections from signal processing: band energies averaged into half-second blocks, a self-similarity matrix, and + * Sections from signal processing: band energies and chroma averaged into half-second blocks, a self-similarity matrix, and * Foote's checkerboard novelty, whose peaks are where the music changes. Boundaries snap to the nearest beat. */ @@ -47,14 +47,25 @@ function standardizeBands(blocks: number[][]): void { } } -/** Band energies averaged over blocks, each band standardized over time and each block scaled to unit length. */ -function blockFeatures({ fps, bands }: BandFrames): number[][] { +/** Block means of the band energies and of the log chroma, each part unit length within a block. */ +function blockParts({ fps, bands, chroma, chromaFps }: BandFrames): number[][] { const per = Math.max(1, Math.round(BLOCK_SECONDS * fps)); + const chromaPer = Math.max(1, Math.round(BLOCK_SECONDS * chromaFps)); const blocks: number[][] = []; - for (let start = 0; start + per <= bands.length; start += per) - blocks.push(meanOf(bands.slice(start, start + per))); + for (let start = 0, b = 0; start + per <= bands.length; start += per, b++) { + const pitch = meanOf(chroma.slice(b * chromaPer, (b + 1) * chromaPer)).map(Math.log1p); + blocks.push([...meanOf(bands.slice(start, start + per)), ...unit(pitch)]); + } + return blocks; +} + +function blockFeatures(frames: BandFrames): number[][] { + const blocks = blockParts(frames); standardizeBands(blocks); - return blocks.map(unit); + const split = frames.bands[0]?.length ?? 0; + return blocks.map((block) => + [...unit(block.slice(0, split)), ...unit(block.slice(split))].map((v) => v * Math.SQRT1_2), + ); } function unit(vector: number[]): number[] { diff --git a/packages/core/src/beats/beatDetection.test.ts b/packages/core/src/beats/beatDetection.test.ts index d8e4244db2..841180c175 100644 --- a/packages/core/src/beats/beatDetection.test.ts +++ b/packages/core/src/beats/beatDetection.test.ts @@ -1,6 +1,7 @@ import { describe, expect, it, vi } from "vitest"; -import type { AudioSignal } from "../audioAnalysis"; +import { signalProvider, type AudioSignal } from "../audioAnalysis"; +import { STEADY_BEAT_CONFIDENCE, rhythm } from "../audioRhythm"; import { analyzeMusicFromBuffer, analyzeMusicFromSignal } from "./beatDetection"; const RATE = 22_050; @@ -105,6 +106,17 @@ describe("analyzeMusicFromSignal", () => { } }); + it("refuses an ambient pad in the tracker itself, so every caller of the signal tracker agrees", async () => { + const ambient = mono(pad(30)); + const { beats, downbeats } = rhythm(ambient); + expect(beats).toMatchObject({ times: [], reason: "no steady beat" }); + expect(beats.confidence).toBeLessThan(STEADY_BEAT_CONFIDENCE); + expect(downbeats.times).toEqual([]); + await expect(signalProvider.analyze("beats", ambient, {})).resolves.toMatchObject({ + times: [], + }); + }); + it("waits on the caller's pause before each long stage", async () => { const samples = new Float32Array(8 * RATE); clicks(samples, grid(0.25, 0.5, 7.5), 0.6); diff --git a/packages/core/src/beats/beatDetection.ts b/packages/core/src/beats/beatDetection.ts index f133bd8e14..63a271fc03 100644 --- a/packages/core/src/beats/beatDetection.ts +++ b/packages/core/src/beats/beatDetection.ts @@ -1,5 +1,5 @@ import { energyMix, type AudioSignal } from "../audioAnalysis"; -import { bandFrames, rhythmOf, tempoOf } from "../audioRhythm"; +import { bandFramesOf, rhythm, tempoOf } from "../audioRhythm"; export interface MusicBeatAnalysis { beatTimes: number[]; @@ -40,8 +40,6 @@ export function strengthAtTime( return Math.min(1, computeRmsAt(analysis.channelData, analysis.sampleRate, time) / analysis.peak); } -/** Below this tracker confidence the grid is more likely the tracker's than the music's, so none is reported. */ -const MIN_CONFIDENCE = 0.2; /** At or above this the tempo is reported as "high" confidence. */ const HIGH_CONFIDENCE = 0.35; @@ -62,13 +60,13 @@ export async function analyzeMusicFromSignal( { pause }: MusicAnalysisOptions = {}, ): Promise { await pause?.(); - const frames = bandFrames(signal); + bandFramesOf(signal); await pause?.(); - const { beats } = rhythmOf(frames, signal.sampleRate); + const { beats } = rhythm(signal); await pause?.(); const channelData = energyMix(signal); const confidence = beats.confidence ?? 0; - const times = confidence >= MIN_CONFIDENCE ? beats.times : []; + const times = beats.times; const { strengths, peak } = beatStrengths(times, channelData, signal.sampleRate); const bpm = tempoOf(times).value; return {