/** * Beats, downbeats and tempo from signal processing alone: a log-band spectral-flux onset curve, an autocorrelation * tempo with a log-normal prior, and dynamic-programming beat placement (Ellis 2007). Times are seconds into the signal. */ import type { AudioSignal } from "./audioAnalysis"; import { resample } from "./audioResample"; import { fft } from "./fft"; export interface Beats { /** Empty when there is no steady beat, with `reason` saying so. */ times: number[]; /** 0–1: how strongly the onsets repeat at the beat period, scaled by how much stronger they are on the beats. */ confidence?: number; reason?: string; } export interface Downbeats { times: number[]; /** Beats per bar, 3 or 4; null when there are too few beats to tell. */ meter: number | null; /** "low": a guess from where the harmony changes and the low end hits. */ confidence?: "low"; } export interface TempoPoint { t: number; bpm: number; } export interface Tempo { /** Beats per minute from the median beat interval; null without at least two beats. */ value: number | null; /** The tempo over the beats around each point, every few seconds, for music that speeds up or slows down. */ curve: TempoPoint[]; } export interface Rhythm { beats: Beats; downbeats: Downbeats; } /** Log energy per frame in log-spaced bands, `fps` frames per second, and a slower pitch-class profile. */ export interface BandFrames { fps: number; bands: Float64Array[]; chroma: Float64Array[]; chromaFps: number; } /** Every signal is analyzed at this rate and as the mean of its channels' power, so decode rate and layout cannot change a result. */ const ANALYSIS_RATE = 22050; const FRAME_SIZE = 1024; const CHROMA_SIZE = 4096; const CHROMA_FPS = 10; const FRAMES_PER_SECOND = 100; const LOW_BAND_HZ = 150; const SLOWEST_BPM = 40; const FASTEST_BPM = 240; const TIGHTNESS = 100; /** The onset curve peaks this much ahead of where listeners place the beat, so beats move this much later. */ const ONSET_LAG_SECONDS = 0.015; const round = (t: number) => Math.round(t * 1000) / 1000; function bandEdges(): number[] { const binHz = ANALYSIS_RATE / FRAME_SIZE; const edges: number[] = []; for (let hz = 30; hz < 9000; hz *= 2 ** (1 / 5)) edges.push(Math.max(1, Math.round(hz / binHz))); return edges; } const hann = (size: number) => Float64Array.from({ length: size }, (_, i) => 0.5 - 0.5 * Math.cos((2 * Math.PI * i) / size)); /** Add each band's power in the Hann-windowed frame centred on `center` into `into`. */ function addFramePower( samples: Float32Array, center: number, window: Float64Array, edges: number[], into: Float64Array, ): void { const size = window.length; const re = new Float64Array(size); const im = new Float64Array(size); const first = center - size / 2; for (let i = 0; i < size; i++) { const at = first + i; re[i] = at >= 0 && at < samples.length ? samples[at]! * window[i]! : 0; } fft(re, im); for (let b = 0; b + 1 < edges.length; b++) { const end = Math.max(edges[b]! + 1, edges[b + 1]!); let power = 0; for (let k = edges[b]!; k < end; k++) power += re[k]! * re[k]! + im[k]! * im[k]!; into[b]! += power / (size * size * (end - edges[b]!)); } } /** Per-band log power of every frame, averaged over channels so a polarity-flipped channel cannot cancel. */ function bandPower(channels: Float32Array[]): Float64Array[] { const window = hann(FRAME_SIZE); const edges = bandEdges(); const hop = ANALYSIS_RATE / FRAMES_PER_SECOND; const bands: Float64Array[] = []; for (let f = 0; Math.round(f * hop) < (channels[0]?.length ?? 0); f++) { const power = new Float64Array(edges.length - 1); for (const channel of channels) addFramePower(channel, Math.round(f * hop), window, edges, power); bands.push(power.map((p) => Math.log1p((1e9 * p) / channels.length))); } return bands; } /** Index of the pitch class, A = 9, of each FFT bin from 55 Hz to 2 kHz; -1 outside that range. */ const PITCH_CLASS = Array.from({ length: CHROMA_SIZE / 2 }, (_, bin) => { const hz = (bin * ANALYSIS_RATE) / CHROMA_SIZE; if (hz < 55 || hz > 2000) return -1; return (((Math.round(12 * Math.log2(hz / 440)) + 9) % 12) + 12) % 12; }); function chromaFrames(channels: Float32Array[]): Float64Array[] { const window = hann(CHROMA_SIZE); const hop = ANALYSIS_RATE / CHROMA_FPS; const re = new Float64Array(CHROMA_SIZE); const im = new Float64Array(CHROMA_SIZE); const frames: Float64Array[] = []; for (let f = 0; Math.round(f * hop) < (channels[0]?.length ?? 0); f++) { const chroma = new Float64Array(12); const first = Math.round(f * hop) - CHROMA_SIZE / 2; for (const channel of channels) { for (let i = 0; i < CHROMA_SIZE; i++) { const at = first + i; re[i] = at >= 0 && at < channel.length ? channel[at]! * window[i]! : 0; im[i] = 0; } fft(re, im); PITCH_CLASS.forEach((pc, bin) => { if (pc >= 0) chroma[pc]! += Math.hypot(re[bin]!, im[bin]!); }); } frames.push(chroma); } return frames; } function bandFrames(signal: AudioSignal): BandFrames { const channels = signal.channels.map((c) => resample(c, signal.sampleRate, ANALYSIS_RATE)); return { fps: FRAMES_PER_SECOND, bands: bandPower(channels), chroma: chromaFrames(channels), chromaFps: CHROMA_FPS, }; } /** Subtract the mean of the surrounding ±`radius` frames and keep what rises above it. */ function aboveLocalMean(curve: Float64Array, radius: number): Float64Array { const sums = new Float64Array(curve.length + 1); for (let i = 0; i < curve.length; i++) sums[i + 1] = sums[i]! + curve[i]!; return curve.map((value, i) => { const from = Math.max(0, i - radius); const to = Math.min(curve.length, i + radius + 1); return Math.max(0, value - (sums[to]! - sums[from]!) / (to - from)); }); } function standardized(curve: Float64Array): Float64Array { const mean = curve.reduce((a, b) => a + b, 0) / (curve.length || 1); const spread = Math.sqrt(curve.reduce((a, v) => a + (v - mean) ** 2, 0) / (curve.length || 1)); return spread > 0 ? curve.map((v) => v / spread) : curve.map(() => 0); } interface OnsetCurves { fps: number; /** Rectified rise summed over every band. */ broad: Float64Array; /** Rectified rise of the bands under 150 Hz, where kick drums and bass mark the bar. */ low: Float64Array; } function onsetCurves({ fps, bands }: BandFrames): OnsetCurves { const lowBands = bandEdges().filter( (bin) => (bin * ANALYSIS_RATE) / FRAME_SIZE < LOW_BAND_HZ, ).length; const broad = new Float64Array(bands.length); const low = new Float64Array(bands.length); for (let f = 1; f < bands.length; f++) { const frame = bands[f]!; const previous = bands[f - 1]!; for (let b = 0; b < frame.length; b++) { const rise = Math.max(0, frame[b]! - previous[b]!); broad[f]! += rise; if (b < lowBands) low[f]! += rise; } } const radius = Math.round(0.4 * fps); return { fps, broad: standardized(aboveLocalMean(broad, radius)), low: standardized(aboveLocalMean(low, radius)), }; } /** Autocorrelation of `curve` at lags 0…`maxLag`, normalized by lag 0. */ function autocorrelation(curve: Float64Array, maxLag: number): Float64Array { const mean = curve.reduce((a, b) => a + b, 0) / (curve.length || 1); const centred = curve.map((v) => v - mean); const ac = new Float64Array(maxLag + 1); for (let lag = 0; lag <= maxLag && lag < centred.length; lag++) { let sum = 0; for (let i = lag; i < centred.length; i++) sum += centred[i]! * centred[i - lag]!; ac[lag] = sum; } const zero = ac[0] || 1; return ac.map((v) => v / zero); } const tempoPrior = (lag: number, fps: number) => Math.exp(-0.5 * (Math.log2(60 / (lag / fps) / 120) / 0.8) ** 2); interface LagRange { shortest: number; longest: number; } const fullRange = (fps: number): LagRange => ({ shortest: Math.max(1, Math.floor((60 / FASTEST_BPM) * fps)), longest: Math.ceil((60 / SLOWEST_BPM) * fps), }); /** The beat period in frames: prior-weighted autocorrelation plus half its value at twice the lag. */ function beatPeriod( curve: Float64Array, fps: number, { shortest, longest }: LagRange = fullRange(fps), ): { period: number; clarity: number } { const ac = autocorrelation(curve, 2 * longest + 1); const at = (lag: number) => ac[Math.min(ac.length - 1, Math.max(0, lag))]!; const score = (lag: number) => tempoPrior(lag, fps) * (at(lag) + 0.5 * at(2 * lag) + 0.25 * (at(2 * lag - 1) + at(2 * lag + 1))); let best = shortest; for (let lag = shortest; lag <= longest; lag++) if (score(lag) > score(best)) best = lag; const [a, b, c] = [score(best - 1), score(best), score(best + 1)]; const curvature = a - 2 * b + c; const shift = curvature ? Math.max(-0.5, Math.min(0.5, (0.5 * (a - c)) / curvature)) : 0; return { period: best + shift, clarity: Math.max(0, Math.min(1, at(best))) }; } /** The onset curve smoothed by a Gaussian a thirty-second of a beat wide. */ function smoothed(curve: Float64Array, period: number): Float64Array { const half = Math.max(1, Math.round(period / 8)); const kernel = Array.from({ length: 2 * half + 1 }, (_, i) => Math.exp(-0.5 * (((i - half) * 32) / period) ** 2), ); return curve.map((_, i) => { let sum = 0; for (let j = 0; j < kernel.length; j++) sum += kernel[j]! * (curve[i + j - half] ?? 0); return sum; }); } /** Ellis's recursion: each frame's best score from a predecessor about one period back, penalized by log deviation. */ function cumulativeScore(local: Float64Array, periodAt: (frame: number) => number) { const score = new Float64Array(local.length); const back = new Int32Array(local.length).fill(-1); for (let i = 0; i < local.length; i++) { const period = periodAt(i); let best = -Infinity; let from = -1; for (let j = Math.max(0, Math.round(i - 2 * period)); j <= i - Math.round(period / 2); j++) { const candidate = score[j]! - TIGHTNESS * Math.log((i - j) / period) ** 2; if (candidate > best) [best, from] = [candidate, j]; } score[i] = local[i]! + Math.max(0, best); back[i] = best > 0 ? from : -1; } return { score, back }; } /** The last local maximum of the cumulative score that reaches half the median of all of them. */ function lastBeat(score: Float64Array): number { const maxima: number[] = []; for (let i = 1; i + 1 < score.length; i++) if (score[i]! > score[i - 1]! && score[i]! >= score[i + 1]!) maxima.push(i); if (!maxima.length) return score.length - 1; const median = maxima.map((i) => score[i]!).sort((a, b) => a - b)[maxima.length >> 1]!; for (let k = maxima.length - 1; k >= 0; k--) if (score[maxima[k]!]! >= 0.5 * median) return maxima[k]!; return maxima[maxima.length - 1]!; } /** Drop beats at either end whose onset strength is under half the RMS of the beats' strengths. */ function trimmed(beats: number[], local: Float64Array): number[] { const rms = Math.sqrt(beats.reduce((a, b) => a + local[b]! ** 2, 0) / (beats.length || 1)); let first = 0; let end = beats.length; while (first < end && local[beats[first]!]! < 0.5 * rms) first++; while (end > first && local[beats[end - 1]!]! < 0.5 * rms) end--; return beats.slice(first, end); } const LOCAL_WINDOW_SECONDS = 8; /** The period through the piece: eight-second windows a second apart, held within 10% of the global period, median-smoothed. */ function localPeriods(curve: Float64Array, fps: number, period: number): (frame: number) => number { const window = Math.round(LOCAL_WINDOW_SECONDS * fps); const hop = Math.round(fps); const range = { shortest: Math.max(fullRange(fps).shortest, Math.floor(period * 0.9)), longest: Math.min(fullRange(fps).longest, Math.ceil(period * 1.1)), }; const centres: number[] = []; const periods: number[] = []; for (let start = 0; start === 0 || start + window <= curve.length; start += hop) { const end = Math.min(curve.length, start + window); periods.push(beatPeriod(curve.subarray(start, end), fps, range).period); centres.push((start + end) / 2); } const median = periods.map((_, i) => { const near = periods.slice(Math.max(0, i - 2), i + 3).sort((a, b) => a - b); return near[near.length >> 1]!; }); return (frame) => interpolate(centres, median, frame); } function interpolate(xs: number[], ys: number[], x: number): number { if (x <= xs[0]!) return ys[0]!; for (let i = 1; i < xs.length; i++) if (x <= xs[i]!) { const t = (x - xs[i - 1]!) / (xs[i]! - xs[i - 1]!); return ys[i - 1]! * (1 - t) + ys[i]! * t; } return ys[ys.length - 1]!; } function beatFrames(curve: Float64Array, fps: number, period: number) { const local = smoothed(curve, period); const { score, back } = cumulativeScore(local, localPeriods(curve, fps, period)); const beats: number[] = []; for (let i = lastBeat(score); i >= 0; i = back[i]!) beats.push(i); return { frames: trimmed(beats.reverse(), local), local }; } /** How much stronger the smoothed onsets are on the beats than on average, as a share: 0 is no stronger. */ function onsetLift(frames: number[], local: Float64Array): number { const overall = local.reduce((a, b) => a + b, 0) / (local.length || 1); const onBeats = frames.reduce((a, f) => a + local[f]!, 0) / (frames.length || 1); return overall > 0 ? onBeats / overall - 1 : 0; } const mean = (values: readonly number[]) => values.reduce((a, b) => a + b, 0) / (values.length || 1); function zScored(values: number[]): number[] { const centre = mean(values); const spread = Math.sqrt(mean(values.map((v) => (v - centre) ** 2))) || 1; return values.map((v) => (v - centre) / spread); } const CHROMA_REACH_SECONDS = CHROMA_SIZE / 2 / ANALYSIS_RATE; /** The unit-length mean chroma from one time to another, in seconds, leaving out frames that already hear the next onset. */ function chromaBetween({ chroma, chromaFps }: BandFrames, from: number, to: number): number[] { const middle = Math.round(((from + to) / 2) * chromaFps); let first = Math.ceil(from * chromaFps); let end = Math.floor((to - CHROMA_REACH_SECONDS) * chromaFps) + 1; if (end <= first) [first, end] = [middle, middle + 1]; const sum = new Array(12).fill(0); for (let f = first; f < end; f++) for (let c = 0; c < 12; c++) sum[c]! += chroma[f]?.[c] ?? 0; const length = Math.hypot(...sum) || 1; return sum.map((v) => v / length); } const cosine = (a: number[], b: number[]) => a.reduce((sum, v, i) => sum + v * b[i]!, 0); /** How much the harmony changes at each beat: the chroma `span` beats before it against the `span` after. */ function harmonicChange(beats: number[], fps: number, frames: BandFrames, span: number): number[] { return beats.map((f, k) => { if (k < span || k + span >= beats.length) return 0; const before = chromaBetween(frames, beats[k - span]! / fps, f / fps); return 1 - cosine(before, chromaBetween(frames, f / fps, beats[k + span]! / fps)); }); } /** Each beat's evidence of starting a bar: a low-end hit, a chord change, and a chord change over half a bar. */ function barStartEvidence(beats: number[], curves: OnsetCurves, frames: BandFrames): number[] { const lowHit = (f: number) => Math.max(...[-1, 0, 1, 2].map((d) => curves.low[f + d] ?? 0)); const cues = [ zScored(beats.map(lowHit)), zScored(harmonicChange(beats, curves.fps, frames, 1)), zScored(harmonicChange(beats, curves.fps, frames, 2)), ]; return beats.map((_, k) => cues.reduce((sum, cue) => sum + cue[k]!, 0)); } /** Most music is in four, so three must win by a clear margin. */ const TRIPLE_METER_HANDICAP = 0.7; function barPhase(beats: number[], curves: OnsetCurves, frames: BandFrames) { const evidence = barStartEvidence(beats, curves, frames); let best = { meter: 4, phase: 0, contrast: -Infinity }; for (const meter of [4, 3]) for (let phase = 0; phase < meter; phase++) { const on = evidence.filter((_, k) => (k - phase) % meter === 0); const off = evidence.filter((_, k) => (k - phase) % meter !== 0); const contrast = (mean(on) - mean(off)) * (meter === 3 ? TRIPLE_METER_HANDICAP : 1); if (contrast > best.contrast) best = { meter, phase, contrast }; } return best; } function downbeatsOf( beatList: number[], times: number[], curves: OnsetCurves, frames: BandFrames, ): Downbeats { const bar = beatList.length >= 6 ? barPhase(beatList, curves, frames) : null; return { times: bar ? times.filter((_, k) => (k - bar.phase) % bar.meter === 0) : [], meter: bar?.meter ?? null, confidence: "low", }; } export function medianInterval(times: readonly number[]): number | null { const gaps = times .slice(1) .map((t, i) => t - times[i]!) .sort((a, b) => a - b); return gaps.length ? gaps[gaps.length >> 1]! : null; } const noRhythm = (confidence: number, reason = "no steady beat"): Rhythm => ({ beats: { times: [], confidence, reason }, downbeats: { times: [], meter: null, confidence: "low" }, }); /** Two bars of 4/4 at 80 BPM: anything shorter has too few beats to call steady. */ const SHORTEST_SECONDS = 6; /** Below this the beats are more likely the tracker's own grid than the music's. */ export const STEADY_BEAT_CONFIDENCE = 0.15; /** Clarity of the period, scaled down when the beats are under three times as strong as the onsets around them. */ const beatConfidence = (clarity: number, lift: number) => round(Math.max(0, Math.min(1, clarity)) * Math.max(0, Math.min(1, lift / 3))); /** Beats and downbeats of `frames`, a signal's band energies and chroma. */ function rhythmOf(frames: BandFrames): Rhythm { const curves = onsetCurves(frames); const { period, clarity } = beatPeriod(curves.broad, curves.fps); if (curves.broad.length < SHORTEST_SECONDS * curves.fps) return noRhythm(0, "too short for a steady beat"); if (!curves.broad.some((v) => v > 0) || curves.broad.length < 2 * period) return noRhythm(0); const { frames: beatList, local } = beatFrames(curves.broad, curves.fps, period); const confidence = beatConfidence(clarity, onsetLift(beatList, local)); if (beatList.length < 4 || confidence < STEADY_BEAT_CONFIDENCE) return noRhythm(confidence); const times = beatList.map((f) => round(f / curves.fps + ONSET_LAG_SECONDS)); return { beats: { times, confidence }, downbeats: downbeatsOf(beatList, times, curves, frames) }; } const TEMPO_STEP_SECONDS = 4; function steadyInterval(times: readonly number[]): number | null { const median = medianInterval(times); if (!median) return null; const near = times .slice(1) .map((t, i) => t - times[i]!) .filter((gap) => Math.abs(gap - median) <= 0.1 * median); return near.reduce((a, b) => a + b, 0) / near.length; } /** Tempo from beat times, by whichever provider placed them: the steady interval, and a curve over ±4 s windows. */ export function tempoOf(times: readonly number[]): Tempo { const interval = steadyInterval(times); if (!interval) return { value: null, curve: [] }; const bpm = (gap: number) => Math.round(600 / gap) / 10; const curve: TempoPoint[] = []; const last = times[times.length - 1]!; for (let t = times[0]!; t <= last; t += TEMPO_STEP_SECONDS) { const gap = steadyInterval(times.filter((b) => Math.abs(b - t) <= TEMPO_STEP_SECONDS)); if (gap) curve.push({ t: round(t), bpm: bpm(gap) }); } return { value: bpm(interval), curve }; } const measured = new WeakMap(); /** Band energies of `signal`, computed once per signal for every capability that reads them. */ export function bandFramesOf(signal: AudioSignal): BandFrames { let entry = measured.get(signal); if (!entry) measured.set(signal, (entry = { frames: bandFrames(signal) })); return entry.frames; } export function rhythm(signal: AudioSignal): Rhythm { const frames = bandFramesOf(signal); const entry = measured.get(signal)!; return (entry.rhythm ??= rhythmOf(frames)); }