feat(core): signal rhythm is rate-independent, refuses grids in the tracker, follows chords (#5644)

* fix(core): signal beats read the same at any decode rate and refuse a grid in the tracker

The tracker resamples every channel to 22.05 kHz and averages their band power, so the
CLI's native-rate decode and Studio's AudioContext decode get the same beats, and a mono
file and the same sound on two channels get identical ones.

The no-steady-beat floor moves from analyzeMusicFromSignal into the tracker, so
hyperframes audio, grep_audio and beats share one floor. It is 0.15: above every one of
66 synthetic ambient, speech, scattered-effect and noise clips (max 0.122), where the
wrapper's 0.2 also refused a third of GTZAN's produced music.

* feat(core): signal downbeats follow chord changes, and sections hear harmony

The tracker now keeps a 10 fps chroma beside its band energies. Downbeats weigh where the
chords change, over one beat and over two, with the low-end hit, instead of the low end
alone. Sections compare timbre and harmony equally.

Tuned on GTZAN's dev half: the tempo prior narrows to 0.8 octave, the local tempo stays
within 10% of the global one, and beats sit 15 ms after the onset peak, where annotators
put them.

* chore(core): trim redundant audio analysis comments

* refactor(core): keep the rhythm files under their comment share

Drops doc comments whose function or field name already says the same thing.

packages/core/src/audioResample.test.ts

@@ -0,0 +1,35 @@
+import { describe, expect, it } from "vitest";
+import { resample } from "./audioResample";
+
+const tone = (hz: number, rate: number, seconds: number) =>
+  Float32Array.from({ length: seconds * rate }, (_, i) => Math.sin((2 * Math.PI * hz * i) / rate));
+
+const rms = (x: Float32Array) => Math.sqrt(x.reduce((a, v) => a + v * v, 0) / x.length);
+
+/** Upward zero crossings per second, over the middle of the signal. */
+function frequency(x: Float32Array, rate: number): number {
+  const from = Math.floor(x.length / 4);
+  const to = Math.floor((3 * x.length) / 4);
+  let crossings = 0;
+  for (let i = from + 1; i < to; i++) if (x[i - 1]! < 0 && x[i]! >= 0) crossings++;
+  return (crossings * rate) / (to - from);
+}
+
+describe("resample", () => {
+  it("keeps a tone's pitch and level from 48 kHz to 22.05 kHz", () => {
+    const out = resample(tone(1000, 48_000, 2), 48_000, 22_050);
+    expect(out.length).toBe(44_100);
+    expect(frequency(out, 22_050)).toBeCloseTo(1000, -1);
+    expect(rms(out.subarray(1000, -1000))).toBeCloseTo(Math.SQRT1_2, 2);
+  });
+
+  it("removes what the lower rate cannot hold instead of folding it down", () => {
+    const out = resample(tone(15_000, 48_000, 1), 48_000, 22_050);
+    expect(rms(out.subarray(1000, -1000))).toBeLessThan(0.01);
+  });
+
+  it("returns the samples untouched at the same rate", () => {
+    const x = tone(440, 22_050, 1);
+    expect(resample(x, 22_050, 22_050)).toBe(x);
+  });
+});

packages/core/src/audioResample.ts

@@ -0,0 +1,38 @@
+/** Band-limited resampling by windowed-sinc interpolation, so analysis reads the same at any decode rate. */
+
+const ZERO_CROSSINGS = 8;
+const STEPS_PER_CROSSING = 256;
+/** Cutoff as a share of the lower Nyquist, leaving the window's transition band below it. */
+const CUTOFF = 0.95;
+
+const KERNEL = Float64Array.from({ length: ZERO_CROSSINGS * STEPS_PER_CROSSING + 2 }, (_, i) => {
+  const x = i / STEPS_PER_CROSSING;
+  if (x >= ZERO_CROSSINGS) return 0;
+  const sinc = x === 0 ? 1 : Math.sin(Math.PI * x) / (Math.PI * x);
+  return sinc * (0.5 + 0.5 * Math.cos((Math.PI * x) / ZERO_CROSSINGS));
+});
+
+function kernelAt(x: number): number {
+  const at = Math.abs(x) * STEPS_PER_CROSSING;
+  const i = Math.floor(at);
+  if (i + 1 >= KERNEL.length) return 0;
+  const frac = at - i;
+  return KERNEL[i]! * (1 - frac) + KERNEL[i + 1]! * frac;
+}
+
+export function resample(samples: Float32Array, from: number, to: number): Float32Array {
+  if (from === to) return samples;
+  const ratio = to / from;
+  const scale = Math.min(1, ratio) * CUTOFF;
+  const reach = ZERO_CROSSINGS / scale;
+  const out = new Float32Array(Math.floor(samples.length * ratio));
+  for (let n = 0; n < out.length; n++) {
+    const t = n / ratio;
+    const last = Math.min(samples.length - 1, Math.floor(t + reach));
+    let sum = 0;
+    for (let k = Math.max(0, Math.ceil(t - reach)); k <= last; k++)
+      sum += samples[k]! * kernelAt((t - k) * scale);
+    out[n] = sum * scale;
+  }
+  return out;
+}

packages/core/src/audioRhythm.test.ts

@@ -15,14 +15,14 @@ function random(seed = 3) {
 }
 
 /** A drum pattern on a known grid: a kick with a low thump on each bar's first beat, a click on the others. */
-function drums(beatTimes: number[], meter: number, seconds: number): AudioSignal {
-  const samples = new Float32Array(Math.round(seconds * RATE));
+function drums(beatTimes: number[], meter: number, seconds: number, rate = RATE): AudioSignal {
+  const samples = new Float32Array(Math.round(seconds * rate));
   const noise = random();
   beatTimes.forEach((t, k) => {
     const downbeat = k % meter === 0;
-    const start = Math.round(t * RATE);
-    for (let i = 0; i < 0.12 * RATE && start + i < samples.length; i++) {
-      const since = i / RATE;
+    const start = Math.round(t * rate);
+    for (let i = 0; i < 0.12 * rate && start + i < samples.length; i++) {
+      const since = i / rate;
       const click = (downbeat ? 0.5 : 0.35) * noise() * Math.exp(-since / 0.02);
       const thump = downbeat
         ? 0.8 * Math.sin(2 * Math.PI * 55 * since) * Math.exp(-since / 0.06)
@@ -30,6 +30,31 @@ function drums(beatTimes: number[], meter: number, seconds: number): AudioSignal
       samples[start + i]! += click + thump;
     }
   });
+  return { sampleRate: rate, channels: [samples] };
+}
+
+/** The same hit on every beat, over chords that change every four beats from beat `firstChange`. */
+function chordsOverEvenHits(
+  beatTimes: number[],
+  firstChange: number,
+  seconds: number,
+): AudioSignal {
+  const samples = new Float32Array(Math.round(seconds * RATE));
+  const noise = random(7);
+  const roots = [220, 175, 262, 196];
+  beatTimes.forEach((t) => {
+    const start = Math.round(t * RATE);
+    for (let i = 0; i < 0.08 * RATE && start + i < samples.length; i++)
+      samples[start + i]! += 0.4 * noise() * Math.exp(-i / RATE / 0.02);
+  });
+  const period = beatTimes[1]! - beatTimes[0]!;
+  for (let i = 0; i < samples.length; i++) {
+    const t = i / RATE;
+    const bar = Math.floor((t - beatTimes[firstChange]!) / (4 * period));
+    const root = roots[((bar % 4) + 4) % 4]!;
+    for (const ratio of [1, 1.26, 1.5])
+      samples[i]! += 0.08 * Math.sin(2 * Math.PI * root * ratio * t);
+  }
   return { sampleRate: RATE, channels: [samples] };
 }
 
@@ -66,6 +91,23 @@ describe("rhythm", () => {
     expect(curve[curve.length - 1]!.bpm).toBeCloseTo(133, -1);
   });
 
+  it("gives the same beats at any sample rate and for mono or the same sound on two channels", () => {
+    const truth = grid(0.25, 0.5, 19.5);
+    const reference = rhythm(drums(truth, 4, 20)).beats.times;
+    const wide = drums(truth, 4, 20, 44_100).channels[0]!;
+    const stereo = rhythm({ sampleRate: 44_100, channels: [wide, wide.slice()] }).beats.times;
+    expect(stereo).toHaveLength(reference.length);
+    stereo.forEach((t, k) => expect(Math.abs(t - reference[k]!)).toBeLessThanOrEqual(0.011));
+  });
+
+  it("puts the bar where the chords change when every beat hits the same", () => {
+    const truth = grid(0.25, 0.5, 23.5);
+    const { downbeats } = rhythm(chordsOverEvenHits(truth, 1, 24));
+    expect(downbeats.meter).toBe(4);
+    const changes = truth.filter((_, k) => k % 4 === 1);
+    expect(recall(downbeats.times, changes)).toBeGreaterThan(0.9);
+  });
+
   it("finds no steady beat in silence or in noise, and none in a clip too short to tell", () => {
     const silent = rhythm({ sampleRate: RATE, channels: [new Float32Array(10 * RATE)] });
     expect(silent.beats).toMatchObject({ times: [], reason: "no steady beat" });
@@ -75,6 +117,7 @@ describe("rhythm", () => {
       channels: [Float32Array.from({ length: 10 * RATE }, () => 0.3 * noise())],
     });
     expect(hiss.beats.times).toEqual([]);
+    expect(hiss.downbeats.times).toEqual([]);
     const short = rhythm(drums(grid(0.25, 0.5, 3), 4, 3));
     expect(short.beats).toMatchObject({ times: [], reason: "too short for a steady beat" });
   });

packages/core/src/audioRhythm.ts

@@ -4,6 +4,7 @@
  */
 
 import type { AudioSignal } from "./audioAnalysis";
+import { resample } from "./audioResample";
 import { fft } from "./fft";
 
 export interface Beats {
@@ -18,7 +19,7 @@ export interface Downbeats {
   times: number[];
   /** Beats per bar, 3 or 4; null when there are too few beats to tell. */
   meter: number | null;
-  /** "low" for a guess from where the low end hits hardest. */
+  /** "low": a guess from where the harmony changes and the low end hits. */
   confidence?: "low";
 }
 
@@ -39,28 +40,39 @@ export interface Rhythm {
   downbeats: Downbeats;
 }
 
-/** Log energy per frame in log-spaced bands, the frames `fps` per second. */
+/** Log energy per frame in log-spaced bands, `fps` frames per second, and a slower pitch-class profile. */
 export interface BandFrames {
   fps: number;
   bands: Float64Array[];
+  chroma: Float64Array[];
+  chromaFps: number;
 }
 
+/** Every signal is analyzed at this rate and as the mean of its channels' power, so decode rate and layout cannot change a result. */
+const ANALYSIS_RATE = 22050;
+const FRAME_SIZE = 1024;
+const CHROMA_SIZE = 4096;
+const CHROMA_FPS = 10;
 const FRAMES_PER_SECOND = 100;
 const LOW_BAND_HZ = 150;
 const SLOWEST_BPM = 40;
 const FASTEST_BPM = 240;
 const TIGHTNESS = 100;
+/** The onset curve peaks this much ahead of where listeners place the beat, so beats move this much later. */
+const ONSET_LAG_SECONDS = 0.015;
 
 const round = (t: number) => Math.round(t * 1000) / 1000;
 
-function bandEdges(sampleRate: number, size: number): number[] {
-  const binHz = sampleRate / size;
+function bandEdges(): number[] {
+  const binHz = ANALYSIS_RATE / FRAME_SIZE;
   const edges: number[] = [];
-  for (let hz = 30; hz < Math.min(9000, sampleRate / 2); hz *= 2 ** (1 / 5))
-    edges.push(Math.max(1, Math.round(hz / binHz)));
+  for (let hz = 30; hz < 9000; hz *= 2 ** (1 / 5)) edges.push(Math.max(1, Math.round(hz / binHz)));
   return edges;
 }
 
+const hann = (size: number) =>
+  Float64Array.from({ length: size }, (_, i) => 0.5 - 0.5 * Math.cos((2 * Math.PI * i) / size));
+
 /** Add each band's power in the Hann-windowed frame centred on `center` into `into`. */
 function addFramePower(
   samples: Float32Array,
@@ -86,24 +98,61 @@ function addFramePower(
   }
 }
 
-/** Per-band log power, summed over channels so a polarity-flipped channel cannot cancel. */
-export function bandFrames(signal: AudioSignal): BandFrames {
-  const { sampleRate, channels } = signal;
-  const hop = Math.max(1, Math.round(sampleRate / FRAMES_PER_SECOND));
-  const size = 2 ** Math.round(Math.log2(sampleRate * 0.046));
-  const window = Float64Array.from(
-    { length: size },
-    (_, i) => 0.5 - 0.5 * Math.cos((2 * Math.PI * i) / size),
-  );
-  const edges = bandEdges(sampleRate, size);
-  const length = channels[0]?.length ?? 0;
+/** Per-band log power of every frame, averaged over channels so a polarity-flipped channel cannot cancel. */
+function bandPower(channels: Float32Array[]): Float64Array[] {
+  const window = hann(FRAME_SIZE);
+  const edges = bandEdges();
+  const hop = ANALYSIS_RATE / FRAMES_PER_SECOND;
   const bands: Float64Array[] = [];
-  for (let center = 0; center < length; center += hop) {
-    const power = new Float64Array(Math.max(0, edges.length - 1));
-    for (const channel of channels) addFramePower(channel, center, window, edges, power);
-    bands.push(power.map((p) => Math.log1p(1e9 * p)));
+  for (let f = 0; Math.round(f * hop) < (channels[0]?.length ?? 0); f++) {
+    const power = new Float64Array(edges.length - 1);
+    for (const channel of channels)
+      addFramePower(channel, Math.round(f * hop), window, edges, power);
+    bands.push(power.map((p) => Math.log1p((1e9 * p) / channels.length)));
   }
-  return { fps: sampleRate / hop, bands };
+  return bands;
+}
+
+/** Index of the pitch class, A = 9, of each FFT bin from 55 Hz to 2 kHz; -1 outside that range. */
+const PITCH_CLASS = Array.from({ length: CHROMA_SIZE / 2 }, (_, bin) => {
+  const hz = (bin * ANALYSIS_RATE) / CHROMA_SIZE;
+  if (hz < 55 || hz > 2000) return -1;
+  return (((Math.round(12 * Math.log2(hz / 440)) + 9) % 12) + 12) % 12;
+});
+
+function chromaFrames(channels: Float32Array[]): Float64Array[] {
+  const window = hann(CHROMA_SIZE);
+  const hop = ANALYSIS_RATE / CHROMA_FPS;
+  const re = new Float64Array(CHROMA_SIZE);
+  const im = new Float64Array(CHROMA_SIZE);
+  const frames: Float64Array[] = [];
+  for (let f = 0; Math.round(f * hop) < (channels[0]?.length ?? 0); f++) {
+    const chroma = new Float64Array(12);
+    const first = Math.round(f * hop) - CHROMA_SIZE / 2;
+    for (const channel of channels) {
+      for (let i = 0; i < CHROMA_SIZE; i++) {
+        const at = first + i;
+        re[i] = at >= 0 && at < channel.length ? channel[at]! * window[i]! : 0;
+        im[i] = 0;
+      }
+      fft(re, im);
+      PITCH_CLASS.forEach((pc, bin) => {
+        if (pc >= 0) chroma[pc]! += Math.hypot(re[bin]!, im[bin]!);
+      });
+    }
+    frames.push(chroma);
+  }
+  return frames;
+}
+
+function bandFrames(signal: AudioSignal): BandFrames {
+  const channels = signal.channels.map((c) => resample(c, signal.sampleRate, ANALYSIS_RATE));
+  return {
+    fps: FRAMES_PER_SECOND,
+    bands: bandPower(channels),
+    chroma: chromaFrames(channels),
+    chromaFps: CHROMA_FPS,
+  };
 }
 
 /** Subtract the mean of the surrounding ±`radius` frames and keep what rises above it. */
@@ -131,10 +180,9 @@ interface OnsetCurves {
   low: Float64Array;
 }
 
-function onsetCurves({ fps, bands }: BandFrames, sampleRate: number): OnsetCurves {
-  const size = 2 ** Math.round(Math.log2(sampleRate * 0.046));
-  const lowBands = bandEdges(sampleRate, size).filter(
-    (bin) => (bin * sampleRate) / size < LOW_BAND_HZ,
+function onsetCurves({ fps, bands }: BandFrames): OnsetCurves {
+  const lowBands = bandEdges().filter(
+    (bin) => (bin * ANALYSIS_RATE) / FRAME_SIZE < LOW_BAND_HZ,
   ).length;
   const broad = new Float64Array(bands.length);
   const low = new Float64Array(bands.length);
@@ -170,7 +218,7 @@ function autocorrelation(curve: Float64Array, maxLag: number): Float64Array {
 }
 
 const tempoPrior = (lag: number, fps: number) =>
-  Math.exp(-0.5 * Math.log2(60 / (lag / fps) / 120) ** 2);
+  Math.exp(-0.5 * (Math.log2(60 / (lag / fps) / 120) / 0.8) ** 2);
 
 interface LagRange {
   shortest: number;
@@ -256,13 +304,13 @@ function trimmed(beats: number[], local: Float64Array): number[] {
 
 const LOCAL_WINDOW_SECONDS = 8;
 
-/** The period through the piece: eight-second windows a second apart, held near the global period, median-smoothed. */
+/** The period through the piece: eight-second windows a second apart, held within 10% of the global period, median-smoothed. */
 function localPeriods(curve: Float64Array, fps: number, period: number): (frame: number) => number {
   const window = Math.round(LOCAL_WINDOW_SECONDS * fps);
   const hop = Math.round(fps);
   const range = {
-    shortest: Math.max(fullRange(fps).shortest, Math.floor(period * 0.7)),
-    longest: Math.min(fullRange(fps).longest, Math.ceil(period * 1.4)),
+    shortest: Math.max(fullRange(fps).shortest, Math.floor(period * 0.9)),
+    longest: Math.min(fullRange(fps).longest, Math.ceil(period * 1.1)),
   };
   const centres: number[] = [];
   const periods: number[] = [];
@@ -303,25 +351,81 @@ function onsetLift(frames: number[], local: Float64Array): number {
   return overall > 0 ? onBeats / overall - 1 : 0;
 }
 
-/** The meter and bar phase whose beats carry the most low-band onset energy against the others. */
-function barPhase(beats: number[], curves: OnsetCurves): { meter: number; phase: number } {
-  const strength = (f: number) =>
-    Math.max(
-      ...[-1, 0, 1, 2].map((d) => (curves.low[f + d] ?? 0) + 0.3 * (curves.broad[f + d] ?? 0)),
-    );
-  const levels = beats.map(strength);
+const mean = (values: readonly number[]) =>
+  values.reduce((a, b) => a + b, 0) / (values.length || 1);
+
+function zScored(values: number[]): number[] {
+  const centre = mean(values);
+  const spread = Math.sqrt(mean(values.map((v) => (v - centre) ** 2))) || 1;
+  return values.map((v) => (v - centre) / spread);
+}
+
+const CHROMA_REACH_SECONDS = CHROMA_SIZE / 2 / ANALYSIS_RATE;
+
+/** The unit-length mean chroma from one time to another, in seconds, leaving out frames that already hear the next onset. */
+function chromaBetween({ chroma, chromaFps }: BandFrames, from: number, to: number): number[] {
+  const middle = Math.round(((from + to) / 2) * chromaFps);
+  let first = Math.ceil(from * chromaFps);
+  let end = Math.floor((to - CHROMA_REACH_SECONDS) * chromaFps) + 1;
+  if (end <= first) [first, end] = [middle, middle + 1];
+  const sum = new Array<number>(12).fill(0);
+  for (let f = first; f < end; f++) for (let c = 0; c < 12; c++) sum[c]! += chroma[f]?.[c] ?? 0;
+  const length = Math.hypot(...sum) || 1;
+  return sum.map((v) => v / length);
+}
+
+const cosine = (a: number[], b: number[]) => a.reduce((sum, v, i) => sum + v * b[i]!, 0);
+
+/** How much the harmony changes at each beat: the chroma `span` beats before it against the `span` after. */
+function harmonicChange(beats: number[], fps: number, frames: BandFrames, span: number): number[] {
+  return beats.map((f, k) => {
+    if (k < span || k + span >= beats.length) return 0;
+    const before = chromaBetween(frames, beats[k - span]! / fps, f / fps);
+    return 1 - cosine(before, chromaBetween(frames, f / fps, beats[k + span]! / fps));
+  });
+}
+
+/** Each beat's evidence of starting a bar: a low-end hit, a chord change, and a chord change over half a bar. */
+function barStartEvidence(beats: number[], curves: OnsetCurves, frames: BandFrames): number[] {
+  const lowHit = (f: number) => Math.max(...[-1, 0, 1, 2].map((d) => curves.low[f + d] ?? 0));
+  const cues = [
+    zScored(beats.map(lowHit)),
+    zScored(harmonicChange(beats, curves.fps, frames, 1)),
+    zScored(harmonicChange(beats, curves.fps, frames, 2)),
+  ];
+  return beats.map((_, k) => cues.reduce((sum, cue) => sum + cue[k]!, 0));
+}
+
+/** Most music is in four, so three must win by a clear margin. */
+const TRIPLE_METER_HANDICAP = 0.7;
+
+function barPhase(beats: number[], curves: OnsetCurves, frames: BandFrames) {
+  const evidence = barStartEvidence(beats, curves, frames);
   let best = { meter: 4, phase: 0, contrast: -Infinity };
   for (const meter of [4, 3])
     for (let phase = 0; phase < meter; phase++) {
-      const on = levels.filter((_, k) => (k - phase) % meter === 0);
-      const off = levels.filter((_, k) => (k - phase) % meter !== 0);
-      const mean = (v: number[]) => v.reduce((a, b) => a + b, 0) / (v.length || 1);
-      const contrast = (mean(on) - mean(off)) * (meter === 3 ? 0.85 : 1);
+      const on = evidence.filter((_, k) => (k - phase) % meter === 0);
+      const off = evidence.filter((_, k) => (k - phase) % meter !== 0);
+      const contrast = (mean(on) - mean(off)) * (meter === 3 ? TRIPLE_METER_HANDICAP : 1);
       if (contrast > best.contrast) best = { meter, phase, contrast };
     }
   return best;
 }
 
+function downbeatsOf(
+  beatList: number[],
+  times: number[],
+  curves: OnsetCurves,
+  frames: BandFrames,
+): Downbeats {
+  const bar = beatList.length >= 6 ? barPhase(beatList, curves, frames) : null;
+  return {
+    times: bar ? times.filter((_, k) => (k - bar.phase) % bar.meter === 0) : [],
+    meter: bar?.meter ?? null,
+    confidence: "low",
+  };
+}
+
 export function medianInterval(times: readonly number[]): number | null {
   const gaps = times
     .slice(1)
@@ -339,15 +443,15 @@ const noRhythm = (confidence: number, reason = "no steady beat"): Rhythm => ({
 const SHORTEST_SECONDS = 6;
 
 /** Below this the beats are more likely the tracker's own grid than the music's. */
-export const STEADY_BEAT_CONFIDENCE = 0.05;
+export const STEADY_BEAT_CONFIDENCE = 0.15;
 
 /** Clarity of the period, scaled down when the beats are under three times as strong as the onsets around them. */
 const beatConfidence = (clarity: number, lift: number) =>
   round(Math.max(0, Math.min(1, clarity)) * Math.max(0, Math.min(1, lift / 3)));
 
-/** Beats and downbeats of `frames`, the band energies of a signal at `sampleRate`. */
-export function rhythmOf(frames: BandFrames, sampleRate: number): Rhythm {
-  const curves = onsetCurves(frames, sampleRate);
+/** Beats and downbeats of `frames`, a signal's band energies and chroma. */
+function rhythmOf(frames: BandFrames): Rhythm {
+  const curves = onsetCurves(frames);
   const { period, clarity } = beatPeriod(curves.broad, curves.fps);
   if (curves.broad.length < SHORTEST_SECONDS * curves.fps)
     return noRhythm(0, "too short for a steady beat");
@@ -355,16 +459,8 @@ export function rhythmOf(frames: BandFrames, sampleRate: number): Rhythm {
   const { frames: beatList, local } = beatFrames(curves.broad, curves.fps, period);
   const confidence = beatConfidence(clarity, onsetLift(beatList, local));
   if (beatList.length < 4 || confidence < STEADY_BEAT_CONFIDENCE) return noRhythm(confidence);
-  const times = beatList.map((f) => round(f / curves.fps));
-  const bar = beatList.length >= 6 ? barPhase(beatList, curves) : null;
-  return {
-    beats: { times, confidence },
-    downbeats: {
-      times: bar ? times.filter((_, k) => (k - bar.phase) % bar.meter === 0) : [],
-      meter: bar?.meter ?? null,
-      confidence: "low",
-    },
-  };
+  const times = beatList.map((f) => round(f / curves.fps + ONSET_LAG_SECONDS));
+  return { beats: { times, confidence }, downbeats: downbeatsOf(beatList, times, curves, frames) };
 }
 
 const TEMPO_STEP_SECONDS = 4;
@@ -405,5 +501,5 @@ export function bandFramesOf(signal: AudioSignal): BandFrames {
 export function rhythm(signal: AudioSignal): Rhythm {
   const frames = bandFramesOf(signal);
   const entry = measured.get(signal)!;
-  return (entry.rhythm ??= rhythmOf(frames, signal.sampleRate));
+  return (entry.rhythm ??= rhythmOf(frames));
 }

packages/core/src/audioSections.ts

@@ -1,5 +1,5 @@
 /**
- * Sections from signal processing: band energies averaged into half-second blocks, a self-similarity matrix, and
+ * Sections from signal processing: band energies and chroma averaged into half-second blocks, a self-similarity matrix, and
  * Foote's checkerboard novelty, whose peaks are where the music changes. Boundaries snap to the nearest beat.
  */
 
@@ -47,14 +47,25 @@ function standardizeBands(blocks: number[][]): void {
   }
 }
 
-/** Band energies averaged over blocks, each band standardized over time and each block scaled to unit length. */
-function blockFeatures({ fps, bands }: BandFrames): number[][] {
+/** Block means of the band energies and of the log chroma, each part unit length within a block. */
+function blockParts({ fps, bands, chroma, chromaFps }: BandFrames): number[][] {
   const per = Math.max(1, Math.round(BLOCK_SECONDS * fps));
+  const chromaPer = Math.max(1, Math.round(BLOCK_SECONDS * chromaFps));
   const blocks: number[][] = [];
-  for (let start = 0; start + per <= bands.length; start += per)
-    blocks.push(meanOf(bands.slice(start, start + per)));
+  for (let start = 0, b = 0; start + per <= bands.length; start += per, b++) {
+    const pitch = meanOf(chroma.slice(b * chromaPer, (b + 1) * chromaPer)).map(Math.log1p);
+    blocks.push([...meanOf(bands.slice(start, start + per)), ...unit(pitch)]);
+  }
+  return blocks;
+}
+
+function blockFeatures(frames: BandFrames): number[][] {
+  const blocks = blockParts(frames);
   standardizeBands(blocks);
-  return blocks.map(unit);
+  const split = frames.bands[0]?.length ?? 0;
+  return blocks.map((block) =>
+    [...unit(block.slice(0, split)), ...unit(block.slice(split))].map((v) => v * Math.SQRT1_2),
+  );
 }
 
 function unit(vector: number[]): number[] {

packages/core/src/beats/beatDetection.test.ts

@@ -1,6 +1,7 @@
 import { describe, expect, it, vi } from "vitest";
 
-import type { AudioSignal } from "../audioAnalysis";
+import { signalProvider, type AudioSignal } from "../audioAnalysis";
+import { STEADY_BEAT_CONFIDENCE, rhythm } from "../audioRhythm";
 import { analyzeMusicFromBuffer, analyzeMusicFromSignal } from "./beatDetection";
 
 const RATE = 22_050;
@@ -105,6 +106,17 @@ describe("analyzeMusicFromSignal", () => {
     }
   });
 
+  it("refuses an ambient pad in the tracker itself, so every caller of the signal tracker agrees", async () => {
+    const ambient = mono(pad(30));
+    const { beats, downbeats } = rhythm(ambient);
+    expect(beats).toMatchObject({ times: [], reason: "no steady beat" });
+    expect(beats.confidence).toBeLessThan(STEADY_BEAT_CONFIDENCE);
+    expect(downbeats.times).toEqual([]);
+    await expect(signalProvider.analyze("beats", ambient, {})).resolves.toMatchObject({
+      times: [],
+    });
+  });
+
   it("waits on the caller's pause before each long stage", async () => {
     const samples = new Float32Array(8 * RATE);
     clicks(samples, grid(0.25, 0.5, 7.5), 0.6);

packages/core/src/beats/beatDetection.ts

@@ -1,5 +1,5 @@
 import { energyMix, type AudioSignal } from "../audioAnalysis";
-import { bandFrames, rhythmOf, tempoOf } from "../audioRhythm";
+import { bandFramesOf, rhythm, tempoOf } from "../audioRhythm";
 
 export interface MusicBeatAnalysis {
   beatTimes: number[];
@@ -40,8 +40,6 @@ export function strengthAtTime(
   return Math.min(1, computeRmsAt(analysis.channelData, analysis.sampleRate, time) / analysis.peak);
 }
 
-/** Below this tracker confidence the grid is more likely the tracker's than the music's, so none is reported. */
-const MIN_CONFIDENCE = 0.2;
 /** At or above this the tempo is reported as "high" confidence. */
 const HIGH_CONFIDENCE = 0.35;
 
@@ -62,13 +60,13 @@ export async function analyzeMusicFromSignal(
   { pause }: MusicAnalysisOptions = {},
 ): Promise<MusicBeatAnalysis> {
   await pause?.();
-  const frames = bandFrames(signal);
+  bandFramesOf(signal);
   await pause?.();
-  const { beats } = rhythmOf(frames, signal.sampleRate);
+  const { beats } = rhythm(signal);
   await pause?.();
   const channelData = energyMix(signal);
   const confidence = beats.confidence ?? 0;
-  const times = confidence >= MIN_CONFIDENCE ? beats.times : [];
+  const times = beats.times;
   const { strengths, peak } = beatStrengths(times, channelData, signal.sampleRate);
   const bpm = tempoOf(times).value;
   return {