Files
hyperframes/packages/core/src/audioSections.ts
Xuanru Li 695110e46b feat(core): signal rhythm is rate-independent, refuses grids in the tracker, follows chords (#5644)
* fix(core): signal beats read the same at any decode rate and refuse a grid in the tracker

The tracker resamples every channel to 22.05 kHz and averages their band power, so the
CLI's native-rate decode and Studio's AudioContext decode get the same beats, and a mono
file and the same sound on two channels get identical ones.

The no-steady-beat floor moves from analyzeMusicFromSignal into the tracker, so
hyperframes audio, grep_audio and beats share one floor. It is 0.15: above every one of
66 synthetic ambient, speech, scattered-effect and noise clips (max 0.122), where the
wrapper's 0.2 also refused a third of GTZAN's produced music.

* feat(core): signal downbeats follow chord changes, and sections hear harmony

The tracker now keeps a 10 fps chroma beside its band energies. Downbeats weigh where the
chords change, over one beat and over two, with the low-end hit, instead of the low end
alone. Sections compare timbre and harmony equally.

Tuned on GTZAN's dev half: the tempo prior narrows to 0.8 octave, the local tempo stays
within 10% of the global one, and beats sit 15 ms after the onset peak, where annotators
put them.

* chore(core): trim redundant audio analysis comments

* refactor(core): keep the rhythm files under their comment share

Drops doc comments whose function or field name already says the same thing.
2026-10-15 07:31:04 +00:00

151 lines
6.2 KiB
TypeScript
Raw Permalink Blame History

This file contains ambiguous Unicode characters
This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
/**
* Sections from signal processing: band energies and chroma averaged into half-second blocks, a self-similarity matrix, and
* Foote's checkerboard novelty, whose peaks are where the music changes. Boundaries snap to the nearest beat.
*/
import type { AudioSignal } from "./audioAnalysis";
import { bandFramesOf, medianInterval, rhythm, type BandFrames } from "./audioRhythm";
import { snapper, type SnapReport } from "./audioSnap";
export interface Section {
start: number;
end: number;
/** Sections that sound alike share a letter. */
label: string;
}
export interface Sections {
spans: Section[];
snap?: SnapReport;
}
export interface SectionOptions {
/** Shortest section, in seconds. */
minSeconds?: number;
}
const BLOCK_SECONDS = 0.5;
/** Checkerboard half-width: eight seconds of music either side of a candidate boundary. */
const KERNEL_BLOCKS = 16;
const SAME_SECTION = 0.9;
const round = (t: number) => Math.round(t * 1000) / 1000;
function meanOf(frames: ArrayLike<number>[]): number[] {
const sum = new Array<number>(frames[0]?.length ?? 0).fill(0);
for (const frame of frames) for (let b = 0; b < sum.length; b++) sum[b]! += frame[b]!;
return sum.map((v) => v / (frames.length || 1));
}
/** Each band moved to zero mean and unit spread over time, in place. */
function standardizeBands(blocks: number[][]): void {
for (let b = 0; b < (blocks[0]?.length ?? 0); b++) {
const column = blocks.map((block) => block[b]!);
const mean = column.reduce((a, v) => a + v, 0) / column.length;
const spread = Math.sqrt(column.reduce((a, v) => a + (v - mean) ** 2, 0) / column.length);
for (const block of blocks) block[b] = (block[b]! - mean) / (spread || 1);
}
}
/** Block means of the band energies and of the log chroma, each part unit length within a block. */
function blockParts({ fps, bands, chroma, chromaFps }: BandFrames): number[][] {
const per = Math.max(1, Math.round(BLOCK_SECONDS * fps));
const chromaPer = Math.max(1, Math.round(BLOCK_SECONDS * chromaFps));
const blocks: number[][] = [];
for (let start = 0, b = 0; start + per <= bands.length; start += per, b++) {
const pitch = meanOf(chroma.slice(b * chromaPer, (b + 1) * chromaPer)).map(Math.log1p);
blocks.push([...meanOf(bands.slice(start, start + per)), ...unit(pitch)]);
}
return blocks;
}
function blockFeatures(frames: BandFrames): number[][] {
const blocks = blockParts(frames);
standardizeBands(blocks);
const split = frames.bands[0]?.length ?? 0;
return blocks.map((block) =>
[...unit(block.slice(0, split)), ...unit(block.slice(split))].map((v) => v * Math.SQRT1_2),
);
}
function unit(vector: number[]): number[] {
const length = Math.hypot(...vector) || 1;
return vector.map((v) => v / length);
}
const dot = (a: number[], b: number[]) => a.reduce((sum, v, i) => sum + v * b[i]!, 0);
/** The checkerboard kernel, offsets -KERNEL_BLOCKS…KERNEL_BLOCKS-1 each way: + within a side, − across, tapered. */
const CHECKERBOARD = (() => {
const offsets = Array.from({ length: 2 * KERNEL_BLOCKS }, (_, k) => k - KERNEL_BLOCKS);
const taper = (k: number) =>
Math.exp(-0.5 * (((k < 0 ? -k - 1 : k) + 0.5) / (KERNEL_BLOCKS / 2)) ** 2);
return offsets.map((a) => offsets.map((b) => (a < 0 === b < 0 ? 1 : -1) * taper(a) * taper(b)));
})();
/** Foote novelty at block `i`: similarity within each side of it minus similarity across it. */
function noveltyAt(blocks: number[][], i: number): number {
let sum = 0;
CHECKERBOARD.forEach((row, a) => {
const x = blocks[i + a - KERNEL_BLOCKS];
if (!x) return;
row.forEach((weight, b) => {
const y = blocks[i + b - KERNEL_BLOCKS];
if (y) sum += weight * dot(x, y);
});
});
return sum;
}
const novelty = (blocks: number[][]) => blocks.map((_, i) => noveltyAt(blocks, i));
/** Blocks where novelty peaks over its neighbourhood and clears the mean by a standard deviation. */
function boundaryBlocks(curve: number[], minBlocks: number): number[] {
const mean = curve.reduce((a, b) => a + b, 0) / (curve.length || 1);
const spread = Math.sqrt(curve.reduce((a, v) => a + (v - mean) ** 2, 0) / (curve.length || 1));
const peaks: number[] = [];
for (let i = minBlocks; i <= curve.length - minBlocks; i++) {
const neighbourhood = curve.slice(Math.max(0, i - minBlocks), i + minBlocks + 1);
if (curve[i]! < Math.max(...neighbourhood) || curve[i]! <= mean + spread) continue;
if (!peaks.length || i - peaks[peaks.length - 1]! >= minBlocks) peaks.push(i);
}
return peaks;
}
/** A letter per span: the letter of the first earlier span whose mean features are this alike, else a new one. */
function labels(blocks: number[][], edges: number[]): string[] {
const means: number[][] = [];
const letters: string[] = [];
for (let s = 0; s + 1 < edges.length; s++) {
const part = blocks.slice(edges[s]!, edges[s + 1]!);
const mean = unit(part[0]!.map((_, b) => part.reduce((a, block) => a + block[b]!, 0)));
const match = means.findIndex((other) => dot(other, mean) >= SAME_SECTION);
letters.push(
match >= 0 ? letters[match]! : String.fromCharCode(65 + (new Set(letters).size % 26)),
);
means.push(mean);
}
return letters;
}
/** Half the median beat interval: a boundary moves to a beat, never past one. */
const beatTolerance = (beats: number[]) => (medianInterval(beats) ?? 0) / 2;
export function sections(signal: AudioSignal, options: SectionOptions = {}): Sections {
const duration = round((signal.channels[0]?.length ?? 0) / signal.sampleRate);
const blocks = blockFeatures(bandFramesOf(signal));
const minBlocks = Math.max(1, Math.round((options.minSeconds ?? 4) / BLOCK_SECONDS));
if (blocks.length < 2 * minBlocks) return { spans: [{ start: 0, end: duration, label: "A" }] };
const edges = [0, ...boundaryBlocks(novelty(blocks), minBlocks), blocks.length];
const letters = labels(blocks, edges);
const beats = rhythm(signal).beats.times;
const onBeat = snapper(beats, beatTolerance(beats), "signal beats");
const times = edges.map((e, i) =>
i === 0 ? 0 : i === edges.length - 1 ? duration : round(onBeat.snap(e * BLOCK_SECONDS)),
);
return {
spans: letters.map((label, s) => ({ start: times[s]!, end: times[s + 1]!, label })),
snap: onBeat.report(),
};
}