hyperframes/packages/core/src/audioAnalysis.test.ts

import { describe, expect, it } from "vitest";
import {
  classify,
  envelope,
  loudness,
  moments,
  onsets,
  runLadder,
  signalProvider,
  silence,
  type AudioAnalysisProvider,
  type AudioSignal,
} from "./audioAnalysis";

const RATE = 48_000;

/** A seeded noise source, so every run measures the same signal. */
function noise(seed = 1) {
  let state = seed;
  return () => {
    state = (state * 1_103_515_245 + 12_345) % 2_147_483_648;
    return (state / 2_147_483_648) * 2 - 1;
  };
}

function signal(
  seconds: number,
  fill: (t: number, random: () => number) => number,
  channels = 1,
): AudioSignal {
  const random = noise();
  const samples = Float32Array.from({ length: Math.round(seconds * RATE) }, (_, i) =>
    fill(i / RATE, random),
  );
  return { sampleRate: RATE, channels: Array.from({ length: channels }, () => samples) };
}

/** Silence, then a noise burst that peaks `rise` seconds after `at` and decays exponentially. */
const impact =
  (at: number, rise: number, decay = 0.08) =>
  (t: number, random: () => number) => {
    if (t < at) return 0;
    const since = t - at;
    const gain = since < rise ? since / rise : Math.exp(-(since - rise) / decay);
    return 0.8 * gain * random();
  };

describe("moments", () => {
  it("finds where an impact starts, rises fastest, peaks and fades", () => {
    const hit = moments(signal(1, impact(0.3, 0.02)))!;
    expect(hit.onset).toBeGreaterThanOrEqual(0.3);
    expect(hit.onset).toBeLessThan(0.31);
    expect(hit.peak).toBeCloseTo(0.32, 2);
    expect(hit.steepest).toBeGreaterThanOrEqual(hit.onset);
    expect(hit.steepest).toBeLessThanOrEqual(hit.peak);
    expect(hit.tailEnd).toBeGreaterThan(0.6);
  });

  it("puts a slow swell's peak at its end, well after its steepest rise", () => {
    const swell = moments(
      signal(1.5, (t, random) => (t < 1.2 ? 0.8 * (t / 1.2) ** 3 * random() : 0)),
    )!;
    expect(swell.peak).toBeGreaterThan(1.1);
    expect(swell.onset).toBeGreaterThan(0.6);
    expect(swell.onset).toBeLessThan(swell.peak);
  });

  it("has no moments in silence", () => {
    expect(moments(signal(0.5, () => 0))).toBeNull();
  });
});

describe("opposite polarity", () => {
  const reversed = signal(3, impact(1, 0.01, 0.05));
  reversed.channels.push(Float32Array.from(reversed.channels[0]!, (sample) => -sample));

  it("still hears a channel that is the reverse of the other", () => {
    expect(moments(reversed)?.peak).toBeCloseTo(1.01, 2);
    for (const method of ["spectral-flux", "energy"] as const)
      expect(
        onsets(reversed, { method }).events.some((event) => Math.abs(event.t - 1) < 0.02),
      ).toBe(true);
    expect(Math.max(...envelope(reversed, { hopMs: 10 }).values)).toBeGreaterThan(-40);
    expect(silence(reversed).some((span) => span.start <= 1.01 && span.end > 1.05)).toBe(false);
  });
});

describe("onsets", () => {
  it("finds each hit of a three-hit effect, the loudest strongest, by spectral flux and by energy", () => {
    const hits = [0.2, 0.55, 0.9];
    const gains = [1, 0.4, 0.7];
    const fill = (t: number, random: () => number) =>
      hits.reduce((sum, at, i) => sum + gains[i]! * impact(at, 0.003, 0.05)(t, random), 0);
    for (const method of ["spectral-flux", "energy"] as const) {
      const found = onsets(signal(1.2, fill), { method });
      expect(found.events.map((e) => e.t).every((t, i) => Math.abs(t - hits[i]!) < 0.015)).toBe(
        true,
      );
      expect(found.events).toHaveLength(3);
      expect(found.events[0]!.strength).toBe(1);
    }
  });

  it("finds a hit 16 dB under the loudest one", () => {
    const fill = (t: number, random: () => number) =>
      impact(0.2, 0.003, 0.05)(t, random) + 0.16 * impact(0.6, 0.003, 0.05)(t, random);
    const found = onsets(signal(1, fill)).events.map((e) => e.t);
    expect(found.some((t) => Math.abs(t - 0.6) < 0.015)).toBe(true);
  });

  it("finds nothing in silence", () => {
    expect(onsets(signal(0.5, () => 0)).events).toEqual([]);
  });
});

describe("envelope", () => {
  it("reads a steady tone at its RMS level, in dB or linear", () => {
    const tone = signal(0.5, (t) => 0.5 * Math.sin(2 * Math.PI * 440 * t));
    const db = envelope(tone, { hopMs: 10 });
    expect(db.values.length).toBe(50);
    expect(db.values[25]).toBeCloseTo(20 * Math.log10(0.5 / Math.SQRT2), 0);
    expect(envelope(tone, { scale: "linear" }).values[50]).toBeCloseTo(0.5 / Math.SQRT2, 2);
  });

  it("measures only the band asked for", () => {
    const tone = signal(0.5, (t) => 0.5 * Math.sin(2 * Math.PI * 440 * t));
    const inBand = envelope(tone, { band: [300, 600], hopMs: 10 }).values[25]!;
    const outOfBand = envelope(tone, { band: [4000, 8000], hopMs: 10 }).values[25]!;
    expect(inBand).toBeCloseTo(20 * Math.log10(0.5 / Math.SQRT2), 0);
    expect(outOfBand).toBeLessThan(inBand - 40);
  });
});

describe("loudness", () => {
  it("reads a full-scale 997 Hz sine in one channel at -3.01 LUFS, as BS.1770 specifies", () => {
    const sine = signal(3, (t) => Math.sin(2 * Math.PI * 997 * t));
    const measured = loudness(sine);
    expect(measured.lufsIntegrated).toBeCloseTo(-3.0, 1);
    expect(measured.samplePeakDb).toBeCloseTo(0, 1);
    expect(measured.clipping.count).toBeGreaterThan(0);
  });

  it("sums channels: the same sine in two channels is 3 dB louder", () => {
    const one = loudness(signal(3, (t) => 0.5 * Math.sin(2 * Math.PI * 997 * t)));
    const two = loudness(signal(3, (t) => 0.5 * Math.sin(2 * Math.PI * 997 * t), 2));
    expect(two.lufsIntegrated! - one.lufsIntegrated!).toBeCloseTo(3.01, 1);
  });

  it("sees an inter-sample peak above the sample peak", () => {
    const quarter = signal(1, (t) => Math.sin(2 * Math.PI * (RATE / 4) * t + Math.PI / 4));
    const measured = loudness(quarter);
    expect(measured.truePeakDb).toBeGreaterThan(measured.samplePeakDb + 2);
  });

  it("has no integrated loudness for silence", () => {
    expect(loudness(signal(1, () => 0)).lufsIntegrated).toBeNull();
  });
});

describe("silence", () => {
  it("returns the quiet spans long enough to count", () => {
    const gaps = silence(
      signal(2, (t, random) => (t < 0.5 || (t > 1.2 && t < 1.25) ? 0 : 0.3 * random())),
    );
    expect(gaps).toHaveLength(1);
    expect(gaps[0]!.start).toBe(0);
    expect(gaps[0]!.end).toBeCloseTo(0.5, 1);
  });
});

describe("classify", () => {
  const gaps = noise(7);
  const syllables: number[] = [];
  for (let at = 0.1; at < 10; at += 0.12 + 0.25 * Math.abs(gaps())) syllables.push(at);
  /** Noise in irregular syllable-length bursts over a quiet room tone. */
  const speechLike = (t: number, random: () => number) => {
    const into = syllables.map((at) => t - at).find((d) => d >= 0 && d < 0.15);
    const voiced = into === undefined ? 0 : 0.5 * Math.sin((Math.PI * into) / 0.15);
    return (voiced + 0.02) * random();
  };

  it("calls a short sound an effect", () => {
    expect(classify(signal(1, impact(0.1, 0.01))).value).toBe("sfx");
  });

  it("calls a long sound that only fades away an effect, not a bed", () => {
    expect(classify(signal(10, impact(0.1, 0.01, 1.2))).value).toBe("sfx");
  });

  it("calls a steady beat over a held pad music", () => {
    const beat = (t: number, random: () => number) =>
      0.15 * Math.sin(2 * Math.PI * 220 * t) + impact(0, 0.003, 0.04)(t % 0.5, random);
    expect(classify(signal(10, beat)).value).toBe("music");
  });

  it("calls irregular syllable-length bursts voice, in a long take and in a short line", () => {
    expect(classify(signal(10, speechLike)).value).toBe("voice");
    expect(classify(signal(3, speechLike)).value).toBe("voice");
  });

  it("reports what the guess was made from", () => {
    expect(Object.keys(classify(signal(3, speechLike)).features)).toEqual([
      "duration",
      "regularity",
      "syllables",
      "sustain",
    ]);
  });
});

describe("runLadder", () => {
  const tone = signal(0.2, (t) => 0.5 * Math.sin(2 * Math.PI * 440 * t));
  const service: AudioAnalysisProvider<AudioSignal> = {
    id: "service",
    local: false,
    capabilities: ["moments"],
    analyze: async () => ({ onset: 0, steepest: 0, peak: 0, tailEnd: 0 }),
  };
  const broken: AudioAnalysisProvider<AudioSignal> = {
    id: "broken",
    local: true,
    capabilities: ["moments"],
    analyze: async () => {
      throw new Error("model missing");
    },
  };

  it("uses the first provider allowed at the quality asked for", async () => {
    const providers = [service, signalProvider];
    expect((await runLadder("moments", tone, providers)).provenance).toBe("service");
    const local = await runLadder("moments", tone, providers, { quality: "local" });
    expect(local.provenance).toBe("signal");
    expect(local.steps[0]).toEqual({
      provider: "service",
      outcome: "excluded",
      reason: "quality local keeps audio on this machine",
    });
  });

  it("falls through a failing provider and says so", async () => {
    const result = await runLadder("moments", tone, [broken, signalProvider]);
    expect(result.provenance).toBe("signal");
    expect(result.warnings).toEqual(["moments: used signal; broken failed (model missing)"]);
  });

  it("falls through a provider that cannot run here, or whose check throws, and says why", async () => {
    const keyless = { ...service, available: () => ({ ok: false as const, reason: "no key" }) };
    const throwing = {
      ...broken,
      id: "throwing",
      available: () => {
        throw new Error("probe crashed");
      },
    };
    const result = await runLadder("moments", tone, [keyless, throwing, signalProvider]);
    expect(result.provenance).toBe("signal");
    expect(result.steps.map((s) => [s.provider, s.outcome, s.reason])).toEqual([
      ["service", "unavailable", "no key"],
      ["throwing", "unavailable", "probe crashed"],
      ["signal", "answered", undefined],
    ]);
    expect(result.warnings).toEqual([
      "moments: used signal; service unavailable (no key); throwing unavailable (probe crashed)",
    ]);
  });

  it("follows a ladder given in place of the default, and reports a capability no provider offers", async () => {
    const result = await runLadder("moments", tone, [service, signalProvider], {
      ladder: ["signal", "service"],
    });
    expect(result.provenance).toBe("signal");
    const described = await runLadder("describe", tone, [signalProvider]);
    expect(described).toMatchObject({ value: null, provenance: null });
    expect(described.warnings).toEqual(["describe: no provider answered (none offers it)"]);
  });
});