fix(cli): audio --read reports noise beds as ambience, not music (#6536)

* fix(cli): audio --read reports noise beds as ambience, not music

A stretch of the non-voice stems counted as music whenever it was above
-55 dB, so room tone and engine rumble separated into "other" printed as
a music piece. A stretch is now music only with a beat, clear tonal chroma
or more than one leading instrument; otherwise it is ambience with its
level in dBFS, called quiet under -45 dBFS, and breaks lie only between
two runs of music. The header lists only stems that carry sound, with
their level, instead of the separator's six fixed names.

* refactor(cli): audio --read gate carries its meaning in names, not comments

* fix(cli): keep ambience out of downstream music measurements

* fix(cli): measure music beat evidence within each stretch

docs/packages/cli.mdx

@@ -769,6 +769,12 @@ npx hyperframes audio refs/episode.mp4 --read --range 60,70   # only what overla
 - **music**: the bed split at its cut-outs, each part naming its piece, the
   instruments that carry it and its tempo, whether it starts, resumes after a
   break, returns or is new, and where it fades.
+- **ambience**: stretches of the bed without detected musical evidence, reported
+  with their level in dBFS and called quiet under -45 dBFS. The gate counts a
+  beat detected within that stretch, concentrated pitch-class energy, or multiple
+  leading stems as evidence of music. This is a signal heuristic, not a guarantee: tonal or
+  rhythmic noise can count as music, and music with weak evidence can count as
+  ambience. Breaks are reported only between adjacent music stretches.
 
 Sentences come from a transcript: `--transcript <file>`, or
 `<recording>.transcript.json` beside the recording. Without one, voice lines
@@ -782,7 +788,8 @@ same origin's direct-upload endpoints. The separation request waits for the
 stems; an in-progress response or dropped connection retries the same request
 with the same idempotency key (`HYPERFRAMES_AUDIO_STEMS_TIMEOUT_MS`, 30 minutes
 by default). Effects are read
-from the drums stem and the music from the other four. Without stems, effects are
+from the drums stem and the music and ambience from the other four; the header
+lists only the stems that carry sound, with their level. Without stems, effects are
 read from the mix outside speech and the music is not read; with neither stems nor
 a transcript, voice and effects are not read either, because the mix alone does
 not tell speech from music. Each case says so in a warning.
@@ -791,6 +798,15 @@ The result is computed once per file content and kept in `.hyperframes/read-audi
 under the project, with the stems; the text output ends with the path of the
 whole recording's text, and `--json` returns the analysis.
 
+Read schema version 3 changes `analysis.stems` from names to `{ name, db }[]`,
+ordered loudest first. Only stems with frames at or above -55 dBFS are listed;
+`db` is the rounded power-average level of those frames, not an average of dB
+values or the level across silent gaps. `analysis.ambience` contains
+`{ start, end, db }` stretches, or `null` when stems were not read. Music energy
+and effect levels over music exclude stretches classified as ambience. Version 2
+cached analyses are recomputed using existing cached stems; JSON consumers must
+use `stem.name` instead of treating each stem as a string.
+
 Every measurement comes from a provider ladder, tried in this order until one
 answers:
 

packages/cli/src/audio/read/effects.ts

@@ -267,6 +267,7 @@ export interface EffectInput {
   sampleRate: number;
   /** The music without drums, to say how far an effect stands over it. */
   music: Float32Array | null;
+  musicSegments?: readonly Span[];
   /** The whole recording, when the source is a stem: an effect far under it is separation residue nobody hears. */
   heard: Float32Array | null;
   beats: readonly number[];
@@ -308,9 +309,14 @@ function effectOf(
   group: string,
   kind: EffectKind,
   musicDb: Float32Array | null,
+  musicSegments: readonly Span[] | undefined,
 ): Effect {
   const musicLevel = musicDb?.[Math.round(m.peak / HOP)];
-  const musicPlays = musicLevel !== undefined && musicLevel > MUSIC_OUT_DB;
+  const musicPlays =
+    musicLevel !== undefined &&
+    musicLevel > MUSIC_OUT_DB &&
+    (musicSegments === undefined ||
+      musicSegments.some((span) => m.peak >= span.start && m.peak < span.end));
   return {
     t: round(m.t),
     end: round(m.end),
@@ -349,7 +355,10 @@ export function findEffects(input: EffectInput): Effects {
     if (previous?.group === group && m.t - previous.end <= TRAIN_GAP) {
       previous.end = round(m.end);
       previous.count++;
-    } else events.push(effectOf(m, group, kindOfGroup.get(group) ?? m.kind, musicDb));
+    } else
+      events.push(
+        effectOf(m, group, kindOfGroup.get(group) ?? m.kind, musicDb, input.musicSegments),
+      );
   }
   const drumHits = [...drums].reduce((n, id) => n + members.get(id)!.length, 0);
   return { events, groups, drumHits };

packages/cli/src/audio/read/format.ts

@@ -1,9 +1,20 @@
 import type { Span } from "@hyperframes/core/audio-analysis";
 import { semitones } from "./pitch.js";
-import type { Effect, Entry, MusicSegment, ReadAudio, Sentence, Voice } from "./types.js";
+import type {
+  Ambience,
+  Effect,
+  Effects,
+  Entry,
+  Music,
+  MusicSegment,
+  ReadAudio,
+  Sentence,
+  Voice,
+} from "./types.js";
 
 const PITCH_NOTE_SEMITONES = 3;
-const KIND_WIDTH = 7;
+const KIND_WIDTH = 8;
+const QUIET_DB = -45;
 
 interface Line {
   start: number;
@@ -58,8 +69,8 @@ function effectLine(effect: Effect, pauses: readonly Span[], musicRead: boolean)
 
 const ENTRY: Record<Entry, string> = {
   start: "",
-  new: ", first appearance after a break",
-  resumes: ", resumes after a break",
+  new: ", first appearance",
+  resumes: ", resumes",
   returns: ", returns",
 };
 
@@ -68,12 +79,27 @@ function musicLine(segment: MusicSegment, analysis: ReadAudio): Line {
   const about = [piece.instruments.join(", "), segment.bpm ? `${segment.bpm} BPM` : null]
     .filter(Boolean)
     .join(" · ");
+  const afterBreak =
+    (segment.entry === "new" || segment.entry === "resumes") &&
+    analysis.music!.breaks.some((gap) => gap.end === segment.start)
+      ? " after a break"
+      : "";
   const fade = segment.fadeFrom !== null ? `, fades from ${s(segment.fadeFrom)}` : "";
   return {
     start: segment.start,
     end: segment.end,
     kind: "music",
-    text: `piece ${piece.id}${about ? ` (${about})` : ""}${ENTRY[segment.entry]}${fade}`,
+    text: `piece ${piece.id}${about ? ` (${about})` : ""}${ENTRY[segment.entry]}${afterBreak}${fade}`,
+  };
+}
+
+function ambienceLine(ambience: Ambience): Line {
+  const level = `${ambience.db} dBFS`;
+  return {
+    start: ambience.start,
+    end: ambience.end,
+    kind: "ambience",
+    text: ambience.db < QUIET_DB ? `quiet, ${level}` : level,
   };
 }
 
@@ -82,34 +108,45 @@ function overlaps(line: Line, range?: [number, number]): boolean {
   return line.start < range[1] && (line.end ?? line.start) >= range[0];
 }
 
+function separation(analysis: ReadAudio): string {
+  if (analysis.separation !== "stems") return "not separated";
+  const heard = analysis.stems.map((stem) => `${stem.name} ${stem.db} dB`).join(", ");
+  return `stems: ${heard || "all silent"}`;
+}
+
+function voiceSummary(voice: Voice): string {
+  const rate = voice.meanRate !== null ? `mean ${voice.meanRate} ${voice.unit}/s` : null;
+  const pitch = voice.medianPitchHz !== null ? `median pitch ${voice.medianPitchHz} Hz` : null;
+  return `voice: ${voice.sentences.length} lines, ${voice.pauses.length} pauses${[rate, pitch]
+    .filter(Boolean)
+    .map((p) => `, ${p}`)
+    .join("")}`;
+}
+
+function effectsSummary(effects: Effects): string {
+  const top = effects.groups
+    .slice(0, 6)
+    .map((g) => `${g.id} ${g.kind} ×${g.count}`)
+    .join(", ");
+  return `effects: ${effects.events.length} in ${effects.groups.length} sounds${top ? ` (${top})` : ""}${
+    effects.drumHits ? `; ${effects.drumHits} drum hits on the beat left out` : ""
+  }`;
+}
+
+const musicSummary = (music: Music) =>
+  music.pieces.length
+    ? `music: ${music.pieces.length} pieces, ${music.breaks.length} breaks`
+    : "music: none";
+
 function header(analysis: ReadAudio): string[] {
-  const { voice, effects, music } = analysis;
-  const separated =
-    analysis.separation === "stems" ? `stems: ${analysis.stems.join(", ")}` : "not separated";
-  const lines = [`${analysis.file} · ${s(analysis.duration)} s · ${separated}`];
-  if (voice) {
-    const rate = voice.meanRate !== null ? `mean ${voice.meanRate} ${voice.unit}/s` : null;
-    const pitch = voice.medianPitchHz !== null ? `median pitch ${voice.medianPitchHz} Hz` : null;
-    lines.push(
-      `voice: ${voice.sentences.length} lines, ${voice.pauses.length} pauses${[rate, pitch]
-        .filter(Boolean)
-        .map((p) => `, ${p}`)
-        .join("")}`,
-    );
-  }
-  if (effects) {
-    const top = effects.groups
-      .slice(0, 6)
-      .map((g) => `${g.id} ${g.kind} ×${g.count}`)
-      .join(", ");
-    lines.push(
-      `effects: ${effects.events.length} in ${effects.groups.length} sounds${top ? ` (${top})` : ""}${
-        effects.drumHits ? `; ${effects.drumHits} drum hits on the beat left out` : ""
-      }`,
-    );
-  }
-  if (music) lines.push(`music: ${music.pieces.length} pieces, ${music.breaks.length} breaks`);
-  return lines;
+  const { voice, effects, music, ambience } = analysis;
+  return [
+    `${analysis.file} · ${s(analysis.duration)} s · ${separation(analysis)}`,
+    voice && voiceSummary(voice),
+    effects && effectsSummary(effects),
+    music && musicSummary(music),
+    ambience?.length && `ambience: ${ambience.length} stretches`,
+  ].filter((line): line is string => typeof line === "string");
 }
 
 /** The recording as time-stamped lines, all of it or the events overlapping `range`. */
@@ -132,6 +169,7 @@ export function formatRead(analysis: ReadAudio, range?: [number, number]): strin
       kind: "music",
       text: `break ${(b.end - b.start).toFixed(1)} s`,
     })) ?? []),
+    ...(analysis.ambience?.map(ambienceLine) ?? []),
   ]
     .filter((line) => overlaps(line, range))
     .sort((a, b) => a.start - b.start || (a.end ?? a.start) - (b.end ?? b.start));

packages/cli/src/audio/read/music.test.ts

@@ -1,4 +1,6 @@
 import { describe, expect, it } from "vitest";
+import { formatRead } from "./format.js";
+import { READ_VERSION } from "./types.js";
 import { readMusic } from "./music.js";
 import { READ_RATE } from "./signal.js";
 
@@ -15,7 +17,56 @@ function piano(seconds: number, parts: [number, number, (t: number) => number][]
   return out;
 }
 
+let seed = 11;
+const noise = () => ((seed = (seed * 16807) % 2147483647) / 2147483647) * 2 - 1;
+
+function rumble(seconds: number, from = 0): Float32Array {
+  const out = new Float32Array(Math.round(seconds * READ_RATE));
+  let low = 0;
+  for (let i = Math.round(from * READ_RATE); i < out.length; i++)
+    out[i] = 0.4 * (low = 0.995 * low + 0.02 * noise());
+  return out;
+}
+
+const kick = (t: number) => {
+  const since = t % 0.5;
+  return 0.6 * Math.exp(-since / 0.05) * Math.sin(2 * Math.PI * 60 * since);
+};
+
 describe("readMusic", () => {
+  it.each([0, 0.01, 0.37, 1.03])("keeps silent %s s clips finite and empty", (seconds) => {
+    const result = readMusic({
+      stems: { other: new Float32Array(Math.round(seconds * READ_RATE)) },
+      drums: null,
+      sampleRate: READ_RATE,
+    });
+    expect(result.music.segments).toEqual([]);
+    expect(result.music.breaks).toEqual([]);
+    expect(result.ambience).toEqual([]);
+    expect(result.music.energy).toEqual([]);
+  });
+
+  it("bounds short ambience and music to the actual recording", () => {
+    for (const samples of [
+      rumble(0.01),
+      rumble(0.037),
+      rumble(0.37),
+      piano(1.036, [[0, 1.036, C_MAJOR]]),
+    ]) {
+      const { music, ambience } = readMusic({
+        stems: { other: samples },
+        drums: null,
+        sampleRate: READ_RATE,
+      });
+      const spans = [...music.segments, ...ambience, ...music.energy];
+      expect(spans.length).toBeGreaterThan(0);
+      for (const span of [...ambience, ...music.energy])
+        expect(Number.isFinite(span.db)).toBe(true);
+      for (const span of spans)
+        expect(span.end).toBeLessThanOrEqual(samples.length / READ_RATE + 0.0005);
+    }
+  });
+
   it("splits the bed at its cut-outs and tells a new piece from one that returns", () => {
     const stem = piano(26, [
       [0, 8, C_MAJOR],
@@ -50,4 +101,123 @@ describe("readMusic", () => {
     expect(music.pieces[0]!.instruments).toEqual(["piano", "guitar"]);
     expect(music.segments[0]!.fadeFrom).toBeGreaterThan(6.5);
   });
+
+  it("keeps a tonal bed on a beat as music, alone in one stem", () => {
+    const song = piano(20, [[0, 20, (t) => C_MAJOR(t) + kick(t)]]);
+    const { music, ambience } = readMusic({
+      stems: { other: song, bass: new Float32Array(song.length) },
+      drums: null,
+      sampleRate: READ_RATE,
+    });
+    expect(ambience).toEqual([]);
+    expect(music.segments).toHaveLength(1);
+    expect(music.segments[0]!.bpm).toBe(120);
+  });
+
+  it("reads noise in a single stem as ambience with its level, not as a piece", () => {
+    const { music, ambience } = readMusic({
+      stems: { other: rumble(10), bass: new Float32Array(10 * READ_RATE) },
+      drums: null,
+      sampleRate: READ_RATE,
+    });
+    expect(music.segments).toEqual([]);
+    expect(music.pieces).toEqual([]);
+    expect(ambience).toEqual([{ start: 0, end: expect.closeTo(10, 0), db: expect.any(Number) }]);
+    expect(ambience[0]!.db).toBeGreaterThan(-40);
+    expect(ambience[0]!.db).toBeLessThan(-10);
+  });
+
+  it.each(["bed", "drums", "lead-in"])(
+    "does not borrow beat evidence from other stretches (%s)",
+    (placement) => {
+      const parts: [number, number, (t: number) => number][] =
+        placement === "lead-in"
+          ? [[41, 60, C_MAJOR]]
+          : [
+              [0, 20, C_MAJOR],
+              [41, 60, C_MAJOR],
+            ];
+      const pitched = piano(
+        60,
+        parts.map(([start, end, sound]) => [
+          start,
+          end,
+          (t) => sound(t) + (placement === "bed" ? kick(t) : 0),
+        ]),
+      );
+      const drums =
+        placement === "bed"
+          ? null
+          : piano(
+              60,
+              parts.map(([start, end]) => [start, end, kick]),
+            );
+      const noisy = rumble(60, 21);
+      noisy.fill(0, 40 * READ_RATE);
+      const { music, ambience } = readMusic({
+        stems: { piano: pitched, other: noisy },
+        drums,
+        sampleRate: READ_RATE,
+      });
+      expect(ambience.map((span) => [Math.round(span.start), Math.round(span.end)])).toEqual([
+        [21, 40],
+      ]);
+      expect(music.segments.map((span) => [Math.round(span.start), Math.round(span.end)])).toEqual(
+        placement === "lead-in"
+          ? [[41, 60]]
+          : [
+              [0, 20],
+              [41, 60],
+            ],
+      );
+      expect(music.breaks).toEqual([]);
+    },
+  );
+
+  it("does not describe a resumption after intervening ambience as a break", () => {
+    const stem = piano(26, [
+      [0, 8, C_MAJOR],
+      [18, 26, C_MAJOR],
+    ]);
+    const noisy = rumble(26, 9);
+    noisy.fill(0, 17 * READ_RATE);
+    const { music, ambience } = readMusic({
+      stems: { piano: stem, other: noisy },
+      drums: null,
+      sampleRate: READ_RATE,
+    });
+    expect(music.segments).toHaveLength(2);
+    expect(music.segments[1]!.entry).toBe("resumes");
+    expect(ambience).toHaveLength(1);
+    expect(music.breaks).toEqual([]);
+    const text = formatRead({
+      version: READ_VERSION,
+      file: "bed.wav",
+      sha256: "test",
+      duration: 26,
+      separation: "stems",
+      stems: [],
+      transcript: null,
+      voice: null,
+      effects: null,
+      music,
+      ambience,
+      warnings: [],
+    });
+    expect(text).toContain("resumes");
+    expect(text).not.toContain("after a break");
+  });
+
+  it("does not call the gap between music and ambience a break", () => {
+    const bed = piano(18, [[0, 8, C_MAJOR]]);
+    const noisy = rumble(18, 9);
+    const { music, ambience } = readMusic({
+      stems: { piano: bed, other: noisy },
+      drums: null,
+      sampleRate: READ_RATE,
+    });
+    expect(music.segments.map((s) => [Math.round(s.start), Math.round(s.end)])).toEqual([[0, 8]]);
+    expect(ambience.map((a) => [Math.round(a.start), Math.round(a.end)])).toEqual([[9, 18]]);
+    expect(music.breaks).toEqual([]);
+  });
 });

packages/cli/src/audio/read/music.ts

@@ -1,9 +1,13 @@
-import { bandFramesOf, medianInterval, rhythm } from "@hyperframes/core/audio-analysis";
-import { asSignal, levels, median, mixed, round } from "./signal.js";
-import type { Entry, Music, MusicSegment, Piece } from "./types.js";
+import {
+  bandFramesOf,
+  medianInterval,
+  rhythm,
+  type AudioSignal,
+} from "@hyperframes/core/audio-analysis";
+import { ABSENT_DB, asSignal, levels, median, mixed, round } from "./signal.js";
+import type { Ambience, Entry, Music, MusicSegment, Piece } from "./types.js";
 
 const HOP = 0.1;
-const ABSENT_DB = -55;
 const CUT_DB = 25;
 const RUNNING_FRAMES = 200;
 const LEVEL_WINDOW = 0.4;
@@ -21,6 +25,7 @@ const FADE_DB = 10;
 const FADE_WINDOW = 3;
 const ENERGY_WINDOW = 30;
 const LEADING_SHARE = 0.2;
+const TONAL_SHARE = 0.4;
 
 export interface MusicInput {
   /** Every pitched stem, by name: bass, guitar, piano, other. */
@@ -126,6 +131,18 @@ function chromaOf(chroma: Float64Array[], fps: number, run: Run): Float64Array {
   return norm ? sum.map((v) => v / norm) : sum;
 }
 
+function medianTopThreePitchShare(chroma: Float64Array[], fps: number, run: Run): number {
+  const a = Math.floor(run.start * fps);
+  const b = Math.min(chroma.length, Math.ceil(run.end * fps));
+  const held: number[] = [];
+  for (let f = a; f < b; f++) {
+    const sorted = [...chroma[f]!].sort((x, y) => y - x);
+    const total = sorted.reduce((n, v) => n + v, 0);
+    if (total > 0) held.push((sorted[0]! + sorted[1]! + sorted[2]!) / total);
+  }
+  return median(held);
+}
+
 const dot = (a: Float64Array, b: Float64Array) => a.reduce((n, v, k) => n + v * b[k]!, 0);
 
 function energyOf(samples: Float32Array, sampleRate: number, run: Run): number {
@@ -154,6 +171,17 @@ function bpmOf(beats: readonly number[], run: Run): number | null {
   return interval ? Math.round(60 / interval) : null;
 }
 
+function beatsInRun(signal: AudioSignal, run: Run): number[] {
+  const samples = signal.channels[0]!;
+  if (run.start === 0 && run.end === samples.length / signal.sampleRate)
+    return rhythm(signal).beats.times;
+  const excerpt = samples.subarray(
+    Math.floor(run.start * signal.sampleRate),
+    Math.ceil(run.end * signal.sampleRate),
+  );
+  return rhythm(asSignal(excerpt, signal.sampleRate)).beats.times;
+}
+
 const sameTempo = (a: number | null, b: number | null) =>
   a === null || b === null || Math.abs(a - b) / Math.max(a, b) <= SAME_TEMPO;
 
@@ -175,8 +203,18 @@ interface Heard {
   print: Float64Array;
   mix: Record<string, number>;
   bpm: number | null;
+  tonality: number;
 }
 
+const leading = (mix: Record<string, number>) =>
+  Object.entries(mix)
+    .filter(([, share]) => share >= LEADING_SHARE)
+    .sort((a, b) => b[1] - a[1])
+    .map(([name]) => name);
+
+const isMusic = (heard: Heard) =>
+  heard.bpm !== null || heard.tonality >= TONAL_SHARE || leading(heard.mix).length > 1;
+
 const alike = (p: Known, heard: Heard, piece: number, mix: number) =>
   dot(p.print, heard.print) >= piece &&
   mixDistance(p.mix, heard.mix) <= mix &&
@@ -201,10 +239,7 @@ function pieceOf(
 function newPiece(id: number, heard: Heard): Known {
   return {
     id,
-    instruments: Object.entries(heard.mix)
-      .filter(([, share]) => share >= LEADING_SHARE)
-      .sort((a, b) => b[1] - a[1])
-      .map(([name]) => name),
+    instruments: leading(heard.mix),
     bpm: heard.bpm,
     firstAt: round(heard.run.start),
     print: heard.print,
@@ -212,37 +247,76 @@ function newPiece(id: number, heard: Heard): Known {
   };
 }
 
-function energyCurve(melodic: Float32Array, sampleRate: number, duration: number) {
+function liesBetweenMusic(found: Run[], gap: Run, musical: Set<Run>): boolean {
+  const after = found.findIndex((run) => run.start >= gap.end);
+  return after > 0 && musical.has(found[after - 1]!) && musical.has(found[after]!);
+}
+
+function energyCurve(
+  melodic: Float32Array,
+  sampleRate: number,
+  duration: number,
+  musical: Set<Run>,
+) {
   const energy = [];
   for (let start = 0; start < duration; start += ENERGY_WINDOW) {
     const end = Math.min(duration, start + ENERGY_WINDOW);
-    const power = energyOf(melodic, sampleRate, { start, end });
+    const overlaps = [...musical]
+      .map((run) => ({ start: Math.max(start, run.start), end: Math.min(end, run.end) }))
+      .filter((run) => run.end > run.start);
+    if (!overlaps.length) continue;
+    const power =
+      overlaps.reduce(
+        (sum, run) => sum + energyOf(melodic, sampleRate, run) * (run.end - run.start),
+        0,
+      ) /
+      (end - start);
     energy.push({
       start,
-      end: round(end),
+      end: Math.min(round(end), round(duration, 3)),
       db: power > 0 ? Math.round(10 * Math.log10(power)) : -120,
     });
   }
   return energy;
 }
 
-export function readMusic(input: MusicInput): { music: Music; beats: number[] } {
+export function readMusic(input: MusicInput): {
+  music: Music;
+  ambience: Ambience[];
+  beats: number[];
+} {
   const melodic = mixed(Object.values(input.stems));
   const db = levels(melodic, input.sampleRate, HOP, LEVEL_WINDOW);
+  const duration = melodic.length / input.sampleRate;
   const { segments: found, breaks } = runs(db);
+  for (const run of found) run.end = Math.min(run.end, duration);
   const frames = bandFramesOf(asSignal(melodic, input.sampleRate));
-  const beats = rhythm(
-    asSignal(input.drums ? mixed([melodic, input.drums]) : melodic, input.sampleRate),
-  ).beats.times;
+  const rhythmic = asSignal(
+    input.drums ? mixed([melodic, input.drums]) : melodic,
+    input.sampleRate,
+  );
+  const beats = rhythm(rhythmic).beats.times;
   const pieces: Known[] = [];
   const segments: MusicSegment[] = [];
+  const ambience: Ambience[] = [];
+  const musical = new Set<Run>();
   for (const run of found) {
     const heard: Heard = {
       run,
       print: chromaOf(frames.chroma, frames.chromaFps, run),
       mix: shares(input, run),
-      bpm: bpmOf(beats, run),
+      bpm: bpmOf(beatsInRun(rhythmic, run), { start: 0, end: run.end - run.start }),
+      tonality: medianTopThreePitchShare(frames.chroma, frames.chromaFps, run),
     };
+    if (!isMusic(heard)) {
+      ambience.push({
+        start: round(run.start),
+        end: Math.min(round(run.end), round(duration, 3)),
+        db: Math.round(10 * Math.log10(Math.max(1e-12, energyOf(melodic, input.sampleRate, run)))),
+      });
+      continue;
+    }
+    musical.add(run);
     const previous = segments[segments.length - 1];
     let piece = pieceOf(pieces, previous, heard);
     let entry: Entry;
@@ -254,7 +328,7 @@ export function readMusic(input: MusicInput): { music: Music; beats: number[] }
     }
     segments.push({
       start: round(run.start),
-      end: round(run.end),
+      end: Math.min(round(run.end), round(duration, 3)),
       piece: piece.id,
       entry,
       bpm: heard.bpm,
@@ -265,9 +339,12 @@ export function readMusic(input: MusicInput): { music: Music; beats: number[] }
     music: {
       segments,
       pieces: pieces.map(({ print: _print, mix: _mix, ...piece }) => piece),
-      breaks: breaks.map((b) => ({ start: round(b.start), end: round(b.end) })),
-      energy: energyCurve(melodic, input.sampleRate, db.length * HOP),
+      breaks: breaks
+        .filter((b) => liesBetweenMusic(found, b, musical))
+        .map((b) => ({ start: round(b.start), end: round(b.end) })),
+      energy: energyCurve(melodic, input.sampleRate, duration, musical),
     },
+    ambience,
     beats,
   };
 }

packages/cli/src/audio/read/read.test.ts

@@ -1,11 +1,14 @@
 import { createServer, type IncomingMessage, type Server } from "node:http";
 import type { AddressInfo } from "node:net";
-import { existsSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from "node:fs";
+import { existsSync, mkdirSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from "node:fs";
+import { createHash } from "node:crypto";
 import { tmpdir } from "node:os";
 import { join } from "node:path";
 import { afterAll, afterEach, beforeAll, beforeEach, describe, expect, it, vi } from "vitest";
 import { findFfBinary } from "@hyperframes/parsers/ff-binaries";
 import { readAudio } from "./read.js";
+import { formatRead } from "./format.js";
+import { READ_VERSION } from "./types.js";
 import { cachedStems, fetchStems } from "./stems.js";
 
 const RATE = 22_050;
@@ -149,8 +152,8 @@ describe.skipIf(!ffmpeg)("readAudio", () => {
     expect(first.analysis.voice!.medianPitchHz).toBeGreaterThan(170);
     expect(first.analysis.warnings.join("\n")).toContain("no HeyGen credential");
     const text = readFileSync(first.textPath, "utf8");
-    expect(text).toContain('voice   "一口气了解。"');
-    expect(text).toContain("pause   1.0 s");
+    expect(text).toContain('voice    "一口气了解。"');
+    expect(text).toContain("pause    1.0 s");
     const again = await readAudio(path, "talk.wav", { cacheDir, quality: "best" });
     expect(again.cached).toBe(true);
     writeFileSync(
@@ -182,7 +185,7 @@ describe.skipIf(!ffmpeg)("readAudio", () => {
     expect(stemKeys[1]).toBe(stemKeys[0]);
     expect(analysis).toMatchObject({
       separation: "stems",
-      stems: ["bass", "drums", "guitar", "other", "piano", "vocals"],
+      stems: ["vocals", "drums", "piano"].map((name) => ({ name, db: expect.any(Number) })),
       warnings: [expect.stringContaining("no transcript")],
     });
     expect(analysis.voice!.source).toBe("activity");
@@ -192,12 +195,95 @@ describe.skipIf(!ffmpeg)("readAudio", () => {
     expect(analysis.effects!.events[0]!.overMusicDb).toBeGreaterThan(5);
     expect(analysis.music!.breaks.map((b) => Math.round(b.start))).toEqual([5]);
     expect(analysis.music!.segments.map((s) => s.entry)).toEqual(["start", "resumes"]);
-    expect(readFileSync(textPath, "utf8")).toContain("effect  boom (S1): low");
+    const text = readFileSync(textPath, "utf8");
+    expect(text).toMatch(/stems: vocals -\d+ dB, drums -\d+ dB, piano -\d+ dB\n/);
+    expect(text).toContain("effect   boom (S1): low");
+    expect(analysis.ambience).toEqual([]);
     expect(existsSync(join(cacheDir, "stems"))).toBe(true);
     await readAudio(path, "episode.wav", { cacheDir, quality: "best" });
     expect(stemCalls).toBe(2);
   });
 
+  it("reads a bed of noise as ambience, not music, and names only the stems that carry sound", async () => {
+    let seed = 7;
+    const noise = () => ((seed = (seed * 16807) % 2147483647) / 2147483647) * 2 - 1;
+    const roomTone = new Float32Array(Math.round(6 * RATE)).map(() => 0.006 * noise());
+    const path = wav("room.wav", 6, (t) => roomTone[Math.round(t * RATE)] ?? 0);
+    const cacheDir = join(dir, "cache-room");
+    const sha = createHash("sha256").update(readFileSync(path)).digest("hex");
+    const stemDir = join(cacheDir, "stems", sha.slice(0, 12));
+    mkdirSync(stemDir, { recursive: true });
+    writeFileSync(join(stemDir, "other.wav"), readFileSync(path));
+    writeFileSync(join(stemDir, "silence.wav"), readFileSync(wav("room-silence.wav", 6, () => 0)));
+    const names = ["vocals", "drums", "bass", "guitar", "piano"];
+    writeFileSync(
+      join(stemDir, "stems.json"),
+      JSON.stringify({
+        other: "other.wav",
+        ...Object.fromEntries(names.map((n) => [n, "silence.wav"])),
+      }),
+    );
+    const { analysis, textPath, jsonPath } = await readAudio(path, "room.wav", {
+      cacheDir,
+      quality: "best",
+    });
+    expect(analysis.stems).toEqual([{ name: "other", db: expect.any(Number) }]);
+    expect(analysis.music!.segments).toEqual([]);
+    expect(analysis.ambience).toEqual([
+      { start: 0, end: expect.closeTo(6, 0), db: expect.any(Number) },
+    ]);
+    expect(analysis.ambience![0]!.db).toBeLessThan(-45);
+    const text = readFileSync(textPath, "utf8");
+    expect(text).toMatch(/stems: other -\d+ dB\n/);
+    expect(text).toMatch(/ambience quiet, -\d+ dBFS/);
+    expect(text).not.toContain("piece");
+    expect(formatRead(analysis, [2, 3])).toContain("ambience quiet");
+    expect(formatRead(analysis, [8, 9])).not.toContain("ambience quiet");
+    expect(analysis.music!.energy).toEqual([]);
+    const old = { analysis: { ...analysis, version: 2, stems: ["other"] }, transcriptKey: "none" };
+    writeFileSync(jsonPath, JSON.stringify(old));
+    const refreshed = await readAudio(path, "room.wav", { cacheDir });
+    expect(refreshed.cached).toBe(false);
+    expect(refreshed.analysis.version).toBe(READ_VERSION);
+    expect(refreshed.analysis.stems).toEqual(analysis.stems);
+    expect((await readAudio(path, "room.wav", { cacheDir })).cached).toBe(true);
+  });
+
+  it("does not measure an effect over music when its bed is only ambience", async () => {
+    let seed = 7;
+    const noise = () => ((seed = (seed * 16807) % 2147483647) / 2147483647) * 2 - 1;
+    const samples = new Float32Array(6 * RATE).map(() => 0.006 * noise());
+    const other = wav("effect-room.wav", 6, (t) => samples[Math.round(t * RATE)] ?? 0);
+    const path = wav(
+      "effect-in-room.wav",
+      6,
+      (t) => (samples[Math.round(t * RATE)] ?? 0) + boom(t),
+    );
+    const cacheDir = join(dir, "cache-effect-room");
+    const sha = createHash("sha256").update(readFileSync(path)).digest("hex");
+    const stemDir = join(cacheDir, "stems", sha.slice(0, 12));
+    mkdirSync(stemDir, { recursive: true });
+    writeFileSync(join(stemDir, "other.wav"), readFileSync(other));
+    writeFileSync(join(stemDir, "drums.wav"), readFileSync(wav("room-boom.wav", 6, boom)));
+    writeFileSync(
+      join(stemDir, "silence.wav"),
+      readFileSync(wav("effect-silence.wav", 6, () => 0)),
+    );
+    writeFileSync(
+      join(stemDir, "stems.json"),
+      JSON.stringify({
+        other: "other.wav",
+        drums: "drums.wav",
+        ...Object.fromEntries(["vocals", "bass", "guitar", "piano"].map((n) => [n, "silence.wav"])),
+      }),
+    );
+    const { analysis, textPath } = await readAudio(path, "effect-in-room.wav", { cacheDir });
+    expect(analysis.ambience).toHaveLength(1);
+    expect(analysis.effects!.events).toHaveLength(1);
+    expect(analysis.effects!.events[0]!.overMusicDb).toBeNull();
+    expect(readFileSync(textPath, "utf8")).not.toContain("over music");
+  });
+
   it("falls back on partial stems and retries instead of caching unmeasured layers as silence", async () => {
     for (const name of Object.keys(stemFiles)) delete stemFiles[name];
     stemFiles.vocals = wav("partial-vocals.wav", 2, voiced);

packages/cli/src/audio/read/read.ts

@@ -14,9 +14,9 @@ import { findEffects } from "./effects.js";
 import { formatRead } from "./format.js";
 import { readMusic } from "./music.js";
 import { pitchTrack } from "./pitch.js";
-import { PITCH_RATE, READ_RATE, decodeMono, mixed, round } from "./signal.js";
+import { PITCH_RATE, READ_RATE, decodeMono, levelWhileHeardDb, mixed, round } from "./signal.js";
 import { DRUM_STEM, VOICE_STEM, cachedStems, fetchStems, stemsAccess } from "./stems.js";
-import { READ_VERSION, type ReadAudio } from "./types.js";
+import { READ_VERSION, type ReadAudio, type AudibleStem } from "./types.js";
 import { voiceFromActivity, voiceFromWords, type TimedWord } from "./voice.js";
 
 export interface ReadOptions {
@@ -92,25 +92,43 @@ async function stemsFor(
   }
 }
 
-/** Effects from the drums and the music from the other non-voice stems, measured against the whole recording. */
-async function fromStems(path: string, stemPaths: Record<string, string>) {
+function heardStems(tracks: Record<string, Float32Array>): AudibleStem[] {
+  return Object.entries(tracks)
+    .map(([name, samples]) => ({ name, db: levelWhileHeardDb(samples, READ_RATE) }))
+    .filter((stem): stem is AudibleStem => stem.db !== null)
+    .sort((a, b) => b.db - a.db || a.name.localeCompare(b.name));
+}
+
+async function decodedStems(stemPaths: Record<string, string>) {
   const pitched: Record<string, Float32Array> = {};
   for (const [name, stemPath] of Object.entries(stemPaths))
     if (name !== VOICE_STEM && name !== DRUM_STEM) pitched[name] = await decodeMono(stemPath);
   const drums = stemPaths[DRUM_STEM] ? await decodeMono(stemPaths[DRUM_STEM]) : null;
+  return { pitched, drums };
+}
+
+/** Effects from the drums and the music from the other non-voice stems, measured against the whole recording. */
+async function fromStems(path: string, stemPaths: Record<string, string>, voice: Float32Array) {
+  const { pitched, drums } = await decodedStems(stemPaths);
   const bed = Object.keys(pitched).length ? mixed(Object.values(pitched)) : null;
-  const { music, beats } = bed
+  const { music, ambience, beats } = bed
     ? readMusic({ stems: pitched, drums, sampleRate: READ_RATE })
-    : { music: null, beats: [] };
+    : { music: null, ambience: null, beats: [] };
   const effects = findEffects({
     source: drums ?? bed ?? new Float32Array(0),
     sampleRate: READ_RATE,
     music: bed,
+    musicSegments: music?.segments ?? [],
     heard: await decodeMono(path),
     beats,
     speech: [],
   });
-  return { effects, music };
+  const stems = heardStems({
+    ...pitched,
+    ...(drums ? { [DRUM_STEM]: drums } : {}),
+    ...(stemPaths[VOICE_STEM] ? { [VOICE_STEM]: voice } : {}),
+  });
+  return { stems, effects, music, ambience };
 }
 
 const transcribeHint = (file: string) => `run hyperframes transcribe ${file} and pass --transcript`;
@@ -133,7 +151,7 @@ async function analyze(
     warnings.push(
       `no transcript and no stems: voice and effects are not read, because the mix alone does not tell speech from music; ${transcribeHint(file)}`,
     );
-    return { ...about, voice: null, effects: null, music: null, warnings };
+    return { ...about, voice: null, effects: null, music: null, ambience: null, warnings };
   }
   if (!words.length)
     warnings.push(`no transcript: voice lines carry no words; ${transcribeHint(file)}`);
@@ -141,7 +159,8 @@ async function analyze(
   const voice = words.length
     ? voiceFromWords(words, pitch)
     : voiceFromActivity(voiceSamples, READ_RATE, pitch);
-  if (stemPaths) return { ...about, voice, ...(await fromStems(path, stemPaths)), warnings };
+  if (stemPaths)
+    return { ...about, voice, ...(await fromStems(path, stemPaths, voiceSamples)), warnings };
   warnings.push("without stems, effects under speech are not found and music is not read");
   const effects = findEffects({
     source: voiceSamples,
@@ -151,7 +170,7 @@ async function analyze(
     beats: [],
     speech: voice.sentences,
   });
-  return { ...about, voice, effects, music: null, warnings };
+  return { ...about, voice, effects, music: null, ambience: null, warnings };
 }
 
 function base(
@@ -160,14 +179,14 @@ function base(
   duration: number,
   stemPaths: Record<string, string> | null,
   transcript: string | null,
-): Omit<ReadAudio, "voice" | "effects" | "music" | "warnings"> {
+): Omit<ReadAudio, "voice" | "effects" | "music" | "ambience" | "warnings"> {
   return {
     version: READ_VERSION,
     file,
     sha256: sha,
     duration,
     separation: stemPaths ? "stems" : "mix",
-    stems: stemPaths ? Object.keys(stemPaths).sort() : [],
+    stems: [],
     transcript: transcript ? basename(transcript) : null,
   };
 }

packages/cli/src/audio/read/signal.test.ts

@@ -1,5 +1,35 @@
-import { describe, expect, it } from "vitest";
-import { levels, READ_RATE } from "./signal.js";
+import { describe, expect, it, vi } from "vitest";
+import { decodeAudio } from "../analysis/decode.js";
+import { decodeMono, levelWhileHeardDb, levels, READ_RATE } from "./signal.js";
+
+vi.mock("../analysis/decode.js", () => ({ decodeAudio: vi.fn() }));
+
+describe("read signal boundaries", () => {
+  it.each([NaN, Infinity, -Infinity])("rejects decoded non-finite PCM (%s)", async (value) => {
+    vi.mocked(decodeAudio).mockResolvedValue({
+      signal: { channels: [Float32Array.of(0, value, 0)], sampleRate: READ_RATE },
+      stream: { channels: 1, sampleRate: READ_RATE, duration: 3 / READ_RATE },
+    });
+    await expect(decodeMono("invalid.wav")).rejects.toThrow("non-finite audio samples");
+  });
+
+  it("omits silence, empty tracks and levels below the audibility floor", () => {
+    expect(levelWhileHeardDb(new Float32Array(), READ_RATE)).toBeNull();
+    expect(levelWhileHeardDb(new Float32Array(READ_RATE), READ_RATE)).toBeNull();
+    expect(levelWhileHeardDb(new Float32Array(READ_RATE).fill(0.001), READ_RATE)).toBeNull();
+    expect(levelWhileHeardDb(new Float32Array(READ_RATE).fill(0.002), READ_RATE)).toBe(-54);
+  });
+
+  it("averages audible power rather than dB, without diluting a stem with silence", () => {
+    const samples = new Float32Array(20 * READ_RATE);
+    samples.fill(0.1, 0, 10 * READ_RATE);
+    samples.fill(0.01, 10 * READ_RATE);
+    expect(levelWhileHeardDb(samples, READ_RATE)).toBe(-23);
+    const sparse = new Float32Array(30 * READ_RATE);
+    sparse.fill(0.1, 10 * READ_RATE, 20 * READ_RATE);
+    expect(levelWhileHeardDb(sparse, READ_RATE)).toBe(-20);
+  });
+});
 
 describe("levels", () => {
   it("keeps fractional sample hops aligned late in a recording", () => {

packages/cli/src/audio/read/signal.ts

@@ -5,11 +5,15 @@ import { decodeAudio } from "../analysis/decode.js";
 export const READ_RATE = 22050;
 export const PITCH_RATE = 8000;
 const SILENT_DB = -120;
+export const ABSENT_DB = -55;
 
 export async function decodeMono(path: string, sampleRate = READ_RATE): Promise<Float32Array> {
   const decoded = await decodeAudio(path, {}, undefined, { sampleRate, mono: true });
   if (!decoded) throw new Error(`${path} has no audio stream`);
-  return decoded.signal.channels[0]!;
+  const samples = decoded.signal.channels[0]!;
+  if (samples.some((sample) => !Number.isFinite(sample)))
+    throw new Error(`${path} contains non-finite audio samples`);
+  return samples;
 }
 
 export const asSignal = (samples: Float32Array, sampleRate = READ_RATE): AudioSignal => ({
@@ -47,6 +51,17 @@ export function levels(
   return out;
 }
 
+export function levelWhileHeardDb(samples: Float32Array, sampleRate: number): number | null {
+  let sum = 0;
+  let heard = 0;
+  for (const db of levels(samples, sampleRate, 0.1, 0.4))
+    if (db >= ABSENT_DB) {
+      sum += 10 ** (db / 10);
+      heard++;
+    }
+  return heard ? Math.round(10 * Math.log10(sum / heard)) : null;
+}
+
 export function quantile(values: ArrayLike<number>, q: number): number {
   if (!values.length) return NaN;
   const sorted = Float64Array.from(values).sort();

packages/cli/src/audio/read/types.ts

@@ -1,6 +1,6 @@
 import type { Span } from "@hyperframes/core/audio-analysis";
 
-export const READ_VERSION = 2;
+export const READ_VERSION = 3;
 
 export type Separation = "stems" | "mix";
 
@@ -88,17 +88,29 @@ export interface Music {
   energy: { start: number; end: number; db: number }[];
 }
 
+export interface Ambience {
+  start: number;
+  end: number;
+  db: number;
+}
+
+export interface AudibleStem {
+  name: string;
+  db: number;
+}
+
 export interface ReadAudio {
   version: typeof READ_VERSION;
   file: string;
   sha256: string;
   duration: number;
   separation: Separation;
-  stems: string[];
+  stems: AudibleStem[];
   transcript: string | null;
   /** Null when the mix alone cannot tell speech from music: no stems and no transcript. */
   voice: Voice | null;
   effects: Effects | null;
   music: Music | null;
+  ambience: Ambience[] | null;
   warnings: string[];
 }