Files
hyperframes/packages/cli/src/audio/read/read.ts
T
Michael Wang 9701f0639c fix(cli): the audio caches never write or read through links in .hyperframes (#7825)
* fix(cli): the audio caches never write or read through links in .hyperframes

The read-audio cache, its stems, mix and descriptions, and the audio-kind cache
now use only real folders under the project's .hyperframes. A link or a file in
place of a cache folder stops the write with a clear message. Every cache file is
written to an exclusive temp file with a random name and renamed into place, so a
link at a temp or final name is replaced, never written through. ffmpeg writes the
voice-free mix into a private staging folder. Stem downloads use the new
downloadToFile replaceLink option. Cached files are read only when they are plain
files, and a stem list may only name files in its own folder.

* fix(cli): the kind cache reads no entry through a linked cache folder

readKindCache now checks that .hyperframes and audio-kind are real folders before
it reads an entry, as writeKindCache already does before it writes one.

* test(cli): read the replaced download before checking its type and mode

The replaceLink test now reads the file's bytes and lstat result once each, so
no read follows a check on the same path.
2026-11-15 17:31:16 +00:00

505 lines
17 KiB
TypeScript

import { createHash } from "node:crypto";
import { createReadStream, existsSync, readFileSync, renameSync, rmSync } from "node:fs";
import { basename, dirname, extname, join, relative } from "node:path";
import { findFfBinary } from "@hyperframes/parsers/ff-binaries";
import { createStagingDir, replaceFileAtomically } from "@hyperframes/core/atomic-file";
import { loudness, type AnalysisQuality, type Span } from "@hyperframes/core/audio-analysis";
import { loadTranscript } from "../../whisper/normalize.js";
import { readPlainFile } from "../../utils/projectRecords.js";
import { runFile } from "../analysis/providers/shared.js";
import { isPlainFile, makeRealFolder } from "../cacheFolder.js";
import { describeAccess, describeStems } from "./describe.js";
import { findEffects, type EffectSource } from "./effects.js";
import { formatRead } from "./format.js";
import { activeSpans, readMusic } from "./music.js";
import { pitchTrack } from "./pitch.js";
import {
PITCH_RATE,
READ_RATE,
asSignal,
decodeMono,
levelWhileHeardDb,
mixed,
round,
spanDb,
} from "./signal.js";
import { DRUM_STEM, VOICE_STEM, cachedStems, fetchStems, stemsAccess } from "./stems.js";
import {
READ_VERSION,
type AudibleStem,
type Layer,
type Music,
type ReadAudio,
type Voice,
} from "./types.js";
import { voiceFromActivity, voiceFromWords, type TimedWord } from "./voice.js";
export interface ReadOptions {
/** Where analyses and stems are kept, normally the project's `.hyperframes/read-audio`. */
cacheDir: string;
/** A transcript of the recording; `<file>.transcript.json` next to it is used when absent. */
transcript?: string;
quality?: AnalysisQuality;
layer?: Layer;
}
export interface ReadResult {
analysis: ReadAudio;
/** The whole recording as text, for grep. */
textPath: string;
jsonPath: string;
cached: boolean;
}
async function sha256(path: string): Promise<string> {
const hash = createHash("sha256");
for await (const chunk of createReadStream(path)) hash.update(chunk as Buffer);
return hash.digest("hex");
}
function transcriptFor(path: string, given?: string): string | null {
if (given) return given;
const sidecar = `${path}.transcript.json`;
return existsSync(sidecar) ? sidecar : null;
}
function wordsOf(transcript: string): TimedWord[] {
return loadTranscript(transcript).words.map(({ text, start, end }) => ({ text, start, end }));
}
const fingerprint = (path: string | null) =>
path ? createHash("sha256").update(readFileSync(path)).digest("hex").slice(0, 16) : "none";
const CACHE_REVISION = 1;
const MAX_ENTRY_BYTES = 256 * 1024 * 1024;
interface CacheEntry {
revision: number;
analysis: ReadAudio;
transcriptKey: string;
}
function cachedEntry(jsonPath: string, textPath: string): CacheEntry | null {
if (!isPlainFile(jsonPath) || !isPlainFile(textPath)) return null;
try {
return JSON.parse(readPlainFile(jsonPath, MAX_ENTRY_BYTES)) as CacheEntry;
} catch {
return null;
}
}
async function stemsFor(
path: string,
dir: string,
quality: AnalysisQuality,
warnings: string[],
): Promise<Record<string, string> | null> {
const cached = cachedStems(dir);
if (cached) return cached;
if (quality !== "best") {
warnings.push(
`not separated into stems: quality ${quality} keeps audio on this machine; --quality best separates it through HeyGen`,
);
return null;
}
const access = await stemsAccess();
if (typeof access === "string") {
warnings.push(`not separated into stems: ${access}`);
return null;
}
try {
return await fetchStems(path, dir, access);
} catch (error) {
warnings.push(`not separated into stems: ${error instanceof Error ? error.message : error}`);
return null;
}
}
function heardStems(tracks: Record<string, Float32Array>): AudibleStem[] {
return Object.entries(tracks)
.flatMap(([name, samples]) => {
const db = levelWhileHeardDb(samples, READ_RATE);
return db === null
? []
: [{ name, db, spans: activeSpans(samples, READ_RATE), description: null }];
})
.sort((a, b) => b.db - a.db || a.name.localeCompare(b.name));
}
const VOICE_FREE = "no-vocals.wav";
/** Stems summed at full quality into one file next to them, made once. */
async function mixedFile(paths: string[], out: string, warnings: string[]) {
if (isPlainFile(out)) return out;
const ffmpeg = findFfBinary("ffmpeg", { configuredMustExist: true });
if (!ffmpeg) {
warnings.push(`no ${basename(out)}: ffmpeg is required; run: npx hyperframes doctor`);
return null;
}
let stage: string | undefined;
try {
stage = createStagingDir(dirname(out), ".hf-mix-");
const temporary = join(stage, VOICE_FREE);
await runFile(
ffmpeg,
[
...["-v", "error", "-y"],
...paths.flatMap((path) => ["-i", path]),
...["-filter_complex", `amix=inputs=${paths.length}:normalize=0`],
...["-c:a", "pcm_f32le", temporary],
],
{ windowsHide: true, maxBuffer: 8 * 1024 * 1024 },
);
renameSync(temporary, out);
return out;
} catch (error) {
warnings.push(`no ${basename(out)}: ${error instanceof Error ? error.message : error}`);
return null;
} finally {
if (stage) rmSync(stage, { recursive: true, force: true });
}
}
const voiceFreeMix = (stemPaths: Record<string, string>, warnings: string[]) =>
mixedFile(
Object.entries(stemPaths)
.filter(([name]) => name !== VOICE_STEM)
.map(([, path]) => path),
join(dirname(stemPaths[VOICE_STEM]!), VOICE_FREE),
warnings,
);
function tempoOf(music: Music | null): number | null {
const longest = [...(music?.segments ?? [])]
.filter((m) => m.bpm !== null)
.sort((a, b) => b.end - b.start - (a.end - a.start))[0];
return longest?.bpm ?? null;
}
async function decodedStems(stemPaths: Record<string, string>) {
const pitched: Record<string, Float32Array> = {};
for (const [name, stemPath] of Object.entries(stemPaths))
if (name !== VOICE_STEM && name !== DRUM_STEM) pitched[name] = await decodeMono(stemPath);
const drums = stemPaths[DRUM_STEM] ? await decodeMono(stemPaths[DRUM_STEM]) : null;
return { pitched, drums };
}
const present = (tracks: (Float32Array | null)[]) =>
tracks.filter((t): t is Float32Array => t !== null);
function musicOf(
pitched: Record<string, Float32Array>,
drums: Float32Array | null,
bed: Float32Array | null,
) {
return bed
? readMusic({ stems: pitched, drums, sampleRate: READ_RATE })
: { music: null, ambience: null, beats: [] };
}
function separatedStems(
stemPaths: Record<string, string>,
pitched: Record<string, Float32Array>,
drums: Float32Array | null,
vocals: Float32Array,
): AudibleStem[] {
return heardStems({
...pitched,
...(drums ? { [DRUM_STEM]: drums } : {}),
...(stemPaths[VOICE_STEM] ? { [VOICE_STEM]: vocals } : {}),
});
}
const recording = (path: string, separated: boolean, voiceSamples: Float32Array) =>
separated ? decodeMono(path) : Promise.resolve(voiceSamples);
function excerpt(samples: Float32Array, spans: readonly Span[]): Float32Array {
const parts = spans.map((span) =>
samples.subarray(Math.round(span.start * READ_RATE), Math.round(span.end * READ_RATE)),
);
const out = new Float32Array(parts.reduce((n, p) => n + p.length, 0));
let at = 0;
for (const part of parts) {
out.set(part, at);
at += part.length;
}
return out;
}
function againstMusic(
voice: Voice,
vocals: Float32Array,
music: Float32Array,
segments: readonly Span[],
): Voice {
const sentences = voice.sentences.map((sentence) => {
const under = segments
.map((m) => ({
start: Math.max(m.start, sentence.start),
end: Math.min(m.end, sentence.end),
}))
.filter((span) => span.end > span.start);
if (!under.length) return sentence;
const voiceLufs = loudness(asSignal(excerpt(vocals, under))).lufsIntegrated;
const musicLufs = loudness(asSignal(excerpt(music, under))).lufsIntegrated;
return voiceLufs === null || musicLufs === null
? sentence
: { ...sentence, againstMusic: { voiceLufs, musicLufs } };
});
return { ...voice, sentences };
}
function effectSources(
pitched: Record<string, Float32Array>,
drums: Float32Array | null,
bed: Float32Array | null,
music: Music | null,
ambience: Span[] | null,
): EffectSource[] {
const segments = music?.segments ?? [];
const playing = [...segments, ...(music?.breaks ?? [])];
return [
...(drums ? [{ samples: drums, stem: DRUM_STEM, playsIn: playing }] : []),
...(bed ? [{ samples: bed, heldBy: [...segments, ...(ambience ?? [])], parts: pitched }] : []),
];
}
/** Effects from the drums and from the bed where it plays neither music nor ambience, each level from the whole recording. */
async function fromStems(
stemPaths: Record<string, string>,
voice: Voice,
vocals: Float32Array,
mix: Float32Array,
) {
const { pitched, drums } = await decodedStems(stemPaths);
const bed = Object.keys(pitched).length ? mixed(Object.values(pitched)) : null;
const { music, ambience, beats } = musicOf(pitched, drums, bed);
const segments = music?.segments ?? [];
const everythingElse = mixed(present([bed, drums]));
const effects = findEffects({
sources: effectSources(pitched, drums, bed, music, ambience),
sampleRate: READ_RATE,
mix,
music: bed && everythingElse,
musicSegments: segments,
beats,
speech: [],
});
const stems = separatedStems(stemPaths, pitched, drums, vocals).filter((stem) =>
heardLongEnough(stem, vocals.length / READ_RATE),
);
const read = music ? againstMusic(voice, vocals, everythingElse, music.segments) : voice;
return { voice: read, stems, effects, music: music && onlyThe(stems, music), ambience };
}
const SHORTEST_STEM = 2;
const SMALLEST_STEM_SHARE = 0.05;
/** A stem heard for under two seconds, or under 5% of the recording, is left out of the reading. */
function heardLongEnough(stem: AudibleStem, duration: number): boolean {
const heard = stem.spans.reduce((n, span) => n + span.end - span.start, 0);
return heard >= SHORTEST_STEM && heard >= SMALLEST_STEM_SHARE * duration;
}
function onlyThe(stems: readonly AudibleStem[], music: Music): Music {
const kept = new Set(stems.map((stem) => stem.name));
return {
...music,
pieces: music.pieces.map((p) => ({
...p,
instruments: p.instruments.filter((n) => kept.has(n)),
})),
segments: music.segments.map((m) => ({ ...m, stems: m.stems.filter((s) => kept.has(s.name)) })),
};
}
const transcribeHint = (file: string) => `run hyperframes transcribe ${file} and pass --transcript`;
async function described(
stemPaths: Record<string, string>,
read: ReadAudio,
cacheDir: string,
quality: AnalysisQuality,
): Promise<ReadAudio> {
const warnings = [...read.warnings];
const project = dirname(dirname(cacheDir));
const mix = await voiceFreeMix(stemPaths, warnings);
const located = {
...read,
stemsDir: `${relative(project, dirname(stemPaths[VOICE_STEM]!))}/`,
voiceFreeMix: mix && relative(project, mix),
warnings,
};
if (quality === "fast") return located;
const { stems, description } = await describeStems(
stemPaths,
{
duration: read.duration,
bpm: read.bpm,
stems: read.stems,
breaks: read.music?.breaks ?? [],
},
cacheDir,
warnings,
);
return { ...located, stems, description };
}
async function analyze(
path: string,
file: string,
sha: string,
transcript: string | null,
stemDir: string,
quality: AnalysisQuality,
cacheDir: string,
): Promise<ReadAudio> {
const warnings: string[] = [];
const stemPaths = await stemsFor(path, stemDir, quality, warnings);
const words = transcript ? wordsOf(transcript) : [];
const voicePath = stemPaths?.[VOICE_STEM] ?? path;
const voiceSamples = await decodeMono(voicePath);
const about = base(file, sha, round(voiceSamples.length / READ_RATE, 3), stemPaths, transcript);
if (!stemPaths && !words.length) {
warnings.push(
`no transcript and no stems: voice and effects are not read, because the mix alone does not tell speech from music; ${transcribeHint(file)}`,
);
return { ...about, voice: null, effects: null, music: null, ambience: null, warnings };
}
if (!words.length)
warnings.push(`no transcript: voice lines carry no words; ${transcribeHint(file)}`);
const pitch = pitchTrack(await decodeMono(voicePath, PITCH_RATE));
const mix = await recording(path, stemPaths !== null, voiceSamples);
const levelOf = (span: Span) => spanDb(mix, READ_RATE, span.start, span.end);
const voice = words.length
? voiceFromWords(words, pitch, levelOf)
: voiceFromActivity(voiceSamples, READ_RATE, pitch, levelOf);
if (stemPaths) {
const read = await fromStems(stemPaths, voice, voiceSamples, mix);
const analysis = { ...about, ...read, bpm: tempoOf(read.music), warnings };
return described(stemPaths, analysis, cacheDir, quality);
}
warnings.push("without stems, effects under speech are not found and music is not read");
const effects = findEffects({
sources: [{ samples: voiceSamples }],
sampleRate: READ_RATE,
mix,
music: null,
beats: [],
speech: voice.sentences,
});
return { ...about, voice, effects, music: null, ambience: null, warnings };
}
async function analyzeLayer(
path: string,
file: string,
sha: string,
transcript: string | null,
layer: Layer,
): Promise<ReadAudio> {
const samples = await decodeMono(path);
const about = {
...base(file, sha, round(samples.length / READ_RATE, 3), null, transcript),
layer,
};
const none = { voice: null, effects: null, music: null, ambience: null, warnings: [] };
if (layer === "music") {
const { music, ambience } = musicOf({ music: samples }, null, samples);
return { ...about, ...none, music, ambience, bpm: tempoOf(music) };
}
if (layer === "sfx") {
const effects = findEffects({
sources: [{ samples }],
sampleRate: READ_RATE,
mix: samples,
music: null,
beats: [],
speech: [],
});
return { ...about, ...none, effects };
}
const words = transcript ? wordsOf(transcript) : [];
const pitch = pitchTrack(await decodeMono(path, PITCH_RATE));
const levelOf = (span: Span) => spanDb(samples, READ_RATE, span.start, span.end);
const voice = words.length
? voiceFromWords(words, pitch, levelOf)
: voiceFromActivity(samples, READ_RATE, pitch, levelOf);
const warnings = words.length
? []
: [`no transcript: voice lines carry no words; ${transcribeHint(file)}`];
return { ...about, ...none, voice, warnings };
}
function base(
file: string,
sha: string,
duration: number,
stemPaths: Record<string, string> | null,
transcript: string | null,
): Omit<ReadAudio, "voice" | "effects" | "music" | "ambience" | "warnings"> {
return {
version: READ_VERSION,
file,
sha256: sha,
duration,
separation: stemPaths ? "stems" : "mix",
stems: [],
stemsDir: null,
voiceFreeMix: null,
bpm: null,
description: null,
transcript: transcript ? basename(transcript) : null,
};
}
/** Read a recording as time-stamped voice, effect and music events, computed once per file content. */
export async function readAudio(
path: string,
file: string,
options: ReadOptions,
): Promise<ReadResult> {
const quality = options.quality ?? "local";
const { layer } = options;
const sha = await sha256(path);
const transcript = transcriptFor(path, options.transcript);
const transcriptKey = fingerprint(transcript);
const stem = `${basename(file, extname(file)).replace(/[^\w.-]/g, "_")}-${sha.slice(0, 12)}`;
const cacheDir = makeRealFolder(options.cacheDir, 2);
const jsonPath = join(cacheDir, `${stem}.json`);
const textPath = join(cacheDir, `${stem}.txt`);
const stemDir = join(cacheDir, "stems", sha.slice(0, 12));
const entry = cachedEntry(jsonPath, textPath);
if (entry) {
const stemsNowPossible =
!layer &&
entry.analysis.separation === "mix" &&
quality === "best" &&
(cachedStems(stemDir) !== null || typeof (await stemsAccess()) !== "string");
if (
(entry.analysis.separation !== "stems" || cachedStems(stemDir) !== null) &&
entry.analysis.version === READ_VERSION &&
entry.analysis.layer === layer &&
entry.revision === CACHE_REVISION &&
entry.transcriptKey === transcriptKey &&
!stemsNowPossible &&
!(
entry.analysis.separation === "stems" &&
!entry.analysis.description &&
quality !== "fast" &&
typeof (await describeAccess()) !== "string"
)
)
return { analysis: entry.analysis, textPath, jsonPath, cached: true };
}
const analysis = layer
? await analyzeLayer(path, file, sha, transcript, layer)
: await analyze(path, file, sha, transcript, stemDir, quality, cacheDir);
replaceFileAtomically(
jsonPath,
JSON.stringify({ revision: CACHE_REVISION, analysis, transcriptKey } satisfies CacheEntry),
);
replaceFileAtomically(textPath, formatRead(analysis));
return { analysis, textPath, jsonPath, cached: false };
}