Files
hyperframes/packages/cli/src/utils/checkAudio.ts
T
Miguel Ángel d83bf997b0 feat(cli): check catches audio the render would lose or play as silence (#6439)
* feat(cli): check catches audio the render would lose or play as silence

check gains an Audio section, read from the media list the render mixes:
- error: a clip's source has no audio the render can read (its mix fails,
  the film ships without sound)
- warning: a clip starts past the end of its file, or the part it plays
  is silent
- info: the film has no sound at all

The bundle and media list are now built once and shared with the HDR
inspection; the audio pass runs while the browser pass does.

Fixes #2775

* fix(cli): check reports an unreadable audio source the way render handles it

A source with no audio track is refused by render's own media-type rule, now exported from the producer and called by check; a source that does not decode as audio fails the mix and stops the render.

* perf(cli): check samples a long clip at a few spots before reading all of it for silence

Sound at any spot ends the decode; a clip is only called silent after its whole played part is read.

* chore(cli): drop a comment the shared finding helper's name already says
2026-10-26 17:02:57 +00:00

217 lines
7.9 KiB
TypeScript

import { existsSync } from "node:fs";
import { parseHTML } from "linkedom";
import { normalizeRateSpec, sourceTimeAt } from "@hyperframes/core";
import { silence } from "@hyperframes/core/audio-analysis";
import {
extractAudioMetadata,
extractMediaMetadata,
probeMediaProfile,
type AudioElement,
type AudioMetadata,
} from "@hyperframes/engine";
import { decodeAudio } from "../audio/analysis/decode.js";
import { createProbeGate } from "../timeline/describeProject.js";
import { findingAtRoot } from "./checkFindings.js";
import { loadProducer } from "./producer.js";
import type { CheckFinding } from "./checkTypes.js";
import { mediaPath, type ProjectMedia } from "./projectMedia.js";
export interface AudioCheckDependencies {
/** Render's own media-type rule for a mixed clip; `null` when the file does not probe. */
renderAccepts(path: string): Promise<boolean | null>;
probeAudio(path: string): Promise<AudioMetadata>;
videoHasAudio(path: string): Promise<boolean>;
/** Whether `seconds` of the file from `from` are silent all through. */
isSilent(path: string, from: number, seconds: number, probed: AudioMetadata): Promise<boolean>;
}
/** Silence is a level question: a low mono rate decodes fast and keeps it. */
const SILENCE_SAMPLE_RATE = 8_000;
/** The silence measure's last analysis window can stop short of the decoded end. */
const EDGE_SECONDS = 0.02;
/** A long clip is first sampled at a few short spots; sound in any one ends the decode. */
const SPOT_SECONDS = 1;
const SPOTS = 3;
const PROBE_CONCURRENCY = 4;
const DEFAULT_DEPENDENCIES: AudioCheckDependencies = {
async renderAccepts(path) {
const profile = await probeMediaProfile(path).catch(() => null);
if (!profile) return null;
const { mediaProfileMatchesElementType } = await loadProducer();
return mediaProfileMatchesElementType("audio", profile);
},
probeAudio: (path) => extractAudioMetadata(path),
videoHasAudio: async (path) => (await extractMediaMetadata(path)).hasAudio,
async isSilent(path, from, seconds, probed) {
if (seconds > SPOT_SECONDS * SPOTS) {
const step = (seconds - SPOT_SECONDS) / (SPOTS - 1);
for (let spot = 0; spot < SPOTS; spot++)
if (!(await silentPart(path, from + spot * step, SPOT_SECONDS, probed))) return false;
}
return silentPart(path, from, seconds, probed);
},
};
async function silentPart(
path: string,
from: number,
seconds: number,
probed: AudioMetadata,
): Promise<boolean> {
const decoded = await decodeAudio(
path,
{ from, seconds },
{ sampleRate: probed.sampleRate, channels: probed.channels, duration: probed.durationSeconds },
{ mono: true, sampleRate: SILENCE_SAMPLE_RATE },
);
const samples = decoded?.signal.channels[0]?.length ?? 0;
if (!decoded || samples === 0) return true;
const [quiet, ...rest] = silence(decoded.signal);
return (
!!quiet &&
rest.length === 0 &&
quiet.start === 0 &&
quiet.end >= samples / SILENCE_SAMPLE_RATE - EDGE_SECONDS
);
}
/** Findings for the clips the render mixes: unreadable source, start past the file's end, silent
* played part; or a film with no sound at all. */
export async function inspectAudio(
projectDir: string,
sourceFile: string,
media: ProjectMedia,
dependencies: AudioCheckDependencies = DEFAULT_DEPENDENCIES,
): Promise<CheckFinding[]> {
const withSlot = createProbeGate(PROBE_CONCURRENCY);
// Parsed only for a clip with a finding, to name the composition file it was written in.
let bundled: Document | undefined;
const fileOf = (id: string) => {
bundled ??= parseHTML(media.html).document;
const host = bundled.getElementById(id)?.closest("[data-composition-file]");
return host?.getAttribute("data-composition-file") || sourceFile;
};
const findings = await Promise.all(
media.audios.map((clip) => withSlot(() => inspectClip(projectDir, fileOf, clip, dependencies))),
);
if (
media.audios.length === 0 &&
!(await videoSoundsAutoDetected(projectDir, media, dependencies))
)
return [
findingAtRoot(
"audio_none",
"info",
"The film has no sound: no clip in it plays audio.",
sourceFile,
),
];
return findings.flat();
}
/** Render adds an unmuted video's own sound when its file has an audio stream. */
async function videoSoundsAutoDetected(
projectDir: string,
media: ProjectMedia,
dependencies: AudioCheckDependencies,
): Promise<boolean> {
for (const video of media.videos) {
if (!video.hasAudio) continue;
if (await dependencies.videoHasAudio(mediaPath(video.src, projectDir)).catch(() => false))
return true;
}
return false;
}
type ClipProblem = Pick<CheckFinding, "code" | "severity" | "message" | "fixHint">;
async function inspectClip(
projectDir: string,
fileOf: (id: string) => string,
clip: AudioElement,
dependencies: AudioCheckDependencies,
): Promise<CheckFinding[]> {
const path = mediaPath(clip.src, projectDir);
// ponytail: remote sources are not fetched; missing local files are lint's audio_src_not_found.
if (path === clip.src || !existsSync(path)) return [];
const problem = await clipProblem(path, clip, dependencies);
if (!problem) return [];
const id = clip.type === "video" ? clip.id.replace(/-audio$/, "") : clip.id;
const played: Partial<CheckFinding> =
clip.end > clip.start ? { timeRange: [clip.start, clip.end] } : {};
return [
{
...findingAtRoot(problem.code, problem.severity, problem.message, fileOf(id)),
selector: `#${id}`,
time: clip.start,
...played,
fixHint: problem.fixHint,
},
];
}
async function clipProblem(
path: string,
clip: AudioElement,
dependencies: AudioCheckDependencies,
): Promise<ClipProblem | null> {
if ((await dependencies.renderAccepts(path)) === false) return rejected(clip);
let probed: AudioMetadata;
try {
probed = await dependencies.probeAudio(path);
} catch (error) {
return unreadable(clip, error);
}
const fileSeconds = probed.durationSeconds;
if (clip.mediaStart >= fileSeconds) {
return {
code: "audio_starts_past_source_end",
severity: "warning",
message: `${clip.src} is ${round(fileSeconds)} s long but the clip starts ${round(clip.mediaStart)} s into it, so it plays nothing.`,
fixHint: "Lower data-media-start below the file's length.",
};
}
const from = clip.mediaStart;
const seconds = Math.min(playedSourceSeconds(clip), fileSeconds - from);
if (!(await dependencies.isSilent(path, from, seconds, probed))) return null;
return {
code: "audio_clip_silent",
severity: "warning",
message: `${clip.src} is silent for the whole part this clip plays (${round(from)}-${round(from + seconds)} s of the file).`,
fixHint: "Check the file has sound there, or move data-media-start to where it does.",
};
}
function rejected(clip: AudioElement): ClipProblem {
return {
code: "audio_source_rejected",
severity: "error",
message: `${clip.src} has no audio track, so render stops before it starts.`,
fixHint: fixSource(clip),
};
}
function unreadable(clip: AudioElement, error: unknown): ClipProblem {
return {
code: "audio_source_unreadable",
severity: "error",
message: `${clip.src} cannot be read as audio (${error instanceof Error ? error.message : String(error)}), so the render's audio mix fails and the render stops.`,
fixHint: fixSource(clip),
};
}
function fixSource(clip: AudioElement): string {
return clip.type === "video"
? 'Remove data-has-audio="true" from the video, or point it at a file with an audio track.'
: "Point src at a file with an audio track, or re-export this one.";
}
/** Seconds of its file a clip plays; unbounded clips play to the end of the file. */
function playedSourceSeconds(clip: AudioElement): number {
if (clip.end <= clip.start) return Infinity;
return sourceTimeAt(normalizeRateSpec(clip.playbackRate), clip.end - clip.start);
}
const round = (seconds: number) => Math.round(seconds * 100) / 100;