fix(skills): bake a short edge fade into every faceless-explainer voice clip

Per-frame voice clips are butt-joined back-to-back with zero gap by the
cumulative-start layout (adjacent clips touch exactly, by construction),
but were mounted with no fade at all — a hard cut at both edges of every
line, producing audible word-jumps and abrupt cuts at scene seams.

applyVoiceEdgeFade() bakes an ~11ms in/out fade into each voice file
before it's mounted, the same way ensureBgmCovers() already bakes BGM's
own loop-fade — same ffprobe-then-ffmpeg shape, same extension/codec
discipline (explicit PCM output so a .wav-named file is actually WAV).
Each edge is capped to half the clip's own duration so a very short
clip still fades smoothly instead of going silent for its whole length.
Degrades to the untouched clip (with an anomaly note) when ffmpeg/
ffprobe are unavailable, matching ensureBgmCovers()'s own degrade path.

Cost: one extra ffprobe + ffmpeg subprocess pair per voice clip, each a
fast audio-only pass over a few seconds of speech — negligible next to
the render pipeline's own screenshot/encode time.

Co-Authored-By: Miguel Ángel <miguel.sierra@heygen.com>
This commit is contained in:
miga-heygen
2026-09-16 13:26:51 +00:00
co-authored by Miguel Ángel
parent 08d48ab753
commit b437da6963
3 changed files with 183 additions and 2 deletions
+1 -1
View File
@@ -6,7 +6,7 @@
"files": 142
},
"faceless-explainer": {
"hash": "7ff529b85f588ffc",
"hash": "49d84ce2eede2b2e",
"files": 24
},
"figma": {
@@ -123,6 +123,54 @@ function ensureBgmCovers(relPath, hyperframesDir, total) {
return { looped: true, rel: relOut, from: dur };
}
// Voice clips are butt-joined back-to-back with zero gap by the cumulative-start
// layout below (see "cumulative starts" comment) — by construction, with no edge
// fade at all, so consecutive lines cut/word-jump at the join. Bake a short in/out
// fade into each voice file itself, the same way ensureBgmCovers() bakes BGM's
// loop-fade, into a sibling *.faded.wav; mounting still just points
// data-start/duration at the same slot, only the audio content changes. Needs
// ffprobe+ffmpeg (present in the render env); degrades to the original file + an
// anomaly note when they're absent, matching ensureBgmCovers()'s degrade path.
const VOICE_FADE_SECONDS = 0.011;
function applyVoiceEdgeFade(relPath, hyperframesDir) {
const abs = join(hyperframesDir, relPath);
const probe = spawnSync(
"ffprobe",
["-v", "error", "-show_entries", "format=duration", "-of", "csv=p=0", "--", abs],
{ encoding: "utf8" },
);
if (probe.status !== 0) return { faded: false, reason: "ffprobe unavailable" };
const dur = parseFloat(String(probe.stdout || "").trim());
if (!Number.isFinite(dur) || dur <= 0) return { faded: false, reason: "unreadable duration" };
// Each edge gets at most half the clip (dur is already confirmed > 0 above,
// so this is always > 0 too), so a clip shorter than 2 fade windows fades
// smoothly in and out instead of being silent for its entire length.
const fade = Math.min(VOICE_FADE_SECONDS, dur / 2);
// Always emit .wav via an explicit PCM codec, matching ensureBgmCovers()'s own
// extension/codec discipline just above — the contract's voice files are
// "assets/voice/NN.wav", so the output must actually be WAV, not smuggle a
// different container in a .wav-named file.
const relOut = relPath.replace(/\.([^./]+)$/, ".faded.wav");
const absOut = join(hyperframesDir, relOut);
const fadeOutStart = Math.max(0, dur - fade);
const ff = spawnSync(
"ffmpeg",
[
"-y",
"-i",
abs,
"-af",
`afade=t=in:st=0:d=${fade},afade=t=out:st=${fadeOutStart}:d=${fade}`,
"-c:a",
"pcm_s16le",
absOut,
],
{ encoding: "utf8" },
);
if (ff.status !== 0 || !existsSync(absOut)) return { faded: false, reason: "ffmpeg unavailable" };
return { faded: true, rel: relOut };
}
const hyperframesDir = resolve(flag("hyperframes", "."));
const storyboardPath = resolve(flag("storyboard", join(hyperframesDir, "STORYBOARD.md")));
const audioMetaPath = resolve(flag("audio-meta", join(hyperframesDir, "audio_meta.json")));
@@ -394,10 +442,17 @@ for (const m of mounted) {
const v = m.frame.number != null ? voiceByFrame.get(m.frame.number) : undefined;
if (v?.path) {
if (existsSync(join(hyperframesDir, v.path))) {
let voiceSrc = v.path;
const fade = applyVoiceEdgeFade(v.path, hyperframesDir);
if (fade.faded) {
voiceSrc = fade.rel;
} else {
anomalies.push(`${m.compId}: voice edge fade skipped (${fade.reason}) — using raw clip`);
}
body.push(
` <audio`,
` id="el-${m.compId}-voice"`,
` src="${v.path}"`,
` src="${voiceSrc}"`,
` data-start="${m.start}"`,
` data-duration="${m.durationSeconds}"`,
` data-track-index="10"`,
@@ -125,3 +125,129 @@ test(
assert.equal(probe.stdout.trim(), "mp3");
},
);
// ── voice clip edge fades ─────────────────────────────────────────────────────
// Voice clips are butt-joined with zero gap by the cumulative-start layout, and
// (pre-fix) mounted with zero edge fade — a hard cut at both edges, causing
// audible word-jumps/abrupt cuts at scene seams. This measures the actual RMS
// level in a short window at the very start of the mounted voice audio: a
// constant-amplitude tone should measure near its full level pre-fix, and a
// clearly quieter level once an in-fade is baked in.
function rmsDbAt(path, startSec, durSec) {
const r = spawnSync(
"ffmpeg",
[
"-i",
path,
"-af",
`atrim=start=${startSec}:end=${startSec + durSec},astats=metadata=1:reset=1`,
"-f",
"null",
"-",
],
{ encoding: "utf8" },
);
// Mono input reports the same RMS level per-channel and "Overall" — take the
// last occurrence (the "Overall" summary line) rather than anchor to it by
// position, since every astats line carries its own `[Parsed_astats_N @ ...]`
// prefix rather than being indented under a bare "Overall" heading.
const matches = [...r.stderr.matchAll(/RMS level dB:\s*(-?[\d.]+|-inf)/g)];
assert.ok(matches.length > 0, `expected an RMS level dB reading in ffmpeg stderr:\n${r.stderr}`);
const value = matches.at(-1)[1];
return value === "-inf" ? -Infinity : parseFloat(value);
}
test("voice clips get a short edge fade baked in, not a hard cut", { skip: !HAS_FFMPEG }, () => {
const { dir, r } = assembleWith({
audioMeta: { bgm: null, voices: [{ frame: 1, path: "assets/voice/01.wav" }], sfx: [] },
beforeAssemble: (projectDir) => {
mkdirSync(join(projectDir, "assets", "voice"), { recursive: true });
// A constant-amplitude tone spanning the full 3s frame duration: with no
// fade, the start/end windows measure the same RMS level as the middle.
const gen = spawnSync(
"ffmpeg",
[
"-y",
"-f",
"lavfi",
"-i",
"sine=frequency=440:duration=3",
join(projectDir, "assets", "voice", "01.wav"),
],
{ encoding: "utf8" },
);
assert.equal(gen.status, 0, gen.stderr);
},
});
assert.equal(r.status, 0, r.stderr);
const html = readFileSync(join(dir, "index.html"), "utf8");
const voiceEl = html.match(/id="el-[^"]*-voice"[\s\S]*?src="([^"]+)"/);
assert.ok(voiceEl, "expected a voice <audio> element in index.html");
const voiceSrc = voiceEl[1];
assert.match(voiceSrc, /\.faded\.wav$/, `voice src should end in .faded.wav, got "${voiceSrc}"`);
const outPath = join(dir, voiceSrc);
assert.equal(existsSync(outPath), true);
const rawMidDb = rmsDbAt(join(dir, "assets", "voice", "01.wav"), 1.5, 0.005);
const fadedStartDb = rmsDbAt(outPath, 0, 0.002);
const fadedMidDb = rmsDbAt(outPath, 1.5, 0.005);
// The clip's steady middle is unaffected by the edge fade.
assert.ok(
Math.abs(fadedMidDb - rawMidDb) < 1,
`mid-clip level should be ~unchanged: raw ${rawMidDb} dB vs faded ${fadedMidDb} dB`,
);
// The very start of the faded clip should read markedly quieter than the
// steady-state level — the fade-in ramp, not a hard cut at full volume.
assert.ok(
fadedStartDb < fadedMidDb - 6,
`start-of-clip level should be well below mid-clip level: start ${fadedStartDb} dB vs mid ${fadedMidDb} dB`,
);
});
test(
"a clip shorter than two fade windows still fades, without going silent for its whole length",
{ skip: !HAS_FFMPEG },
() => {
const { dir, r } = assembleWith({
audioMeta: { bgm: null, voices: [{ frame: 1, path: "assets/voice/01.wav" }], sfx: [] },
beforeAssemble: (projectDir) => {
mkdirSync(join(projectDir, "assets", "voice"), { recursive: true });
// Shorter than 2 * VOICE_FADE_SECONDS (0.011s) would ever be in practice,
// but proves the half-duration clamp rather than assuming it.
const gen = spawnSync(
"ffmpeg",
[
"-y",
"-f",
"lavfi",
"-i",
"sine=frequency=440:duration=0.01",
join(projectDir, "assets", "voice", "01.wav"),
],
{ encoding: "utf8" },
);
assert.equal(gen.status, 0, gen.stderr);
},
});
assert.equal(r.status, 0, r.stderr);
const html = readFileSync(join(dir, "index.html"), "utf8");
const voiceEl = html.match(/id="el-[^"]*-voice"[\s\S]*?src="([^"]+)"/);
assert.ok(voiceEl, "expected a voice <audio> element in index.html");
const outPath = join(dir, voiceEl[1]);
assert.equal(existsSync(outPath), true);
const probe = spawnSync(
"ffprobe",
["-v", "error", "-show_entries", "format=duration", "-of", "csv=p=0", "--", outPath],
{ encoding: "utf8" },
);
const outDur = parseFloat(probe.stdout.trim());
assert.ok(
outDur > 0.005,
`faded clip should not have collapsed to near-zero length: ${outDur}s`,
);
},
);