mirror of
https://github.com/heygen-com/hyperframes.git
synced 2026-11-17 23:47:33 +00:00
fix(skills): bake a short edge fade into every faceless-explainer voice clip
Per-frame voice clips are butt-joined back-to-back with zero gap by the cumulative-start layout (adjacent clips touch exactly, by construction), but were mounted with no fade at all — a hard cut at both edges of every line, producing audible word-jumps and abrupt cuts at scene seams. applyVoiceEdgeFade() bakes an ~11ms in/out fade into each voice file before it's mounted, the same way ensureBgmCovers() already bakes BGM's own loop-fade — same ffprobe-then-ffmpeg shape, same extension/codec discipline (explicit PCM output so a .wav-named file is actually WAV). Each edge is capped to half the clip's own duration so a very short clip still fades smoothly instead of going silent for its whole length. Degrades to the untouched clip (with an anomaly note) when ffmpeg/ ffprobe are unavailable, matching ensureBgmCovers()'s own degrade path. Cost: one extra ffprobe + ffmpeg subprocess pair per voice clip, each a fast audio-only pass over a few seconds of speech — negligible next to the render pipeline's own screenshot/encode time. Co-Authored-By: Miguel Ángel <miguel.sierra@heygen.com>
This commit is contained in:
co-authored by
Miguel Ángel
parent
08d48ab753
commit
b437da6963
@@ -6,7 +6,7 @@
|
||||
"files": 142
|
||||
},
|
||||
"faceless-explainer": {
|
||||
"hash": "7ff529b85f588ffc",
|
||||
"hash": "49d84ce2eede2b2e",
|
||||
"files": 24
|
||||
},
|
||||
"figma": {
|
||||
|
||||
@@ -123,6 +123,54 @@ function ensureBgmCovers(relPath, hyperframesDir, total) {
|
||||
return { looped: true, rel: relOut, from: dur };
|
||||
}
|
||||
|
||||
// Voice clips are butt-joined back-to-back with zero gap by the cumulative-start
|
||||
// layout below (see "cumulative starts" comment) — by construction, with no edge
|
||||
// fade at all, so consecutive lines cut/word-jump at the join. Bake a short in/out
|
||||
// fade into each voice file itself, the same way ensureBgmCovers() bakes BGM's
|
||||
// loop-fade, into a sibling *.faded.wav; mounting still just points
|
||||
// data-start/duration at the same slot, only the audio content changes. Needs
|
||||
// ffprobe+ffmpeg (present in the render env); degrades to the original file + an
|
||||
// anomaly note when they're absent, matching ensureBgmCovers()'s degrade path.
|
||||
const VOICE_FADE_SECONDS = 0.011;
|
||||
function applyVoiceEdgeFade(relPath, hyperframesDir) {
|
||||
const abs = join(hyperframesDir, relPath);
|
||||
const probe = spawnSync(
|
||||
"ffprobe",
|
||||
["-v", "error", "-show_entries", "format=duration", "-of", "csv=p=0", "--", abs],
|
||||
{ encoding: "utf8" },
|
||||
);
|
||||
if (probe.status !== 0) return { faded: false, reason: "ffprobe unavailable" };
|
||||
const dur = parseFloat(String(probe.stdout || "").trim());
|
||||
if (!Number.isFinite(dur) || dur <= 0) return { faded: false, reason: "unreadable duration" };
|
||||
// Each edge gets at most half the clip (dur is already confirmed > 0 above,
|
||||
// so this is always > 0 too), so a clip shorter than 2 fade windows fades
|
||||
// smoothly in and out instead of being silent for its entire length.
|
||||
const fade = Math.min(VOICE_FADE_SECONDS, dur / 2);
|
||||
// Always emit .wav via an explicit PCM codec, matching ensureBgmCovers()'s own
|
||||
// extension/codec discipline just above — the contract's voice files are
|
||||
// "assets/voice/NN.wav", so the output must actually be WAV, not smuggle a
|
||||
// different container in a .wav-named file.
|
||||
const relOut = relPath.replace(/\.([^./]+)$/, ".faded.wav");
|
||||
const absOut = join(hyperframesDir, relOut);
|
||||
const fadeOutStart = Math.max(0, dur - fade);
|
||||
const ff = spawnSync(
|
||||
"ffmpeg",
|
||||
[
|
||||
"-y",
|
||||
"-i",
|
||||
abs,
|
||||
"-af",
|
||||
`afade=t=in:st=0:d=${fade},afade=t=out:st=${fadeOutStart}:d=${fade}`,
|
||||
"-c:a",
|
||||
"pcm_s16le",
|
||||
absOut,
|
||||
],
|
||||
{ encoding: "utf8" },
|
||||
);
|
||||
if (ff.status !== 0 || !existsSync(absOut)) return { faded: false, reason: "ffmpeg unavailable" };
|
||||
return { faded: true, rel: relOut };
|
||||
}
|
||||
|
||||
const hyperframesDir = resolve(flag("hyperframes", "."));
|
||||
const storyboardPath = resolve(flag("storyboard", join(hyperframesDir, "STORYBOARD.md")));
|
||||
const audioMetaPath = resolve(flag("audio-meta", join(hyperframesDir, "audio_meta.json")));
|
||||
@@ -394,10 +442,17 @@ for (const m of mounted) {
|
||||
const v = m.frame.number != null ? voiceByFrame.get(m.frame.number) : undefined;
|
||||
if (v?.path) {
|
||||
if (existsSync(join(hyperframesDir, v.path))) {
|
||||
let voiceSrc = v.path;
|
||||
const fade = applyVoiceEdgeFade(v.path, hyperframesDir);
|
||||
if (fade.faded) {
|
||||
voiceSrc = fade.rel;
|
||||
} else {
|
||||
anomalies.push(`${m.compId}: voice edge fade skipped (${fade.reason}) — using raw clip`);
|
||||
}
|
||||
body.push(
|
||||
` <audio`,
|
||||
` id="el-${m.compId}-voice"`,
|
||||
` src="${v.path}"`,
|
||||
` src="${voiceSrc}"`,
|
||||
` data-start="${m.start}"`,
|
||||
` data-duration="${m.durationSeconds}"`,
|
||||
` data-track-index="10"`,
|
||||
|
||||
@@ -125,3 +125,129 @@ test(
|
||||
assert.equal(probe.stdout.trim(), "mp3");
|
||||
},
|
||||
);
|
||||
|
||||
// ── voice clip edge fades ─────────────────────────────────────────────────────
|
||||
// Voice clips are butt-joined with zero gap by the cumulative-start layout, and
|
||||
// (pre-fix) mounted with zero edge fade — a hard cut at both edges, causing
|
||||
// audible word-jumps/abrupt cuts at scene seams. This measures the actual RMS
|
||||
// level in a short window at the very start of the mounted voice audio: a
|
||||
// constant-amplitude tone should measure near its full level pre-fix, and a
|
||||
// clearly quieter level once an in-fade is baked in.
|
||||
function rmsDbAt(path, startSec, durSec) {
|
||||
const r = spawnSync(
|
||||
"ffmpeg",
|
||||
[
|
||||
"-i",
|
||||
path,
|
||||
"-af",
|
||||
`atrim=start=${startSec}:end=${startSec + durSec},astats=metadata=1:reset=1`,
|
||||
"-f",
|
||||
"null",
|
||||
"-",
|
||||
],
|
||||
{ encoding: "utf8" },
|
||||
);
|
||||
// Mono input reports the same RMS level per-channel and "Overall" — take the
|
||||
// last occurrence (the "Overall" summary line) rather than anchor to it by
|
||||
// position, since every astats line carries its own `[Parsed_astats_N @ ...]`
|
||||
// prefix rather than being indented under a bare "Overall" heading.
|
||||
const matches = [...r.stderr.matchAll(/RMS level dB:\s*(-?[\d.]+|-inf)/g)];
|
||||
assert.ok(matches.length > 0, `expected an RMS level dB reading in ffmpeg stderr:\n${r.stderr}`);
|
||||
const value = matches.at(-1)[1];
|
||||
return value === "-inf" ? -Infinity : parseFloat(value);
|
||||
}
|
||||
|
||||
test("voice clips get a short edge fade baked in, not a hard cut", { skip: !HAS_FFMPEG }, () => {
|
||||
const { dir, r } = assembleWith({
|
||||
audioMeta: { bgm: null, voices: [{ frame: 1, path: "assets/voice/01.wav" }], sfx: [] },
|
||||
beforeAssemble: (projectDir) => {
|
||||
mkdirSync(join(projectDir, "assets", "voice"), { recursive: true });
|
||||
// A constant-amplitude tone spanning the full 3s frame duration: with no
|
||||
// fade, the start/end windows measure the same RMS level as the middle.
|
||||
const gen = spawnSync(
|
||||
"ffmpeg",
|
||||
[
|
||||
"-y",
|
||||
"-f",
|
||||
"lavfi",
|
||||
"-i",
|
||||
"sine=frequency=440:duration=3",
|
||||
join(projectDir, "assets", "voice", "01.wav"),
|
||||
],
|
||||
{ encoding: "utf8" },
|
||||
);
|
||||
assert.equal(gen.status, 0, gen.stderr);
|
||||
},
|
||||
});
|
||||
|
||||
assert.equal(r.status, 0, r.stderr);
|
||||
const html = readFileSync(join(dir, "index.html"), "utf8");
|
||||
const voiceEl = html.match(/id="el-[^"]*-voice"[\s\S]*?src="([^"]+)"/);
|
||||
assert.ok(voiceEl, "expected a voice <audio> element in index.html");
|
||||
const voiceSrc = voiceEl[1];
|
||||
assert.match(voiceSrc, /\.faded\.wav$/, `voice src should end in .faded.wav, got "${voiceSrc}"`);
|
||||
|
||||
const outPath = join(dir, voiceSrc);
|
||||
assert.equal(existsSync(outPath), true);
|
||||
|
||||
const rawMidDb = rmsDbAt(join(dir, "assets", "voice", "01.wav"), 1.5, 0.005);
|
||||
const fadedStartDb = rmsDbAt(outPath, 0, 0.002);
|
||||
const fadedMidDb = rmsDbAt(outPath, 1.5, 0.005);
|
||||
|
||||
// The clip's steady middle is unaffected by the edge fade.
|
||||
assert.ok(
|
||||
Math.abs(fadedMidDb - rawMidDb) < 1,
|
||||
`mid-clip level should be ~unchanged: raw ${rawMidDb} dB vs faded ${fadedMidDb} dB`,
|
||||
);
|
||||
// The very start of the faded clip should read markedly quieter than the
|
||||
// steady-state level — the fade-in ramp, not a hard cut at full volume.
|
||||
assert.ok(
|
||||
fadedStartDb < fadedMidDb - 6,
|
||||
`start-of-clip level should be well below mid-clip level: start ${fadedStartDb} dB vs mid ${fadedMidDb} dB`,
|
||||
);
|
||||
});
|
||||
|
||||
test(
|
||||
"a clip shorter than two fade windows still fades, without going silent for its whole length",
|
||||
{ skip: !HAS_FFMPEG },
|
||||
() => {
|
||||
const { dir, r } = assembleWith({
|
||||
audioMeta: { bgm: null, voices: [{ frame: 1, path: "assets/voice/01.wav" }], sfx: [] },
|
||||
beforeAssemble: (projectDir) => {
|
||||
mkdirSync(join(projectDir, "assets", "voice"), { recursive: true });
|
||||
// Shorter than 2 * VOICE_FADE_SECONDS (0.011s) would ever be in practice,
|
||||
// but proves the half-duration clamp rather than assuming it.
|
||||
const gen = spawnSync(
|
||||
"ffmpeg",
|
||||
[
|
||||
"-y",
|
||||
"-f",
|
||||
"lavfi",
|
||||
"-i",
|
||||
"sine=frequency=440:duration=0.01",
|
||||
join(projectDir, "assets", "voice", "01.wav"),
|
||||
],
|
||||
{ encoding: "utf8" },
|
||||
);
|
||||
assert.equal(gen.status, 0, gen.stderr);
|
||||
},
|
||||
});
|
||||
|
||||
assert.equal(r.status, 0, r.stderr);
|
||||
const html = readFileSync(join(dir, "index.html"), "utf8");
|
||||
const voiceEl = html.match(/id="el-[^"]*-voice"[\s\S]*?src="([^"]+)"/);
|
||||
assert.ok(voiceEl, "expected a voice <audio> element in index.html");
|
||||
const outPath = join(dir, voiceEl[1]);
|
||||
assert.equal(existsSync(outPath), true);
|
||||
const probe = spawnSync(
|
||||
"ffprobe",
|
||||
["-v", "error", "-show_entries", "format=duration", "-of", "csv=p=0", "--", outPath],
|
||||
{ encoding: "utf8" },
|
||||
);
|
||||
const outDur = parseFloat(probe.stdout.trim());
|
||||
assert.ok(
|
||||
outDur > 0.005,
|
||||
`faded clip should not have collapsed to near-zero length: ${outDur}s`,
|
||||
);
|
||||
},
|
||||
);
|
||||
|
||||
Reference in New Issue
Block a user