mirror of
https://github.com/heygen-com/hyperframes.git
synced 2026-09-10 22:20:14 +00:00
feat(whisper+captions): language detection, audio-reactive captions, multilingual defaults (#175)
## Summary **Whisper improvements:** - Auto-detect language and switch from `.en` to multilingual model when needed - Detect speech onset in WAV to strip hallucinated words before speech begins - Merge whisper-cpp token fragments: contractions (`didn` + `'t` → `didn't`), split capitals (`C` + `aught` → `Caught`), dropped-g (`shin` + `in'` → `shinin'`) - Interpolate zero-duration word clusters for reliable karaoke timing **Captions skill updates (folded from #176):** - Rewrite script-to-style mapping as an energy detection table (high → low) with mandatory animation requirements: karaoke baseline, 2+ highlight techniques, kinetic exits - Replace `tl.call()` per-frame audio-reactive pattern with group-level GSAP tweens — read peak bass/treble for each group's time range and modulate entrance intensity at build time - Add transcript quality check with automatic retry rules (>20% music tokens = retry with larger model) - Add caption word structure lint rule (`.caption-group` + `<span>`) for studio editor compatibility **Multilingual defaults (folded from #186):** - Default whisper model changed from `small.en` to `small` to prevent silent translation of non-English audio - Added non-negotiable language rule to captions skill ## Test plan - [ ] `pnpm test` passes (contraction merging, fragment merging, zero-duration interpolation, speech onset) - [ ] Transcribe non-English audio — verify it transcribes in original language, not translates - [ ] Skill files render correctly, cross-references resolve - [ ] `dynamic-techniques.md` audio-reactive section uses `tl.to()`/`tl.set()` only, no `tl.call()` loops 🤖 Generated with [Claude Code](https://claude.com/claude-code)
This commit is contained in:
@@ -103,7 +103,8 @@ async function transcribeAudio(
|
||||
opts: { model?: string; language?: string; json?: boolean },
|
||||
): Promise<void> {
|
||||
const { transcribe } = await import("../whisper/transcribe.js");
|
||||
const { loadTranscript, patchCaptionHtml } = await import("../whisper/normalize.js");
|
||||
const { loadTranscript, patchCaptionHtml, stripBeforeOnset } =
|
||||
await import("../whisper/normalize.js");
|
||||
|
||||
const model = opts.model ?? DEFAULT_MODEL;
|
||||
const spin = opts.json ? null : clack.spinner();
|
||||
@@ -116,7 +117,19 @@ async function transcribeAudio(
|
||||
onProgress: spin ? (msg) => spin.message(msg) : undefined,
|
||||
});
|
||||
|
||||
const { words } = loadTranscript(result.transcriptPath);
|
||||
let { words } = loadTranscript(result.transcriptPath);
|
||||
|
||||
if (result.speechOnsetSeconds != null) {
|
||||
const before = words.length;
|
||||
words = stripBeforeOnset(words, result.speechOnsetSeconds);
|
||||
const stripped = before - words.length;
|
||||
if (stripped > 0 && !opts.json) {
|
||||
spin?.message(
|
||||
`Stripped ${stripped} words before speech onset at ${result.speechOnsetSeconds.toFixed(1)}s`,
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
writeFileSync(result.transcriptPath, JSON.stringify(words, null, 2));
|
||||
patchCaptionHtml(dir, words);
|
||||
|
||||
@@ -127,13 +140,18 @@ async function transcribeAudio(
|
||||
model,
|
||||
wordCount: words.length,
|
||||
durationSeconds: result.durationSeconds,
|
||||
speechOnsetSeconds: result.speechOnsetSeconds,
|
||||
transcriptPath: result.transcriptPath,
|
||||
}),
|
||||
);
|
||||
} else {
|
||||
spin!.stop(
|
||||
const onsetNote =
|
||||
result.speechOnsetSeconds != null
|
||||
? ` — speech detected at ${result.speechOnsetSeconds.toFixed(1)}s`
|
||||
: "";
|
||||
spin?.stop(
|
||||
c.success(
|
||||
`Transcribed ${c.accent(String(words.length))} words (${result.durationSeconds.toFixed(1)}s)`,
|
||||
`Transcribed ${c.accent(String(words.length))} words (${result.durationSeconds.toFixed(1)}s${onsetNote})`,
|
||||
),
|
||||
);
|
||||
}
|
||||
@@ -142,7 +160,7 @@ async function transcribeAudio(
|
||||
if (opts.json) {
|
||||
console.log(JSON.stringify({ ok: false, error: message }));
|
||||
} else {
|
||||
spin!.stop(c.error(`Transcription failed: ${message}`));
|
||||
spin?.stop(c.error(`Transcription failed: ${message}`));
|
||||
}
|
||||
process.exit(1);
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user