mirror of
https://github.com/heygen-com/hyperframes.git
synced 2026-09-03 04:38:33 +00:00
feat(whisper+captions): language detection, audio-reactive captions, multilingual defaults (#175)
## Summary **Whisper improvements:** - Auto-detect language and switch from `.en` to multilingual model when needed - Detect speech onset in WAV to strip hallucinated words before speech begins - Merge whisper-cpp token fragments: contractions (`didn` + `'t` → `didn't`), split capitals (`C` + `aught` → `Caught`), dropped-g (`shin` + `in'` → `shinin'`) - Interpolate zero-duration word clusters for reliable karaoke timing **Captions skill updates (folded from #176):** - Rewrite script-to-style mapping as an energy detection table (high → low) with mandatory animation requirements: karaoke baseline, 2+ highlight techniques, kinetic exits - Replace `tl.call()` per-frame audio-reactive pattern with group-level GSAP tweens — read peak bass/treble for each group's time range and modulate entrance intensity at build time - Add transcript quality check with automatic retry rules (>20% music tokens = retry with larger model) - Add caption word structure lint rule (`.caption-group` + `<span>`) for studio editor compatibility **Multilingual defaults (folded from #186):** - Default whisper model changed from `small.en` to `small` to prevent silent translation of non-English audio - Added non-negotiable language rule to captions skill ## Test plan - [ ] `pnpm test` passes (contraction merging, fragment merging, zero-duration interpolation, speech onset) - [ ] Transcribe non-English audio — verify it transcribes in original language, not translates - [ ] Skill files render correctly, cross-references resolve - [ ] `dynamic-techniques.md` audio-reactive section uses `tl.to()`/`tl.set()` only, no `tl.call()` loops 🤖 Generated with [Claude Code](https://claude.com/claude-code)
This commit is contained in:
@@ -55,6 +55,59 @@ export const captionRules: Array<(ctx: LintContext) => HyperframeLintFinding[]>
|
||||
return findings;
|
||||
},
|
||||
|
||||
// caption_transcript_not_inline
|
||||
({ scripts, styles, options }) => {
|
||||
const findings: HyperframeLintFinding[] = [];
|
||||
// Only check files that look like caption compositions
|
||||
const isCaptionFile =
|
||||
(options.filePath && /caption/i.test(options.filePath)) ||
|
||||
styles.some((s) => /\.caption[-_]?(?:group|word)/i.test(s.content));
|
||||
if (!isCaptionFile) return findings;
|
||||
|
||||
const allScript = scripts.map((s) => s.content).join("\n");
|
||||
const hasInlineTranscript = /(?:const|let|var)\s+(?:TRANSCRIPT|script)\s*=\s*\[/.test(
|
||||
allScript,
|
||||
);
|
||||
const hasFetchTranscript = /fetch\s*\(\s*["'][^"']*transcript/i.test(allScript);
|
||||
|
||||
if (!hasInlineTranscript && hasFetchTranscript) {
|
||||
findings.push({
|
||||
code: "caption_transcript_not_inline",
|
||||
severity: "warning",
|
||||
message:
|
||||
"Captions composition loads transcript via fetch(). The studio caption editor " +
|
||||
"requires an inline `var TRANSCRIPT = [...]` array to detect and edit captions.",
|
||||
fixHint:
|
||||
'Embed the transcript as `var TRANSCRIPT = [{ "text": "...", "start": 0, "end": 1 }, ...]` ' +
|
||||
"with JSON-quoted property keys. See the captions skill for details.",
|
||||
});
|
||||
}
|
||||
|
||||
if (hasInlineTranscript) {
|
||||
// Verify the inline transcript can be parsed
|
||||
const varPattern = /(?:const|let|var)\s+(?:TRANSCRIPT|script)\s*=\s*(\[[\s\S]*?\]);/;
|
||||
const match = allScript.match(varPattern);
|
||||
if (match?.[1]) {
|
||||
try {
|
||||
JSON.parse(match[1]);
|
||||
} catch {
|
||||
findings.push({
|
||||
code: "caption_transcript_parse_error",
|
||||
severity: "warning",
|
||||
message:
|
||||
"Inline TRANSCRIPT array is not valid JSON. The studio caption editor may fail " +
|
||||
"to parse it. Common cause: unquoted property keys with apostrophes in text.",
|
||||
fixHint:
|
||||
'Use JSON-quoted keys: { "text": "don\'t", "start": 0, "end": 1 } instead of ' +
|
||||
'{ text: "don\'t", start: 0, end: 1 }.',
|
||||
});
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return findings;
|
||||
},
|
||||
|
||||
// caption_container_relative_position
|
||||
({ styles }) => {
|
||||
const findings: HyperframeLintFinding[] = [];
|
||||
|
||||
Reference in New Issue
Block a user