mirror of
https://github.com/heygen-com/hyperframes.git
synced 2026-09-05 00:56:23 +00:00
feat(media-use): v2 media OS core (resolve cascade, providers, local generation, telemetry) + retire hyperframes-media
This commit is contained in:
Miguel Angel Simon Sierra
@@ -0,0 +1,35 @@
|
||||
# SFX Credits
|
||||
|
||||
All sound effects in this directory are sourced from [Pixabay](https://pixabay.com/sound-effects/) and used under the [Pixabay Content License](https://pixabay.com/service/license-summary/).
|
||||
|
||||
The Pixabay license allows free use for commercial and non-commercial purposes without attribution, but attribution is appreciated and given here for transparency.
|
||||
|
||||
## Files
|
||||
|
||||
The following `.mp3` files are bundled with this skill:
|
||||
|
||||
- `chime.mp3`
|
||||
- `click.mp3` / `click-soft.mp3`
|
||||
- `error.mp3`
|
||||
- `glitch-1.mp3` / `glitch-2.mp3` / `glitch-3.mp3`
|
||||
- `impact-bass-1.mp3` / `impact-bass-2.mp3`
|
||||
- `key-press.mp3`
|
||||
- `notification.mp3`
|
||||
- `ping.mp3`
|
||||
- `pop.mp3`
|
||||
- `riser.mp3`
|
||||
- `sparkle.mp3`
|
||||
- `typing.mp3`
|
||||
- `whoosh.mp3` / `whoosh-short.mp3` / `whoosh-cinematic.mp3`
|
||||
|
||||
See `manifest.json` for per-file metadata (duration, energy character, recommended use).
|
||||
|
||||
## License
|
||||
|
||||
All files are distributed under the [Pixabay Content License](https://pixabay.com/service/license-summary/), which permits:
|
||||
|
||||
- Commercial and non-commercial use
|
||||
- Modification and remixing
|
||||
- Redistribution as part of derivative works (such as videos rendered with HyperFrames)
|
||||
|
||||
without any attribution requirement.
|
||||
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
@@ -0,0 +1,97 @@
|
||||
{
|
||||
"chime": {
|
||||
"file": "chime.mp3",
|
||||
"duration": 2.5,
|
||||
"description": "Soft melodic chime — gentle positive beat: success/confirmation or a lighthearted transition. Sync to the visual moment."
|
||||
},
|
||||
"click-soft": {
|
||||
"file": "click-soft.mp3",
|
||||
"duration": 0.37,
|
||||
"description": "Quiet short click — low-key UI tap / soft selection. Short accent, sync exactly to the on-screen action."
|
||||
},
|
||||
"click": {
|
||||
"file": "click.mp3",
|
||||
"duration": 0.37,
|
||||
"description": "Crisp UI click — button press, toggle, selection. Short accent, sync exactly to the on-screen action."
|
||||
},
|
||||
"error": {
|
||||
"file": "error.mp3",
|
||||
"duration": 1.62,
|
||||
"description": "Negative / error tone — failure state, a 'wrong' beat, or a glitchy interruption."
|
||||
},
|
||||
"glitch-1": {
|
||||
"file": "glitch-1.mp3",
|
||||
"duration": 2.64,
|
||||
"description": "Punchy digital glitch — hard-cut accent or sudden reveal. Trigger on the hit; let the decay bleed into the next shot (J-cut)."
|
||||
},
|
||||
"glitch-2": {
|
||||
"file": "glitch-2.mp3",
|
||||
"duration": 3.5,
|
||||
"description": "Harsh, longer glitch — chaotic / jarring transition or a distorted reveal."
|
||||
},
|
||||
"glitch-3": {
|
||||
"file": "glitch-3.mp3",
|
||||
"duration": 3.1,
|
||||
"description": "Low-key glitch texture — subtle digital shift, minimal transition that sits under other audio."
|
||||
},
|
||||
"impact-bass-1": {
|
||||
"file": "impact-bass-1.mp3",
|
||||
"duration": 2.12,
|
||||
"description": "Bass impact hit — logo/hero snap, headline slam. Trigger on the visual landing; decay carries into the next shot (J-cut)."
|
||||
},
|
||||
"impact-bass-2": {
|
||||
"file": "impact-bass-2.mp3",
|
||||
"duration": 2.59,
|
||||
"description": "Bass impact with a short swell — brief anticipation then a deep hit. Place so the peak lands on the reveal."
|
||||
},
|
||||
"key-press": {
|
||||
"file": "key-press.mp3",
|
||||
"duration": 0.4,
|
||||
"description": "Single key press — one keystroke / terminal-input beat. Short accent, sync to the typed character."
|
||||
},
|
||||
"notification": {
|
||||
"file": "notification.mp3",
|
||||
"duration": 2.46,
|
||||
"description": "Notification chime — alert, message-in, toast/badge appears. Sync to the element entering."
|
||||
},
|
||||
"ping": {
|
||||
"file": "ping.mp3",
|
||||
"duration": 1.32,
|
||||
"description": "Sharp electronic ping — punchy accent on a key reveal or data point. Sync to the beat."
|
||||
},
|
||||
"pop": {
|
||||
"file": "pop.mp3",
|
||||
"duration": 0.72,
|
||||
"description": "Quick pop — element appear/spawn, chip/tag/badge in. Small precise accent, sync to the pop-in."
|
||||
},
|
||||
"riser": {
|
||||
"file": "riser.mp3",
|
||||
"duration": 10.03,
|
||||
"description": "Long cinematic riser (~10s build, peak at the end). Trigger at (climax_time − 10.03s) so it crests exactly on the reveal."
|
||||
},
|
||||
"sparkle": {
|
||||
"file": "sparkle.mp3",
|
||||
"duration": 1.8,
|
||||
"description": "Bright sparkle / shimmer — magical reveal or 'shine' highlight on a hero element. Sync to the highlight."
|
||||
},
|
||||
"typing": {
|
||||
"file": "typing.mp3",
|
||||
"duration": 1.5,
|
||||
"description": "Typing burst (~1.5s of keys) — keyboard / code typing reveal, text-being-typed beat. Start as the text begins typing."
|
||||
},
|
||||
"whoosh-cinematic": {
|
||||
"file": "whoosh-cinematic.mp3",
|
||||
"duration": 5.54,
|
||||
"description": "Cinematic whoosh build (~5.5s) — sweeping scene transition. Align so the swell peaks on the cut."
|
||||
},
|
||||
"whoosh-short": {
|
||||
"file": "whoosh-short.mp3",
|
||||
"duration": 0.57,
|
||||
"description": "Short whoosh — quick swipe/slide accent, fast element move, snappy transition. Sync to the motion."
|
||||
},
|
||||
"whoosh": {
|
||||
"file": "whoosh.mp3",
|
||||
"duration": 0.57,
|
||||
"description": "Punchy whoosh/impact — fast reveal or hard transition accent. Sync to the motion."
|
||||
}
|
||||
}
|
||||
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
@@ -0,0 +1,72 @@
|
||||
# Background music (BGM)
|
||||
|
||||
One music bed per composition, produced by the shared audio engine (`scripts/audio.mjs` → `scripts/lib/bgm.mjs`). Two routes, chosen by the engine's one switch — whether a HeyGen credential is present:
|
||||
|
||||
- **HeyGen retrieval — the default when credentialed.** Search HeyGen's music catalog by mood, download the top track. No generation; same `~/.heygen` / `$HEYGEN_API_KEY` credential as TTS.
|
||||
- **Local generation (Lyria → MusicGen) — the fallback when there is no credential** (or when asked for explicitly). Generate a WAV from a mood prompt. There is **no `npx hyperframes bgm` command**; the engine spawns `scripts/lyria-recipe.py` or an inline MusicGen script directly.
|
||||
|
||||
> **Run the Preflight first — no credential is not a green light to silently generate locally.** Before generating, complete the sign-in **Preflight** (see `../SKILL.md` → Preflight): run `npx hyperframes auth status`, recommend signing in, and **STOP for the user's choice** (sign in for HeyGen's music library, or continue offline with local generation). This applies to a one-off "generate a BGM" request just as much as inside a full workflow.
|
||||
|
||||
## Driving it from the request
|
||||
|
||||
`audio_request.json` → `bgm: { mode?, query?, prompt? }`:
|
||||
|
||||
- **`mode`** — `retrieve | generate | none`. Omit for **auto** (retrieve when credentialed, else generate). An **explicit** `retrieve` is strict: no credential ⇒ skip, never a detached generate (so a caller with no `wait-bgm` step, e.g. product-launch, can't get a pending job it won't await).
|
||||
- **`query`** — the mood, used for retrieval and as a fallback prompt seed (e.g. a storyboard's `music:` field, falling back to `message` → `arc` → `"calm cinematic underscore"`).
|
||||
- **`prompt`** — an explicit full prompt for generation; omit and the engine infers one (see Mood inference). Optional `blob` / `archetype` / `arc` feed that inference.
|
||||
|
||||
## HeyGen retrieval (default)
|
||||
|
||||
`searchSounds(query, "music", { limit: 5 })` → `GET /audio/sounds?query=<mood>&type=music&limit=5`. Take the top result (ranked by `score`), download its presigned `audio_url` → `assets/bgm/track.mp3`. Synchronous. No match → skip (BGM is optional; never fail the render over it). Cue written to `audio_meta.json`:
|
||||
|
||||
```jsonc
|
||||
{
|
||||
"path": "assets/bgm/track.mp3",
|
||||
"volume": 0.8,
|
||||
"mode": "retrieve",
|
||||
"query": "calm cinematic underscore",
|
||||
"duration_s": 42.0,
|
||||
}
|
||||
```
|
||||
|
||||
`volume` is 0.8 under narration, 0.9 for a silent film (no voice). `bgm_pending` is `false` — the file is on disk when the engine returns.
|
||||
|
||||
## Local generation (fallback) — Lyria → MusicGen
|
||||
|
||||
Spawned **detached** so voice work isn't blocked; `audio_meta.bgm_pending: true` and `bgm_pid` / `bgm_log` are set until it finishes. **Run `scripts/wait-bgm.mjs` before assembling** — it polls the output file / process / log, detects crashes, and writes `bgm_status.json` (`status: ready | failed | timeout | disabled`). A failed/absent track is simply omitted; it never blocks voice/SFX.
|
||||
|
||||
| Order | Provider | Env / deps | Speed | Quality |
|
||||
| ----- | ------------------------------------ | ------------------------------------------------------------------------------------- | --------------------------------------- | --------------------------- |
|
||||
| 1 | Google Lyria RealTime | `$GEMINI_API_KEY` or `$GOOGLE_API_KEY` + `google-genai` (auto-installed on demand) | Real-time stream (≈ requested duration) | Production-grade |
|
||||
| 2 | MusicGen (`facebook/musicgen-small`) | Python `transformers + torch + soundfile + numpy` (~300 MB first run; auto-installed) | Slow on CPU; fast on Apple MPS / CUDA | Decent; prompt-only control |
|
||||
|
||||
Output → `assets/bgm/track.wav`, target = total voice duration. MusicGen generates **one** seed clip (≤28–30s, under the decoder's positional limit) then crossfade-loops it up to the target (or trims down if shorter), avoiding per-segment seams. Backend selection is by what can actually **run**: Lyria only when `import google.genai` succeeds, else MusicGen; if neither can be made to run, BGM is skipped (voice + SFX still render).
|
||||
|
||||
## Mood inference (the generate prompt)
|
||||
|
||||
`inferBgmPrompt()` in `scripts/lib/bgm.mjs`: an explicit `prompt` wins; otherwise industry-keyword **base** → narrative-**archetype** shape → emotional-**arc** tiebreaker.
|
||||
|
||||
| Match in `blob` / `query` | Base prompt | BPM |
|
||||
| ------------------------------------------------------ | --------------------------------------------------------------------------- | --- |
|
||||
| `crypto / nft / web3 / defi / token / blockchain` | atmospheric electronic, deep bass, futuristic synths, restrained percussion | 100 |
|
||||
| `finance / fintech / bank / payment / invest / wealth` | calm cinematic, soft strings, subtle piano, restrained percussion | 92 |
|
||||
| `creative / agency / design / studio / art / brand` | playful electronic, warm pads, light percussion | 115 |
|
||||
| _(default: SaaS / tech / platform)_ | uplifting corporate tech, bright modern piano with synth pads | 108 |
|
||||
|
||||
Archetype then reshapes the arc — PAS → "MINOR to MAJOR" build; BAB / future-pacing → aspirational rising; feature-cascade → +10 BPM driving; demo-loop → −8 BPM minimal. The emotional arc breaks remaining ties (tension→relief, excitement, trust/reassurance).
|
||||
|
||||
## Lyria knobs (direct recipe use)
|
||||
|
||||
The engine bakes BPM / scale into the **prompt text** (via the inference above) and passes only `--output` / `--duration` / `--prompt` to the recipe. If you invoke `scripts/lyria-recipe.py` directly you can also set: `--bpm` (90–110 calm, 110–130 energetic), `--brightness` (0–1, ≥0.7 promotional), `--density` (0–1, higher = fuller), `--scale` (`MAJOR` / `MINOR` / `PENTATONIC` / …), `--negative-prompt` (styles to exclude). MusicGen ignores all of these — put the mood in the prompt.
|
||||
|
||||
## Failure modes
|
||||
|
||||
| Failure | Behavior |
|
||||
| --------------------------------------------- | ---------------------------------------------------------------------------------------- |
|
||||
| No music match (retrieve) | `bgm: null`, anomaly logged. Render proceeds without BGM. |
|
||||
| Explicit `retrieve`, no credential | Skipped (no silent generate fallback). Use `mode: generate` or omit `mode` for auto. |
|
||||
| Neither Lyria nor MusicGen can run (generate) | `bgm` disabled with a `pip install …` hint. Voice + SFX still render. |
|
||||
| Generate still rendering at assemble time | `bgm_pending: true`; `wait-bgm.mjs` waits/checks and writes `bgm_status.json` first. |
|
||||
| Generate crashed | `wait-bgm.mjs` → `bgm_status.json { status: "failed" }`; the `<audio>` track is omitted. |
|
||||
|
||||
BGM failure never blocks a render.
|
||||
@@ -0,0 +1,159 @@
|
||||
# Captions
|
||||
|
||||
Before authoring: confirm the transcript came from the right Whisper model. CLI default `small.en` silently translates non-English audio — see [`../transcribe.md`](../transcribe.md) → "Language Rule" and [`transcript-handling.md`](transcript-handling.md) for the mandatory quality check.
|
||||
|
||||
Analyze spoken content to determine caption style. If user specifies a style, use that. Otherwise, detect tone from the transcript.
|
||||
|
||||
## Transcript Source
|
||||
|
||||
```json
|
||||
[
|
||||
{ "id": "w0", "text": "Hello", "start": 0.0, "end": 0.5 },
|
||||
{ "id": "w1", "text": "world.", "start": 0.6, "end": 1.2 }
|
||||
]
|
||||
```
|
||||
|
||||
`id` (`w0`, `w1`, …) is the stable reference for per-word overrides and is added by `hyperframes transcribe`. It's optional for backwards compatibility with hand-authored transcripts. See [`../transcribe.md`](../transcribe.md) → "Output Shape" for how this is produced, and [`transcript-handling.md`](transcript-handling.md) for cleanup before consumption.
|
||||
|
||||
## Style Detection (When No Style Specified)
|
||||
|
||||
Read the full transcript before choosing. Four dimensions:
|
||||
|
||||
**1. Visual feel** — corporate→clean; energetic→bold; storytelling→elegant; technical→precise; social→playful.
|
||||
|
||||
**2. Color palette** — dark+bright for energy; muted for professional; high contrast for clarity; one accent color.
|
||||
|
||||
**3. Font mood** — heavy/condensed for impact; clean sans for modern; rounded for friendly; serif for elegance.
|
||||
|
||||
**4. Animation character** — scale-pop for punchy; gentle fade for calm; word-by-word for emphasis; typewriter for technical.
|
||||
|
||||
## Per-Word Styling
|
||||
|
||||
Scan for words deserving distinct treatment:
|
||||
|
||||
- **Brand/product names** — larger size, unique color
|
||||
- **ALL CAPS** — scale boost, flash, accent color
|
||||
- **Numbers/statistics** — bold weight, accent color
|
||||
- **Emotional keywords** — exaggerated animation (overshoot, bounce)
|
||||
- **Call-to-action** — highlight, underline, color pop
|
||||
- **Marker highlight** — for beyond-color emphasis (highlight sweep, circle, burst, scribble, sketchout), see `hyperframes-animation/rules/css-marker-patterns.md`.
|
||||
|
||||
## Script-to-Style Mapping
|
||||
|
||||
| Tone | Font mood | Animation | Color | Size |
|
||||
| ------------ | ------------------------ | ---------------------------------- | --------------------------- | ------- |
|
||||
| Hype/launch | Heavy condensed, 800-900 | Scale-pop, back.out(1.7), 0.1-0.2s | Bright on dark | 72-96px |
|
||||
| Corporate | Clean sans, 600-700 | Fade+slide, power3.out, 0.3s | White/neutral, muted accent | 56-72px |
|
||||
| Tutorial | Mono/clean sans, 500-600 | Typewriter/fade, 0.4-0.5s | High contrast, minimal | 48-64px |
|
||||
| Storytelling | Serif/elegant, 400-500 | Slow fade, power2.out, 0.5-0.6s | Warm muted tones | 44-56px |
|
||||
| Social | Rounded sans, 700-800 | Bounce, elastic.out, word-by-word | Playful, colored pills | 56-80px |
|
||||
|
||||
## Word Grouping
|
||||
|
||||
- **High energy:** 2-3 words. Quick turnover.
|
||||
- **Conversational:** 3-5 words. Natural phrases.
|
||||
- **Measured/calm:** 4-6 words. Longer groups.
|
||||
|
||||
Break on sentence boundaries, 150ms+ pauses, or max word count.
|
||||
|
||||
## Positioning
|
||||
|
||||
- **Landscape (1920x1080):** Bottom 80-120px, centered
|
||||
- **Portrait (1080x1920):** Lower middle ~600-700px from bottom, centered
|
||||
- Never cover the subject's face
|
||||
- `position: absolute` — never relative
|
||||
- One caption group visible at a time
|
||||
|
||||
## Text Overflow Prevention
|
||||
|
||||
Use `window.__hyperframes.fitTextFontSize()`:
|
||||
|
||||
```js
|
||||
var result = window.__hyperframes.fitTextFontSize(group.text.toUpperCase(), {
|
||||
fontFamily: "Outfit",
|
||||
fontWeight: 900,
|
||||
maxWidth: 1600,
|
||||
});
|
||||
el.style.fontSize = result.fontSize + "px";
|
||||
```
|
||||
|
||||
Options: `maxWidth` (1600 landscape, 900 portrait), `baseFontSize` (78), `minFontSize` (42), `fontWeight`, `fontFamily`, `step` (2).
|
||||
|
||||
CSS safety nets: `max-width` on container, `overflow: visible` (**not** `hidden` — hidden clips scaled emphasis words and glow effects), `position: absolute`, explicit `height`. When per-word styling uses `scale > 1.0`, compute `maxWidth = safeWidth / maxScale` to leave headroom.
|
||||
|
||||
**Container pattern:** Full-width absolute container, centered. Do **not** use `left: 50%; transform: translateX(-50%)` — causes clipping at composition edges.
|
||||
|
||||
## Caption Exit Guarantee
|
||||
|
||||
Every group **must** have a hard kill after exit animation:
|
||||
|
||||
```js
|
||||
tl.to(groupEl, { opacity: 0, scale: 0.95, duration: 0.12, ease: "power2.in" }, group.end - 0.12);
|
||||
// `tl.set` is an instant flip, not a tween — safe to set `visibility` here (core's "no animating
|
||||
// visibility" rule applies to tweens, which can't smoothly interpolate non-numeric values anyway).
|
||||
tl.set(groupEl, { opacity: 0, visibility: "hidden" }, group.end);
|
||||
```
|
||||
|
||||
Self-lint after building timeline — place **before** `window.__timelines[id] = tl` so it runs at composition init:
|
||||
|
||||
```js
|
||||
GROUPS.forEach(function (group, gi) {
|
||||
var el = document.getElementById("cg-" + gi);
|
||||
if (!el) return;
|
||||
tl.seek(group.end + 0.01);
|
||||
var computed = window.getComputedStyle(el);
|
||||
if (computed.opacity !== "0" && computed.visibility !== "hidden") {
|
||||
console.warn(
|
||||
"[caption-lint] group " + gi + " still visible at t=" + (group.end + 0.01).toFixed(2) + "s",
|
||||
);
|
||||
}
|
||||
});
|
||||
tl.seek(0);
|
||||
```
|
||||
|
||||
## Pre-Built Caption Components
|
||||
|
||||
Before building caption styles from scratch, check the registry — 15 ready-to-use caption components cover the most common styles. Install with `npx hyperframes add <name>` and wire as a sub-composition via `data-composition-src` (see `hyperframes-registry`).
|
||||
|
||||
```bash
|
||||
npx hyperframes catalog --tag caption-style # list all caption components
|
||||
npx hyperframes add caption-highlight # install a specific one
|
||||
```
|
||||
|
||||
| Style | Component | Best for |
|
||||
| ------------------------- | ---------------------------- | ---------------------------- |
|
||||
| TikTok-style highlight | `caption-highlight` | Social, high-energy |
|
||||
| Karaoke pill | `caption-pill-karaoke` | Music, lyric videos |
|
||||
| Cinematic editorial | `caption-editorial-emphasis` | Documentary, storytelling |
|
||||
| Glitch / cyber | `caption-glitch-rgb` | Tech, gaming |
|
||||
| Full-screen slam | `caption-kinetic-slam` | Hype, announcements |
|
||||
| Neon glow | `caption-neon-glow` | Night, club, neon aesthetics |
|
||||
| Neon accent (multi-color) | `caption-neon-accent` | Colorful, playful |
|
||||
| Wipe reveal | `caption-clip-wipe` | Clean, modern |
|
||||
| Gradient fill | `caption-gradient-fill` | Vibrant, eye-catching |
|
||||
| Matrix decode | `caption-matrix-decode` | Sci-fi, tech reveals |
|
||||
| Emoji pop | `caption-emoji-pop` | Social, casual |
|
||||
| Parallax layers | `caption-parallax-layers` | Depth, cinematic |
|
||||
| Particle burst | `caption-particle-burst` | Celebration, impact keywords |
|
||||
| Lava texture | `caption-texture` | Bold, dramatic |
|
||||
| Weight shift | `caption-weight-shift` | Elegant, typographic |
|
||||
|
||||
Related: `caption-blend-difference` (tagged `text` / `blend-mode`, not `caption-style`, so it won't appear under the filter above) auto-inverts text against any background via `mix-blend-mode: difference` — useful when the background is busy or unpredictable.
|
||||
|
||||
Browse all with previews: [hyperframes.heygen.com/catalog](https://hyperframes.heygen.com/catalog)
|
||||
|
||||
Caption components ship with transparent backgrounds — they're pure overlays. If the underlying video is bright or busy, add a contrast layer (e.g. a semi-transparent dark div) in the host composition beneath the caption sub-composition, not inside the component itself.
|
||||
|
||||
## Further References
|
||||
|
||||
- [`motion.md`](motion.md) — karaoke, marker effects, audio-reactive modulation, scatter exits.
|
||||
- [`transcript-handling.md`](transcript-handling.md) — input formats, quality checks, cleaning, external API fallback.
|
||||
- `hyperframes-animation/rules/css-marker-patterns.md` — marker highlighting (deterministic, fully seekable).
|
||||
|
||||
## Constraints
|
||||
|
||||
- Deterministic. No `Math.random()`, no `Date.now()`.
|
||||
- Sync to transcript timestamps.
|
||||
- One group visible at a time.
|
||||
- Every group must have a hard `tl.set` kill at `group.end`.
|
||||
- Fonts: the compiler auto-embeds only its **built-in mapped set** (Inter, Roboto, Montserrat, …) — for those, just declare `font-family` in CSS. Any **other** font (a brand/custom font like `TT Norms Pro`, or a non-Latin CJK/Devanagari family) is **not** auto-supplied: it needs an `@font-face` pointing at a real `.woff2` shipped with the project, or the text silently falls back to a generic font in the render. Don't assume a `font-family` you can see locally will render — the render machine is a clean headless Chrome with no installed fonts.
|
||||
@@ -0,0 +1,87 @@
|
||||
# Dynamic Caption Techniques
|
||||
|
||||
You are here because SKILL.md told you to read this file before writing animation code. Pick your technique combination from the table below based on the energy level you detected from the transcript, then implement using standard GSAP patterns.
|
||||
|
||||
## Technique Selection by Energy
|
||||
|
||||
| Energy level | Highlight | Exit | Cycle pattern |
|
||||
| ------------ | ------------------------------------- | ------------------- | ----------------------------------------- |
|
||||
| High | Karaoke with accent glow + scale pop | Scatter or drop | Alternate highlight styles every 2 groups |
|
||||
| Medium-high | Karaoke with color pop | Scatter or collapse | Alternate every 3 groups |
|
||||
| Medium | Karaoke (subtle, white only) | Fade + slide | Alternate every 3 groups |
|
||||
| Medium-low | Karaoke (minimal scale change) | Fade | Single style, vary ease per group |
|
||||
| Low | Karaoke (warm tones, slow transition) | Collapse | Alternate every 4 groups |
|
||||
|
||||
**All energy levels use karaoke highlight as the baseline.** The difference is intensity — high energy gets accent color + glow + 15% scale pop on active words, low energy gets a gentle white shift with 3% scale.
|
||||
|
||||
**Emphasis words always break the pattern.** When a word is flagged as emphasis (emotional keyword, ALL CAPS, brand name), give it a stronger animation than surrounding words (larger scale, accent color, overshoot ease). This creates contrast.
|
||||
|
||||
**Marker highlight modes add a visual layer on top of karaoke.** For emphasis words that need more than color/scale, add a marker-style effect: highlight sweep, circle, burst, scribble, or sketchout. See `hyperframes-animation/rules/css-marker-patterns.md` for implementation details. Match mode to energy: burst for hype, circle for key terms, highlight for standard, scribble for subtle.
|
||||
|
||||
## Audio-Reactive Captions (Mandatory for Music)
|
||||
|
||||
**If the source audio is music (vocals over instrumentation, beats, any musical content), you MUST extract audio data and add audio-reactive animations.** This is not optional — music without audio reactivity looks disconnected. Even low-energy ballads get subtle bass pulse and treble glow.
|
||||
|
||||
No special wiring is needed. The group loop already iterates over every caption group to build entrance, karaoke, and exit tweens. At that point, read the audio data for each group's time range and use it to modulate the group's animation intensity with regular GSAP tweens.
|
||||
|
||||
```js
|
||||
// Load audio data inline (same pattern as TRANSCRIPT)
|
||||
var AUDIO = JSON.parse(audioDataJson); // { fps, totalFrames, frames: [{ bands: [...] }] }
|
||||
|
||||
GROUPS.forEach(function (group, gi) {
|
||||
var groupEl = document.getElementById("cg-" + gi);
|
||||
if (!groupEl) return;
|
||||
|
||||
// Read peak energy for this group's time range
|
||||
var startFrame = Math.floor(group.start * AUDIO.fps);
|
||||
var endFrame = Math.min(Math.floor(group.end * AUDIO.fps), AUDIO.totalFrames - 1);
|
||||
var peakBass = 0;
|
||||
var peakTreble = 0;
|
||||
for (var f = startFrame; f <= endFrame; f++) {
|
||||
var frame = AUDIO.frames[f];
|
||||
if (!frame) continue;
|
||||
peakBass = Math.max(peakBass, frame.bands[0] || 0, frame.bands[1] || 0);
|
||||
peakTreble = Math.max(peakTreble, frame.bands[6] || 0, frame.bands[7] || 0);
|
||||
}
|
||||
|
||||
// Modulate entrance — louder groups enter bigger and glowier
|
||||
tl.to(
|
||||
groupEl,
|
||||
{
|
||||
scale: 1 + peakBass * 0.06,
|
||||
textShadow:
|
||||
"0 0 " + Math.round(peakTreble * 12) + "px rgba(255,255,255," + peakTreble * 0.4 + ")",
|
||||
duration: 0.3,
|
||||
ease: "power2.out",
|
||||
},
|
||||
group.start,
|
||||
);
|
||||
|
||||
// Reset at exit so audio-driven values don't persist
|
||||
tl.set(groupEl, { scale: 1, textShadow: "none" }, group.end - 0.15);
|
||||
});
|
||||
```
|
||||
|
||||
This shapes the animation at build time, not playback time — no per-frame callbacks, no `tl.call()` loops, no async fetch timing issues. Loud groups come in with more weight and glow; quiet groups come in soft. The audio data modulates _how much_, the content determines _what_.
|
||||
|
||||
Keep audio reactivity subtle — 3-6% scale variation and soft glow. Heavy pulsing makes text unreadable.
|
||||
|
||||
To generate the audio data file:
|
||||
|
||||
```bash
|
||||
python3 skills/hyperframes-creative/scripts/extract-audio-data.py audio.mp3 --fps 30 --bands 8 -o audio-data.json
|
||||
```
|
||||
|
||||
## Combining Techniques
|
||||
|
||||
Don't use the same highlight animation on every group — cycle through styles using the group index. Don't combine multiple competing animations on the same word at the same timestamp. Vary techniques across groups to match the content's pace changes.
|
||||
|
||||
**Marker highlight effects** layer well with karaoke — use karaoke for the word-by-word reveal, then add a marker effect on emphasis words only. For example: karaoke highlights each word in white, but brand names get a yellow highlight sweep and stats get a red circle. Cycle marker modes across groups for visual variety.
|
||||
|
||||
## Runtime Tools
|
||||
|
||||
Caption motion uses standard HyperFrames runtime APIs. Use the canonical sources:
|
||||
|
||||
- **GSAP timeline + tween syntax** — `hyperframes-animation/adapters/gsap.md` (eases, position parameter, performance)
|
||||
- **`window.__hyperframes.fitTextFontSize` / `pretext`** — `hyperframes-core/references/determinism-rules.md` → Layout Contract (overflow prevention, per-frame text measurement)
|
||||
- **Audio data extraction** — generate via `python3 skills/hyperframes-creative/scripts/extract-audio-data.py audio.mp3 --fps 30 --bands 8 -o audio-data.json`, then load inline as shown in "Audio-Reactive Captions" above
|
||||
@@ -0,0 +1,97 @@
|
||||
# Transcript Guide
|
||||
|
||||
For the `transcribe` CLI invocation, the `.en`-translates-non-English rule, and whisper model selection, see [`../transcribe.md`](../transcribe.md). This file covers what to do with the resulting transcript when authoring captions: input formats, mandatory quality checks, cleaning code, external-API fallbacks.
|
||||
|
||||
## Supported Input Formats
|
||||
|
||||
The CLI auto-detects and normalizes these formats:
|
||||
|
||||
| Format | Extension | Source | Word-level? |
|
||||
| --------------------- | --------- | --------------------------------------------------------------------------- | ----------------- |
|
||||
| whisper.cpp JSON | `.json` | `hyperframes init --video`, `hyperframes transcribe` | Yes |
|
||||
| OpenAI Whisper API | `.json` | `openai.audio.transcriptions.create({ timestamp_granularities: ["word"] })` | Yes |
|
||||
| SRT subtitles | `.srt` | Video editors, subtitle tools, YouTube | No (phrase-level) |
|
||||
| VTT subtitles | `.vtt` | Web players, YouTube, transcription services | No (phrase-level) |
|
||||
| Normalized word array | `.json` | Pre-processed by any tool | Yes |
|
||||
|
||||
**Word-level timestamps produce better captions.** SRT/VTT give phrase-level timing, which works but can't do per-word animation effects.
|
||||
|
||||
## Transcript Quality Check (Mandatory)
|
||||
|
||||
After every transcription, **read the transcript and check for quality issues before proceeding.** Bad transcripts produce nonsensical captions. Never skip this step.
|
||||
|
||||
### What to look for
|
||||
|
||||
| Signal | Example | Cause |
|
||||
| ---------------------------- | -------------------------------------- | ---------------------------------------------------------------------------- |
|
||||
| Music note tokens (`♪`, `�`) | `{ "text": "♪" }` or `{ "text": "�" }` | Whisper detected music, not speech |
|
||||
| Garbled / nonsense words | "Do a chin", "Get so gay", "huh" | Model misheard lyrics or background noise |
|
||||
| Long gaps with no words | 20+ seconds of only `♪` tokens | Instrumental section — expected, but high ratio means speech is being missed |
|
||||
| Repeated filler | Many "huh", "uh", "oh" entries | Model is hallucinating on music |
|
||||
| Very short word spans | Words with `end - start < 0.05` | Unreliable timestamp alignment |
|
||||
|
||||
### Automatic retry rules
|
||||
|
||||
**If more than 20% of entries are `♪`/`�` tokens, or the transcript contains obvious nonsense words, the transcription failed.** Do not proceed with the bad transcript. Instead:
|
||||
|
||||
1. **Retry with `medium.en`** if the original used `small.en` or smaller:
|
||||
```bash
|
||||
npx hyperframes transcribe audio.mp3 --model medium.en
|
||||
```
|
||||
2. **If `medium.en` also fails** (still >20% music tokens or garbled), tell the user the audio is too noisy for local transcription and suggest:
|
||||
- Providing lyrics manually as an SRT/VTT file
|
||||
- Using an external API (OpenAI or Groq Whisper — see below)
|
||||
3. **Always clean the transcript** before building captions — filter out `♪`/`�` tokens and entries where `text` is a single non-word character. Only real words should reach the caption composition.
|
||||
|
||||
### Cleaning a transcript
|
||||
|
||||
After transcription (even with a good model), strip non-word entries:
|
||||
|
||||
```js
|
||||
var raw = JSON.parse(transcriptJson);
|
||||
var words = raw.filter(function (w) {
|
||||
if (!w.text || w.text.trim().length === 0) return false;
|
||||
if (/^[♪�\u266a\u266b\u266c\u266d\u266e\u266f]+$/.test(w.text)) return false;
|
||||
if (/^(huh|uh|um|ah|oh)$/i.test(w.text) && w.end - w.start < 0.1) return false;
|
||||
return true;
|
||||
});
|
||||
```
|
||||
|
||||
For model-selection guidance by content type, see [`../transcribe.md`](../transcribe.md) → "Picking a model by content type".
|
||||
|
||||
## Using External Transcription APIs
|
||||
|
||||
For the best accuracy, use an external API and import the result:
|
||||
|
||||
**OpenAI Whisper API** (recommended for quality):
|
||||
|
||||
```bash
|
||||
# Generate with word timestamps, then import
|
||||
curl https://api.openai.com/v1/audio/transcriptions \
|
||||
-H "Authorization: Bearer $OPENAI_API_KEY" \
|
||||
-F file=@audio.mp3 -F model=whisper-1 \
|
||||
-F response_format=verbose_json \
|
||||
-F "timestamp_granularities[]=word" \
|
||||
-o transcript-openai.json
|
||||
|
||||
npx hyperframes transcribe transcript-openai.json
|
||||
```
|
||||
|
||||
**Groq Whisper API** (fast, free tier available):
|
||||
|
||||
```bash
|
||||
curl https://api.groq.com/openai/v1/audio/transcriptions \
|
||||
-H "Authorization: Bearer $GROQ_API_KEY" \
|
||||
-F file=@audio.mp3 -F model=whisper-large-v3 \
|
||||
-F response_format=verbose_json \
|
||||
-F "timestamp_granularities[]=word" \
|
||||
-o transcript-groq.json
|
||||
|
||||
npx hyperframes transcribe transcript-groq.json
|
||||
```
|
||||
|
||||
## If No Transcript Exists
|
||||
|
||||
1. Check the project root for `transcript.json`, `.srt`, or `.vtt` files.
|
||||
2. If none found, run [`../transcribe.md`](../transcribe.md) — pick the starting model from "Picking a model by content type" there.
|
||||
3. Run the quality check above. If it fails, retry with a larger model or fall back to manual lyrics / external API.
|
||||
@@ -0,0 +1,143 @@
|
||||
# Background Removal
|
||||
|
||||
Make a transparent overlay (typical: a talking head over an arbitrary scene). Uses `u2net_human_seg` (MIT).
|
||||
|
||||
```bash
|
||||
npx hyperframes remove-background subject.mp4 -o transparent.webm # default: VP9 + alpha
|
||||
npx hyperframes remove-background subject.mp4 -o transparent.mov # ProRes 4444 (editing)
|
||||
npx hyperframes remove-background portrait.jpg -o cutout.png # single-image cutout
|
||||
npx hyperframes remove-background subject.mp4 -o subject.webm \
|
||||
--background-output plate.webm # both layers, one pass
|
||||
npx hyperframes remove-background subject.mp4 -o transparent.webm --device cpu
|
||||
npx hyperframes remove-background --info # detected providers
|
||||
```
|
||||
|
||||
## Output Format
|
||||
|
||||
- **`.webm` (VP9 alpha)** — default. Plug straight into `<video>` for Chrome-native transparent playback (~1 MB / 4s @ 1080p).
|
||||
- **`.mov` (ProRes 4444)** — round-trip in editors (Premiere / Resolve / DaVinci). ~50 MB / 4s.
|
||||
- **`.png`** — single-image cutout.
|
||||
|
||||
## Quality (`--quality`)
|
||||
|
||||
Controls VP9 encoder CRF only — segmentation quality is fixed. Higher quality keeps the cutout's RGB closer to the source MP4 (important when overlaying the cutout on its own source).
|
||||
|
||||
| Preset | CRF | When |
|
||||
| ---------- | --- | --------------------------------------------- |
|
||||
| `fast` | 30 | Iterating, smaller files, looser color match |
|
||||
| `balanced` | 18 | **Default**; visually identical for most uses |
|
||||
| `best` | 12 | Master / final delivery, tightest color match |
|
||||
|
||||
## Device (`--device`)
|
||||
|
||||
`auto` (default) picks CoreML on Apple Silicon, CUDA when available, otherwise CPU. Force with `--device cpu | coreml | cuda`. CUDA requires `HYPERFRAMES_CUDA=1` plus a GPU-enabled `onnxruntime-node` build. Use `--info` to inspect detected providers without rendering.
|
||||
|
||||
## Compositing patterns — pick the right one
|
||||
|
||||
The cutout WebM is a **re-encoded copy** of the source MP4's RGB. What sits behind it matters.
|
||||
|
||||
| Pattern | Behind the cutout | Result |
|
||||
| -------------------------------------------------------- | --------------------------------------- | --------------------------------------------------------------------------------------------------------------- |
|
||||
| **Cutout over a different scene** (most common) | Static image, gradient, unrelated video | Looks great. Single RGB source for the subject. |
|
||||
| **Cutout over its own source mp4** (text-behind-subject) | Same mp4 the cutout came from | At `balanced` doubling is barely visible; at `fast` you'll see color shift / edge halo. Use `best` for masters. |
|
||||
| **Cutout over a different take of the same person** | Footage of the same subject | **Two overlapping people. Don't do this.** |
|
||||
|
||||
## Text-behind-subject pattern (two non-obvious rules)
|
||||
|
||||
Putting a headline behind a presenter cutout:
|
||||
|
||||
```html
|
||||
<video
|
||||
src="presenter.mp4"
|
||||
id="bg"
|
||||
data-start="0"
|
||||
data-duration="6"
|
||||
data-track-index="0"
|
||||
muted
|
||||
playsinline
|
||||
></video>
|
||||
|
||||
<h1 id="headline" style="z-index:2; ...">MAKE IT IN HYPERFRAMES</h1>
|
||||
|
||||
<div class="cutout-wrap" style="position:absolute; inset:0; z-index:3; opacity:0">
|
||||
<video
|
||||
src="presenter.webm"
|
||||
data-start="0"
|
||||
data-duration="6"
|
||||
data-track-index="1"
|
||||
muted
|
||||
playsinline
|
||||
></video>
|
||||
</div>
|
||||
```
|
||||
|
||||
```js
|
||||
// Flip the wrapper's opacity at the cut, NOT the video's
|
||||
tl.set(".cutout-wrap", { opacity: 1 }, 3.3);
|
||||
```
|
||||
|
||||
Two rules that are easy to miss:
|
||||
|
||||
1. **Wrap the cutout `<video>` in a non-timed `<div>` and animate the wrapper's opacity, not the video element's.** The framework forces `opacity: 1` on active clips (any element with `data-start` / `data-duration`), so animating the video's opacity directly is silently overridden. The wrapper has no `data-*` attributes, so it's owned by your CSS / GSAP.
|
||||
2. **Both videos use `data-start="0"` and `data-media-start="0"`** so the framework decodes them in sync from t=0. Late-mounting the cutout (`data-start=3.3`) introduces a seek + warm-up that lands a frame off the base mp4 — visible as one frame of misalignment at the cut.
|
||||
|
||||
## Layer separation (`--background-output`)
|
||||
|
||||
Emits a **second** transparent video alongside the cutout: same source RGB, alpha is `255 - mask` instead of `mask`. The cutout has the subject opaque; the plate has the surroundings opaque (with a transparent hole where the subject was). Use it when text / graphics need to live **between** the two layers.
|
||||
|
||||
| File | Alpha is… | Use it for |
|
||||
| -------------------------------- | ------------------------------------------------------- | ---------------------------------------------------------------- |
|
||||
| `-o subject.webm` | mask — subject opaque, background transparent | Foreground layer (top) |
|
||||
| `--background-output plate.webm` | inverse mask — surroundings opaque, subject transparent | Bottom layer; place text / graphics between this and the subject |
|
||||
|
||||
Both share the same `--quality` and run from a single inference pass — only encode cost roughly doubles. Only valid for video inputs with `.webm` / `.mov` outputs.
|
||||
|
||||
**Hole-cut, not inpainted.** The subject region in `plate.webm` is fully transparent — composite something opaque under it to fill the hole.
|
||||
|
||||
**Single test for whether `--background-output` is the right tool:** _will anything ever be visible through the subject's silhouette where the subject used to be?_ If no, you don't need the plate — `subject.webm` alone over a different background is enough.
|
||||
|
||||
### Use case → right tool
|
||||
|
||||
| Use case | Right tool |
|
||||
| ----------------------------------------------------------------------------------- | ---------------------------------------------------------------------------------- |
|
||||
| Text/graphics between the cutout and the plate (this command's reason for existing) | **Hole-cut** (`--background-output`) |
|
||||
| Subject onto an unrelated scene | Just `subject.webm`; ignore the plate |
|
||||
| Show the room _without_ the person, alone over no other content | **Clean plate** — needs an inpainter (LaMa, ProPainter, E2FGVI). Not this command. |
|
||||
| Replace the subject with a different subject | **Clean plate** — same as above |
|
||||
|
||||
### Canonical 3-layer template (plate + content + cutout)
|
||||
|
||||
Ship just the two transparent layers and let arbitrary content live between them — no original mp4 needed:
|
||||
|
||||
```html
|
||||
<!-- z=1 plate: surroundings opaque, subject silhouette transparent -->
|
||||
<video
|
||||
src="plate.webm"
|
||||
data-start="0"
|
||||
data-duration="6"
|
||||
data-track-index="0"
|
||||
muted
|
||||
playsinline
|
||||
></video>
|
||||
|
||||
<!-- z=2 your content lives between the layers -->
|
||||
<h1 id="headline" style="z-index:2; ...">MAKE IT IN HYPERFRAMES</h1>
|
||||
|
||||
<!-- z=3 cutout floats the subject back on top -->
|
||||
<div class="cutout-wrap" style="position:absolute; inset:0; z-index:3">
|
||||
<video
|
||||
src="subject.webm"
|
||||
data-start="0"
|
||||
data-duration="6"
|
||||
data-track-index="1"
|
||||
muted
|
||||
playsinline
|
||||
></video>
|
||||
</div>
|
||||
```
|
||||
|
||||
Functionally equivalent to the text-behind-subject pattern above, but doesn't require shipping the original mp4 — the plate replaces it. Use this when delivering just the two transparent layers as a reusable asset.
|
||||
|
||||
## When `remove-background` is NOT the right tool
|
||||
|
||||
If a user asks for "the room **without** the person, displayed standalone" (no subject anywhere, no compositing on top), `--background-output` is wrong — its plate has a transparent hole, not a filled-in clean plate. They need an **inpainter**: LaMa, ProPainter, or E2FGVI. Tell them this command can't do it.
|
||||
@@ -0,0 +1,29 @@
|
||||
# Requirements & Caches
|
||||
|
||||
## Credential & key priority
|
||||
|
||||
Run `npx hyperframes auth status` to see what's configured and which engines a workflow will use (see the skill's **Preflight** section). Keys resolve in this order — **first match wins**:
|
||||
|
||||
| Provider | Resolution order (first non-empty wins) | Local deps when used |
|
||||
| ------------------------------------ | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | ------------------------------------------------ |
|
||||
| **HeyGen** (TTS + BGM/SFX retrieval) | `$HEYGEN_API_KEY` → `$HYPERFRAMES_API_KEY` → `~/.heygen/credentials` (shared with heygen-cli; `$HEYGEN_CONFIG_DIR` overrides the dir; written by `hyperframes auth login`) | none (REST) |
|
||||
| **ElevenLabs** (TTS fallback) | `$ELEVENLABS_API_KEY` | `pip install elevenlabs` |
|
||||
| **Lyria** (BGM fallback) | `$GEMINI_API_KEY` → `$GOOGLE_API_KEY` | `pip install google-genai` |
|
||||
| **Kokoro** (TTS, no key) | always — final voice fallback | `pip install kokoro-onnx soundfile` |
|
||||
| **MusicGen** (BGM, no key) | always — final music fallback | `pip install transformers torch soundfile numpy` |
|
||||
|
||||
`hyperframes auth login` (browser OAuth) is the recommended setup: one sign-in, every project, no per-repo `.env`. An OAuth login is sent as `Authorization: Bearer`; an API key as `X-Api-Key`. With no HeyGen credential, voice/BGM run fully locally (Kokoro / MusicGen) — `hyperframes auth status` and `hyperframes doctor` both report whether those local deps are installed.
|
||||
|
||||
## Model caches & system dependencies
|
||||
|
||||
Each command downloads its own model on first run and caches it under `~/.cache/hyperframes/`:
|
||||
|
||||
- **TTS (HeyGen)** — no local deps; needs a HeyGen credential + `ffmpeg` on PATH (to transcode the mp3 response to `.wav`). Credential resolves like the CLI: `$HEYGEN_API_KEY` → `$HYPERFRAMES_API_KEY` → `~/.heygen/credentials` (shared with heygen-cli; run `npx hyperframes auth login`). An OAuth login is sent as `Authorization: Bearer`; an API key as `X-Api-Key`.
|
||||
- **TTS (ElevenLabs)** — same as HeyGen: API key + `ffmpeg`.
|
||||
- **TTS (Kokoro)** — Kokoro-82M (~311 MB) + voices (~27 MB) in `tts/`. Requires Python 3.8+ with `kokoro-onnx` and `soundfile` (`pip install kokoro-onnx soundfile`). Non-English text also needs `espeak-ng` system-wide.
|
||||
- **BGM (Lyria)** — needs `$GEMINI_API_KEY` or `$GOOGLE_API_KEY` + `pip install google-genai`. No local model cache.
|
||||
- **BGM (MusicGen)** — `pip install transformers torch soundfile`. `facebook/musicgen-small` (~300 MB) cached under `~/.cache/huggingface/` on first run.
|
||||
- **Transcribe** — Whisper model size depending on choice (75 MB – 3.1 GB) in `whisper/`. Bundles `whisper.cpp`.
|
||||
- **Remove-background** — `u2net_human_seg` (~168 MB ONNX) in `background-removal/models/`. Peak inference RAM ~1.5 GB.
|
||||
|
||||
Run `npx hyperframes doctor` if a command fails because of a missing dependency.
|
||||
@@ -0,0 +1,42 @@
|
||||
# Sound effects (SFX)
|
||||
|
||||
Named sound effects, produced by the shared audio engine (`scripts/audio.mjs` → `scripts/lib/sfx.mjs`). **Provider-gated** by the engine's one switch — whether a HeyGen credential is present, decided once (not per cue):
|
||||
|
||||
- **HeyGen credential present → retrieve every cue** from HeyGen's audio library (`/v3/audio/sounds`, `type=sound_effects`, `min_score=0.4`). Search-and-download, **not** generation. The bundled library is NOT consulted.
|
||||
- **No credential → the bundled 21-file library** (`assets/sfx/` + `manifest.json`): match each cue name, copy the matched file into the project. Offline, deterministic, free.
|
||||
|
||||
There is no `npx hyperframes sfx` command. SFX is never generated — it is retrieved (online) or taken from the bundled library (offline).
|
||||
|
||||
## Cues — request → meta
|
||||
|
||||
Each line names the effects it wants: `lines[].sfx: ["whoosh", "ui click"]`. The engine flattens these into cues, resolves them per the switch, dedupes identical `(id, name)` pairs (the same effect named twice downloads/copies once), and writes `audio_meta.sfx[]`:
|
||||
|
||||
```jsonc
|
||||
{
|
||||
"id": "3", // joins the cue to the caller's model (frame / scene / segment)
|
||||
"name": "whoosh",
|
||||
"file": "assets/sfx/whoosh.mp3", // downloaded or copied, relative to project root
|
||||
"source": "heygen" | "local", // which route resolved it
|
||||
"offset_s": 0, // delay from the line's start
|
||||
"duration_s": 0.57,
|
||||
"volume": 0.35 // SFX sit UNDER voice + BGM
|
||||
}
|
||||
```
|
||||
|
||||
A cue that matches nothing is **skipped** (recorded as an anomaly); SFX never blocks a render.
|
||||
|
||||
## HeyGen retrieval (credentialed)
|
||||
|
||||
`searchSounds(name, "sound_effects", { limit: 3, minScore: 0.4 })` → top hit → `assets/sfx/<slug>.mp3`. Results are ranked by `score` (each carries a presigned `audio_url`, `duration`, `description`). The floor is **0.4** because good SFX hits score ~0.5–0.67 — below the API's default `0.7`, which would silently drop most named cues (only whoosh/swoosh-family clears 0.7). `duration_s` comes from the result (else 1.0). Name effects concretely (`glass shatter`, not `dramatic sound`); a vague query returns a poor match.
|
||||
|
||||
## Bundled library (no credential)
|
||||
|
||||
21 curated files in `assets/sfx/`, indexed by `manifest.json` — `{ file, duration, description }` per key (e.g. `whoosh`, `pop`, `click`, `chime`, `riser`, `impact-bass-1`, `glitch-1`, `typing`, …). A cue name resolves by **manifest key, file basename, or slug**, so `whoosh`, `whoosh.mp3`, or `"ui click"` (→ slug) all match. Matched files are copied into the project's `assets/sfx/`; `duration_s` comes from the manifest, so timing is known **offline** — e.g. `riser` is 10.03s, so trigger it at `climax − 10.03s`. The manifest's `description` field carries placement hints per effect; read `assets/sfx/manifest.json` for the full set and usage.
|
||||
|
||||
## Rules
|
||||
|
||||
- **Volume ~0.35.** SFX must sit under narration and BGM, not fight them.
|
||||
- **No match → skip, don't fail.** A missing effect logs an anomaly and moves on; never a render blocker.
|
||||
- **Retrieval (credentialed) or bundled library (offline) — never generation.** You search HeyGen by text, or match a name against the 21-file manifest.
|
||||
- **One asset per distinct name.** Reuse across lines is deduped to a single download/copy, many cues.
|
||||
- **The switch is global, not per cue.** With a credential, retrieval handles even the long tail (effects not in the 21); without one, only the 21 bundled names resolve.
|
||||
@@ -0,0 +1,52 @@
|
||||
# Transcription
|
||||
|
||||
Create normalized word-level timestamps. **Always specify `--model` explicitly** — the CLI default is `small.en`, which silently translates non-English audio into English.
|
||||
|
||||
```bash
|
||||
npx hyperframes transcribe audio.mp3 --model small.en # known English
|
||||
npx hyperframes transcribe video.mp4 --model small --language es # known Spanish
|
||||
npx hyperframes transcribe audio.mp3 --model small # unknown language (auto-detect)
|
||||
npx hyperframes transcribe subtitles.srt # import existing
|
||||
npx hyperframes transcribe subtitles.vtt
|
||||
npx hyperframes transcribe openai-response.json
|
||||
```
|
||||
|
||||
## Language Rule (Non-Negotiable)
|
||||
|
||||
`.en` models (`tiny.en` / `base.en` / `small.en` / `medium.en`) **translate** non-English audio into English. This silently destroys the original language.
|
||||
|
||||
1. **Known English** → `--model small.en` (or `medium.en` for music / noisy audio)
|
||||
2. **Known non-English** → `--model small --language <iso-code>` (no `.en` suffix)
|
||||
3. **Unknown language** → `--model small` (whisper auto-detects)
|
||||
|
||||
**CLI default is `small.en`** — do not rely on it; always pass `--model` to make the choice explicit. `--language` also filters out non-target-language segments from mixed-language audio.
|
||||
|
||||
## Model Sizes
|
||||
|
||||
| Model | Size | Speed | When |
|
||||
| ---------- | ------ | -------- | ------------------------------------- |
|
||||
| `tiny` | 75 MB | Fastest | Quick previews, smoke tests |
|
||||
| `base` | 142 MB | Fast | Short clips, clear audio |
|
||||
| `small` | 466 MB | Moderate | Default for most multilingual content |
|
||||
| `medium` | 1.5 GB | Slow | Music with vocals, noisy audio |
|
||||
| `large-v3` | 3.1 GB | Slowest | Production quality |
|
||||
|
||||
### Picking a model by content type
|
||||
|
||||
1. Speech over silence / light background → `small.en`
|
||||
2. Speech over music, or music with vocals → start with `medium.en`
|
||||
3. Produced music track (vocals + full instrumentation) → start with `medium.en`; expect to need manual lyrics or an external API ([`captions/transcript-handling.md`](captions/transcript-handling.md) → "Using External Transcription APIs")
|
||||
4. Multilingual → `medium` or `large-v3` (no `.en` suffix), pair with `--language`
|
||||
|
||||
## Output Shape
|
||||
|
||||
Compositions consume a flat array of word objects. The `id` (`w0`, `w1`, …) is added during normalization for stable references in caption overrides; optional for backwards compatibility.
|
||||
|
||||
```json
|
||||
[
|
||||
{ "id": "w0", "text": "Hello", "start": 0.0, "end": 0.5 },
|
||||
{ "id": "w1", "text": "world.", "start": 0.6, "end": 1.2 }
|
||||
]
|
||||
```
|
||||
|
||||
For mandatory caption-quality checks, retry rules, and the OpenAI/Groq Whisper API import path, see `captions/transcript-handling.md`.
|
||||
@@ -0,0 +1,24 @@
|
||||
# TTS → Captions
|
||||
|
||||
When no recorded voiceover exists, generate one and obtain word-level caption timing. Two paths depending on which TTS provider is in use:
|
||||
|
||||
## Path A — HeyGen (single call, no Whisper)
|
||||
|
||||
HeyGen returns word timestamps in the same response as the audio. Pass `--words` and you're done:
|
||||
|
||||
```bash
|
||||
npx hyperframes tts script.txt --provider heygen --output narration.wav --words narration.words.json
|
||||
```
|
||||
|
||||
`narration.words.json` is already in the `[{ id, text, start, end }]` shape the captions pipeline consumes — no separate transcribe pass.
|
||||
|
||||
## Path B — ElevenLabs / Kokoro (TTS → Whisper)
|
||||
|
||||
These providers don't return word data. Generate the audio, then transcribe:
|
||||
|
||||
```bash
|
||||
npx hyperframes tts script.txt --voice af_heart --output narration.wav
|
||||
npx hyperframes transcribe narration.wav --model small.en # voice af_heart is American English
|
||||
```
|
||||
|
||||
Whisper extracts precise word boundaries from the generated audio, so caption timing matches delivery without hand-tuning. Match `--model` to the voice's language (use `small.en` for `a`/`b` prefixes, `small --language <code>` otherwise). Then consume `transcript.json` via the caption references in `captions/`.
|
||||
@@ -0,0 +1,135 @@
|
||||
# Text To Speech
|
||||
|
||||
`npx hyperframes tts` auto-detects a provider from env vars; explicit override via `--provider`.
|
||||
|
||||
> **Run the Preflight first — no credential is not a green light to silently use the local voice.** Before generating a voiceover, complete the sign-in **Preflight** (see `../SKILL.md` → Preflight): run `npx hyperframes auth status`, recommend signing in, and **STOP for the user's choice** (sign in for HeyGen voices, or continue offline with local Kokoro). This applies to a one-off "generate a voiceover" request just as much as inside a full workflow.
|
||||
|
||||
## Provider chain
|
||||
|
||||
| Order | Provider | Env trigger | Voice IDs | Word timestamps | Audio format |
|
||||
| ----- | ----------------- | ------------------------------------------- | ------------------------------------------- | ----------------------------------------- | -------------------- |
|
||||
| 1 | HeyGen (Starfish) | `$HEYGEN_API_KEY` / `~/.heygen/credentials` | UUIDs from `GET /v3/voices?engine=starfish` | **Yes** (`word_timestamps[]` in response) | mp3 → wav via ffmpeg |
|
||||
| 2 | ElevenLabs | `$ELEVENLABS_API_KEY` | UUIDs from elevenlabs.io dashboard | No | mp3 → wav via ffmpeg |
|
||||
| 3 | Kokoro-82M | always (local fallback) | `am_michael`, `af_heart`, … (54 voices) | No | wav direct |
|
||||
|
||||
```bash
|
||||
# Auto-detect (HeyGen if key set, else ElevenLabs, else Kokoro)
|
||||
npx hyperframes tts "Welcome to HyperFrames" -o narration.wav
|
||||
|
||||
# Pin the provider explicitly
|
||||
npx hyperframes tts "Hello" --provider kokoro
|
||||
npx hyperframes tts "Hello" --provider heygen --voice <heygen-uuid>
|
||||
npx hyperframes tts "Hello" --provider elevenlabs --voice 21m00Tcm4TlvDq8ikWAM
|
||||
|
||||
# HeyGen path: capture word timestamps in one call (skips a Whisper pass)
|
||||
npx hyperframes tts "Hi there" --words narration.words.json
|
||||
```
|
||||
|
||||
## Self-contained HeyGen (no CLI) — `scripts/heygen-tts.mjs`
|
||||
|
||||
The published `hyperframes tts` CLI synthesizes locally with Kokoro only. When you
|
||||
want HeyGen specifically — best quality **plus** word timestamps in one call — use
|
||||
the skill's bundled script, which calls the HeyGen v3 REST API directly and needs
|
||||
no CLI provider plumbing:
|
||||
|
||||
The script resolves a HeyGen credential the same way the CLI does — first source
|
||||
wins: `$HEYGEN_API_KEY` → `$HYPERFRAMES_API_KEY` → a project `.env` (auto-loaded,
|
||||
walks up ≤5 dirs) → `~/.heygen/credentials` (shared with heygen-cli;
|
||||
`$HEYGEN_CONFIG_DIR` overrides the dir). An OAuth login is sent as
|
||||
`Authorization: Bearer`; an API key as `X-Api-Key`. If the only credential is an
|
||||
expired OAuth token it stops with a hint to run `npx hyperframes auth refresh`.
|
||||
|
||||
```bash
|
||||
# Only needed if you haven't run `npx hyperframes auth login`:
|
||||
export HEYGEN_API_KEY=... # or put it in a project .env
|
||||
|
||||
# Synthesize + capture word timestamps in one call (skips a Whisper pass)
|
||||
node skills/media-use/audio/scripts/heygen-tts.mjs \
|
||||
"Welcome to HyperFrames." -o narration.wav --words narration.words.json
|
||||
|
||||
node skills/media-use/audio/scripts/heygen-tts.mjs ./script.txt -o narration.wav
|
||||
node skills/media-use/audio/scripts/heygen-tts.mjs --list # public starfish voices
|
||||
```
|
||||
|
||||
- **Voice:** `--voice <id>` must be a **starfish** voice_id (`--list`, or `GET /v3/voices?engine=starfish`). v2-catalog ids are rejected with HTTP 400. Omit `--voice` (English) and it defaults to **Marcia** (`05f19352e8f74b0392a8f411eba40de1`, a fixed default so the choice is deterministic). Non-English with no `--voice` falls back to the first matching catalog voice.
|
||||
- **Output:** `.wav` → transcoded to 44.1k mono via ffmpeg; `.mp3` → raw bytes (no ffmpeg needed).
|
||||
- **Words:** `--words <path>` writes the flat `[{id,text,start,end}]` shape below, drop-in for the captions pipeline. HeyGen's `<start>`/`<end>` boundary sentinels are filtered out and ids are re-contiguous.
|
||||
- **Non-English:** `--lang <code>` (anything but `en`) is sent as the request `language`.
|
||||
|
||||
## When to use which provider
|
||||
|
||||
| Goal | Use |
|
||||
| --------------------------------------------------------- | --------------------------------------------------- |
|
||||
| Best voice quality + word timestamps in one call | **HeyGen** |
|
||||
| Drop-in cloud TTS, big voice catalog | **ElevenLabs** |
|
||||
| Offline, no API key, fast iteration | **Kokoro** |
|
||||
| Non-English multilingual with deterministic phonemization | **Kokoro** (`ef_dora`, `jf_alpha`, `zf_xiaobei`, …) |
|
||||
|
||||
## ffmpeg requirement
|
||||
|
||||
HeyGen + ElevenLabs return mp3. The CLI transcodes to wav when `--output` ends in `.wav` (the default and what downstream `ffprobe` + Whisper expect). If you'd rather skip the transcode, pass `-o file.mp3`. Without `ffmpeg` on PATH, `.wav` output from the cloud providers fails — install ffmpeg or use `.mp3`.
|
||||
|
||||
## Voice selection (Kokoro)
|
||||
|
||||
Default `af_heart`. Curated picks:
|
||||
|
||||
| Content type | Voice |
|
||||
| ----------------- | ---------------------- |
|
||||
| Product demo | `af_heart`, `af_nova` |
|
||||
| Tutorial / how-to | `am_adam`, `bf_emma` |
|
||||
| Marketing / promo | `af_sky`, `am_michael` |
|
||||
| Documentation | `bf_emma`, `bm_george` |
|
||||
| Casual / social | `af_heart`, `af_sky` |
|
||||
|
||||
Run `npx hyperframes tts --list` for the bundled set.
|
||||
|
||||
## Multilingual (Kokoro voice prefix → language)
|
||||
|
||||
The first letter of a Kokoro voice ID picks the phonemizer language; `--lang` overrides auto-detection.
|
||||
|
||||
| Prefix | Language |
|
||||
| ------ | -------------------- |
|
||||
| `a` | American English |
|
||||
| `b` | British English |
|
||||
| `e` | Spanish |
|
||||
| `f` | French |
|
||||
| `h` | Hindi |
|
||||
| `i` | Italian |
|
||||
| `j` | Japanese |
|
||||
| `p` | Brazilian Portuguese |
|
||||
| `z` | Mandarin |
|
||||
|
||||
```bash
|
||||
npx hyperframes tts "La reunión empieza a las nueve" --voice ef_dora --provider kokoro
|
||||
npx hyperframes tts "Today is a nice day" --voice af_heart --provider kokoro
|
||||
```
|
||||
|
||||
Valid `--lang` codes (only needed to override the voice's auto-detected language): `en-us`, `en-gb`, `es`, `fr-fr`, `hi`, `it`, `pt-br`, `ja`, `zh`.
|
||||
|
||||
Non-English phonemization requires `espeak-ng` system-wide (`brew install espeak-ng` / `apt-get install espeak-ng`).
|
||||
|
||||
## Speed
|
||||
|
||||
- `0.7-0.8` — tutorial, complex content, accessibility
|
||||
- `1.0` — natural pace (default)
|
||||
- `1.1-1.2` — intros, transitions, upbeat content
|
||||
- `1.5+` — rarely appropriate, test carefully
|
||||
|
||||
Honored by Kokoro + HeyGen; ElevenLabs ignores `--speed` (use voice settings on their dashboard).
|
||||
|
||||
## Long scripts
|
||||
|
||||
Past a few paragraphs, write the text to a `.txt` file and pass the path. Inputs over ~5 minutes of speech may benefit from splitting into segments.
|
||||
|
||||
## HeyGen word-timestamp shape
|
||||
|
||||
When `--words <path>` is passed to a HeyGen call, the file is written in the same flat shape `transcribe` produces — drop-in compatible with the captions pipeline:
|
||||
|
||||
```json
|
||||
[
|
||||
{ "id": "w0", "text": "Hi", "start": 0.0, "end": 0.21 },
|
||||
{ "id": "w1", "text": "there", "start": 0.22, "end": 0.55 }
|
||||
]
|
||||
```
|
||||
|
||||
For ElevenLabs / Kokoro, run `npx hyperframes transcribe narration.wav --model small.en` to get the same shape.
|
||||
@@ -0,0 +1,293 @@
|
||||
#!/usr/bin/env node
|
||||
// audio.mjs — the shared HyperFrames audio engine. ONE implementation of TTS +
|
||||
// BGM + SFX for every video workflow (product-launch, general-video, pr-to-video,
|
||||
// …). Workflows do NOT vendor a copy: they write a neutral `audio_request.json`
|
||||
// (a tiny per-workflow adapter maps their storyboard/scenes into it) and call:
|
||||
//
|
||||
// node <MEDIA_DIR>/scripts/audio.mjs --request ./audio_request.json --hyperframes . --out ./audio_meta.json
|
||||
//
|
||||
// The three capabilities degrade on ONE switch — whether HeyGen is configured
|
||||
// (credential present, NOT the CLI). This mirrors the table in ../SKILL.md:
|
||||
//
|
||||
// TTS : HeyGen REST → ElevenLabs → Kokoro (CLI)
|
||||
// BGM : HeyGen retrieve → (no credential) Lyria/MusicGen generate
|
||||
// SFX : HeyGen retrieve → (no credential) bundled 19-file library
|
||||
//
|
||||
// ── audio_request.json (input) ────────────────────────────────────────────────
|
||||
// {
|
||||
// "provider": "auto", // auto|heygen|elevenlabs|kokoro (override: --provider)
|
||||
// "lang": "en", "speed": 1.0,
|
||||
// "lines": [ // one TTS unit each; id joins back to the caller's model
|
||||
// { "id": "01", "text": "...", "sfx": ["whoosh", "ui click"] }
|
||||
// ],
|
||||
// "bgm": { "mode": "retrieve", // retrieve|generate|none (override: --bgm-mode / --no-bgm)
|
||||
// "query": "calm cinematic underscore", // mood for retrieval
|
||||
// "prompt": null, // full prompt for generation (else inferred)
|
||||
// "blob": "...", "archetype": "...", "arc": "..." } // optional mood-inference hints
|
||||
// }
|
||||
//
|
||||
// ── audio_meta.json (output, id-keyed) ───────────────────────────────────────
|
||||
// { tts_provider, voice_id,
|
||||
// bgm: { path, volume, mode, query?, duration_s? } | null,
|
||||
// bgm_pending, bgm_provider, bgm_pid, bgm_log, bgm_mode, bgm_target_duration_s, …,
|
||||
// voices: [ { id, path, duration_s, words: [{id,text,start,end}] } ],
|
||||
// sfx: [ { id, name, file, source, offset_s, duration_s, volume } ],
|
||||
// total_duration_s }
|
||||
//
|
||||
// --only tts,bgm,sfx runs a subset and MERGES into an existing --out (so a
|
||||
// workflow can do TTS+BGM early, then SFX later once cues exist). When BGM uses
|
||||
// the generate path it is spawned detached (bgm_pending:true) — run wait-bgm.mjs
|
||||
// before assembling.
|
||||
|
||||
import { existsSync, mkdirSync, readFileSync, writeFileSync } from "node:fs";
|
||||
import { dirname, join, resolve } from "node:path";
|
||||
import { fileURLToPath } from "node:url";
|
||||
import { heygenAuthHeaders, heygenCredential, loadEnvFromDir } from "./lib/heygen.mjs";
|
||||
import {
|
||||
ffprobeDuration,
|
||||
pickProvider,
|
||||
resolveVoiceId,
|
||||
synthesizeOne,
|
||||
transcribeWav,
|
||||
withWordIds,
|
||||
} from "./lib/tts.mjs";
|
||||
import { generateBgmDetached, inferBgmPrompt, retrieveBgm } from "./lib/bgm.mjs";
|
||||
import { resolveSfx } from "./lib/sfx.mjs";
|
||||
import { mapWithConcurrency } from "./lib/concurrency.mjs";
|
||||
|
||||
const HERE = dirname(fileURLToPath(import.meta.url));
|
||||
const argv = process.argv.slice(2);
|
||||
const flag = (name, def) => {
|
||||
const i = argv.indexOf(`--${name}`);
|
||||
return i >= 0 && i + 1 < argv.length ? argv[i + 1] : def;
|
||||
};
|
||||
const has = (name) => argv.includes(`--${name}`);
|
||||
const die = (m) => {
|
||||
console.error(`✗ audio engine: ${m}`);
|
||||
process.exit(1);
|
||||
};
|
||||
const r3 = (x) => Number(x.toFixed(3));
|
||||
|
||||
// Two independent reports of an unbounded Promise.all over TTS lines
|
||||
// overwhelming a machine: one OOM'd 12/13 concurrent Kokoro TTS +
|
||||
// whisper-transcribe lines on a resource-constrained laptop, the other saw
|
||||
// 7/8 lines fail on first run (concurrent cold-start model loads) and pass on
|
||||
// retry once the model was cached. Kokoro/Whisper each load their own local
|
||||
// model per subprocess, so firing every line at once multiplies that cost by
|
||||
// the line count. mapWithConcurrency caps how many run at once — still
|
||||
// parallel, just bounded.
|
||||
const ttsConcurrency = Math.max(1, Number(process.env.HYPERFRAMES_TTS_CONCURRENCY) || 4);
|
||||
|
||||
const hyperframesDir = resolve(flag("hyperframes", "."));
|
||||
const requestPath = resolve(flag("request", join(hyperframesDir, "audio_request.json")));
|
||||
const outPath = resolve(flag("out", join(hyperframesDir, "audio_meta.json")));
|
||||
const sfxLibDir = resolve(flag("sfx-lib", join(HERE, "..", "assets", "sfx")));
|
||||
const lyriaRecipe = resolve(flag("lyria-recipe", join(HERE, "lyria-recipe.py")));
|
||||
const onlyArg = flag("only", "tts,bgm,sfx");
|
||||
const only = new Set(
|
||||
onlyArg
|
||||
.split(",")
|
||||
.map((s) => s.trim())
|
||||
.filter(Boolean),
|
||||
);
|
||||
const providerOverride = flag("provider", null);
|
||||
const bgmModeOverride = flag("bgm-mode", null);
|
||||
const noBgm = has("no-bgm");
|
||||
const voiceOverride = flag("voice", null);
|
||||
const speedOverride = flag("speed", null);
|
||||
const langOverride = flag("lang", null);
|
||||
const seedSeconds = Number(flag("seed-seconds", "28")) || 28;
|
||||
|
||||
if (!existsSync(requestPath)) die(`audio_request.json not found at ${requestPath}`);
|
||||
let request;
|
||||
try {
|
||||
request = JSON.parse(readFileSync(requestPath, "utf8"));
|
||||
} catch (e) {
|
||||
die(`audio_request.json parse: ${e.message}`);
|
||||
}
|
||||
const lines = Array.isArray(request.lines) ? request.lines : [];
|
||||
const lang = langOverride || request.lang || "en";
|
||||
const speed = Number(speedOverride ?? request.speed ?? 1.0) || 1.0;
|
||||
|
||||
// ── env + HeyGen availability (the single switch) ─────────────────────────────
|
||||
loadEnvFromDir(hyperframesDir);
|
||||
const heygenOK = heygenCredential() !== null;
|
||||
const headers = heygenOK ? heygenAuthHeaders() : null;
|
||||
|
||||
// ── merge base: preserve sections not selected by --only ──────────────────────
|
||||
const prev = existsSync(outPath) ? JSON.parse(readFileSync(outPath, "utf8")) : {};
|
||||
const anomalies = [];
|
||||
|
||||
// ── TTS ───────────────────────────────────────────────────────────────────────
|
||||
let voices = prev.voices ?? [];
|
||||
let ttsProvider = prev.tts_provider ?? null;
|
||||
let voiceId = prev.voice_id ?? null;
|
||||
if (only.has("tts") && lines.length) {
|
||||
try {
|
||||
ttsProvider = pickProvider(
|
||||
providerOverride || (request.provider === "auto" ? null : request.provider),
|
||||
);
|
||||
} catch (e) {
|
||||
die(e.message);
|
||||
}
|
||||
voiceId = await resolveVoiceId({
|
||||
provider: ttsProvider,
|
||||
userVoice: voiceOverride || request.voice,
|
||||
lang,
|
||||
});
|
||||
console.error(`· tts: ${ttsProvider} · voice ${voiceId} · ${lines.length} line(s)`);
|
||||
const synthLine = async (line) => {
|
||||
const id = String(line.id);
|
||||
const text = String(line.text ?? "").trim();
|
||||
if (!text) {
|
||||
anomalies.push(`line ${id}: empty text — skipped`);
|
||||
return null;
|
||||
}
|
||||
const rel = `assets/voice/${id}.wav`;
|
||||
const abs = join(hyperframesDir, rel);
|
||||
const { ok, words } = await synthesizeOne({
|
||||
provider: ttsProvider,
|
||||
text,
|
||||
voiceId,
|
||||
lang,
|
||||
speed,
|
||||
wavAbs: abs,
|
||||
hyperframesDir,
|
||||
});
|
||||
if (!ok) {
|
||||
anomalies.push(`line ${id}: TTS failed — omitted`);
|
||||
return null;
|
||||
}
|
||||
let wordArr = words; // heygen: native; else transcribe
|
||||
if (!wordArr) wordArr = await transcribeWav({ wavRel: rel, lang, hyperframesDir });
|
||||
const dur = ffprobeDuration(abs);
|
||||
if (!isFinite(dur) || dur <= 0) {
|
||||
anomalies.push(`line ${id}: bad voice duration — omitted`);
|
||||
return null;
|
||||
}
|
||||
return { id, path: rel, duration_s: r3(dur), words: withWordIds(wordArr) };
|
||||
};
|
||||
const results = await mapWithConcurrency(lines, ttsConcurrency, synthLine);
|
||||
voices = results.filter(Boolean);
|
||||
for (const v of voices)
|
||||
console.error(` voice ${v.id}: ${v.path} (${v.duration_s}s, ${v.words.length} words)`);
|
||||
}
|
||||
const hasVoice = voices.length > 0;
|
||||
const totalDuration = r3(voices.reduce((a, v) => a + (v.duration_s || 0), 0));
|
||||
|
||||
// ── BGM ─────────────────────────────────────────────────────────────────────
|
||||
let bgm = prev.bgm ?? null;
|
||||
const bgmFields = {
|
||||
bgm_pending: prev.bgm_pending ?? false,
|
||||
bgm_provider: prev.bgm_provider ?? null,
|
||||
bgm_pid: prev.bgm_pid ?? null,
|
||||
bgm_log: prev.bgm_log ?? null,
|
||||
bgm_mode: prev.bgm_mode ?? null,
|
||||
bgm_target_duration_s: prev.bgm_target_duration_s ?? null,
|
||||
bgm_seed_duration_s: prev.bgm_seed_duration_s ?? null,
|
||||
bgm_loop_count: prev.bgm_loop_count ?? null,
|
||||
};
|
||||
if (only.has("bgm")) {
|
||||
bgm = null;
|
||||
Object.keys(bgmFields).forEach((k) => (bgmFields[k] = k === "bgm_pending" ? false : null));
|
||||
// Mode resolution. An EXPLICIT mode (flag or request.bgm.mode) is strict:
|
||||
// "retrieve" means retrieve-or-nothing — it never silently starts a detached
|
||||
// generate (a caller with no wait-bgm step, e.g. product-launch, must not get
|
||||
// a pending job it can't await). Only the UNSET/auto default picks generate
|
||||
// when HeyGen is absent.
|
||||
const explicitMode = bgmModeOverride || request.bgm?.mode || null;
|
||||
let mode = noBgm ? "none" : explicitMode || (heygenOK ? "retrieve" : "generate");
|
||||
if (mode === "retrieve" && !heygenOK) {
|
||||
anomalies.push(
|
||||
"bgm: retrieve requires a HeyGen credential — skipped (no generate fallback for an explicit retrieve)",
|
||||
);
|
||||
mode = "none";
|
||||
}
|
||||
|
||||
if (mode === "none") {
|
||||
console.error(`· bgm: disabled`);
|
||||
} else if (mode === "retrieve") {
|
||||
try {
|
||||
bgm = await retrieveBgm({ query: request.bgm?.query, headers, hyperframesDir, hasVoice });
|
||||
if (bgm) {
|
||||
bgmFields.bgm_provider = "heygen";
|
||||
bgmFields.bgm_mode = "retrieve";
|
||||
console.error(` bgm: ${bgm.path} (retrieve "${bgm.query}")`);
|
||||
} else {
|
||||
anomalies.push(`bgm: no music match for "${request.bgm?.query ?? ""}" — skipped`);
|
||||
}
|
||||
} catch (e) {
|
||||
anomalies.push(`bgm retrieve failed: ${e.message} — skipped`);
|
||||
}
|
||||
} else {
|
||||
// generate
|
||||
const prompt = inferBgmPrompt({
|
||||
userPrompt: request.bgm?.prompt,
|
||||
blob: request.bgm?.blob || request.bgm?.query,
|
||||
archetype: request.bgm?.archetype,
|
||||
arc: request.bgm?.arc,
|
||||
});
|
||||
const gen = generateBgmDetached({
|
||||
prompt,
|
||||
durationS: totalDuration || 30,
|
||||
hyperframesDir,
|
||||
lyriaRecipe: existsSync(lyriaRecipe) ? lyriaRecipe : null,
|
||||
seedSeconds,
|
||||
hasVoice,
|
||||
});
|
||||
if (gen.disabled) {
|
||||
anomalies.push(`bgm: ${gen.reason}`);
|
||||
} else {
|
||||
bgm = { path: gen.path, volume: gen.volume, mode: gen.mode, duration_s: null };
|
||||
bgmFields.bgm_pending = true;
|
||||
bgmFields.bgm_provider = gen.provider;
|
||||
bgmFields.bgm_pid = gen.pid;
|
||||
bgmFields.bgm_log = gen.log;
|
||||
bgmFields.bgm_mode = gen.mode;
|
||||
bgmFields.bgm_target_duration_s = gen.target_duration_s ?? null;
|
||||
bgmFields.bgm_seed_duration_s = gen.seed_duration_s ?? null;
|
||||
bgmFields.bgm_loop_count = gen.loop_count ?? null;
|
||||
console.error(` bgm: launched ${gen.provider} (detached, pid ${gen.pid}) → ${gen.path}`);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// ── SFX ─────────────────────────────────────────────────────────────────────
|
||||
let sfx = prev.sfx ?? [];
|
||||
if (only.has("sfx")) {
|
||||
const cues = lines.flatMap((l) =>
|
||||
(Array.isArray(l.sfx) ? l.sfx : [])
|
||||
.map((name) => ({ id: String(l.id), name: String(name).trim() }))
|
||||
.filter((c) => c.name),
|
||||
);
|
||||
const res = await resolveSfx({ cues, heygenOK, headers, hyperframesDir, sfxLibDir });
|
||||
sfx = res.sfx;
|
||||
anomalies.push(...res.anomalies);
|
||||
console.error(
|
||||
`· sfx: ${sfx.length} cue(s) resolved (${heygenOK ? "heygen retrieval" : "bundled library"})`,
|
||||
);
|
||||
}
|
||||
|
||||
// ── write audio_meta.json ─────────────────────────────────────────────────────
|
||||
const meta = {
|
||||
tts_provider: ttsProvider,
|
||||
voice_id: voiceId,
|
||||
bgm,
|
||||
...bgmFields,
|
||||
voices,
|
||||
sfx,
|
||||
total_duration_s: totalDuration,
|
||||
};
|
||||
mkdirSync(dirname(outPath), { recursive: true });
|
||||
writeFileSync(outPath, JSON.stringify(meta, null, 2));
|
||||
|
||||
console.log(`✓ audio engine → ${outPath}`);
|
||||
console.log(` heygen: ${heygenOK ? "yes" : "no"} · ran: ${[...only].join(",")}`);
|
||||
console.log(
|
||||
` voices: ${voices.length} · bgm: ${bgm ? `${bgmFields.bgm_provider}${bgmFields.bgm_pending ? " (pending)" : ""}` : "none"} · sfx: ${sfx.length}`,
|
||||
);
|
||||
console.log(` total voice duration: ${totalDuration}s`);
|
||||
if (anomalies.length) {
|
||||
console.log(`\nanomalies (non-fatal):`);
|
||||
for (const a of anomalies) console.log(` - ${a}`);
|
||||
}
|
||||
@@ -0,0 +1,49 @@
|
||||
import { strict as assert } from "node:assert";
|
||||
import { test } from "node:test";
|
||||
import { mkdtempSync, rmSync, existsSync } from "node:fs";
|
||||
import { join, dirname } from "node:path";
|
||||
import { tmpdir } from "node:os";
|
||||
import { fileURLToPath } from "node:url";
|
||||
import { resolveSfx } from "./lib/sfx.mjs";
|
||||
|
||||
// Proves the relocated engine (skills/media-use/audio/) still resolves its
|
||||
// bundled SFX library from the moved location — the path most likely to break
|
||||
// on a subtree move. Offline (heygenOK:false), no network.
|
||||
|
||||
const HERE = dirname(fileURLToPath(import.meta.url));
|
||||
const sfxLibDir = join(HERE, "..", "assets", "sfx"); // same offset the engine uses
|
||||
|
||||
test("bundled SFX library resolves from the relocated path", async () => {
|
||||
assert.ok(existsSync(join(sfxLibDir, "manifest.json")), "moved manifest is present");
|
||||
const dir = mkdtempSync(join(tmpdir(), "mu-audio-"));
|
||||
try {
|
||||
const { sfx, anomalies } = await resolveSfx({
|
||||
cues: [{ id: "1", name: "whoosh" }],
|
||||
heygenOK: false,
|
||||
hyperframesDir: dir,
|
||||
sfxLibDir,
|
||||
});
|
||||
assert.equal(sfx.length, 1, `expected 1 resolved cue, got anomalies: ${anomalies.join("; ")}`);
|
||||
assert.equal(sfx[0].source, "local");
|
||||
assert.match(sfx[0].file, /assets\/sfx\//);
|
||||
assert.ok(existsSync(join(dir, sfx[0].file)), "matched SFX copied into the project");
|
||||
} finally {
|
||||
rmSync(dir, { recursive: true, force: true });
|
||||
}
|
||||
});
|
||||
|
||||
test("an unknown cue is reported, not fatal", async () => {
|
||||
const dir = mkdtempSync(join(tmpdir(), "mu-audio-"));
|
||||
try {
|
||||
const { sfx, anomalies } = await resolveSfx({
|
||||
cues: [{ id: "1", name: "definitely-not-a-real-sfx" }],
|
||||
heygenOK: false,
|
||||
hyperframesDir: dir,
|
||||
sfxLibDir,
|
||||
});
|
||||
assert.equal(sfx.length, 0);
|
||||
assert.ok(anomalies.some((a) => /not in bundled library/.test(a)));
|
||||
} finally {
|
||||
rmSync(dir, { recursive: true, force: true });
|
||||
}
|
||||
});
|
||||
+121
@@ -0,0 +1,121 @@
|
||||
#!/usr/bin/env node
|
||||
// Self-contained HeyGen TTS — single text in → one wav (+ optional words JSON)
|
||||
// out. A thin CLI over lib/tts.mjs (the same code the audio engine uses), so the
|
||||
// HeyGen REST call, starfish voice pick, mp3→wav transcode, and word-timestamp
|
||||
// filtering live in exactly one place. Bypasses the `hyperframes` CLI, which in
|
||||
// the published build is Kokoro-only.
|
||||
//
|
||||
// Usage:
|
||||
// node heygen-tts.mjs "Text to speak" -o narration.wav [--words narration.words.json]
|
||||
// node heygen-tts.mjs ./script.txt -o narration.wav --words narration.words.json
|
||||
// node heygen-tts.mjs "Bonjour" -o fr.wav --lang fr --voice <id>
|
||||
// node heygen-tts.mjs --list # list starfish voices and exit
|
||||
//
|
||||
// Flags: -o/--output (.wav → ffmpeg transcode; .mp3 → raw bytes), --words,
|
||||
// --voice (starfish id), --speed, --lang, --list.
|
||||
// Requires: $HEYGEN_API_KEY (or ~/.heygen) and ffmpeg for .wav output.
|
||||
|
||||
import { existsSync, mkdirSync, readFileSync, writeFileSync } from "node:fs";
|
||||
import { dirname, resolve } from "node:path";
|
||||
import { heygenAuthHeaders, heygenJSON, loadEnvFromDir } from "./lib/heygen.mjs";
|
||||
import { ffprobeDuration, resolveVoiceId, synthesizeOne, withWordIds } from "./lib/tts.mjs";
|
||||
|
||||
const argv = process.argv.slice(2);
|
||||
function flag(name, def) {
|
||||
const i = argv.indexOf(`--${name}`);
|
||||
if (i < 0) return def;
|
||||
if (i + 1 >= argv.length) return true;
|
||||
const v = argv[i + 1];
|
||||
return v.startsWith("--") ? true : v;
|
||||
}
|
||||
const die = (m) => {
|
||||
console.error(`✗ heygen-tts: ${m}`);
|
||||
process.exit(1);
|
||||
};
|
||||
|
||||
// First arg that isn't a flag or the -o value is the text / .txt path.
|
||||
const positional = (() => {
|
||||
for (let i = 0; i < argv.length; i++) {
|
||||
const a = argv[i];
|
||||
if (a.startsWith("--")) {
|
||||
const next = argv[i + 1];
|
||||
if (next && !next.startsWith("--")) i++;
|
||||
continue;
|
||||
}
|
||||
if (a === "-o") {
|
||||
i++;
|
||||
continue;
|
||||
}
|
||||
return a;
|
||||
}
|
||||
return null;
|
||||
})();
|
||||
|
||||
const output = resolve(
|
||||
(typeof flag("output") === "string" && flag("output")) ||
|
||||
(argv.includes("-o") && argv[argv.indexOf("-o") + 1]) ||
|
||||
"narration.wav",
|
||||
);
|
||||
const wordsPath = typeof flag("words") === "string" ? resolve(flag("words")) : null;
|
||||
const userVoice = typeof flag("voice") === "string" ? flag("voice") : null;
|
||||
const speedRaw = typeof flag("speed") === "string" ? Number(flag("speed")) : 1.0;
|
||||
const speed = isFinite(speedRaw) && speedRaw > 0 ? speedRaw : 1.0;
|
||||
const lang = typeof flag("lang") === "string" ? flag("lang") : "en";
|
||||
const listOnly = flag("list") === true;
|
||||
|
||||
loadEnvFromDir(process.cwd());
|
||||
let authHeaders;
|
||||
try {
|
||||
authHeaders = heygenAuthHeaders();
|
||||
} catch (e) {
|
||||
die(e.message);
|
||||
}
|
||||
|
||||
// ---------- --list ----------
|
||||
if (listOnly) {
|
||||
const payload = await heygenJSON(`/voices?engine=starfish&type=public&limit=50`, {
|
||||
headers: authHeaders,
|
||||
});
|
||||
for (const v of payload.data ?? payload.voices ?? []) {
|
||||
console.log(`${v.voice_id}\t${v.name}\t${v.language ?? ""}`);
|
||||
}
|
||||
process.exit(0);
|
||||
}
|
||||
|
||||
// ---------- resolve text + voice ----------
|
||||
if (!positional) die("no text given. Pass a string or a .txt path, or use --list.");
|
||||
const text =
|
||||
positional.endsWith(".txt") && existsSync(resolve(positional))
|
||||
? readFileSync(resolve(positional), "utf8").trim()
|
||||
: positional;
|
||||
if (!text) die("input text is empty");
|
||||
|
||||
const voiceId = await resolveVoiceId({ provider: "heygen", userVoice, lang });
|
||||
if (!userVoice) console.error(`· using voice ${voiceId}`);
|
||||
|
||||
// ---------- synthesize (shared engine code) ----------
|
||||
const { ok, words } = await synthesizeOne({
|
||||
provider: "heygen",
|
||||
text,
|
||||
voiceId,
|
||||
lang,
|
||||
speed,
|
||||
wavAbs: output,
|
||||
hyperframesDir: process.cwd(),
|
||||
});
|
||||
if (!ok) die("synthesis failed (HeyGen request/transcode error)");
|
||||
|
||||
let wordCount = 0;
|
||||
if (wordsPath) {
|
||||
if (words && words.length) {
|
||||
mkdirSync(dirname(wordsPath), { recursive: true });
|
||||
writeFileSync(wordsPath, JSON.stringify(withWordIds(words), null, 2));
|
||||
wordCount = words.length;
|
||||
} else {
|
||||
console.error("⚠ no word_timestamps in response — run `hyperframes transcribe` instead");
|
||||
}
|
||||
}
|
||||
|
||||
const dur = ffprobeDuration(output);
|
||||
const durStr = isFinite(dur) ? ` (${dur.toFixed(2)}s)` : "";
|
||||
console.log(`✓ ${output}${durStr}${wordCount ? ` · ${wordsPath} (${wordCount} words)` : ""}`);
|
||||
@@ -0,0 +1,250 @@
|
||||
// bgm.mjs — background music for the media audio engine. Two routes, gated the
|
||||
// same way as TTS/SFX:
|
||||
//
|
||||
// retrieve (default when HeyGen is configured) — search HeyGen's music library
|
||||
// by mood, download the top track. Synchronous. assets/bgm/track.mp3.
|
||||
// generate (the alternative; the automatic choice when HeyGen is absent) —
|
||||
// Lyria (cloud, $GEMINI_API_KEY/$GOOGLE_API_KEY + google-genai) preferred,
|
||||
// else local MusicGen (facebook/musicgen-small via transformers). Spawned
|
||||
// DETACHED so the engine can return while audio renders; the caller marks
|
||||
// bgm_pending and runs wait-bgm.mjs before assembling. assets/bgm/track.wav.
|
||||
//
|
||||
// Missing/failed BGM never blocks a render.
|
||||
|
||||
import { spawn, spawnSync } from "node:child_process";
|
||||
import { existsSync, mkdirSync, openSync, closeSync } from "node:fs";
|
||||
import { join } from "node:path";
|
||||
import { downloadTo, searchSounds } from "./heygen.mjs";
|
||||
import { pythonInvocation } from "./python.mjs";
|
||||
|
||||
const r3 = (x) => Number(x.toFixed(3));
|
||||
const lyriaKey = () => process.env.GEMINI_API_KEY || process.env.GOOGLE_API_KEY || "";
|
||||
|
||||
const BGM_PY_DEPS = ["transformers", "torch", "soundfile", "numpy"];
|
||||
const BGM_PY_PROBE =
|
||||
"import transformers, soundfile, torch, numpy; from transformers import MusicgenForConditionalGeneration";
|
||||
const LYRIA_PY_DEPS = ["google-genai", "python-dotenv"];
|
||||
const LYRIA_PY_PROBE = "import google.genai";
|
||||
|
||||
function pyOk(probe) {
|
||||
const { cmd, args } = pythonInvocation(["-c", probe]);
|
||||
return spawnSync(cmd, args, { stdio: "ignore" }).status === 0;
|
||||
}
|
||||
// `python -m pip`, not a bare `pip` binary: a Homebrew/system Python often
|
||||
// exposes only `python3`/`pip3` on PATH, so a plain `pip` spawn silently
|
||||
// no-ops (ENOENT) and the documented "auto-installed on demand" path never
|
||||
// actually installs. `-m pip` also guarantees the packages land in the SAME
|
||||
// interpreter pyOk() probes — a bare `pip`/`pip3` could resolve to a
|
||||
// different Python installation than `python3` if more than one is on PATH.
|
||||
function pipInstall(deps) {
|
||||
const { cmd, args } = pythonInvocation(["-m", "pip", "install", "-q", ...deps]);
|
||||
return spawnSync(cmd, args, { stdio: "ignore" }).status === 0;
|
||||
}
|
||||
|
||||
// ── retrieval (HeyGen music library) ──────────────────────────────────────────
|
||||
export async function retrieveBgm({ query, headers, hyperframesDir, hasVoice }) {
|
||||
const q = query || "calm cinematic underscore";
|
||||
const results = await searchSounds(q, "music", headers, { limit: 5 });
|
||||
if (!results.length) return null;
|
||||
const top = results[0];
|
||||
const rel = "assets/bgm/track.mp3";
|
||||
await downloadTo(top.audio_url, join(hyperframesDir, rel));
|
||||
return {
|
||||
path: rel,
|
||||
volume: hasVoice ? 0.8 : 0.9,
|
||||
query: q,
|
||||
mode: "retrieve",
|
||||
duration_s: typeof top.duration === "number" ? r3(top.duration) : null,
|
||||
};
|
||||
}
|
||||
|
||||
// ── mood inference (for the generate path's prompt) ──────────────────────────
|
||||
// Industry base → archetype shape → emotional-arc tiebreaker. Exported so a
|
||||
// workflow adapter can build a rich prompt from its own narrative metadata; the
|
||||
// engine also calls it when generate has only a plain mood query.
|
||||
export function inferBgmPrompt({ blob = "", archetype = "", arc = "", userPrompt = "" } = {}) {
|
||||
if (userPrompt) return userPrompt;
|
||||
const b = String(blob).toLowerCase();
|
||||
let base;
|
||||
let bpm;
|
||||
if (/\b(crypto|nft|web3|defi|token|blockchain|exchange|wallet|dao)\b/.test(b)) {
|
||||
base = "atmospheric electronic, deep bass, futuristic synths, restrained percussion";
|
||||
bpm = 100;
|
||||
} else if (/\b(finance|fintech|bank|payment|invest|wealth|insurance|treasury)\b/.test(b)) {
|
||||
base = "calm cinematic, soft strings, subtle piano, restrained percussion";
|
||||
bpm = 92;
|
||||
} else if (/\b(creative|agency|design|studio|art|brand|marketing|content)\b/.test(b)) {
|
||||
base = "playful electronic, warm pads, light percussion";
|
||||
bpm = 115;
|
||||
} else {
|
||||
base = "uplifting corporate tech, bright modern piano with synth pads";
|
||||
bpm = 108;
|
||||
}
|
||||
const at = String(archetype).toLowerCase();
|
||||
const ar = String(arc).toLowerCase();
|
||||
if (/\bpas\b|pain.agitate|pain.+solve/.test(at))
|
||||
return `${base}, starts with subtle tension then builds to resolution, BPM ${bpm}, transitions from MINOR to MAJOR`;
|
||||
if (/\bbab\b|before.after|future.pac|vision/.test(at))
|
||||
return `${base}, cinematic and aspirational, steady build with rising energy, BPM ${bpm}, MAJOR`;
|
||||
if (/cascade|feature.benefit/.test(at))
|
||||
return `${base}, energetic and driving, consistent momentum, BPM ${Math.min(bpm + 10, 128)}, MAJOR`;
|
||||
if (/demo.loop|question.+answer/.test(at))
|
||||
return `${base}, clean and focused, minimal arrangement, BPM ${Math.max(bpm - 8, 88)}`;
|
||||
if (/frustrat|anxiety|overwhelm|tension/.test(ar) && /relief|excite|triumph/.test(ar))
|
||||
return `${base}, builds from understated tension to uplifting resolution, BPM ${bpm}, MINOR to MAJOR`;
|
||||
if (/excit|awe|power|triumph/.test(ar))
|
||||
return `${base}, energetic and confident, BPM ${bpm}, MAJOR`;
|
||||
if (/trust|ease|clarity|reassur/.test(ar))
|
||||
return `${base}, warm and reassuring, BPM ${Math.max(bpm - 5, 85)}`;
|
||||
return `${base}, BPM ${bpm}, MAJOR`;
|
||||
}
|
||||
|
||||
// ── generation (Lyria → MusicGen, detached) ──────────────────────────────────
|
||||
// Returns a bgmMeta the caller folds into audio_meta:
|
||||
// { path, mode, volume, provider, pid, log, target_duration_s, seed_duration_s,
|
||||
// loop_count, pending:true } on success, or { disabled:true, reason }.
|
||||
export function generateBgmDetached({
|
||||
prompt,
|
||||
durationS,
|
||||
hyperframesDir,
|
||||
lyriaRecipe,
|
||||
seedSeconds = 28,
|
||||
hasVoice,
|
||||
}) {
|
||||
const rel = "assets/bgm/track.wav";
|
||||
const abs = join(hyperframesDir, rel);
|
||||
mkdirSync(join(hyperframesDir, "assets", "bgm"), { recursive: true });
|
||||
const log = join(hyperframesDir, "assets", "bgm", `bgm-${Date.now()}.log`);
|
||||
const targetS = Math.max(1, durationS);
|
||||
const baseMeta = { path: rel, mode: null, volume: hasVoice ? 0.8 : 0.9, pending: true };
|
||||
|
||||
const lyriaConfigured = !!lyriaKey() && !!lyriaRecipe && existsSync(lyriaRecipe);
|
||||
|
||||
// Make a backend runnable: prefer Lyria when configured (install google-genai
|
||||
// on demand), else ensure local MusicGen deps. Installs are synchronous here —
|
||||
// generation itself is detached, so the engine still returns promptly.
|
||||
if (lyriaConfigured && !pyOk(LYRIA_PY_PROBE)) pipInstall(LYRIA_PY_DEPS);
|
||||
const useLyria = lyriaConfigured && pyOk(LYRIA_PY_PROBE);
|
||||
if (!useLyria && !pyOk(BGM_PY_PROBE)) pipInstall(BGM_PY_DEPS);
|
||||
|
||||
const fd = openSync(log, "w");
|
||||
if (useLyria) {
|
||||
const { cmd, args } = pythonInvocation([
|
||||
lyriaRecipe,
|
||||
"--output",
|
||||
abs,
|
||||
"--duration",
|
||||
String(targetS),
|
||||
"--prompt",
|
||||
prompt,
|
||||
]);
|
||||
const proc = spawn(cmd, args, { detached: true, stdio: ["ignore", fd, fd] });
|
||||
proc.unref();
|
||||
closeSync(fd);
|
||||
return {
|
||||
...baseMeta,
|
||||
mode: "detached-single",
|
||||
provider: "lyria",
|
||||
pid: proc.pid,
|
||||
log,
|
||||
target_duration_s: r3(targetS),
|
||||
};
|
||||
}
|
||||
|
||||
if (pyOk(BGM_PY_PROBE)) {
|
||||
const seedS = Math.min(Math.max(seedSeconds, 10), 30);
|
||||
const loops = targetS > seedS ? Math.ceil(targetS / seedS) : 1;
|
||||
const script = musicgenScript({ prompt, abs, targetS, seedS });
|
||||
const { cmd, args } = pythonInvocation(["-c", script]);
|
||||
const proc = spawn(cmd, args, { detached: true, stdio: ["ignore", fd, fd] });
|
||||
proc.unref();
|
||||
closeSync(fd);
|
||||
return {
|
||||
...baseMeta,
|
||||
mode: targetS > seedS ? "detached-seed-loop" : "detached-seed-trim",
|
||||
provider: "musicgen",
|
||||
pid: proc.pid,
|
||||
log,
|
||||
target_duration_s: r3(targetS),
|
||||
seed_duration_s: seedS,
|
||||
loop_count: loops,
|
||||
};
|
||||
}
|
||||
|
||||
closeSync(fd);
|
||||
return {
|
||||
disabled: true,
|
||||
reason: lyriaConfigured
|
||||
? `Lyria configured but google-genai uninstallable, and local MusicGen unavailable (pip install ${BGM_PY_DEPS.join(" ")})`
|
||||
: `no Lyria key/recipe and local MusicGen deps unavailable (pip install ${BGM_PY_DEPS.join(" ")})`,
|
||||
};
|
||||
}
|
||||
|
||||
// Inline MusicGen: generate ONE seed clip (≤30s to stay under the decoder's
|
||||
// positional limit), then trim it down or crossfade-loop it up to the target.
|
||||
function musicgenScript({ prompt, abs, targetS, seedS }) {
|
||||
return `
|
||||
import math, os, sys, traceback
|
||||
from pathlib import Path
|
||||
import numpy as np
|
||||
import soundfile as sf
|
||||
from transformers import MusicgenForConditionalGeneration, AutoProcessor
|
||||
|
||||
prompt = ${JSON.stringify(prompt)}
|
||||
out_path = ${JSON.stringify(abs)}
|
||||
target_s = float(${targetS.toFixed(3)})
|
||||
seed_s = float(${seedS.toFixed(3)})
|
||||
token_rate = 50
|
||||
crossfade_s = 0.3
|
||||
|
||||
def apply_fade(arr, sr, fade_in_s=0.08, fade_out_s=0.5):
|
||||
n_in = min(int(round(fade_in_s * sr)), arr.shape[0] // 2)
|
||||
n_out = min(int(round(fade_out_s * sr)), arr.shape[0] // 2)
|
||||
if n_in > 1: arr[:n_in] *= np.linspace(0.0, 1.0, n_in, dtype="float32")
|
||||
if n_out > 1: arr[-n_out:] *= np.linspace(1.0, 0.0, n_out, dtype="float32")
|
||||
return arr
|
||||
|
||||
def loop_crossfade(seed, target_len, xf):
|
||||
if seed.shape[0] >= target_len: return seed[:target_len]
|
||||
xf = min(xf, seed.shape[0] // 2)
|
||||
if xf < 1:
|
||||
reps = int(math.ceil(target_len / seed.shape[0]))
|
||||
return np.tile(seed, reps)[:target_len]
|
||||
t = np.linspace(0.0, 1.0, xf, dtype="float32")
|
||||
fade_out = np.cos(t * (math.pi / 2)); fade_in = np.sin(t * (math.pi / 2))
|
||||
out = seed.copy()
|
||||
while out.shape[0] < target_len:
|
||||
tail = out[-xf:] * fade_out; head = seed[:xf] * fade_in
|
||||
out = np.concatenate([out[:-xf], tail + head, seed[xf:]])
|
||||
return out[:target_len]
|
||||
|
||||
try:
|
||||
Path(os.path.dirname(out_path)).mkdir(parents=True, exist_ok=True)
|
||||
processor = AutoProcessor.from_pretrained("facebook/musicgen-small")
|
||||
model = MusicgenForConditionalGeneration.from_pretrained("facebook/musicgen-small")
|
||||
model.eval()
|
||||
sr = int(model.config.audio_encoder.sampling_rate)
|
||||
gen_s = min(seed_s, target_s)
|
||||
tokens = max(1, int(math.ceil(gen_s * token_rate)))
|
||||
print(f"[musicgen] seed dur={gen_s:.2f}s tokens={tokens}", flush=True)
|
||||
inputs = processor(text=[prompt], padding=True, return_tensors="pt")
|
||||
audio = model.generate(**inputs, max_new_tokens=tokens)
|
||||
seed = audio[0, 0].detach().cpu().numpy().astype("float32")
|
||||
peak = float(np.max(np.abs(seed)))
|
||||
if peak > 1e-6: seed = seed * (0.89 / peak)
|
||||
want = max(1, int(round(target_s * sr)))
|
||||
if seed.shape[0] >= want:
|
||||
final = seed[:want].copy()
|
||||
else:
|
||||
final = loop_crossfade(seed, want, int(round(crossfade_s * sr)))
|
||||
if final.shape[0] < want: final = np.pad(final, (0, want - final.shape[0]))
|
||||
else: final = final[:want]
|
||||
final = apply_fade(final, sr)
|
||||
peak = float(np.max(np.abs(final)))
|
||||
if peak > 1.0: final = final / peak
|
||||
sf.write(out_path, final, sr)
|
||||
print(f"[musicgen] wrote {out_path} samples={final.shape[0]} sr={sr}", flush=True)
|
||||
except Exception:
|
||||
traceback.print_exc(); sys.exit(1)
|
||||
`;
|
||||
}
|
||||
@@ -0,0 +1,14 @@
|
||||
// mapWithConcurrency — run `fn` over `items` with at most `limit` in flight at
|
||||
// once. Preserves input order in the result array regardless of completion order.
|
||||
export async function mapWithConcurrency(items, limit, fn) {
|
||||
const results = new Array(items.length);
|
||||
let next = 0;
|
||||
async function worker() {
|
||||
while (next < items.length) {
|
||||
const i = next++;
|
||||
results[i] = await fn(items[i], i);
|
||||
}
|
||||
}
|
||||
await Promise.all(Array.from({ length: Math.min(limit, items.length) }, worker));
|
||||
return results;
|
||||
}
|
||||
@@ -0,0 +1,41 @@
|
||||
import { test } from "node:test";
|
||||
import assert from "node:assert/strict";
|
||||
import { mapWithConcurrency } from "./concurrency.mjs";
|
||||
|
||||
// Regression: audio.mjs used a bare Promise.all(lines.map(synthLine)) to
|
||||
// synthesize every TTS line at once, spawning one Kokoro/whisper model load
|
||||
// per line concurrently. Two independent reports of this overwhelming a
|
||||
// machine (OOM, and cold-start contention causing spurious failures).
|
||||
// mapWithConcurrency is the extracted cap; test it in isolation since
|
||||
// audio.mjs itself is a script (runs CLI/exit side effects on import).
|
||||
test("processes every item and preserves input order regardless of completion order", async () => {
|
||||
const order = [5, 1, 3, 2, 4];
|
||||
const results = await mapWithConcurrency(order, 2, async (n) => {
|
||||
await new Promise((r) => setTimeout(r, n));
|
||||
return n * 10;
|
||||
});
|
||||
assert.deepEqual(results, [50, 10, 30, 20, 40]);
|
||||
});
|
||||
|
||||
test("never runs more than `limit` at once", async () => {
|
||||
let inFlight = 0;
|
||||
let maxInFlight = 0;
|
||||
const items = Array.from({ length: 10 }, (_, i) => i);
|
||||
await mapWithConcurrency(items, 3, async () => {
|
||||
inFlight++;
|
||||
maxInFlight = Math.max(maxInFlight, inFlight);
|
||||
await new Promise((r) => setTimeout(r, 5));
|
||||
inFlight--;
|
||||
});
|
||||
assert.equal(maxInFlight, 3);
|
||||
});
|
||||
|
||||
test("limit larger than the item count runs everything without hanging", async () => {
|
||||
const results = await mapWithConcurrency([1, 2], 10, async (n) => n * 2);
|
||||
assert.deepEqual(results, [2, 4]);
|
||||
});
|
||||
|
||||
test("empty input resolves to an empty array", async () => {
|
||||
const results = await mapWithConcurrency([], 4, async (n) => n);
|
||||
assert.deepEqual(results, []);
|
||||
});
|
||||
@@ -0,0 +1,130 @@
|
||||
// heygen.mjs — vendored HeyGen REST helpers (auth + transport) for the audio
|
||||
// pipeline. The credential resolver matches the hyperframes CLI auth: first
|
||||
// usable source wins — $HEYGEN_API_KEY / $HYPERFRAMES_API_KEY → a nearby .env → ~/.heygen/
|
||||
// credentials (oauth → Bearer, else api_key → X-Api-Key; $HEYGEN_CONFIG_DIR
|
||||
// overrides the dir). Vendored so the skill ships standalone. Pure node.
|
||||
|
||||
import { existsSync, mkdirSync, readFileSync, writeFileSync } from "node:fs";
|
||||
import { homedir } from "node:os";
|
||||
import { dirname, join, resolve } from "node:path";
|
||||
|
||||
export const HEYGEN_BASE = "https://api.heygen.com/v3";
|
||||
|
||||
// Walk up ≤5 dirs from startDir; load the first .env (shell env always wins).
|
||||
export function loadEnvFromDir(startDir) {
|
||||
let dir = resolve(startDir);
|
||||
for (let i = 0; i < 5; i++) {
|
||||
const envPath = join(dir, ".env");
|
||||
if (existsSync(envPath)) {
|
||||
for (const raw of readFileSync(envPath, "utf8").split("\n")) {
|
||||
let line = raw.trim();
|
||||
if (!line || line.startsWith("#")) continue;
|
||||
if (line.startsWith("export ")) line = line.slice(7).trim();
|
||||
const eq = line.indexOf("=");
|
||||
if (eq < 1) continue;
|
||||
const key = line.slice(0, eq).trim();
|
||||
let val = line.slice(eq + 1).trim();
|
||||
if (val.startsWith('"') || val.startsWith("'")) {
|
||||
const q = val[0];
|
||||
const end = val.indexOf(q, 1);
|
||||
val = end > 0 ? val.slice(1, end) : val.slice(1);
|
||||
}
|
||||
if (!(key in process.env)) process.env[key] = val;
|
||||
}
|
||||
return;
|
||||
}
|
||||
const parent = dirname(dir);
|
||||
if (parent === dir) break;
|
||||
dir = parent;
|
||||
}
|
||||
}
|
||||
|
||||
// → { headers } | { expired: true } | null. Never throws.
|
||||
export function heygenCredential() {
|
||||
const envKey = process.env.HEYGEN_API_KEY || process.env.HYPERFRAMES_API_KEY;
|
||||
if (envKey) return { headers: { "X-Api-Key": envKey } };
|
||||
|
||||
const file = join(process.env.HEYGEN_CONFIG_DIR || join(homedir(), ".heygen"), "credentials");
|
||||
if (!existsSync(file)) return null;
|
||||
const raw = readFileSync(file, "utf8").trim();
|
||||
if (!raw) return null;
|
||||
if (!raw.startsWith("{")) return { headers: { "X-Api-Key": raw } };
|
||||
|
||||
// A malformed credentials file (partial write / wrong shape) must degrade to
|
||||
// "no credential", not crash the engine at startup — this function never throws.
|
||||
let cred;
|
||||
try {
|
||||
cred = JSON.parse(raw);
|
||||
} catch {
|
||||
return null;
|
||||
}
|
||||
const oauth = cred.oauth;
|
||||
if (oauth?.access_token) {
|
||||
const expired = oauth.expires_at && new Date(oauth.expires_at).getTime() - 60_000 < Date.now();
|
||||
if (!expired) return { headers: { Authorization: `Bearer ${oauth.access_token}` } };
|
||||
if (!cred.api_key) return { expired: true };
|
||||
}
|
||||
if (cred.api_key) return { headers: { "X-Api-Key": cred.api_key } };
|
||||
return null;
|
||||
}
|
||||
|
||||
// → auth headers object, or throw with a fix hint.
|
||||
export function heygenAuthHeaders() {
|
||||
const cred = heygenCredential();
|
||||
if (cred?.headers) return cred.headers;
|
||||
if (cred?.expired)
|
||||
throw new Error(
|
||||
"HeyGen OAuth token expired — run `npx hyperframes auth refresh` (or `npx hyperframes auth login`)",
|
||||
);
|
||||
throw new Error(
|
||||
"no HeyGen credentials — set $HEYGEN_API_KEY, or run `npx hyperframes auth login` (writes ~/.heygen/credentials)",
|
||||
);
|
||||
}
|
||||
|
||||
// Authed JSON request against the v3 API; throws on a non-OK status.
|
||||
export async function heygenJSON(path, { method = "GET", headers = {}, body } = {}) {
|
||||
const opts = { method, headers: { ...headers } };
|
||||
if (body !== undefined) {
|
||||
opts.headers["Content-Type"] = "application/json";
|
||||
opts.body = JSON.stringify(body);
|
||||
}
|
||||
const res = await fetch(`${HEYGEN_BASE}${path}`, opts);
|
||||
if (!res.ok) {
|
||||
const detail = await res.text().catch(() => "");
|
||||
throw new Error(
|
||||
`HeyGen ${method} ${path} → HTTP ${res.status}${detail ? `\n${detail.slice(0, 300)}` : ""}`,
|
||||
);
|
||||
}
|
||||
return res.json();
|
||||
}
|
||||
|
||||
// Download a (presigned) URL to destPath; returns byte length.
|
||||
export async function downloadTo(url, destPath) {
|
||||
const res = await fetch(url);
|
||||
if (!res.ok) throw new Error(`download HTTP ${res.status}: ${String(url).slice(0, 80)}`);
|
||||
const bytes = Buffer.from(await res.arrayBuffer());
|
||||
mkdirSync(dirname(destPath), { recursive: true });
|
||||
writeFileSync(destPath, bytes);
|
||||
return bytes.length;
|
||||
}
|
||||
|
||||
// Retrieval search over HeyGen's audio catalog (NOT generation). type =
|
||||
// "music" | "sound_effects". Returns the ranked results array (best first); each
|
||||
// item has a presigned `audio_url` (+ `duration`, `description`, `name`, `score`).
|
||||
// `query` is required (≥1 char, empty → HTTP 400) and `limit` is capped at 50.
|
||||
// `minScore`: omit to use the server default (0.7). That default is TOO HIGH for
|
||||
// sound_effects — good SFX hits score ~0.5–0.67, so callers wanting SFX should
|
||||
// pass a lower floor (~0.4); music scores high and is fine at the default.
|
||||
export async function searchSounds(query, type, headers, { limit = 5, minScore } = {}) {
|
||||
const params = new URLSearchParams({ query, type, limit: String(limit) });
|
||||
if (minScore != null) params.set("min_score", String(minScore));
|
||||
const payload = await heygenJSON(`/audio/sounds?${params.toString()}`, { headers });
|
||||
// `data` comes back as a ranked array (best first). Older responses keyed it by
|
||||
// numeric index ("0","1",…); normalize both shapes to an array (empty → []).
|
||||
const data = payload?.data ?? payload;
|
||||
if (Array.isArray(data)) return data;
|
||||
if (data && typeof data === "object") return Object.values(data);
|
||||
throw new Error(
|
||||
`unexpected /audio/sounds shape — top keys: ${Object.keys(payload ?? {}).join(", ")}`,
|
||||
);
|
||||
}
|
||||
@@ -0,0 +1,63 @@
|
||||
// python.mjs — resolve which Python 3 executable to spawn, per platform.
|
||||
//
|
||||
// The audio engine (tts.mjs, bgm.mjs) shells out to `python3` for ElevenLabs
|
||||
// TTS and the local Lyria/MusicGen BGM paths. `python3` is the right name on
|
||||
// macOS/Linux, but on Windows the python.org installer only creates
|
||||
// `python.exe` plus the `py` launcher — there is no `python3.exe` (only the
|
||||
// Microsoft Store build adds one). So a bare `spawn("python3", …)` ENOENTs on a
|
||||
// standard Windows Python install, silently disabling every Python-backed audio
|
||||
// feature until the user hand-creates a `python3.exe` shim (reported twice).
|
||||
//
|
||||
// Resolve once, per process: probe the platform's candidates in order and take
|
||||
// the first that actually runs. `py` is the launcher, so it needs a `-3` arg to
|
||||
// select Python 3 — hence candidates are argv PREFIXES, not bare names.
|
||||
|
||||
import { spawnSync } from "node:child_process";
|
||||
|
||||
function defaultProbe(cmd, args) {
|
||||
try {
|
||||
return spawnSync(cmd, args, { stdio: "ignore" }).status === 0;
|
||||
} catch {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Pick the argv prefix that launches Python 3 on this platform.
|
||||
* Returns e.g. `["python3"]`, `["python"]`, or `["py", "-3"]`.
|
||||
*
|
||||
* Pure except for `probe` (which runs `<cmd> … --version`); both `platform`
|
||||
* and `probe` are injectable so every branch is unit-testable without spawning.
|
||||
* If nothing probes OK, falls back to the canonical name for the platform so
|
||||
* the eventual spawn fails loudly exactly as it did before — never worse.
|
||||
*/
|
||||
export function resolvePythonCommand(platform = process.platform, probe = defaultProbe) {
|
||||
const candidates =
|
||||
platform === "win32" ? [["python3"], ["python"], ["py", "-3"]] : [["python3"], ["python"]];
|
||||
for (const prefix of candidates) {
|
||||
if (probe(prefix[0], [...prefix.slice(1), "--version"])) return prefix;
|
||||
}
|
||||
return candidates[0];
|
||||
}
|
||||
|
||||
let cached = null;
|
||||
|
||||
/** Cached `resolvePythonCommand()` — probing spawns, so resolve at most once. */
|
||||
export function pythonCommand() {
|
||||
if (!cached) cached = resolvePythonCommand();
|
||||
return cached;
|
||||
}
|
||||
|
||||
/**
|
||||
* Build a `{ cmd, args }` for running Python 3 with `extraArgs`, using the
|
||||
* resolved (or supplied) prefix. Keeps the launcher's `-3` (and any future
|
||||
* prefix args) ahead of the caller's own arguments.
|
||||
*/
|
||||
export function pythonInvocation(extraArgs, prefix = pythonCommand()) {
|
||||
return { cmd: prefix[0], args: [...prefix.slice(1), ...extraArgs] };
|
||||
}
|
||||
|
||||
/** Test-only: clear the cached resolution so a test can re-probe. */
|
||||
export function _resetPythonCommandCacheForTests() {
|
||||
cached = null;
|
||||
}
|
||||
@@ -0,0 +1,68 @@
|
||||
import { test } from "node:test";
|
||||
import assert from "node:assert/strict";
|
||||
import { resolvePythonCommand, pythonInvocation } from "./python.mjs";
|
||||
|
||||
// Regression: on Windows a standard python.org install has no `python3.exe`
|
||||
// (only `python.exe` + the `py` launcher), so `spawn("python3", …)` ENOENTs and
|
||||
// every Python-backed audio feature silently no-ops. resolvePythonCommand takes
|
||||
// injectable platform/probe params so all branches are testable without
|
||||
// spawning a real interpreter.
|
||||
|
||||
// probeFor(names): a probe that reports success only for the given argv-0 names.
|
||||
function probeFor(...names) {
|
||||
const ok = new Set(names);
|
||||
return (cmd) => ok.has(cmd);
|
||||
}
|
||||
|
||||
test("non-win32 uses python3 when it runs", () => {
|
||||
assert.deepEqual(resolvePythonCommand("linux", probeFor("python3")), ["python3"]);
|
||||
assert.deepEqual(resolvePythonCommand("darwin", probeFor("python3")), ["python3"]);
|
||||
});
|
||||
|
||||
test("win32 prefers python3 when the Microsoft Store build provides it", () => {
|
||||
assert.deepEqual(resolvePythonCommand("win32", probeFor("python3", "python", "py")), ["python3"]);
|
||||
});
|
||||
|
||||
test("win32 falls back to python.exe when python3 is absent (python.org install)", () => {
|
||||
// The exact reported scenario: no python3, but `python` exists.
|
||||
assert.deepEqual(resolvePythonCommand("win32", probeFor("python", "py")), ["python"]);
|
||||
});
|
||||
|
||||
test("win32 falls back to the py launcher with -3 when only py exists", () => {
|
||||
assert.deepEqual(resolvePythonCommand("win32", probeFor("py")), ["py", "-3"]);
|
||||
});
|
||||
|
||||
test("py launcher is probed as `py -3 --version`, not bare `py`", () => {
|
||||
const seen = [];
|
||||
const probe = (cmd, args) => {
|
||||
seen.push([cmd, ...args]);
|
||||
return cmd === "py";
|
||||
};
|
||||
resolvePythonCommand("win32", probe);
|
||||
assert.deepEqual(seen.at(-1), ["py", "-3", "--version"]);
|
||||
});
|
||||
|
||||
test("falls back to the canonical name (loud failure, unchanged) when nothing runs", () => {
|
||||
// No interpreter anywhere — must not throw, and must return python3 so the
|
||||
// eventual spawn fails exactly as it did before this fix, never worse.
|
||||
assert.deepEqual(
|
||||
resolvePythonCommand("win32", () => false),
|
||||
["python3"],
|
||||
);
|
||||
assert.deepEqual(
|
||||
resolvePythonCommand("linux", () => false),
|
||||
["python3"],
|
||||
);
|
||||
});
|
||||
|
||||
test("pythonInvocation prepends the resolved prefix ahead of caller args", () => {
|
||||
assert.deepEqual(pythonInvocation(["-c", "import x"], ["python"]), {
|
||||
cmd: "python",
|
||||
args: ["-c", "import x"],
|
||||
});
|
||||
// The py launcher's -3 must stay ahead of the caller's own arguments.
|
||||
assert.deepEqual(pythonInvocation(["-c", "import x"], ["py", "-3"]), {
|
||||
cmd: "py",
|
||||
args: ["-3", "-c", "import x"],
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,143 @@
|
||||
// sfx.mjs — sound effects for the media audio engine. Provider-gated (NOT a
|
||||
// per-cue merge): the decision is made once, by whether HeyGen is configured —
|
||||
// mirroring how TTS and BGM degrade.
|
||||
//
|
||||
// HeyGen credential present → retrieve EVERY cue from HeyGen's audio library
|
||||
// (/v3/audio/sounds, type=sound_effects, min_score=0.4). The bundled
|
||||
// library is NOT consulted.
|
||||
// HeyGen credential absent → resolve cues against the bundled 21-file
|
||||
// library (assets/sfx/manifest.json), copying matched files into the
|
||||
// project. Offline, deterministic, free.
|
||||
//
|
||||
// A cue that matches nothing is skipped (recorded as an anomaly); SFX never
|
||||
// blocks a render. Every cue sits at volume ~0.35, under voice + BGM.
|
||||
|
||||
import { copyFileSync, existsSync, mkdirSync, readFileSync } from "node:fs";
|
||||
import { join } from "node:path";
|
||||
import { downloadTo, searchSounds } from "./heygen.mjs";
|
||||
|
||||
const SFX_VOLUME = 0.35;
|
||||
const slug = (s) =>
|
||||
s
|
||||
.toLowerCase()
|
||||
.replace(/[^a-z0-9]+/g, "-")
|
||||
.replace(/^-+|-+$/g, "")
|
||||
.slice(0, 40) || "x";
|
||||
const r3 = (x) => Number(x.toFixed(3));
|
||||
|
||||
// cues: [{ id, name }] (id = the line/frame/scene the cue fires in). Returns
|
||||
// { sfx: [{ id, name, file, source, offset_s, duration_s, volume }], anomalies }.
|
||||
export async function resolveSfx({ cues, heygenOK, headers, hyperframesDir, sfxLibDir }) {
|
||||
const sfx = [];
|
||||
const anomalies = [];
|
||||
const destDir = join(hyperframesDir, "assets", "sfx");
|
||||
|
||||
// Dedupe identical (id,name) cues — the same effect named twice in one line
|
||||
// downloads/copies once.
|
||||
const seen = new Set();
|
||||
const uniq = cues.filter((c) => {
|
||||
const k = `${c.id}:${c.name}`;
|
||||
if (seen.has(k)) return false;
|
||||
seen.add(k);
|
||||
return true;
|
||||
});
|
||||
|
||||
if (heygenOK) {
|
||||
for (const { id, name } of uniq) {
|
||||
try {
|
||||
// SFX hits score low (~0.5–0.67), below the API's default 0.7 which
|
||||
// silently drops most named cues — floor to 0.4. (BGM/music score high
|
||||
// and keep the default.)
|
||||
const results = await searchSounds(name, "sound_effects", headers, {
|
||||
limit: 3,
|
||||
minScore: 0.4,
|
||||
});
|
||||
if (!results.length) {
|
||||
anomalies.push(`sfx "${name}" (id ${id}): no HeyGen match — skipped`);
|
||||
continue;
|
||||
}
|
||||
const top = results[0];
|
||||
const file = `assets/sfx/${slug(name)}.mp3`;
|
||||
await downloadTo(top.audio_url, join(hyperframesDir, file));
|
||||
sfx.push({
|
||||
id,
|
||||
name,
|
||||
file,
|
||||
source: "heygen",
|
||||
offset_s: 0,
|
||||
duration_s: typeof top.duration === "number" ? r3(top.duration) : 1.0,
|
||||
volume: SFX_VOLUME,
|
||||
});
|
||||
} catch (e) {
|
||||
anomalies.push(`sfx "${name}" (id ${id}): retrieval failed — ${e.message}`);
|
||||
}
|
||||
}
|
||||
return { sfx, anomalies };
|
||||
}
|
||||
|
||||
// ── offline: bundled library ──
|
||||
const manifestPath = join(sfxLibDir, "manifest.json");
|
||||
if (!existsSync(manifestPath)) {
|
||||
if (uniq.length)
|
||||
anomalies.push(`no HeyGen credential and no SFX library at ${sfxLibDir} — all cues dropped`);
|
||||
return { sfx, anomalies };
|
||||
}
|
||||
let manifest;
|
||||
try {
|
||||
manifest = JSON.parse(readFileSync(manifestPath, "utf8"));
|
||||
} catch (e) {
|
||||
anomalies.push(`SFX manifest parse failed (${e.message}) — all cues dropped`);
|
||||
return { sfx, anomalies };
|
||||
}
|
||||
// Build lookups: by manifest key, by file basename, and by slug of either, so
|
||||
// a cue can name "whoosh", "whoosh.mp3", or "ui click" (→ slug match).
|
||||
const byKey = new Map();
|
||||
for (const [key, entry] of Object.entries(manifest)) {
|
||||
if (!entry?.file || !isFinite(entry.duration)) continue;
|
||||
const rec = { key, file: entry.file, duration: entry.duration };
|
||||
byKey.set(key, rec);
|
||||
byKey.set(entry.file, rec);
|
||||
byKey.set(slug(key), rec);
|
||||
byKey.set(slug(entry.file.replace(/\.\w+$/, "")), rec);
|
||||
}
|
||||
mkdirSync(destDir, { recursive: true });
|
||||
for (const { id, name } of uniq) {
|
||||
const hit = byKey.get(name) ?? byKey.get(slug(name));
|
||||
if (!hit) {
|
||||
const known = [...new Set([...byKey.values()].map((v) => v.key))].slice(0, 8).join(", ");
|
||||
anomalies.push(
|
||||
`sfx "${name}" (id ${id}): not in bundled library — skipped (have: ${known}…)`,
|
||||
);
|
||||
continue;
|
||||
}
|
||||
const src = join(sfxLibDir, hit.file);
|
||||
const destRel = `assets/sfx/${hit.file}`;
|
||||
const dest = join(hyperframesDir, destRel);
|
||||
// The bundled library may be incomplete: some installs of the skill ship
|
||||
// manifest.json without the actual mp3s. Pushing an sfx entry that points at
|
||||
// a file we never copied produces a dangling reference that silently drops
|
||||
// downstream ("not on disk"). Surface it as a loud anomaly and skip the cue
|
||||
// instead, so the audio_meta never references a missing file.
|
||||
if (!existsSync(dest)) {
|
||||
if (!existsSync(src)) {
|
||||
anomalies.push(
|
||||
`sfx "${name}" (id ${id}): bundled file ${hit.file} missing from the offline ` +
|
||||
`library (${sfxLibDir}) — skipped. Reinstall the media-use skill to ` +
|
||||
`restore assets/sfx/*.mp3, or configure a HeyGen credential for retrieval.`,
|
||||
);
|
||||
continue;
|
||||
}
|
||||
copyFileSync(src, dest);
|
||||
}
|
||||
sfx.push({
|
||||
id,
|
||||
name,
|
||||
file: destRel,
|
||||
source: "local",
|
||||
offset_s: 0,
|
||||
duration_s: r3(hit.duration),
|
||||
volume: SFX_VOLUME,
|
||||
});
|
||||
}
|
||||
return { sfx, anomalies };
|
||||
}
|
||||
@@ -0,0 +1,69 @@
|
||||
import { test } from "node:test";
|
||||
import assert from "node:assert/strict";
|
||||
import { mkdtempSync, mkdirSync, writeFileSync, existsSync, rmSync } from "node:fs";
|
||||
import { join } from "node:path";
|
||||
import { tmpdir } from "node:os";
|
||||
import { resolveSfx } from "./sfx.mjs";
|
||||
|
||||
// Offline (no HeyGen) SFX resolution: the bundled library may ship manifest.json
|
||||
// without the actual mp3s. The old code copied only when the source existed but
|
||||
// pushed the sfx entry unconditionally — producing a dangling reference that
|
||||
// silently dropped downstream ("not on disk"). These tests lock in the loud
|
||||
// behavior: a present file is copied + referenced; a missing file yields an
|
||||
// anomaly and NO dangling entry.
|
||||
|
||||
async function withDirs(fn) {
|
||||
const root = mkdtempSync(join(tmpdir(), "hf-sfx-"));
|
||||
const libDir = join(root, "lib");
|
||||
const projDir = join(root, "proj");
|
||||
mkdirSync(libDir, { recursive: true });
|
||||
mkdirSync(projDir, { recursive: true });
|
||||
try {
|
||||
// `await` is load-bearing: without it the finally cleanup runs before the
|
||||
// async test body resolves, deleting the temp dir mid-assertion.
|
||||
return await fn({ libDir, projDir });
|
||||
} finally {
|
||||
rmSync(root, { recursive: true, force: true });
|
||||
}
|
||||
}
|
||||
|
||||
test("offline: copies and references a present bundled file", async () => {
|
||||
await withDirs(async ({ libDir, projDir }) => {
|
||||
writeFileSync(
|
||||
join(libDir, "manifest.json"),
|
||||
JSON.stringify({ whoosh: { file: "whoosh.mp3", duration: 0.8 } }),
|
||||
);
|
||||
writeFileSync(join(libDir, "whoosh.mp3"), "ID3-fake-bytes");
|
||||
const { sfx, anomalies } = await resolveSfx({
|
||||
cues: [{ id: "s1", name: "whoosh" }],
|
||||
heygenOK: false,
|
||||
hyperframesDir: projDir,
|
||||
sfxLibDir: libDir,
|
||||
});
|
||||
assert.equal(sfx.length, 1);
|
||||
assert.equal(sfx[0].file, "assets/sfx/whoosh.mp3");
|
||||
assert.equal(sfx[0].source, "local");
|
||||
assert.ok(existsSync(join(projDir, "assets/sfx/whoosh.mp3")), "mp3 copied into project");
|
||||
assert.equal(anomalies.length, 0);
|
||||
});
|
||||
});
|
||||
|
||||
test("offline: a matched-but-missing bundled file yields an anomaly and NO dangling entry", async () => {
|
||||
await withDirs(async ({ libDir, projDir }) => {
|
||||
// Manifest names whoosh.mp3, but the mp3 was never shipped (the reported bug).
|
||||
writeFileSync(
|
||||
join(libDir, "manifest.json"),
|
||||
JSON.stringify({ whoosh: { file: "whoosh.mp3", duration: 0.8 } }),
|
||||
);
|
||||
const { sfx, anomalies } = await resolveSfx({
|
||||
cues: [{ id: "s1", name: "whoosh" }],
|
||||
heygenOK: false,
|
||||
hyperframesDir: projDir,
|
||||
sfxLibDir: libDir,
|
||||
});
|
||||
assert.equal(sfx.length, 0, "no dangling entry for a file that was never copied");
|
||||
assert.equal(anomalies.length, 1);
|
||||
assert.match(anomalies[0], /missing from the offline library/);
|
||||
assert.ok(!existsSync(join(projDir, "assets/sfx/whoosh.mp3")), "nothing copied");
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,326 @@
|
||||
// tts.mjs — multi-provider TTS for the media audio engine. The provider chain,
|
||||
// auto-detected from env, is the one documented in ../SKILL.md:
|
||||
//
|
||||
// 1. HeyGen (Starfish) — $HEYGEN_API_KEY / $HYPERFRAMES_API_KEY / ~/.heygen.
|
||||
// Direct v3 REST (NOT `hyperframes tts`, which in the published build is
|
||||
// Kokoro-only and silently ignores a HeyGen key). Returns word_timestamps
|
||||
// in the same call, so no separate transcribe pass.
|
||||
// 2. ElevenLabs — $ELEVENLABS_API_KEY + `pip install elevenlabs`. No
|
||||
// word timings → caller chains transcribeWav().
|
||||
// 3. Kokoro-82M (local) — always available, via the published `hyperframes tts`
|
||||
// CLI. No word timings → caller chains transcribeWav().
|
||||
//
|
||||
// "HeyGen available" is decided by CREDENTIAL presence (heygenCredential), never
|
||||
// by the CLI — see the note above.
|
||||
|
||||
import { spawn, spawnSync } from "node:child_process";
|
||||
import { existsSync, mkdirSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from "node:fs";
|
||||
import { tmpdir } from "node:os";
|
||||
import { dirname, join } from "node:path";
|
||||
import { heygenAuthHeaders, heygenCredential, heygenJSON } from "./heygen.mjs";
|
||||
import { pythonInvocation } from "./python.mjs";
|
||||
|
||||
// ── provider detection ────────────────────────────────────────────────────────
|
||||
export function heygenAvailable() {
|
||||
return heygenCredential() !== null;
|
||||
}
|
||||
export function elevenlabsAvailable() {
|
||||
if (!process.env.ELEVENLABS_API_KEY) return false;
|
||||
const { cmd, args } = pythonInvocation(["-c", "import elevenlabs"]);
|
||||
const r = spawnSync(cmd, args, {
|
||||
stdio: "ignore",
|
||||
});
|
||||
return r.status === 0;
|
||||
}
|
||||
|
||||
// First available provider wins; an explicit choice is honored (and validated).
|
||||
export function pickProvider(userProvider) {
|
||||
if (userProvider) {
|
||||
if (!["heygen", "elevenlabs", "kokoro"].includes(userProvider))
|
||||
throw new Error(`invalid provider "${userProvider}" (heygen | elevenlabs | kokoro)`);
|
||||
if (userProvider === "heygen" && !heygenAvailable())
|
||||
throw new Error(
|
||||
"provider=heygen but no HeyGen credentials (set $HEYGEN_API_KEY or run `npx hyperframes auth login`)",
|
||||
);
|
||||
if (userProvider === "elevenlabs" && !process.env.ELEVENLABS_API_KEY)
|
||||
throw new Error("provider=elevenlabs but $ELEVENLABS_API_KEY is not set");
|
||||
return userProvider;
|
||||
}
|
||||
return heygenAvailable() ? "heygen" : elevenlabsAvailable() ? "elevenlabs" : "kokoro";
|
||||
}
|
||||
|
||||
// ── voice resolution ──────────────────────────────────────────────────────────
|
||||
// HeyGen /v3/voices/speech only accepts STARFISH voice_ids; auto-pick the first
|
||||
// English public starfish voice when none is pinned. ElevenLabs/Kokoro have
|
||||
// their own defaults.
|
||||
export async function resolveVoiceId({ provider, userVoice, lang = "en" }) {
|
||||
if (userVoice) return userVoice;
|
||||
if (provider === "elevenlabs") return "21m00Tcm4TlvDq8ikWAM"; // Rachel
|
||||
if (provider === "kokoro") {
|
||||
if (lang === "en") return "am_michael";
|
||||
throw new Error("Kokoro non-English needs an explicit --voice (see references/tts.md)");
|
||||
}
|
||||
// heygen — pin a fixed English default so the choice is deterministic. The old
|
||||
// "first English voice the API returns" drifts whenever HeyGen re-sorts the
|
||||
// public catalog. Marcia (mature, low female). Override with --voice / request.voice.
|
||||
if (lang === "en") return "05f19352e8f74b0392a8f411eba40de1"; // Marcia · English · female
|
||||
// Non-English: no fixed default — fall back to the first matching catalog voice.
|
||||
const payload = await heygenJSON(`/voices?engine=starfish&type=public&limit=50`, {
|
||||
headers: heygenAuthHeaders(),
|
||||
});
|
||||
const voices = payload.data ?? payload.voices ?? [];
|
||||
const pick = voices.find((v) => v.language === "English") ?? voices[0];
|
||||
if (!pick) throw new Error("no public starfish voice to default to — pass --voice");
|
||||
return pick.voice_id;
|
||||
}
|
||||
|
||||
// ── helpers ─────────────────────────────────────────────────────────────────
|
||||
export function withWordIds(words) {
|
||||
return (words ?? []).map((w, i) => ({
|
||||
id: `w${i}`,
|
||||
text: w.text,
|
||||
start: w.start,
|
||||
end: w.end,
|
||||
}));
|
||||
}
|
||||
|
||||
// `ffmpeg -i <file>` prints a `Duration: HH:MM:SS.ms` line to stderr even
|
||||
// though it exits non-zero with no output requested. Parsing pulled out as
|
||||
// a pure function so the ENOENT fallback below can be tested without
|
||||
// depending on whether ffprobe/ffmpeg are actually installed on the
|
||||
// machine running the tests.
|
||||
export function parseFfmpegDurationBanner(stderrText) {
|
||||
const match = /Duration:\s*(\d+):(\d+):(\d+(?:\.\d+)?)/.exec(stderrText ?? "");
|
||||
if (!match) return NaN;
|
||||
const [, hours, minutes, seconds] = match;
|
||||
return Number(hours) * 3600 + Number(minutes) * 60 + Number(seconds);
|
||||
}
|
||||
|
||||
// Some "essentials"-style ffmpeg distributions (common on Windows) ship
|
||||
// ffmpeg.exe without ffprobe.exe. ffprobeDuration's caller (audio.mjs)
|
||||
// otherwise reads a spurious NaN as "the WAV file is corrupt" and drops an
|
||||
// already-successfully-synthesized TTS line, rather than "the tool for
|
||||
// measuring it is missing".
|
||||
function ffmpegDurationFallback(absPath) {
|
||||
const r = spawnSync("ffmpeg", ["-i", absPath], { encoding: "utf8" });
|
||||
return parseFfmpegDurationBanner(r.stderr);
|
||||
}
|
||||
|
||||
export function ffprobeDuration(absPath) {
|
||||
const r = spawnSync(
|
||||
"ffprobe",
|
||||
["-v", "error", "-show_entries", "format=duration", "-of", "default=nw=1:nk=1", absPath],
|
||||
{ encoding: "utf8" },
|
||||
);
|
||||
if (r.error?.code === "ENOENT") return ffmpegDurationFallback(absPath);
|
||||
if (r.status !== 0) return NaN;
|
||||
return parseFloat(String(r.stdout).trim());
|
||||
}
|
||||
|
||||
export function resolveNpxCliFromNpmExecPath(
|
||||
npmExecPath = process.env.npm_execpath,
|
||||
pathExists = existsSync,
|
||||
) {
|
||||
if (!npmExecPath) return null;
|
||||
const fileName = npmExecPath.replace(/\\/g, "/").split("/").pop()?.toLowerCase();
|
||||
const npxCliPath =
|
||||
fileName === "npx-cli.js" ? npmExecPath : join(dirname(npmExecPath), "npx-cli.js");
|
||||
return pathExists(npxCliPath) ? npxCliPath : null;
|
||||
}
|
||||
|
||||
export function resolveSpawnCommand(
|
||||
cmd,
|
||||
args,
|
||||
opts = {},
|
||||
platform = process.platform,
|
||||
env = process.env,
|
||||
pathExists = existsSync,
|
||||
) {
|
||||
if (cmd !== "npx" || platform !== "win32") {
|
||||
return { cmd, args, opts: { stdio: "ignore", ...opts } };
|
||||
}
|
||||
|
||||
// On Windows, npx resolves to npx.cmd, which Node cannot execute directly.
|
||||
// Avoid `shell:true` and the .cmd shim entirely by invoking npm's JS CLI with
|
||||
// node, preserving request-provided values as argv data instead of shell text.
|
||||
const npxCliPath = resolveNpxCliFromNpmExecPath(env.npm_execpath, pathExists);
|
||||
if (!npxCliPath) return null;
|
||||
return {
|
||||
cmd: env.npm_node_execpath || process.execPath,
|
||||
args: [npxCliPath, ...args.map((arg) => String(arg))],
|
||||
opts: { stdio: "ignore", windowsHide: true, ...opts },
|
||||
};
|
||||
}
|
||||
|
||||
// `platform`/`spawnFn` params (default process.platform / the real spawn)
|
||||
// exist so tests can exercise the win32 branch without mocking node:child_process
|
||||
// (its ESM exports are non-configurable, so mock.method can't patch it).
|
||||
// One-shot so a whole batch of TTS lines doesn't repeat the same diagnostic.
|
||||
let _warnedNpxResolution = false;
|
||||
/** Test-only: reset the one-shot npx-resolution warning latch. */
|
||||
export function _resetNpxResolutionWarnForTests() {
|
||||
_warnedNpxResolution = false;
|
||||
}
|
||||
|
||||
export function spawnP(
|
||||
cmd,
|
||||
args,
|
||||
opts = {},
|
||||
platform = process.platform,
|
||||
spawnFn = spawn,
|
||||
env = process.env,
|
||||
pathExists = existsSync,
|
||||
) {
|
||||
const resolved = resolveSpawnCommand(cmd, args, opts, platform, env, pathExists);
|
||||
if (!resolved) {
|
||||
// resolveSpawnCommand only returns null for the npx-on-win32 case where
|
||||
// npm_execpath isn't set (e.g. audio.mjs invoked directly with `node`, not
|
||||
// through npm/npx). Without this, every call silently returns status:-1 and
|
||||
// stdio:"ignore" hides why — callers just report "TTS failed - omitted" for
|
||||
// every line. Surface the real reason once so it's diagnosable.
|
||||
if (!_warnedNpxResolution) {
|
||||
_warnedNpxResolution = true;
|
||||
console.error(
|
||||
`[media-use] Cannot run "${cmd}" on Windows: npm_execpath is not set, so the ` +
|
||||
`npx JS CLI can't be located. This happens when this script is run directly with ` +
|
||||
`\`node\` instead of through npm/npx. Every "${cmd}" call is being skipped. ` +
|
||||
`Fix: run via \`npx\`/\`npm run\`, or export npm_execpath pointing at your npm-cli.js.`,
|
||||
);
|
||||
}
|
||||
return Promise.resolve({ status: -1 });
|
||||
}
|
||||
return new Promise((resolve) => {
|
||||
const p = spawnFn(resolved.cmd, resolved.args, resolved.opts);
|
||||
p.on("exit", (code) => resolve({ status: code ?? -1 }));
|
||||
p.on("error", () => resolve({ status: -1 }));
|
||||
});
|
||||
}
|
||||
|
||||
// mp3/whatever bytes → wav 44.1k mono at destWav (ffmpeg detects true format).
|
||||
function transcodeToWav(bytes, destWav) {
|
||||
const td = mkdtempSync(join(tmpdir(), "hf-tts-"));
|
||||
const tmp = join(td, "a.mp3");
|
||||
writeFileSync(tmp, bytes);
|
||||
mkdirSync(dirname(destWav), { recursive: true });
|
||||
const ff = spawnSync(
|
||||
"ffmpeg",
|
||||
["-y", "-loglevel", "error", "-i", tmp, "-ar", "44100", "-ac", "1", destWav],
|
||||
{ stdio: "ignore" },
|
||||
);
|
||||
rmSync(td, { recursive: true, force: true });
|
||||
return ff.status === 0 && existsSync(destWav);
|
||||
}
|
||||
|
||||
const ELEVENLABS_PY = `
|
||||
import os, sys
|
||||
from elevenlabs.client import ElevenLabs
|
||||
from elevenlabs import save
|
||||
client = ElevenLabs(api_key=os.environ["ELEVENLABS_API_KEY"])
|
||||
text = open(sys.argv[1]).read()
|
||||
audio = client.text_to_speech.convert(
|
||||
text=text, voice_id=sys.argv[2],
|
||||
model_id="eleven_multilingual_v2", output_format="mp3_44100_128",
|
||||
)
|
||||
save(audio, sys.argv[3])
|
||||
`;
|
||||
|
||||
// ── synthesize one line ───────────────────────────────────────────────────────
|
||||
// Writes wav at wavAbs. Returns { ok, words } — words is the raw
|
||||
// [{text,start,end}] array for HeyGen (native), or null for ElevenLabs/Kokoro
|
||||
// (caller must transcribeWav). Never throws; failures return { ok:false }.
|
||||
export async function synthesizeOne({
|
||||
provider,
|
||||
text,
|
||||
voiceId,
|
||||
lang = "en",
|
||||
speed = 1.0,
|
||||
wavAbs,
|
||||
hyperframesDir,
|
||||
}) {
|
||||
if (provider === "heygen") return synthesizeHeygen({ text, voiceId, lang, speed, wavAbs });
|
||||
if (provider === "elevenlabs") {
|
||||
const { cmd, args } = pythonInvocation([
|
||||
"-c",
|
||||
ELEVENLABS_PY,
|
||||
writeTmpText(text),
|
||||
voiceId,
|
||||
wavAbs,
|
||||
]);
|
||||
const r = await spawnP(cmd, args, {});
|
||||
return { ok: r.status === 0 && existsSync(wavAbs), words: null };
|
||||
}
|
||||
// kokoro — via the published CLI; --output is relative to the project dir.
|
||||
const wavRel = relTo(hyperframesDir, wavAbs);
|
||||
const args = ["hyperframes", "tts", writeTmpText(text), "--voice", voiceId, "--output", wavRel];
|
||||
if (lang !== "en") args.push("--lang", lang);
|
||||
const r = await spawnP("npx", args, { cwd: hyperframesDir });
|
||||
return { ok: r.status === 0 && existsSync(wavAbs), words: null };
|
||||
}
|
||||
|
||||
async function synthesizeHeygen({ text, voiceId, lang, speed, wavAbs }) {
|
||||
try {
|
||||
const body = { text, voice_id: voiceId, speed };
|
||||
if (lang !== "en") body.language = lang;
|
||||
const payload = await heygenJSON(`/voices/speech`, {
|
||||
method: "POST",
|
||||
headers: heygenAuthHeaders(),
|
||||
body,
|
||||
});
|
||||
const inner = payload.data ?? payload;
|
||||
if (!inner.audio_url) return { ok: false, words: null };
|
||||
const res = await fetch(inner.audio_url);
|
||||
if (!res.ok) return { ok: false, words: null };
|
||||
const bytes = Buffer.from(await res.arrayBuffer());
|
||||
// .wav output → transcode to 44.1k mono; .mp3 → raw bytes (no ffmpeg). The
|
||||
// engine always asks for .wav; the standalone heygen-tts CLI may ask for .mp3.
|
||||
if (wavAbs.endsWith(".wav")) {
|
||||
if (!transcodeToWav(bytes, wavAbs)) return { ok: false, words: null };
|
||||
} else {
|
||||
mkdirSync(dirname(wavAbs), { recursive: true });
|
||||
writeFileSync(wavAbs, bytes);
|
||||
}
|
||||
const words = Array.isArray(inner.word_timestamps)
|
||||
? inner.word_timestamps
|
||||
.filter((w) => w && typeof w.word === "string" && isFinite(w.start) && isFinite(w.end))
|
||||
.filter((w) => !/^<.*>$/.test(w.word.trim())) // drop <start>/<end> sentinels
|
||||
.map((w) => ({ text: w.word, start: w.start, end: w.end }))
|
||||
: [];
|
||||
return { ok: true, words };
|
||||
} catch {
|
||||
return { ok: false, words: null };
|
||||
}
|
||||
}
|
||||
|
||||
// ElevenLabs/Kokoro have no word timings — run Whisper over the wav. Returns the
|
||||
// flat [{id,text,start,end}] word array, or null. Each call uses a throwaway
|
||||
// --dir so parallel scenes don't collide on transcript.json.
|
||||
export async function transcribeWav({ wavRel, lang = "en", hyperframesDir }) {
|
||||
const model = lang === "en" ? "small.en" : "small";
|
||||
const td = mkdtempSync(join(tmpdir(), "hf-trans-"));
|
||||
const args = ["hyperframes", "transcribe", wavRel, "--model", model, "--dir", td];
|
||||
if (lang !== "en") args.push("--language", lang);
|
||||
const r = await spawnP("npx", args, { cwd: hyperframesDir });
|
||||
let words = null;
|
||||
if (r.status === 0) {
|
||||
const src = join(td, "transcript.json");
|
||||
if (existsSync(src)) {
|
||||
try {
|
||||
const arr = JSON.parse(readFileSync(src, "utf8"));
|
||||
if (Array.isArray(arr) && arr.length) words = arr;
|
||||
} catch {}
|
||||
}
|
||||
}
|
||||
rmSync(td, { recursive: true, force: true });
|
||||
return words;
|
||||
}
|
||||
|
||||
// ── tiny local utils ──────────────────────────────────────────────────────────
|
||||
function writeTmpText(text) {
|
||||
const td = mkdtempSync(join(tmpdir(), "hf-txt-"));
|
||||
const p = join(td, "line.txt");
|
||||
writeFileSync(p, text);
|
||||
return p;
|
||||
}
|
||||
function relTo(base, abs) {
|
||||
return abs.startsWith(base + "/") ? abs.slice(base.length + 1) : abs;
|
||||
}
|
||||
@@ -0,0 +1,143 @@
|
||||
import { test } from "node:test";
|
||||
import assert from "node:assert/strict";
|
||||
import { EventEmitter } from "node:events";
|
||||
import {
|
||||
resolveNpxCliFromNpmExecPath,
|
||||
resolveSpawnCommand,
|
||||
spawnP,
|
||||
_resetNpxResolutionWarnForTests,
|
||||
} from "./tts.mjs";
|
||||
|
||||
// Regression: on Windows, npx resolves to npx.cmd, which spawn() cannot exec
|
||||
// without shell:true — it fails ENOENT, silently swallowed as ok:false by the
|
||||
// caller. spawnP takes injectable platform/spawnFn params so this doesn't
|
||||
// need to touch the real process.platform or mock node:child_process (whose
|
||||
// ESM exports are non-configurable).
|
||||
function fakeSpawn(captured) {
|
||||
return (cmd, args, opts) => {
|
||||
captured.push({ cmd, args, opts });
|
||||
const p = new EventEmitter();
|
||||
setImmediate(() => p.emit("exit", 0));
|
||||
return p;
|
||||
};
|
||||
}
|
||||
|
||||
const envWithNpxCli = {
|
||||
npm_execpath: "/opt/node/lib/node_modules/npm/bin/npm-cli.js",
|
||||
npm_node_execpath: "/opt/node/bin/node",
|
||||
};
|
||||
const npxCliPath = "/opt/node/lib/node_modules/npm/bin/npx-cli.js";
|
||||
const pathExists = (path) => path === npxCliPath;
|
||||
|
||||
test("resolveNpxCliFromNpmExecPath finds npx-cli next to npm-cli", () => {
|
||||
assert.equal(resolveNpxCliFromNpmExecPath(envWithNpxCli.npm_execpath, pathExists), npxCliPath);
|
||||
});
|
||||
|
||||
test("resolveSpawnCommand routes npx through node+npx-cli on win32 without shell:true", () => {
|
||||
const resolved = resolveSpawnCommand(
|
||||
"npx",
|
||||
["hyperframes", "tts", "C:\\Users\\Test User\\line.txt", "--voice", "am_michael"],
|
||||
{},
|
||||
"win32",
|
||||
envWithNpxCli,
|
||||
pathExists,
|
||||
);
|
||||
assert.ok(resolved);
|
||||
assert.equal(resolved.cmd, envWithNpxCli.npm_node_execpath);
|
||||
assert.deepEqual(resolved.args, [
|
||||
npxCliPath,
|
||||
"hyperframes",
|
||||
"tts",
|
||||
"C:\\Users\\Test User\\line.txt",
|
||||
"--voice",
|
||||
"am_michael",
|
||||
]);
|
||||
assert.equal(resolved.opts.shell, undefined);
|
||||
});
|
||||
|
||||
test("resolveSpawnCommand preserves Windows npx shell metacharacters as argv data", () => {
|
||||
const resolved = resolveSpawnCommand(
|
||||
"npx",
|
||||
["hyperframes", "tts", "hello & calc"],
|
||||
{},
|
||||
"win32",
|
||||
envWithNpxCli,
|
||||
pathExists,
|
||||
);
|
||||
assert.ok(resolved);
|
||||
assert.deepEqual(resolved.args, [npxCliPath, "hyperframes", "tts", "hello & calc"]);
|
||||
});
|
||||
|
||||
test("spawnP uses the resolved node+npx-cli command for npx on win32", async () => {
|
||||
const captured = [];
|
||||
await spawnP(
|
||||
"npx",
|
||||
["hyperframes", "tts"],
|
||||
{},
|
||||
"win32",
|
||||
fakeSpawn(captured),
|
||||
envWithNpxCli,
|
||||
pathExists,
|
||||
);
|
||||
assert.equal(captured.length, 1);
|
||||
assert.equal(captured[0].cmd, envWithNpxCli.npm_node_execpath);
|
||||
assert.deepEqual(captured[0].args, [npxCliPath, "hyperframes", "tts"]);
|
||||
assert.equal(captured[0].opts.shell, undefined);
|
||||
});
|
||||
|
||||
test("spawnP does not enable shell for npx on darwin/linux", async () => {
|
||||
const captured = [];
|
||||
await spawnP("npx", ["hyperframes", "tts"], {}, "darwin", fakeSpawn(captured));
|
||||
assert.equal(captured[0].cmd, "npx");
|
||||
assert.deepEqual(captured[0].args, ["hyperframes", "tts"]);
|
||||
assert.equal(captured[0].opts.shell, undefined);
|
||||
});
|
||||
|
||||
test("spawnP does not enable shell for non-npx commands even on win32", async () => {
|
||||
const captured = [];
|
||||
await spawnP("python3", ["-c", "pass"], {}, "win32", fakeSpawn(captured));
|
||||
assert.equal(captured[0].cmd, "python3");
|
||||
assert.deepEqual(captured[0].args, ["-c", "pass"]);
|
||||
assert.equal(captured[0].opts.shell, undefined);
|
||||
});
|
||||
|
||||
// Regression: win32 + npx with npm_execpath unset can't locate the npx JS CLI,
|
||||
// so resolveSpawnCommand returns null and spawnP short-circuits. Previously it
|
||||
// returned {status:-1} silently — every TTS line just dropped as "TTS failed -
|
||||
// omitted" with no hint. Now it must surface a clear one-time diagnostic naming
|
||||
// npm_execpath, while still returning {status:-1} without spawning anything.
|
||||
test("spawnP surfaces a clear diagnostic (once) when npx can't be resolved on win32", async () => {
|
||||
_resetNpxResolutionWarnForTests();
|
||||
const errors = [];
|
||||
const originalError = console.error;
|
||||
console.error = (msg) => errors.push(msg);
|
||||
const captured = [];
|
||||
const emptyEnv = {}; // no npm_execpath
|
||||
try {
|
||||
const r1 = await spawnP(
|
||||
"npx",
|
||||
["hyperframes", "tts"],
|
||||
{},
|
||||
"win32",
|
||||
fakeSpawn(captured),
|
||||
emptyEnv,
|
||||
() => false,
|
||||
);
|
||||
const r2 = await spawnP(
|
||||
"npx",
|
||||
["hyperframes", "tts"],
|
||||
{},
|
||||
"win32",
|
||||
fakeSpawn(captured),
|
||||
emptyEnv,
|
||||
() => false,
|
||||
);
|
||||
assert.equal(r1.status, -1);
|
||||
assert.equal(r2.status, -1);
|
||||
assert.equal(captured.length, 0, "must not spawn anything when resolution fails");
|
||||
assert.equal(errors.length, 1, "diagnostic is emitted once for a batch, not per line");
|
||||
assert.match(errors[0], /npm_execpath/);
|
||||
} finally {
|
||||
console.error = originalError;
|
||||
}
|
||||
});
|
||||
@@ -0,0 +1,66 @@
|
||||
import { test } from "node:test";
|
||||
import assert from "node:assert/strict";
|
||||
import { mkdtempSync, writeFileSync, chmodSync, rmSync } from "node:fs";
|
||||
import { join } from "node:path";
|
||||
import { tmpdir } from "node:os";
|
||||
import { parseFfmpegDurationBanner, ffprobeDuration } from "./tts.mjs";
|
||||
|
||||
test("parseFfmpegDurationBanner reads ffmpeg's stderr Duration line", () => {
|
||||
const stderr = [
|
||||
"ffmpeg version 6.0",
|
||||
"Input #0, wav, from 'a.wav':",
|
||||
" Duration: 00:00:03.42, bitrate: 705 kb/s",
|
||||
"At least one output file must be specified",
|
||||
].join("\n");
|
||||
assert.equal(parseFfmpegDurationBanner(stderr), 3.42);
|
||||
});
|
||||
|
||||
test("parseFfmpegDurationBanner handles an hours component", () => {
|
||||
const stderr = " Duration: 01:02:03.50, start: 0.000000, bitrate: 128 kb/s";
|
||||
assert.equal(parseFfmpegDurationBanner(stderr), 3723.5);
|
||||
});
|
||||
|
||||
test("parseFfmpegDurationBanner returns NaN when there is no Duration line", () => {
|
||||
assert.ok(Number.isNaN(parseFfmpegDurationBanner("ffmpeg: command not found")));
|
||||
assert.ok(Number.isNaN(parseFfmpegDurationBanner("")));
|
||||
assert.ok(Number.isNaN(parseFfmpegDurationBanner(undefined)));
|
||||
});
|
||||
|
||||
// Regression for the actual bug: ffprobeDuration used to collapse "ffprobe
|
||||
// binary is missing" (ENOENT — the "essentials"-style Windows ffmpeg build
|
||||
// with no ffprobe.exe) and "file is genuinely unreadable" into the same NaN,
|
||||
// giving audio.mjs no way to tell "measure differently" from "give up".
|
||||
//
|
||||
// Builds an isolated PATH containing only a fake `ffmpeg` stub (no `ffprobe`
|
||||
// at all) so ffprobeDuration's spawnSync("ffprobe", ...) call ENOENTs for
|
||||
// real, then verifies it recovers the duration via the ffmpeg fallback
|
||||
// instead of returning NaN.
|
||||
test("ffprobeDuration falls back to ffmpeg when the ffprobe binary itself is missing", () => {
|
||||
const dir = mkdtempSync(join(tmpdir(), "tts-ffprobe-fallback-"));
|
||||
const fakeFfmpeg = join(dir, "ffmpeg");
|
||||
writeFileSync(
|
||||
fakeFfmpeg,
|
||||
"#!/bin/sh\necho 'Duration: 00:00:02.50, start: 0.000000, bitrate: 128 kb/s' 1>&2\nexit 1\n",
|
||||
);
|
||||
chmodSync(fakeFfmpeg, 0o755);
|
||||
const originalPath = process.env.PATH;
|
||||
try {
|
||||
process.env.PATH = dir; // only the fake ffmpeg resolves; no real ffprobe on this PATH
|
||||
assert.equal(ffprobeDuration("/does/not/matter.wav"), 2.5);
|
||||
} finally {
|
||||
process.env.PATH = originalPath;
|
||||
rmSync(dir, { recursive: true, force: true });
|
||||
}
|
||||
});
|
||||
|
||||
test("ffprobeDuration returns NaN when neither ffprobe nor ffmpeg resolve", () => {
|
||||
const dir = mkdtempSync(join(tmpdir(), "tts-no-binaries-"));
|
||||
const originalPath = process.env.PATH;
|
||||
try {
|
||||
process.env.PATH = dir; // empty directory — nothing resolves
|
||||
assert.ok(Number.isNaN(ffprobeDuration("/does/not/matter.wav")));
|
||||
} finally {
|
||||
process.env.PATH = originalPath;
|
||||
rmSync(dir, { recursive: true, force: true });
|
||||
}
|
||||
});
|
||||
@@ -0,0 +1,128 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Generate BGM using Google Lyria RealTime API.
|
||||
|
||||
Usage:
|
||||
python lyria-recipe.py --output <path> --duration <seconds> [tuning flags]
|
||||
|
||||
Requires:
|
||||
$GOOGLE_API_KEY or $GEMINI_API_KEY environment variable (treated as aliases).
|
||||
pip install google-genai python-dotenv. audio.mjs Step 4b installs these on
|
||||
demand when a key is set but google.genai is not importable; if that install
|
||||
fails it falls back to local MusicGen rather than leaving the video with no BGM.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import asyncio
|
||||
import os
|
||||
import sys
|
||||
import wave
|
||||
from pathlib import Path
|
||||
|
||||
DEFAULT_PROMPT = "Uplifting corporate tech, bright and modern, gentle piano with synth pads"
|
||||
SAMPLE_RATE = 48000
|
||||
CHANNELS = 2
|
||||
SAMPLE_WIDTH = 2 # 16-bit
|
||||
|
||||
|
||||
def parse_args() -> argparse.Namespace:
|
||||
p = argparse.ArgumentParser(description="Generate BGM via Google Lyria RealTime.")
|
||||
p.add_argument("--output", required=True, help="Output WAV path.")
|
||||
p.add_argument("--duration", type=float, required=True, help="Target duration in seconds.")
|
||||
p.add_argument("--prompt", default=DEFAULT_PROMPT, help="Mood / instrumentation prompt.")
|
||||
p.add_argument("--negative-prompt", default=None, help="Styles to exclude (optional).")
|
||||
p.add_argument("--bpm", type=int, default=110)
|
||||
p.add_argument("--brightness", type=float, default=0.8, help="0-1, higher = brighter mood.")
|
||||
p.add_argument("--density", type=float, default=0.5, help="0-1, higher = fuller mix.")
|
||||
p.add_argument(
|
||||
"--scale",
|
||||
default="MAJOR",
|
||||
help="MAJOR / MINOR / PENTATONIC / etc. — see google.genai.types.Scale. Pass empty string for none.",
|
||||
)
|
||||
return p.parse_args()
|
||||
|
||||
|
||||
async def generate_bgm(args: argparse.Namespace) -> dict:
|
||||
from google import genai
|
||||
from google.genai import types
|
||||
|
||||
api_key = os.environ.get("GOOGLE_API_KEY") or os.environ.get("GEMINI_API_KEY") or ""
|
||||
if not api_key:
|
||||
raise RuntimeError("Neither GOOGLE_API_KEY nor GEMINI_API_KEY is set.")
|
||||
|
||||
client = genai.Client(
|
||||
api_key=api_key,
|
||||
http_options={"api_version": "v1alpha"},
|
||||
)
|
||||
|
||||
out_path = Path(args.output)
|
||||
out_path.parent.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
target_bytes = int(args.duration * SAMPLE_RATE * CHANNELS * SAMPLE_WIDTH)
|
||||
|
||||
cfg: dict = {"bpm": args.bpm, "temperature": 1.0}
|
||||
if args.density is not None:
|
||||
cfg["density"] = args.density
|
||||
if args.brightness is not None:
|
||||
cfg["brightness"] = args.brightness
|
||||
if args.scale:
|
||||
scale_enum = getattr(types.Scale, args.scale, None)
|
||||
if scale_enum:
|
||||
cfg["scale"] = scale_enum
|
||||
|
||||
prompts = [types.WeightedPrompt(text=args.prompt, weight=1.0)]
|
||||
if args.negative_prompt:
|
||||
prompts.append(types.WeightedPrompt(text=args.negative_prompt, weight=-1.0))
|
||||
|
||||
buf = bytearray()
|
||||
timeout = args.duration + 8
|
||||
|
||||
async with client.aio.live.music.connect(
|
||||
model="models/lyria-realtime-exp",
|
||||
) as session:
|
||||
await session.set_weighted_prompts(prompts=prompts)
|
||||
await session.set_music_generation_config(
|
||||
config=types.LiveMusicGenerationConfig(**cfg),
|
||||
)
|
||||
await session.play()
|
||||
|
||||
async def collect():
|
||||
while len(buf) < target_bytes:
|
||||
async for msg in session.receive():
|
||||
sc = msg.server_content
|
||||
if sc and sc.audio_chunks:
|
||||
for chunk in sc.audio_chunks:
|
||||
buf.extend(chunk.data)
|
||||
if len(buf) >= target_bytes:
|
||||
return
|
||||
await asyncio.sleep(1e-6)
|
||||
|
||||
try:
|
||||
await asyncio.wait_for(collect(), timeout=timeout)
|
||||
except TimeoutError:
|
||||
print(f"Timeout after {timeout:.0f}s, collected {len(buf)} bytes", file=sys.stderr)
|
||||
|
||||
audio = bytes(buf[:target_bytes])
|
||||
with wave.open(str(out_path), "wb") as wf:
|
||||
wf.setnchannels(CHANNELS)
|
||||
wf.setsampwidth(SAMPLE_WIDTH)
|
||||
wf.setframerate(SAMPLE_RATE)
|
||||
wf.writeframes(audio)
|
||||
|
||||
actual_duration = len(audio) / (SAMPLE_RATE * CHANNELS * SAMPLE_WIDTH)
|
||||
print(f"BGM: {out_path} ({actual_duration:.2f}s)")
|
||||
return {"file": str(out_path), "duration_sec": round(actual_duration, 2)}
|
||||
|
||||
|
||||
def main() -> None:
|
||||
args = parse_args()
|
||||
try:
|
||||
asyncio.run(generate_bgm(args))
|
||||
except RuntimeError as exc:
|
||||
print(f"BGM generation failed: {exc}", file=sys.stderr)
|
||||
sys.exit(1)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,167 @@
|
||||
#!/usr/bin/env node
|
||||
// Phase 4c pre-assemble helper — wait for detached BGM, then write status.
|
||||
//
|
||||
// audio.mjs may launch Lyria / MusicGen in a detached process so voice work can
|
||||
// keep moving. Before assemble-index.mjs decides whether to emit the BGM audio
|
||||
// track, this script gives the background renderer a bounded chance to finish
|
||||
// and converts log/process state into a small bgm_status.json file.
|
||||
//
|
||||
// Always exits 0 for normal pipeline use: missing/failed BGM should not block a
|
||||
// voice/captions/SFX render. Structural invocation errors still exit 1.
|
||||
//
|
||||
// Usage:
|
||||
// node wait-bgm.mjs --audio-meta ./audio_meta.json --hyperframes . \
|
||||
// [--timeout-ms 120000] [--interval-ms 2000] [--out ./bgm_status.json]
|
||||
|
||||
import { existsSync, readFileSync, statSync, writeFileSync } from "node:fs";
|
||||
import { join, resolve } from "node:path";
|
||||
|
||||
const argv = process.argv.slice(2);
|
||||
const flag = (name, def) => {
|
||||
const i = argv.indexOf(`--${name}`);
|
||||
return i >= 0 && i + 1 < argv.length ? argv[i + 1] : def;
|
||||
};
|
||||
|
||||
function die(msg) {
|
||||
console.error(`✗ wait-bgm.mjs: ${msg}`);
|
||||
process.exit(1);
|
||||
}
|
||||
|
||||
const audioMetaPath = resolve(flag("audio-meta", "./audio_meta.json"));
|
||||
const hyperframesDir = resolve(flag("hyperframes", "."));
|
||||
const outPath = resolve(flag("out", join(hyperframesDir, "bgm_status.json")));
|
||||
const timeoutMs = Math.max(0, Number(flag("timeout-ms", "120000")) || 0);
|
||||
const intervalMs = Math.max(250, Number(flag("interval-ms", "2000")) || 2000);
|
||||
|
||||
function sleep(ms) {
|
||||
return new Promise((resolveSleep) => setTimeout(resolveSleep, ms));
|
||||
}
|
||||
|
||||
function isProcessAlive(pid) {
|
||||
if (!pid || !Number.isFinite(Number(pid))) return false;
|
||||
try {
|
||||
process.kill(Number(pid), 0);
|
||||
return true;
|
||||
} catch {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
function readTail(path, maxChars = 6000) {
|
||||
if (!path || !existsSync(path)) return "";
|
||||
const s = statSync(path);
|
||||
const txt = readFileSync(path, "utf8");
|
||||
return txt.slice(Math.max(0, txt.length - Math.min(maxChars, s.size)));
|
||||
}
|
||||
|
||||
function detectFailure(logTail) {
|
||||
if (!logTail) return "";
|
||||
const lines = logTail.split("\n");
|
||||
// Bare "out of range" over-matched benign BGM-renderer logs (e.g. a "sample rate
|
||||
// out of range, resampling" notice), mislabelling a healthy track as failed and
|
||||
// silently dropping the music. Anchor to the actual crash strings instead:
|
||||
// Python "(list) index out of range" and torch "index … out of bounds".
|
||||
const idx = lines.findIndex((line) =>
|
||||
/(Traceback|IndexError|RuntimeError|Exception|Killed|No space left|Cannot allocate|index out of range|out of bounds)/i.test(
|
||||
line,
|
||||
),
|
||||
);
|
||||
if (idx < 0) return "";
|
||||
return lines.slice(idx).join("\n").trim();
|
||||
}
|
||||
|
||||
function writeStatus(status) {
|
||||
const payload = {
|
||||
generated_at: new Date().toISOString(),
|
||||
...status,
|
||||
};
|
||||
writeFileSync(outPath, JSON.stringify(payload, null, 2) + "\n");
|
||||
return payload;
|
||||
}
|
||||
|
||||
if (!existsSync(audioMetaPath)) die(`audio_meta.json missing at ${audioMetaPath}`);
|
||||
|
||||
const audioMeta = JSON.parse(readFileSync(audioMetaPath, "utf8"));
|
||||
const bgmPath = audioMeta.bgm?.path || "";
|
||||
const bgmAbsPath = bgmPath ? join(hyperframesDir, bgmPath) : "";
|
||||
const logPath = audioMeta.bgm_log || "";
|
||||
const pid = audioMeta.bgm_pid || null;
|
||||
|
||||
const base = {
|
||||
enabled: Boolean(audioMeta.bgm_pending && bgmPath),
|
||||
provider: audioMeta.bgm_provider || null,
|
||||
mode: audioMeta.bgm_mode || null,
|
||||
path: bgmPath || null,
|
||||
log: logPath || null,
|
||||
pid,
|
||||
target_duration_s: audioMeta.bgm_target_duration_s || null,
|
||||
seed_duration_s: audioMeta.bgm_seed_duration_s || null,
|
||||
loop_count: audioMeta.bgm_loop_count || null,
|
||||
timeout_ms: timeoutMs,
|
||||
};
|
||||
|
||||
if (!base.enabled) {
|
||||
const status = writeStatus({
|
||||
...base,
|
||||
status: "disabled",
|
||||
ready: false,
|
||||
waited_ms: 0,
|
||||
message: "BGM not requested or disabled in audio_meta.json.",
|
||||
});
|
||||
console.log(`✓ bgm: ${status.status} (${status.message})`);
|
||||
process.exit(0);
|
||||
}
|
||||
|
||||
const started = Date.now();
|
||||
let lastFailure = "";
|
||||
let lastTail = "";
|
||||
|
||||
while (Date.now() - started <= timeoutMs) {
|
||||
if (existsSync(bgmAbsPath)) {
|
||||
const size = statSync(bgmAbsPath).size;
|
||||
writeStatus({
|
||||
...base,
|
||||
status: "ready",
|
||||
ready: true,
|
||||
waited_ms: Date.now() - started,
|
||||
size_bytes: size,
|
||||
message: `BGM ready at ${bgmPath}.`,
|
||||
});
|
||||
console.log(`✓ bgm: ready (${bgmPath}, ${size}B)`);
|
||||
process.exit(0);
|
||||
}
|
||||
|
||||
lastTail = readTail(logPath);
|
||||
lastFailure = detectFailure(lastTail);
|
||||
const alive = isProcessAlive(pid);
|
||||
if (lastFailure || (!alive && logPath && existsSync(logPath))) {
|
||||
const message = lastFailure
|
||||
? `BGM renderer failed; see ${logPath}.`
|
||||
: `BGM renderer exited without writing ${bgmPath}; see ${logPath}.`;
|
||||
const status = writeStatus({
|
||||
...base,
|
||||
status: "failed",
|
||||
ready: false,
|
||||
waited_ms: Date.now() - started,
|
||||
process_alive: alive,
|
||||
message,
|
||||
error_tail: lastFailure || lastTail.slice(-2000),
|
||||
});
|
||||
console.log(`! bgm: failed (${status.message})`);
|
||||
process.exit(0);
|
||||
}
|
||||
|
||||
if (timeoutMs === 0) break;
|
||||
await sleep(Math.min(intervalMs, Math.max(0, timeoutMs - (Date.now() - started))));
|
||||
}
|
||||
|
||||
const status = writeStatus({
|
||||
...base,
|
||||
status: "timeout",
|
||||
ready: false,
|
||||
waited_ms: Date.now() - started,
|
||||
process_alive: isProcessAlive(pid),
|
||||
message: `Timed out waiting for ${bgmPath}; assemble-index will skip BGM if still absent.`,
|
||||
log_tail: lastTail.slice(-2000),
|
||||
});
|
||||
console.log(`! bgm: timeout after ${status.waited_ms}ms (${bgmPath})`);
|
||||
Reference in New Issue
Block a user