From 4fcad1ed74bd629b98f91ef53be8fdf154528924 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Miguel=20=C3=81ngel?= Date: Wed, 7 Oct 2026 07:16:51 +0000 Subject: [PATCH] fix(media-use): use a host app's music tools; fail loudly without the heygen CLI (#5154) * fix(media-use): use a host app's music tools; fail loudly without the heygen CLI * fix(media-use): name the heygen CLI only for music and sound effects; keep the sfx test offline * fix(skills): workflow skills use a host app's own music tools first * fix(skills): say how a host app's music track reaches the narrated workflows * fix(skills): set a host app's music track after the last audio pass * fix(skills): a host-music film with no narration still fetches its sound effects * fix(media-use): keep the stub-failure test off the machine's heygen CLI; sync the skills manifest * fix(skills): audio passes keep a host app's music and sounds; silent needs no sound cues * fix(skills): host audio keeps string frames, stops on a broken audio_meta.json, survives a re-run * fix(skills): report a host audio entry with no path and a broken sidecar instead of crashing * fix(skills): keep host audio written while an audio pass runs * test(cli): serve the telemetry capture while the resolve child waits on it * test(cli): fail the telemetry capture when the resolve child exits non-zero * style(skills): wrap the host audio comment to the 120-column limit --- CLAUDE.md | 2 +- README.md | 2 +- docs/prompting/overview.mdx | 2 +- packages/cli/src/media-use/lib/heygen-cli.mjs | 22 ++- .../cli/src/media-use/lib/heygen-cli.test.mjs | 14 ++ packages/cli/src/media-use/resolve.mjs | 17 +- packages/cli/src/media-use/resolve.test.mjs | 107 +++++++++-- scripts/generate-skill-module-copies.mjs | 4 + skills-manifest.json | 22 +-- skills/faceless-explainer/SKILL.md | 8 +- .../references/story-design.md | 2 +- skills/faceless-explainer/scripts/audio.mjs | 76 ++++++-- .../faceless-explainer/scripts/audio.test.mjs | 177 +++++++++++++++++- .../scripts/lib/host-audio.mjs | 28 +++ skills/general-video/SKILL.md | 2 +- .../hyperframes/references/capability-menu.md | 2 +- skills/media-use/SKILL.md | 10 +- .../audio/scripts/lib/host-audio.mjs | 28 +++ .../audio/scripts/lib/host-audio.test.mjs | 106 +++++++++++ .../media-use/references/setup-providers.md | 5 +- skills/music-to-video/SKILL.md | 2 +- skills/pr-to-video/SKILL.md | 8 +- skills/pr-to-video/references/story-design.md | 2 +- skills/pr-to-video/scripts/audio.mjs | 76 ++++++-- skills/pr-to-video/scripts/lib/host-audio.mjs | 28 +++ skills/product-launch-video/SKILL.md | 8 +- .../references/story-design.md | 2 +- skills/product-launch-video/scripts/audio.mjs | 76 ++++++-- .../scripts/audio.test.mjs | 177 +++++++++++++++++- .../scripts/lib/host-audio.mjs | 28 +++ 30 files changed, 921 insertions(+), 122 deletions(-) create mode 100644 skills/faceless-explainer/scripts/lib/host-audio.mjs create mode 100644 skills/media-use/audio/scripts/lib/host-audio.mjs create mode 100644 skills/media-use/audio/scripts/lib/host-audio.test.mjs create mode 100644 skills/pr-to-video/scripts/lib/host-audio.mjs create mode 100644 skills/product-launch-video/scripts/lib/host-audio.mjs diff --git a/CLAUDE.md b/CLAUDE.md index a7c5ef381a..e0d9b9f7a1 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -41,7 +41,7 @@ Atomic capabilities the creation workflows compose against — pull one when you - `/hyperframes-animation` — all animation knowledge: atomic motion rules, scene blueprints, transitions, runtime adapters (GSAP default, plus Lottie / Three.js / Anime.js / CSS / WAAPI / TypeGPU). - `/hyperframes-keyframes` — seek-safe keyframe authoring across runtimes: GSAP timelines, CSS keyframes, Anime.js, WAAPI, FLIP, paths, masks, SVG morph/draw, text trails, 3D depth; plus `hyperframes keyframes` diagnostics for surfacing and verifying rendered motion. - `/hyperframes-creative` — non-animation creative direction: `frame.md` / `design.md` handling, palettes, typography, narration, beat planning, audio-reactive visuals, composition patterns. -- `/media-use` — the media OS: resolve any media need (BGM, SFX, image, icon, logo, voice, color grade, LUT) into a frozen local file or paste-ready block + ledger record; generate via TTS / music / image models when the catalog misses; transcribe, caption, remove backgrounds, and reuse assets across projects. One shared `scripts/audio.mjs` engine + manifest tracking; keeps search noise on disk. +- `/media-use` — the media OS (a host app's own music and sound-effect tools come first for those): resolve any media need (BGM, SFX, image, icon, logo, voice, color grade, LUT) into a frozen local file or paste-ready block + ledger record; generate via TTS / music / image models when the catalog misses; transcribe, caption, remove backgrounds, and reuse assets across projects. One shared `scripts/audio.mjs` engine + manifest tracking; keeps search noise on disk. - `/hyperframes-audio` — mix the audio already placed in a composition: voiceover carve (dip a music bed only in the bands the voice occupies, static or dynamic, level match included), the effect chain (EQ, compressor, limiter, gate, saturation, delay, reverb, chorus, phaser, bitcrush), automation envelopes on volume or any effect parameter, and submix buses (``) that carry one chain, fader and automation clock for several tracks at once. Sourcing the audio is `/media-use`; this is what happens to it afterwards. - `/hyperframes-cli` — CLI dev loop: `init`, `add`, `lint`, `check`, `snapshot`, `preview`, `render`, `publish`, `doctor`, `lambda` (AWS Lambda cloud rendering). - `/hyperframes-registry` — search, install and wire registry blocks and components into compositions via `hyperframes catalog` / `hyperframes add`. Load it before hand-building any named look, effect, treatment or transition: the search ranks the whole hosted registry with nothing installed. Covers authoring a new block or component to contribute upstream. diff --git a/README.md b/README.md index 41204b5d94..4529ffdda7 100644 --- a/README.md +++ b/README.md @@ -118,7 +118,7 @@ Atomic capabilities the creation workflows compose against — pull one when you | `/hyperframes-animation` | All animation knowledge — atomic motion rules, scene blueprints, transitions, runtime adapters (GSAP / Lottie / Three.js / Anime.js / CSS / WAAPI / TypeGPU). | | `/hyperframes-keyframes` | Seek-safe keyframe authoring across runtimes — GSAP timelines, CSS keyframes, Anime.js, WAAPI, FLIP, paths, masks, SVG morph/draw, 3D depth — plus `hyperframes keyframes` diagnostics for rendered motion. | | `/hyperframes-creative` | Non-animation creative direction — `frame.md` / `design.md`, palettes, typography, narration, beat planning, audio-reactive visuals, composition patterns. | -| `/media-use` | The media OS — resolve any media need (BGM, SFX, image, icon, logo, voice, color grade, LUT) into a frozen local file or paste-ready block + ledger record, generate via TTS/music/image models when the catalog misses, transcribe, caption, remove backgrounds, and reuse assets across projects. One shared audio engine + manifest tracking. | +| `/media-use` | The media OS (a host app's own music and sound-effect tools come first for those) — resolve any media need (BGM, SFX, image, icon, logo, voice, color grade, LUT) into a frozen local file or paste-ready block + ledger record, generate via TTS/music/image models when the catalog misses, transcribe, caption, remove backgrounds, and reuse assets across projects. One shared audio engine + manifest tracking. | | `/hyperframes-cli` | CLI dev loop — `init`, `lint`, `check`, `snapshot`, `preview`, `render`, `publish`, `doctor`, plus HeyGen-hosted cloud rendering (`cloud render`) and AWS Lambda rendering (`lambda deploy / render / progress`). | | `/hyperframes-audio` | Mix the audio already placed in a composition — voiceover carve (dip a music bed only in the bands the voice occupies, static or dynamic, level match included), the effect chain (EQ, compressor, limiter, gate, saturation, delay, reverb, chorus, phaser, bitcrush), automation envelopes on volume or any effect parameter, and submix buses (``) carrying one chain, fader and automation clock for several tracks at once. Sourcing the audio is `/media-use`. | | `/hyperframes-registry` | Search, install and wire registry blocks and components into compositions via `hyperframes catalog` / `hyperframes add`. Load before hand-building any named look, effect, treatment or transition. Authoring a new block or component to contribute upstream. | diff --git a/docs/prompting/overview.mdx b/docs/prompting/overview.mdx index 6626229ba7..87c0e71b57 100644 --- a/docs/prompting/overview.mdx +++ b/docs/prompting/overview.mdx @@ -51,7 +51,7 @@ The installer shows a picker. Select the **core skills** below — every project | `/hyperframes-animation`| All animation — motion rules, scene blueprints, transitions, and the runtime adapters (GSAP, Lottie, Three.js, Anime.js, CSS, WAAPI, TypeGPU) | | `/hyperframes-creative` | Creative direction — design spec, palettes, typography, narration, beats | | `/hyperframes-cli` | Dev-loop CLI — `init`, `lint`, `check`, `preview`, `render`, `doctor` | -| `/media-use` | Media OS — TTS voiceover (`tts`), `transcribe`, `remove-background`, plus BGM / SFX / image resolution | +| `/media-use` | Media OS — TTS voiceover (`tts`), `transcribe`, `remove-background`, plus BGM / SFX / image resolution (a host app's own music and sound tools come first) | | `/hyperframes-registry` | Search the catalog before hand-building a look; install via `hyperframes add` | | `/hyperframes-keyframes`| Seek-safe keyframe authoring across runtimes, plus `hyperframes keyframes` diagnostics | | `/general-video` | The general authoring workflow — multi-scene pieces, reels, montages, remixes, and the home of **companion mode**; the fallback when no workflow below fits | diff --git a/packages/cli/src/media-use/lib/heygen-cli.mjs b/packages/cli/src/media-use/lib/heygen-cli.mjs index bbf40c7ca3..5a8e27c209 100644 --- a/packages/cli/src/media-use/lib/heygen-cli.mjs +++ b/packages/cli/src/media-use/lib/heygen-cli.mjs @@ -68,7 +68,11 @@ function classifyHeygenErrorResult(err) { } const version = firstSemver(text); - if (version && versionLessThan(version, HEYGEN_MIN_VERSION)) { + // A CLI older than the --headers flag rejects it before printing any version. + if ( + (version && versionLessThan(version, HEYGEN_MIN_VERSION)) || + lower.includes("unknown flag: --headers") + ) { return { code: "outdated", message: HEYGEN_OUTDATED_MESSAGE }; } @@ -98,6 +102,22 @@ const pendingFailureTracking = new Set(); // API, move this state into a per-resolve context before reusing that path. let pendingRemediation = null; +const TOOL_WORDS = { bgm: "music", sfx: "sound-effect" }; + +/** A music or sound-effect resolve miss after the heygen CLI was missing or too old: what is missing, the host + * app's own tool, the fix. Other types keep the generic miss. */ +export function heygenMiss(type, { code }) { + const tool = TOOL_WORDS[type]; + if (!tool) return null; + const outdated = code === "outdated"; + const state = outdated ? `older than v${HEYGEN_MIN_VERSION}` : "not installed"; + return { + code: outdated ? "heygen_cli_outdated" : "heygen_cli_missing", + fix: outdated ? HEYGEN_UPDATE_COMMAND : HEYGEN_INSTALL_COMMAND, + error: `${type} needs the heygen CLI, which is ${state}: use your host app's own ${tool} tool if it has one, or ${outdated ? "update" : "install"} the CLI.`, + }; +} + export function consumeHeygenRemediation() { const remediation = pendingRemediation; pendingRemediation = null; diff --git a/packages/cli/src/media-use/lib/heygen-cli.test.mjs b/packages/cli/src/media-use/lib/heygen-cli.test.mjs index 56594ac0d0..f94f4a89d7 100644 --- a/packages/cli/src/media-use/lib/heygen-cli.test.mjs +++ b/packages/cli/src/media-use/lib/heygen-cli.test.mjs @@ -9,6 +9,7 @@ import { HEYGEN_NOT_AUTHENTICATED_MESSAGE, HEYGEN_NOT_FOUND_MESSAGE, HEYGEN_OUTDATED_MESSAGE, + heygenMiss, reportHeygenFailure, } from "./heygen-cli.mjs"; @@ -299,3 +300,16 @@ test("flushHeygenFailureTracking waits for a pending report before resolving", a test("flushHeygenFailureTracking resolves immediately when nothing is pending", async () => { await flushHeygenFailureTracking(); }); + +test("only a music or sound-effect miss names the heygen CLI and the host app's own tool", () => { + for (const type of ["image", "icon", "voice", "video"]) + assert.equal(heygenMiss(type, { code: "not_found" }), null, type); + assert.equal(heygenMiss("bgm", { code: "not_found" }).code, "heygen_cli_missing"); + const sfx = heygenMiss("sfx", { code: "outdated" }); + assert.equal(sfx.code, "heygen_cli_outdated"); + assert.match(sfx.error, /host app's own sound-effect tool/); +}); + +test("a CLI that rejects --headers without printing a version is outdated", () => { + assert.equal(classifyHeygenErrorCode({ stderr: "Error: unknown flag: --headers" }), "outdated"); +}); diff --git a/packages/cli/src/media-use/resolve.mjs b/packages/cli/src/media-use/resolve.mjs index 80bc42ec2e..6dd28b9dd8 100644 --- a/packages/cli/src/media-use/resolve.mjs +++ b/packages/cli/src/media-use/resolve.mjs @@ -49,6 +49,7 @@ import { HEYGEN_MIN_VERSION, HEYGEN_UPDATE_COMMAND, consumeHeygenRemediation, + heygenMiss, firstSemver, flushHeygenFailureTracking, versionLessThan, @@ -530,10 +531,16 @@ async function run() { }); // brand stays local: no frame.md/design.md -> upsell the HyperFrames design // flow rather than reporting a generic miss (B5). - const msg = + const ownFailure = providerFailure instanceof BundledSfxAssetsError || - providerFailure instanceof FfBinarySettingError - ? providerFailure.message + providerFailure instanceof FfBinarySettingError; + const heygen = consumeHeygenRemediation(); + const miss = heygen && !ownFailure ? heygenMiss(type, heygen) : null; + const typed = providerFailure instanceof BundledSfxAssetsError ? providerFailure : miss; + const msg = ownFailure + ? providerFailure.message + : miss + ? miss.error : type === "brand" ? "no brand spec found — add a frame.md or design.md (colors/font/logo) to this project. Run the HyperFrames design flow to create one; brand tokens are read locally for deterministic rendering." : args.provider @@ -543,9 +550,7 @@ async function run() { console.log( JSON.stringify({ ok: false, - ...(providerFailure instanceof BundledSfxAssetsError - ? { code: providerFailure.code, fix: providerFailure.fix } - : {}), + ...(typed ? { code: typed.code, fix: typed.fix } : {}), error: msg, }), ); diff --git a/packages/cli/src/media-use/resolve.test.mjs b/packages/cli/src/media-use/resolve.test.mjs index a4b54ff539..d4c676ab47 100644 --- a/packages/cli/src/media-use/resolve.test.mjs +++ b/packages/cli/src/media-use/resolve.test.mjs @@ -16,7 +16,11 @@ import { execFileSync, spawn, spawnSync } from "node:child_process"; import { appendRecord, findByPrompt, readManifest } from "./lib/manifest.mjs"; import { regenerateIndex } from "./lib/index-gen.mjs"; import { getProvider } from "./lib/providers.mjs"; -import { HEYGEN_NOT_FOUND_MESSAGE } from "./lib/heygen-cli.mjs"; +import { + HEYGEN_INSTALL_COMMAND, + HEYGEN_NOT_FOUND_MESSAGE, + HEYGEN_UPDATE_COMMAND, +} from "./lib/heygen-cli.mjs"; import { freezeLocalFile } from "./lib/freeze.mjs"; import { cachePut, cacheGet, importFromCache } from "./lib/cache.mjs"; import { validateCubeFile } from "./lib/cube-validate.mjs"; @@ -244,6 +248,69 @@ test("explicit local bundled SFX resolution does not advise installation", () => } }); +test("music without the HeyGen CLI fails naming it and the host app's own tool", () => { + setup(); + const result = spawnResolve( + ["--type", "bgm", "--intent", "calm piano", "--project", tmp, "--json"], + { + env: { HOME: tmp, PATH: tmp }, + }, + ); + assert.equal(result.status, 1, result.stderr); + const parsed = JSON.parse(result.stdout); + assert.equal(parsed.ok, false); + assert.equal(parsed.code, "heygen_cli_missing"); + assert.equal(parsed.fix, HEYGEN_INSTALL_COMMAND); + assert.match(parsed.error, /needs the heygen CLI, which is not installed/); + assert.match(parsed.error, /host app's own music tool/); + cleanup(); +}); + +test("a sound effect nothing bundled matches fails the same way without the HeyGen CLI", async () => { + setup(); + // The local media index answers 404, so the miss never depends on the network. + const server = createServer((_req, res) => { + res.writeHead(404); + res.end(); + }); + await new Promise((resolve) => server.listen(0, "127.0.0.1", resolve)); + try { + const result = await spawnResolveAsync( + ["--type", "sfx", "--intent", "dog barking", "--project", tmp, "--json"], + { + env: { + HOME: tmp, + PATH: tmp, + HYPERFRAMES_REGISTRY: `http://127.0.0.1:${server.address().port}`, + }, + }, + ); + assert.equal(result.status, 1, result.stderr); + const parsed = JSON.parse(result.stdout); + assert.equal(parsed.code, "heygen_cli_missing"); + assert.match(parsed.error, /host app's own sound-effect tool/); + } finally { + server.close(); + cleanup(); + } +}); + +test("music with an outdated HeyGen CLI fails naming the update", () => { + setup(); + const binDir = writeFakeHeygen('echo "heygen v0.1.5 does not support --headers" >&2', 1); + const result = spawnResolve( + ["--type", "bgm", "--intent", "calm piano", "--project", tmp, "--json"], + { + env: { HOME: tmp, PATH: binDir }, + }, + ); + assert.equal(result.status, 1, result.stderr); + const parsed = JSON.parse(result.stdout); + assert.equal(parsed.code, "heygen_cli_outdated"); + assert.equal(parsed.fix, HEYGEN_UPDATE_COMMAND); + cleanup(); +}); + test("human bundled fallback prints the install hint once", () => { setup(); const result = spawnResolve(["--type", "sfx", "--intent", "whoosh", "--project", tmp], { @@ -801,9 +868,12 @@ test("missing required args exits 2", () => { test("--json returns error JSON on stub provider failure", () => { setup(); + // A HeyGen CLI that finds nothing: the miss is the generic one on every machine, with or without a real CLI. + const binDir = writeFakeHeygen(`echo '{"data":[]}'`); try { runResolve(["--type", "bgm", "--intent", "stub fail", "--project", tmp, "--json"], { stdio: "pipe", + env: { HOME: tmp, PATH: binDir }, }); assert.fail("should have exited"); } catch (err) { @@ -1280,26 +1350,23 @@ async function captureResolveEvent({ provider, type = "bgm", intent }) { // their real email into this test's local-server payload despite HOME // being sandboxed (HEYGEN_CONFIG_DIR, not HOME, resolves the credentials // path). Every other test in this file keeps its untouched default env. - runResolve(["--type", type, "--intent", intent, "--project", tmp, "--json"], { - env: { - DO_NOT_TRACK: "0", - HYPERFRAMES_NO_TELEMETRY: "0", - CI: "", - NODE_ENV: "test", - HOME: sandboxHome, - HEYGEN_CONFIG_DIR: join(sandboxHome, ".heygen"), - MEDIA_USE_TELEMETRY_HOST: `http://127.0.0.1:${port}`, + // Async, so this process serves the POST while the child waits on it: a blocking spawn froze the server + // until the child gave up on its telemetry timeout, and the event arrived or not by luck. + const run = await spawnResolveAsync( + ["--type", type, "--intent", intent, "--project", tmp, "--json"], + { + env: { + DO_NOT_TRACK: "0", + HYPERFRAMES_NO_TELEMETRY: "0", + CI: "", + NODE_ENV: "test", + HOME: sandboxHome, + HEYGEN_CONFIG_DIR: join(sandboxHome, ".heygen"), + MEDIA_USE_TELEMETRY_HOST: `http://127.0.0.1:${port}`, + }, }, - }); - - // runResolve blocks synchronously (execFileSync) until the child exits, which - // pauses this process's own event loop for that whole span -- the child's - // request to our local server sits accepted-but-unprocessed in the kernel - // backlog until control returns here. Poll briefly to let the event loop - // drain it rather than asserting before the server has had a turn to run. - for (let i = 0; i < 100 && received.length === 0; i++) { - await new Promise((resolve) => setTimeout(resolve, 20)); - } + ); + assert.equal(run.status, 0, run.stderr); } finally { await new Promise((resolve) => server.close(resolve)); rmSync(sandboxHome, { recursive: true, force: true }); diff --git a/scripts/generate-skill-module-copies.mjs b/scripts/generate-skill-module-copies.mjs index 60cf9ef521..6c72a39e2a 100644 --- a/scripts/generate-skill-module-copies.mjs +++ b/scripts/generate-skill-module-copies.mjs @@ -13,6 +13,10 @@ export const skillModuleCopies = [ "skills/media-use/audio/scripts/lib/bgm-volume.mjs", workflows.map((skill) => `skills/${skill}/scripts/lib/bgm-volume.mjs`), ], + [ + "skills/media-use/audio/scripts/lib/host-audio.mjs", + workflows.map((skill) => `skills/${skill}/scripts/lib/host-audio.mjs`), + ], [ "skills/hyperframes/scripts/lib/frame-packets-core.mjs", [...workflows, "general-video"].map( diff --git a/skills-manifest.json b/skills-manifest.json index 00ac68f09b..aa3c795193 100644 --- a/skills-manifest.json +++ b/skills-manifest.json @@ -6,19 +6,19 @@ "files": 144 }, "faceless-explainer": { - "hash": "fd247c01901c395c", - "files": 27 + "hash": "3459707d807fdde5", + "files": 28 }, "figma": { "hash": "f8cf9f2c78ea8374", "files": 2 }, "general-video": { - "hash": "0fdb51f01b751cdb", + "hash": "7194a47a82647a88", "files": 5 }, "hyperframes": { - "hash": "75dfebd2be1e95bb", + "hash": "589ae40d15628917", "files": 29 }, "hyperframes-animation": { @@ -54,24 +54,24 @@ "files": 1 }, "media-use": { - "hash": "9b98931fc057f125", - "files": 107 + "hash": "b07cf8ba10a8ac7b", + "files": 109 }, "motion-graphics": { "hash": "6db77031ad839f6a", "files": 23 }, "music-to-video": { - "hash": "5b365022211eaf0c", + "hash": "b775f27b16efb1f2", "files": 169 }, "pr-to-video": { - "hash": "39e2b3e5613c5eec", - "files": 33 + "hash": "b38d166d7328e4cb", + "files": 34 }, "product-launch-video": { - "hash": "87a8a6895cb90865", - "files": 34 + "hash": "d78ebd2c45fa8bec", + "files": 35 }, "remotion-to-hyperframes": { "hash": "d02be65b7f3f4a63", diff --git a/skills/faceless-explainer/SKILL.md b/skills/faceless-explainer/SKILL.md index ff098235d8..2697f22147 100644 --- a/skills/faceless-explainer/SKILL.md +++ b/skills/faceless-explainer/SKILL.md @@ -7,7 +7,7 @@ description: "Turn arbitrary text — an article, notes, a topic, a brief — in > **First, keep this skill fresh — confirm with the user before running:** `npx hyperframes skills update faceless-explainer`. A fast no-op when everything is current; otherwise it refreshes this skill plus the core domain skills it depends on before you rely on them. -> **media-use**: Before sourcing audio/images/logos, call `/media-use` to resolve BGM/SFX/images from the HeyGen catalog and brand logos from their official sources. Run `--adopt` first to register existing assets. See `/media-use` skill. +> **media-use**: Before sourcing audio/images/logos, call `/media-use` to resolve BGM/SFX/images from the HeyGen catalog and brand logos from their official sources. Run `--adopt` first to register existing assets. See `/media-use` skill. When the host app provides its own music or sound-effect tools, use those for music and sound effects. # Faceless Explainer to HyperFrames @@ -105,13 +105,13 @@ Start audio after Step 3 approval. Run it in the background, then continue to St `node /scripts/audio.mjs --script ./SCRIPT.md --storyboard ./STORYBOARD.md --hyperframes . --out ./audio_meta.json --voice &` -The audio script handles narration, word timings, BGM lookup from HeyGen's music library, and timing metadata. BGM mood comes from the storyboard's `music:` field. This uses the HeyGen Audio API for retrieval, not generation, and the same `~/.heygen` credential as TTS. For provider details, read `../media-use/audio/references/tts.md`. +The audio script handles narration, word timings, BGM lookup from HeyGen's music library, and timing metadata. BGM mood comes from the storyboard's `music:` field. This uses the HeyGen Audio API for retrieval, not generation, and the same `~/.heygen` credential as TTS. For provider details, read `../media-use/audio/references/tts.md`. When the host app's own tools make the music or a sound effect, put the file under assets/ and list it in the audio_meta.json file marked `"source": "host"`: music as `bgm` `{ "path": "assets/…", "source": "host" }` with `music: none` in the storyboard; a sound as an `sfx` entry `{ "frame": , "file": "assets/…", "offset_s": 0, "duration_s": , "source": "host" }`. Every audio pass keeps these, and a frame with a host sound gets no looked-up cue. If there is no narration and no `SCRIPT.md`, skip voice generation. BGM may still run if the storyboard has a music mood. -**The canonical fully-silent marker** (shared across the workflows that reuse this audio model): `music: none` in the STORYBOARD.md top YAML block **and** no `SCRIPT.md`. That combination marks the project silent — no narration, no BGM, no SFX. `audio.mjs` recognizes it and generates nothing (it removes any stale `audio_meta.json`; an absent `audio_meta.json` is what assemble treats as silent), so this step is a clean skip. `music: none` with narration keeps TTS and turns only BGM off. Use exactly this spelling — don't improvise other markers. +**The canonical fully-silent marker** (shared across the workflows that reuse this audio model): `music: none` in the STORYBOARD.md top YAML block, no `SCRIPT.md`, **and** no `sfx:` cues or host audio. That combination marks the project silent — no narration, no BGM, no SFX. `audio.mjs` recognizes it and generates nothing (it removes any stale `audio_meta.json`; an absent `audio_meta.json` is what assemble treats as silent), so this step is a clean skip. Without narration but with `sfx:` cues or host audio the film is not silent: `audio.mjs` writes the audio_meta.json file for `fetch-sfx` to fill. `music: none` with narration keeps TTS and turns only BGM off. Use exactly this spelling — don't improvise other markers. -**Gate:** audio job has started, or the project is marked silent (`music: none` + no `SCRIPT.md`). +**Gate:** audio job has started, or the project is marked silent (`music: none`, no `SCRIPT.md`, no `sfx:` cues or host audio). --- diff --git a/skills/faceless-explainer/references/story-design.md b/skills/faceless-explainer/references/story-design.md index c7fba92e86..29fd26c419 100644 --- a/skills/faceless-explainer/references/story-design.md +++ b/skills/faceless-explainer/references/story-design.md @@ -215,7 +215,7 @@ Do not rewrite the user's words. Segment the script into frame-sized chunks at s The storyboard's top YAML block carries a `music:` field — the BGM mood the audio step retrieves against (e.g. `music: confident minimal tech underscore`). Omitting it falls back to `message:` → `arc:` → a neutral default, so BGM plays unless turned off explicitly. - **`music: none`** — BGM off (narration, if any, still runs). -- **`music: none` + no `SCRIPT.md`** — the canonical **fully-silent marker**: no narration, no BGM, no SFX. `audio.mjs` generates nothing and the audio step is a clean skip. Use exactly this spelling when the user asks for a silent / music-free video. +- **`music: none` + no `SCRIPT.md` + no `sfx:` cues or host audio** — the canonical **fully-silent marker**: no narration, no BGM, no SFX. `audio.mjs` generates nothing and the audio step is a clean skip. Use exactly this spelling when the user asks for a silent / music-free video. ## Frame template diff --git a/skills/faceless-explainer/scripts/audio.mjs b/skills/faceless-explainer/scripts/audio.mjs index 30ca2bd838..5e33feb45d 100644 --- a/skills/faceless-explainer/scripts/audio.mjs +++ b/skills/faceless-explainer/scripts/audio.mjs @@ -26,6 +26,7 @@ import { spawnSync } from "node:child_process"; import { existsSync, readFileSync, rmSync, writeFileSync } from "node:fs"; import { dirname, join, resolve } from "node:path"; import { fileURLToPath } from "node:url"; +import { hostAudio, keepHostAudio } from "./lib/host-audio.mjs"; import { parseStoryboard } from "./lib/storyboard.mjs"; const HERE = dirname(fileURLToPath(import.meta.url)); @@ -150,6 +151,28 @@ function runGenerate(argv) { String(g.extra?.music ?? "") .trim() .toLowerCase() === "none"; + const host = previousHostAudio(outPath, hyperframesDir, die, false); + const scored = sfxCueLines(manifest).length || host.bgm || host.sfx.length; + if (bgmOff && !lines.length && scored) { + // No narration and no looked-up music: the host's entries and any sounds fetch-sfx already found; a stale + // voice or bed in the engine's sidecar goes. + const neutral = neutralPath(outPath); + let looked = []; + try { + looked = JSON.parse(readFileSync(neutral, "utf8")).sfx ?? []; + } catch (err) { + if (err.code !== "ENOENT") die(`${neutral} does not parse: ${err.message}`); + } + const kept = { voices: [], bgm: null, bgm_pending: false, sfx: looked }; + writeFileSync(neutral, JSON.stringify(kept)); + const meta = keepHostAudio( + toProductLaunchMeta(kept), + previousHostAudio(outPath, hyperframesDir, die), + ); + writeFileSync(outPath, JSON.stringify(meta, null, 2)); + console.log(`✓ audio generate: no narration or music to make → ${outPath}`); + return; + } if (bgmOff && !lines.length) { rmSync(outPath, { force: true }); rmSync(neutralPath(outPath), { force: true }); @@ -178,13 +201,46 @@ function runGenerate(argv) { const neutral = neutralPath(outPath); runEngine({ request, hyperframesDir, neutral, only: "tts,bgm" }, die); - const meta = toProductLaunchMeta(JSON.parse(readFileSync(neutral, "utf8"))); + const rebuilt = toProductLaunchMeta(JSON.parse(readFileSync(neutral, "utf8"))); + const meta = keepHostAudio(rebuilt, previousHostAudio(outPath, hyperframesDir, die)); writeFileSync(outPath, JSON.stringify(meta, null, 2)); console.log( `✓ audio generate: ${meta.voices.length} voice + ${meta.bgm ? "1 bgm" : "no bgm"} → ${outPath}`, ); } +// Per-frame `sfx:` cues (comma-separated) → engine lines carrying only sfx. +// `filter(Boolean)` alone is not enough: a storyboard that spells "no SFX here" as +// `sfx: none` reaches the engine as a cue literally NAMED "none", which then fails to +// resolve. The absence sentinels are part of the storyboard vocabulary, so drop them. +const SFX_NONE = new Set(["none", "no", "n/a", "na", "skip", "-", "—", "–"]); +function sfxCueLines(manifest) { + const lines = []; + for (const f of manifest.frames) { + const names = (f.extra?.sfx ?? "") + .split(",") + .map((s) => s.trim()) + .filter((s) => s && !SFX_NONE.has(s.toLowerCase())); + if (names.length && f.number != null) lines.push({ id: pad2(f.number), sfx: names }); + } + return lines; +} + +// The host's own entries in audio_meta.json now (lib/host-audio.mjs). A pass reads them again for its final +// write, since the host may add audio while the engine runs. Only a missing file means none: one that does not +// parse stops the pass. +function previousHostAudio(outPath, hyperframesDir, die, warn = true) { + let previous = null; + try { + previous = JSON.parse(readFileSync(outPath, "utf8")); + } catch (err) { + if (err.code !== "ENOENT") die(`${outPath} does not parse: ${err.message}`); + } + const host = hostAudio(previous, (path) => existsSync(join(hyperframesDir, path))); + if (warn) for (const why of host.dropped) console.warn(`⚠ audio: host audio dropped: ${why}`); + return host; +} + // ── fetch-sfx ──────────────────────────────────────────────────────────────── function runFetchSfx(argv) { const die = (m) => { @@ -198,19 +254,8 @@ function runFetchSfx(argv) { if (!existsSync(storyboardPath)) die(`STORYBOARD.md not found at ${storyboardPath}`); const manifest = parseStoryboard(readFileSync(storyboardPath, "utf8")); - // Per-frame `sfx:` cues (comma-separated) → engine lines carrying only sfx. - // `filter(Boolean)` alone is not enough: a storyboard that spells "no SFX here" as - // `sfx: none` reaches the engine as a cue literally NAMED "none", which then fails to - // resolve. The absence sentinels are part of the storyboard vocabulary, so drop them. - const SFX_NONE = new Set(["none", "no", "n/a", "na", "skip", "-", "—", "–"]); - const lines = []; - for (const f of manifest.frames) { - const names = (f.extra?.sfx ?? "") - .split(",") - .map((s) => s.trim()) - .filter((s) => s && !SFX_NONE.has(s.toLowerCase())); - if (names.length && f.number != null) lines.push({ id: pad2(f.number), sfx: names }); - } + const host = previousHostAudio(outPath, hyperframesDir, die, false); + const lines = sfxCueLines(manifest).filter((line) => !host.frames.has(Number(line.id))); const neutral = neutralPath(outPath); const request = { lines, bgm: { mode: "none" } }; @@ -219,7 +264,8 @@ function runFetchSfx(argv) { // voices/bgm written by the earlier generate (--only tts,bgm) pass are preserved. runEngine({ request, hyperframesDir, neutral, only: "sfx" }, die); - const meta = toProductLaunchMeta(JSON.parse(readFileSync(neutral, "utf8"))); + const rebuilt = toProductLaunchMeta(JSON.parse(readFileSync(neutral, "utf8"))); + const meta = keepHostAudio(rebuilt, previousHostAudio(outPath, hyperframesDir, die)); writeFileSync(outPath, JSON.stringify(meta, null, 2)); console.log(`✓ audio fetch-sfx: ${meta.sfx.length} SFX cue(s) → ${outPath}`); // This pass rewrites audio_meta.json from the neutral sidecar. If a detached BGM generate is diff --git a/skills/faceless-explainer/scripts/audio.test.mjs b/skills/faceless-explainer/scripts/audio.test.mjs index cdad04f621..d1f3333fd2 100644 --- a/skills/faceless-explainer/scripts/audio.test.mjs +++ b/skills/faceless-explainer/scripts/audio.test.mjs @@ -1,7 +1,7 @@ import assert from "node:assert/strict"; -import { existsSync, mkdtempSync, readFileSync, writeFileSync } from "node:fs"; +import { existsSync, mkdirSync, mkdtempSync, readFileSync, writeFileSync } from "node:fs"; import { tmpdir } from "node:os"; -import { join } from "node:path"; +import { dirname, join } from "node:path"; import { spawnSync } from "node:child_process"; import test from "node:test"; @@ -24,12 +24,22 @@ test("audio.mjs is byte-identical to pr-to-video's copy (the stated contract)", // the storyboard's top YAML block + no SCRIPT.md ⇒ generate nothing; with // narration ⇒ TTS runs, BGM off. -function runAudioRaw({ storyboard, scriptMd = null, preexistingMeta = null }) { +function runAudioRaw({ + storyboard, + scriptMd = null, + preexistingMeta = null, + files = [], + duringEngine = "", +}) { const dir = mkdtempSync(join(tmpdir(), "faceless-audio-")); const engine = join(dir, "engine.mjs"); writeFileSync(join(dir, "STORYBOARD.md"), storyboard); if (scriptMd != null) writeFileSync(join(dir, "SCRIPT.md"), scriptMd); if (preexistingMeta != null) writeFileSync(join(dir, "audio_meta.json"), preexistingMeta); + for (const file of files) { + mkdirSync(dirname(join(dir, file)), { recursive: true }); + writeFileSync(join(dir, file), ""); + } writeFileSync( engine, `import { readFileSync, writeFileSync } from "node:fs"; @@ -38,7 +48,7 @@ const flag = (name) => argv[argv.indexOf(name) + 1]; const request = JSON.parse(readFileSync(flag("--request"), "utf8")); writeFileSync(new URL("request.json", import.meta.url), JSON.stringify(request)); writeFileSync(flag("--out"), JSON.stringify({ voices: [], bgm: null, sfx: [] })); -`, +${duringEngine}`, ); const result = spawnSync( process.execPath, @@ -95,10 +105,22 @@ test("a storyboard music mood still retrieves BGM (marker is exact, not fuzzy)", // (linear.app site showcase, 2026-07-30). /** Runs the fetch-sfx subcommand. `neutralOut` is what the stub engine writes to --out. */ -function runFetchSfx({ storyboard, neutralOut = { voices: [], bgm: null, sfx: [] } }) { +function runFetchSfx({ + storyboard, + neutralOut = { voices: [], bgm: null, sfx: [] }, + preexistingMeta = null, + files = [], + duringEngine = "", +}) { const dir = mkdtempSync(join(tmpdir(), "product-launch-sfx-")); const engine = join(dir, "engine.mjs"); writeFileSync(join(dir, "STORYBOARD.md"), storyboard); + if (preexistingMeta != null) + writeFileSync(join(dir, "audio_meta.json"), JSON.stringify(preexistingMeta)); + for (const file of files) { + mkdirSync(dirname(join(dir, file)), { recursive: true }); + writeFileSync(join(dir, file), ""); + } writeFileSync( engine, `import { readFileSync, writeFileSync } from "node:fs"; @@ -107,7 +129,7 @@ const flag = (name) => argv[argv.indexOf(name) + 1]; const request = JSON.parse(readFileSync(flag("--request"), "utf8")); writeFileSync(new URL("request.json", import.meta.url), JSON.stringify(request)); writeFileSync(flag("--out"), ${JSON.stringify(JSON.stringify(neutralOut))}); -`, +${duringEngine}`, ); const result = spawnSync( process.execPath, @@ -174,3 +196,146 @@ test("fetch-sfx: a resolved bed reports bgm_pending false and no warning", () => assert.equal(meta.bgm.path, "assets/bgm/track.mp3"); assert.doesNotMatch(result.stderr, /pending/i); }); + +// ── host audio (lib/host-audio.mjs) ───────────────────────────────────────── +// A host app's own music or sound, listed in audio_meta.json as "source": "host", survives every audio pass. + +const HOST_WHOOSH = { + frame: 1, + file: "assets/sfx/custom-whoosh.mp3", + offset_s: 0, + duration_s: 2, + source: "host", +}; +const HOST_BED = { path: "assets/bgm/host.mp3", source: "host" }; + +test("fetch-sfx keeps a host sound and looks up no cue for its frame", () => { + const { dir, result } = runFetchSfx({ + storyboard: FRAME_WITH_SFX("whoosh"), + preexistingMeta: { bgm: null, voices: [], sfx: [HOST_WHOOSH] }, + files: [HOST_WHOOSH.file], + neutralOut: { voices: [], bgm: null, sfx: [{ id: "01", file: "assets/sfx/whoosh.mp3" }] }, + }); + + assert.equal(result.status, 0, result.stderr); + assert.deepEqual(JSON.parse(readFileSync(join(dir, "request.json"), "utf8")).lines, []); + assert.deepEqual(JSON.parse(readFileSync(join(dir, "audio_meta.json"), "utf8")).sfx, [ + HOST_WHOOSH, + ]); +}); + +test("fetch-sfx keeps a host bed the storyboard does not look up", () => { + const { dir, result } = runFetchSfx({ + storyboard: FRAME_WITH_SFX("whoosh"), + preexistingMeta: { bgm: HOST_BED, voices: [], sfx: [] }, + files: [HOST_BED.path], + }); + + assert.equal(result.status, 0, result.stderr); + assert.deepEqual(JSON.parse(readFileSync(join(dir, "audio_meta.json"), "utf8")).bgm, HOST_BED); +}); + +test("music: none + no SCRIPT.md with a sound cue is not silent: audio_meta.json stays for fetch-sfx", () => { + const { dir, result } = runAudioRaw({ + storyboard: + "---\nmessage: Test\nmusic: none\n---\n\n## Frame 1 — Hook\n- duration: 3s\n- sfx: whoosh\n", + }); + + assert.equal(result.status, 0, result.stderr); + assert.doesNotMatch(result.stdout, /marked silent/); + assert.equal(existsSync(join(dir, "request.json")), false); + assert.deepEqual(JSON.parse(readFileSync(join(dir, "audio_meta.json"), "utf8")).sfx, []); +}); + +test("generate keeps a host bed when there is no narration and music: none", () => { + const { dir, result } = runAudioRaw({ + storyboard: "---\nmessage: Test\nmusic: none\n---\n", + preexistingMeta: JSON.stringify({ bgm: HOST_BED, voices: [], sfx: [] }), + files: [HOST_BED.path], + }); + + assert.equal(result.status, 0, result.stderr); + assert.deepEqual(JSON.parse(readFileSync(join(dir, "audio_meta.json"), "utf8")).bgm, HOST_BED); +}); + +test("fetch-sfx stops on an audio_meta.json that does not parse, so host entries are not lost", () => { + const { dir } = runFetchSfx({ storyboard: FRAME_WITH_SFX("whoosh") }); + writeFileSync( + join(dir, "audio_meta.json"), + '{ "bgm": { "path": "assets/bgm/host.mp3", "source": "host" }, }', + ); + const again = spawnSync( + process.execPath, + [script, "fetch-sfx", "--hyperframes", dir, "--storyboard", join(dir, "STORYBOARD.md")], + { encoding: "utf8", env: { ...process.env, HF_MEDIA_ENGINE: join(dir, "engine.mjs") } }, + ); + + assert.equal(again.status, 1); + assert.match(again.stderr, /does not parse/); + assert.match(readFileSync(join(dir, "audio_meta.json"), "utf8"), /host\.mp3/); +}); + +test("re-running generate with no narration keeps the sounds fetch-sfx found", () => { + const storyboard = + "---\nmessage: Test\nmusic: none\n---\n\n## Frame 1 — Hook\n- duration: 3s\n- sfx: whoosh\n"; + const { dir, result } = runAudioRaw({ storyboard }); + assert.equal(result.status, 0, result.stderr); + writeFileSync( + join(dir, "audio_engine_meta.json"), + JSON.stringify({ + voices: [{ id: "01", path: "old.wav" }], + bgm: { path: "old.mp3" }, + sfx: [{ id: "01", file: "assets/sfx/whoosh.mp3" }], + }), + ); + const again = spawnSync( + process.execPath, + [script, "--hyperframes", dir, "--storyboard", join(dir, "STORYBOARD.md")], + { + encoding: "utf8", + }, + ); + + assert.equal(again.status, 0, again.stderr); + const meta = JSON.parse(readFileSync(join(dir, "audio_meta.json"), "utf8")); + assert.deepEqual( + meta.sfx.map((s) => s.file), + ["assets/sfx/whoosh.mp3"], + ); + assert.deepEqual(meta.voices, []); + assert.equal(meta.bgm, null); +}); + +// What a host app writes into audio_meta.json while a pass's engine is still running. +const HOST_WRITES_MID_PASS = ` +const { mkdirSync } = await import("node:fs"); +for (const dir of ["assets/bgm/", "assets/sfx/"]) mkdirSync(new URL(dir, import.meta.url), { recursive: true }); +writeFileSync(new URL("${HOST_BED.path}", import.meta.url), ""); +writeFileSync(new URL("${HOST_WHOOSH.file}", import.meta.url), ""); +writeFileSync(new URL("audio_meta.json", import.meta.url), ${JSON.stringify(JSON.stringify({ bgm: HOST_BED, voices: [], sfx: [HOST_WHOOSH] }))}); +`; + +test("generate keeps host audio written while its engine runs", () => { + const { dir, result } = runAudioRaw({ + storyboard: "---\nmessage: Test\nmusic: calm piano\n---\n", + duringEngine: HOST_WRITES_MID_PASS, + }); + + assert.equal(result.status, 0, result.stderr); + const meta = JSON.parse(readFileSync(join(dir, "audio_meta.json"), "utf8")); + assert.deepEqual(meta.bgm, HOST_BED); + assert.deepEqual(meta.sfx, [HOST_WHOOSH]); +}); + +test("fetch-sfx keeps host audio written while its engine runs, and the host's sound wins its frame", () => { + const { dir, result } = runFetchSfx({ + storyboard: FRAME_WITH_SFX("whoosh"), + neutralOut: { voices: [], bgm: null, sfx: [{ id: "01", file: "assets/sfx/whoosh.mp3" }] }, + duringEngine: HOST_WRITES_MID_PASS, + }); + + assert.equal(result.status, 0, result.stderr); + const meta = JSON.parse(readFileSync(join(dir, "audio_meta.json"), "utf8")); + assert.deepEqual(meta.bgm, HOST_BED); + assert.deepEqual(meta.sfx, [HOST_WHOOSH]); +}); diff --git a/skills/faceless-explainer/scripts/lib/host-audio.mjs b/skills/faceless-explainer/scripts/lib/host-audio.mjs new file mode 100644 index 0000000000..fa45826a82 --- /dev/null +++ b/skills/faceless-explainer/scripts/lib/host-audio.mjs @@ -0,0 +1,28 @@ +// Audio the host app supplied survives every pass that rebuilds audio_meta.json from the storyboard: an entry +// marked "source": "host" whose file exists. A frame the host scored takes no looked-up cue sound. +/** The host's own entries in an earlier audio_meta.json, and why any other host-marked entry was dropped; `exists` + * resolves a path from the project root. A frame written as "1" is frame 1. */ +export function hostAudio(previous, exists) { + const marked = [[previous?.bgm, "path"], ...(previous?.sfx ?? []).map((s) => [s, "file"])].filter( + ([entry]) => entry?.source === "host", + ); + const kept = new Set(); + const dropped = []; + for (const [entry, key] of marked) { + const path = entry[key]; + if (typeof path !== "string") dropped.push(`a host entry has no "${key}"`); + else if (!exists(path)) dropped.push(`${path} is not on disk`); + else kept.add(entry); + } + const bgm = kept.has(previous?.bgm) ? previous.bgm : null; + const sfx = (previous?.sfx ?? []) + .filter((s) => kept.has(s)) + .map((s) => ({ ...s, frame: Number(s.frame) })); + return { bgm, sfx, frames: new Set(sfx.map((s) => s.frame)), dropped }; +} + +/** `rebuilt` with the host's bed and sounds put back; the host's bed wins over a looked-up one. */ +export function keepHostAudio(rebuilt, host) { + const looked = (rebuilt.sfx ?? []).filter((s) => !host.frames.has(s.frame)); + return { ...rebuilt, bgm: host.bgm ?? rebuilt.bgm, sfx: [...host.sfx, ...looked] }; +} diff --git a/skills/general-video/SKILL.md b/skills/general-video/SKILL.md index f1f5f624f8..2fec78bf88 100644 --- a/skills/general-video/SKILL.md +++ b/skills/general-video/SKILL.md @@ -92,7 +92,7 @@ These reads are mandatory when their condition matches: | Any non-trivial creation or visual treatment | `/hyperframes-creative` → `references/house-style.md` and `references/video-composition.md` | | Any motion, animation, or scene transition | `/hyperframes-animation`; follow its routing to the matching rules, adapters, blueprints, or transition references | | `storyboard: yes` | `../hyperframes/references/storyboard-format.md` and `../hyperframes/references/review-loop.md` | -| Any media asset or operation, including narration, BGM, SFX, captions, grading, or transforms | `/media-use`; for framework playback and placement also read `/hyperframes-core` → `references/variables-and-media.md` | +| Any media asset or operation, including narration, BGM, SFX, captions, grading, or transforms | `/media-use` (a host app's own music and sound-effect tools come first for those); for framework playback and placement also read `/hyperframes-core` → `references/variables-and-media.md` | | Multi-scene assembly | `../hyperframes/references/production-loop.md` | | `flow: companion`, before the first plan | `/hyperframes-creative` → `references/story-spine.md` and `references/house-style.md`; the nearest genre lens and the full `../hyperframes/references/capability-menu.md` — the ceiling treatment is designed from these, not recalled | | A companion capability offer, capture, beat grid, generative video, map, publishing, or cross-workflow capability | `../hyperframes/references/capability-menu.md` | diff --git a/skills/hyperframes/references/capability-menu.md b/skills/hyperframes/references/capability-menu.md index 98c3662d9c..1bf663e4d3 100644 --- a/skills/hyperframes/references/capability-menu.md +++ b/skills/hyperframes/references/capability-menu.md @@ -12,7 +12,7 @@ Each row's last column reads **home → entry → what you get**: the owning ski | **Website capture** — headless-Chrome crawl of a real site: screenshots, brand tokens, assets | "I can capture your site and build from its real look" | CLI → `npx hyperframes capture -o ./capture` → `capture/` (screenshots, extracted tokens/text/assets); doctrine: `/product-launch-video` | | **Beat analysis & audio-reactive motion** — a deterministic beat / energy map of a track; cuts land on the grid, elements pulse with the music | "if there's music, I can cut the video to its beat — and make elements move with it" | grid: `/music-to-video` → `scripts/analyze-beatgrid.py` → `audiomap.json` (beats, energy, sections); element reaction: `/hyperframes-creative` → `references/audio-reactive.md` + `scripts/extract-audio-data.py` | | **Motion blueprints** — proven scene shapes (reveals, counters, charts, dioramas) picked per beat | "each scene gets a proven motion treatment, not improvised movement" | `/hyperframes-animation` → `blueprints-index.md` + `rules-index.md` → per-beat `blueprint:` ids the build reads | -| **Voice, music, SFX, images, logos, media treatments** — resolved media plus source-aware color, effects, privacy, reveals, and justified overlays | "narration, music, sound effects, real assets — and footage polished or stylized to fit the story" | `/media-use` → deterministic resolve/operate tools plus `references/media-treatments.md`; shader pixels persist through `hyperframes media-treatment`, optional overlays come from Registry, and finite motion uses the host GSAP timeline | +| **Voice, music, SFX, images, logos, media treatments** — resolved media plus source-aware color, effects, privacy, reveals, and justified overlays | "narration, music, sound effects, real assets — and footage polished or stylized to fit the story" | `/media-use` (a host app's own music and sound-effect tools come first for those) → deterministic resolve/operate tools plus `references/media-treatments.md`; shader pixels persist through `hyperframes media-treatment`, optional overlays come from Registry, and finite motion uses the host GSAP timeline | | **Generative video** — an AI presenter delivers the script; a still photo becomes a talking clip; a finished video gets dubbed into another language | "an AI presenter can read your script on camera; I can animate a photo into a talking clip, or dub the video" | `/media-use` → `references/operations.md` § Generate: video (`heygen video create` / `video-translate`; OAuth free allowance where eligible) → an mp4 clip adopted into `assets/` + manifest record | | **Transcription & captions** — word-timed transcripts; styled caption skins on the finished video | "accurate captions, styled to match" | `/media-use` → `scripts/transcribe.mjs` → word-timed transcript; caption machinery `captions.mjs` (home: `/faceless-explainer`) → the caption track | | **Cut footage by its transcript** — trim a clip by choosing sentences, not timecodes | "I can trim your clip by picking the sentences to keep" | `/media-use` → `scripts/transcript-cut.mjs` → the trimmed clip + updated transcript | diff --git a/skills/media-use/SKILL.md b/skills/media-use/SKILL.md index ec0d30ec1e..f6bbb921aa 100644 --- a/skills/media-use/SKILL.md +++ b/skills/media-use/SKILL.md @@ -1,6 +1,6 @@ --- name: media-use -description: Agent Media OS, the single skill for every media need in a HyperFrames project. Resolve BGM, SFX, image, icon, brand logo, voice, color grade, or LUT into a frozen local file or paste-ready block + ledger record (one verb, `resolve`); generate via TTS / music / image models when the catalog misses; produce voiceover, transcription, captions, and background removal through one shared audio engine; operate on media (cut / reframe / transform); and reuse assets across projects. Also use for vague feedback that real footage looks dark, flat, boring, should feel retro/camcorder/print/ASCII, needs privacy, or needs a media reveal. +description: Agent Media OS for a HyperFrames project. Resolve BGM, SFX, image, icon, brand logo, voice, color grade, or LUT into a frozen local file or paste-ready block + ledger record (one verb, `resolve`); generate via TTS / music / image models when the catalog misses; produce voiceover, transcription, captions, and background removal through one shared audio engine; operate on media (cut / reframe / transform); and reuse assets across projects. Also use for vague feedback that real footage looks dark, flat, boring, should feel retro/camcorder/print/ASCII, needs privacy, or needs a media reveal. When the host app provides its own music or sound-effect tools, use those for music and sound effects; `resolve --type bgm|sfx` needs the heygen CLI. --- **Plugin installs:** Before setup or freshness commands, follow [plugin execution rules](../hyperframes/references/plugin-installation.md) when this skill is inside a HyperFrames plugin. Standalone installs keep the update instructions below. @@ -9,7 +9,9 @@ description: Agent Media OS, the single skill for every media need in a HyperFra The media OS for HyperFrames: resolve · generate · operate · remember — every media type, one skill, zero context noise. -First run: install and sign in to the `heygen` CLI (the free-usage path), then verify with `npx hyperframes media-use resolve --doctor`. Setup and providers: `references/setup-providers.md`. +First run, when you will use HeyGen media (catalog search, TTS, avatar video): install and sign in to the `heygen` CLI (the free-usage path), then verify with `npx hyperframes media-use resolve --doctor`. Setup and providers: `references/setup-providers.md`. + +Music and sound effects inside a host app: when the app you run in gives you its own music or sound-effect tools, use those. `resolve --type bgm` and `--type sfx` search the HeyGen catalog through the `heygen` CLI; without it they fail and say so (`sfx` still answers from its bundled library). Before generating a voiceover or an avatar video, tell the person: signing in to the heygen CLI with OAuth (`heygen auth login --oauth`) gives a free allowance for TTS voiceover and avatar videos, while an API key bills API credits. @@ -23,8 +25,8 @@ Returns one line: `resolved → (, )`. All search no | Type | One-line intent | | ------- | -------------------------------------------------------------------------------- | -| `bgm` | background music (HeyGen catalog, 10k+ tracks) | -| `sfx` | sound effects (bundled 19-file library + catalog) | +| `bgm` | background music (HeyGen catalog via the `heygen` CLI, 10k+ tracks) | +| `sfx` | sound effects (bundled 19-file library + catalog via the `heygen` CLI) | | `image` | photos, backgrounds (HeyGen asset search, 75k+ vectors) | | `icon` | icons, symbols (transparent) | | `logo` | official brand marks (theSVG → GitHub avatar → favicon; never redrawn) | diff --git a/skills/media-use/audio/scripts/lib/host-audio.mjs b/skills/media-use/audio/scripts/lib/host-audio.mjs new file mode 100644 index 0000000000..fa45826a82 --- /dev/null +++ b/skills/media-use/audio/scripts/lib/host-audio.mjs @@ -0,0 +1,28 @@ +// Audio the host app supplied survives every pass that rebuilds audio_meta.json from the storyboard: an entry +// marked "source": "host" whose file exists. A frame the host scored takes no looked-up cue sound. +/** The host's own entries in an earlier audio_meta.json, and why any other host-marked entry was dropped; `exists` + * resolves a path from the project root. A frame written as "1" is frame 1. */ +export function hostAudio(previous, exists) { + const marked = [[previous?.bgm, "path"], ...(previous?.sfx ?? []).map((s) => [s, "file"])].filter( + ([entry]) => entry?.source === "host", + ); + const kept = new Set(); + const dropped = []; + for (const [entry, key] of marked) { + const path = entry[key]; + if (typeof path !== "string") dropped.push(`a host entry has no "${key}"`); + else if (!exists(path)) dropped.push(`${path} is not on disk`); + else kept.add(entry); + } + const bgm = kept.has(previous?.bgm) ? previous.bgm : null; + const sfx = (previous?.sfx ?? []) + .filter((s) => kept.has(s)) + .map((s) => ({ ...s, frame: Number(s.frame) })); + return { bgm, sfx, frames: new Set(sfx.map((s) => s.frame)), dropped }; +} + +/** `rebuilt` with the host's bed and sounds put back; the host's bed wins over a looked-up one. */ +export function keepHostAudio(rebuilt, host) { + const looked = (rebuilt.sfx ?? []).filter((s) => !host.frames.has(s.frame)); + return { ...rebuilt, bgm: host.bgm ?? rebuilt.bgm, sfx: [...host.sfx, ...looked] }; +} diff --git a/skills/media-use/audio/scripts/lib/host-audio.test.mjs b/skills/media-use/audio/scripts/lib/host-audio.test.mjs new file mode 100644 index 0000000000..0cabc26fad --- /dev/null +++ b/skills/media-use/audio/scripts/lib/host-audio.test.mjs @@ -0,0 +1,106 @@ +import assert from "node:assert/strict"; +import { readFileSync } from "node:fs"; +import test from "node:test"; +import { hostAudio, keepHostAudio } from "./host-audio.mjs"; + +const onDisk = new Set([ + "assets/bgm/host.mp3", + "assets/sfx/custom-whoosh.mp3", + "assets/sfx/whoosh.mp3", +]); +const exists = (path) => onDisk.has(path); +const hostWhoosh = { + frame: 1, + file: "assets/sfx/custom-whoosh.mp3", + offset_s: 0, + duration_s: 2, + source: "host", +}; + +test("a host sound and bed survive a rebuild, and the host's frame takes no looked-up sound", () => { + const previous = { + bgm: { path: "assets/bgm/host.mp3", source: "host" }, + sfx: [hostWhoosh, { frame: 2, file: "assets/sfx/whoosh.mp3", offset_s: 0 }], + }; + const rebuilt = { + bgm: null, + voices: [], + sfx: [ + { frame: 1, file: "assets/sfx/whoosh.mp3", offset_s: 0 }, + { frame: 2, file: "assets/sfx/whoosh.mp3", offset_s: 0 }, + ], + }; + const kept = keepHostAudio(rebuilt, hostAudio(previous, exists)); + assert.deepEqual(kept.bgm, previous.bgm); + assert.deepEqual(kept.sfx, [hostWhoosh, rebuilt.sfx[1]]); +}); + +test("an unmarked entry or a missing file is not the host's", () => { + const previous = { + bgm: { path: "assets/bgm/gone.mp3", source: "host" }, + sfx: [ + { frame: 1, file: "assets/sfx/whoosh.mp3", offset_s: 0 }, + { frame: 3, file: "assets/sfx/gone.mp3", source: "host" }, + ], + }; + const host = hostAudio(previous, exists); + assert.equal(host.bgm, null); + assert.deepEqual(host.sfx, []); + const rebuilt = { bgm: { path: "assets/bgm/looked.mp3" }, voices: [], sfx: [] }; + assert.deepEqual(keepHostAudio(rebuilt, host), rebuilt); +}); + +test("no earlier audio_meta.json keeps the rebuild as it is", () => { + const rebuilt = { bgm: null, voices: [], sfx: [{ frame: 1, file: "assets/sfx/whoosh.mp3" }] }; + assert.deepEqual(keepHostAudio(rebuilt, hostAudio(null, exists)), rebuilt); +}); + +// The narrated workflows tell the agent the same host-audio contract and the same meaning of silent. +const workflowText = (skill, file) => + readFileSync(new URL(`../../../../${skill}/${file}`, import.meta.url), "utf8"); + +test("each narrated workflow tells a host app's music and sounds the way in, and when a film is silent", () => { + for (const skill of ["product-launch-video", "pr-to-video", "faceless-explainer"]) { + const steps = workflowText(skill, "SKILL.md"); + assert.match( + steps, + /host app's own tools make the music or a sound effect[^\n]+"source": "host"/, + skill, + ); + assert.match(steps, /"duration_s": [^\n]+Every audio pass keeps these/, skill); + assert.match(steps, /no `SCRIPT.md`, \*\*and\*\* no `sfx:` cues or host audio/, skill); + assert.match( + steps, + /marked silent \(`music: none`, no `SCRIPT.md`, no `sfx:` cues or host audio\)/, + skill, + ); + assert.match( + workflowText(skill, "references/story-design.md"), + /no `SCRIPT.md` \+ no `sfx:` cues or host audio\*\* — the canonical/, + skill, + ); + } +}); + +test("a frame written as a string is the same frame, and a gone host file is reported", () => { + const previous = { + bgm: { path: "assets/bgm/gone.mp3", source: "host" }, + sfx: [{ ...hostWhoosh, frame: "1" }], + }; + const host = hostAudio(previous, exists); + assert.deepEqual(host.sfx, [hostWhoosh]); + assert.ok(host.frames.has(1)); + assert.deepEqual(host.dropped, ["assets/bgm/gone.mp3 is not on disk"]); + const rebuilt = { bgm: null, voices: [], sfx: [{ frame: 1, file: "assets/sfx/whoosh.mp3" }] }; + assert.deepEqual(keepHostAudio(rebuilt, host).sfx, [hostWhoosh]); +}); + +test("a host entry with no path is reported, not thrown", () => { + const throwsOnNonString = (path) => { + if (typeof path !== "string") throw new TypeError("path must be a string"); + return onDisk.has(path); + }; + const host = hostAudio({ bgm: { src: "x.mp3", source: "host" }, sfx: [] }, throwsOnNonString); + assert.equal(host.bgm, null); + assert.deepEqual(host.dropped, ['a host entry has no "path"']); +}); diff --git a/skills/media-use/references/setup-providers.md b/skills/media-use/references/setup-providers.md index 42ce0d5ae6..c6a8acd958 100644 --- a/skills/media-use/references/setup-providers.md +++ b/skills/media-use/references/setup-providers.md @@ -19,7 +19,8 @@ npx hyperframes media-use resolve --doctor media-use holds no keys; every external tool owns its auth. Generation is centered on the HeyGen CLI free-usage path. Install and authenticate `heygen` -before resolving bgm/sfx/image/icon/voice/avatar-video. Local tools are opt-in +before resolving bgm/sfx/image/icon/voice/avatar-video; music and sound effects from +a host app's own tools need none of it. Local tools are opt-in alternatives where they exist: mflux for image, Kokoro for voice, Parakeet for transcription, and LTX for local video generation. `resolve` spec-checks AVAILABLE RAM for those local ladders (`describeModelLadder`); the agent can @@ -27,7 +28,7 @@ see the ladder and override. | Type | Provider / path | | --------- | ----------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | -| bgm/sfx | heygen catalog free-usage path | +| bgm/sfx | heygen catalog free-usage path; a host app's own music and sound tools come first | | image | heygen search free-usage path; optional local mflux; codex `image_gen` upsell | | voice | heygen tts free-usage path; optional local **Kokoro** (free, on-device) | | icon | heygen asset search free-usage path | diff --git a/skills/music-to-video/SKILL.md b/skills/music-to-video/SKILL.md index 6acd406123..ea7c085dfe 100644 --- a/skills/music-to-video/SKILL.md +++ b/skills/music-to-video/SKILL.md @@ -30,7 +30,7 @@ Goal: Establish the music source, create the HyperFrames project, and note any u **The brief starts at the intent layer.** Opening rule, in order: **(1)** `BRIEF.md` exists → read it and ask nothing it answers — its `flow`/`storyboard` derive the mode (brief contract § 1). **(2)** No `BRIEF.md` but the project exists → resume from what's on disk; never re-interrogate. **(3)** A fresh creation request that arrived here directly → read `/hyperframes` and run its intent layer (`references/intent-interview.md`): it confirms this route's must-haves (the music source, destination → aspect — `../hyperframes/references/routes/music-to-video.md`) and announces what stays deferred — brand and genre are chosen at Step 3 by design. Write `BRIEF.md` immediately after init (never before — `init` refuses a non-empty directory) and record the preference-backed answers (`brief-format.md`). Edit requests skip all of this. -The **music is the spine** — establish one track before anything else. This skill is tuned for **fast, high-energy BGM**: a strong beat grid drives the cuts (calm tracks work, but pace by phrase rather than beat). If the user supplied audio — a music file, or a video to pull audio from — use it. Otherwise choose the mood from the request and generate a track through `/media-use` (`audio/references/bgm.md`). Before the first authenticated provider action, run `npx hyperframes auth status` and relay its output verbatim. If signed out, apply one branch: +The **music is the spine** — establish one track before anything else. This skill is tuned for **fast, high-energy BGM**: a strong beat grid drives the cuts (calm tracks work, but pace by phrase rather than beat). If the user supplied audio — a music file, or a video to pull audio from — use it. Otherwise choose the mood from the request and generate a track through `/media-use` (`audio/references/bgm.md`); when the host app provides its own music tool, use that. Before the first authenticated provider action, run `npx hyperframes auth status` and relay its output verbatim. If signed out, apply one branch: - **Collaborative:** wait for sign-in or an explicit choice to continue offline with the local provider. - **Autonomous:** state the status and continue through the available local provider. diff --git a/skills/pr-to-video/SKILL.md b/skills/pr-to-video/SKILL.md index decd09ff60..fe399ecb53 100644 --- a/skills/pr-to-video/SKILL.md +++ b/skills/pr-to-video/SKILL.md @@ -7,7 +7,7 @@ description: "Turn a GitHub pull request (a PR URL, owner/repo#N, or 'this PR' i > **First, keep this skill fresh — confirm with the user before running:** `npx hyperframes skills update pr-to-video`. A fast no-op when everything is current; otherwise it refreshes this skill plus the core domain skills it depends on before you rely on them. -> **media-use**: Before sourcing audio/images/logos, call `/media-use` to resolve BGM/SFX/images from the HeyGen catalog and brand logos from their official sources. Run `--adopt` first to register existing assets. See `/media-use` skill. +> **media-use**: Before sourcing audio/images/logos, call `/media-use` to resolve BGM/SFX/images from the HeyGen catalog and brand logos from their official sources. Run `--adopt` first to register existing assets. See `/media-use` skill. When the host app provides its own music or sound-effect tools, use those for music and sound effects. # PR to HyperFrames @@ -133,13 +133,13 @@ Start audio after Step 3 approval. Run it in the background, then continue to St `node /scripts/audio.mjs --script ./SCRIPT.md --storyboard ./STORYBOARD.md --hyperframes . --out ./audio_meta.json --voice &` -The audio script handles narration, word timings, BGM lookup from HeyGen's music library, and timing metadata. BGM mood comes from the storyboard's `music:` field. This uses the HeyGen Audio API for retrieval, not generation, and the same `~/.heygen` credential as TTS. For provider details, read `../media-use/audio/references/tts.md`. +The audio script handles narration, word timings, BGM lookup from HeyGen's music library, and timing metadata. BGM mood comes from the storyboard's `music:` field. This uses the HeyGen Audio API for retrieval, not generation, and the same `~/.heygen` credential as TTS. For provider details, read `../media-use/audio/references/tts.md`. When the host app's own tools make the music or a sound effect, put the file under assets/ and list it in the audio_meta.json file marked `"source": "host"`: music as `bgm` `{ "path": "assets/…", "source": "host" }` with `music: none` in the storyboard; a sound as an `sfx` entry `{ "frame": , "file": "assets/…", "offset_s": 0, "duration_s": , "source": "host" }`. Every audio pass keeps these, and a frame with a host sound gets no looked-up cue. If there is no narration and no `SCRIPT.md`, skip voice generation. BGM may still run if the storyboard has a music mood. -**The canonical fully-silent marker** (shared across the workflows that reuse this audio model): `music: none` in the STORYBOARD.md top YAML block **and** no `SCRIPT.md`. That combination marks the project silent — no narration, no BGM, no SFX. `audio.mjs` recognizes it and generates nothing (it removes any stale `audio_meta.json`; an absent `audio_meta.json` is what assemble treats as silent), so this step is a clean skip. `music: none` with narration keeps TTS and turns only BGM off. Use exactly this spelling — don't improvise other markers. +**The canonical fully-silent marker** (shared across the workflows that reuse this audio model): `music: none` in the STORYBOARD.md top YAML block, no `SCRIPT.md`, **and** no `sfx:` cues or host audio. That combination marks the project silent — no narration, no BGM, no SFX. `audio.mjs` recognizes it and generates nothing (it removes any stale `audio_meta.json`; an absent `audio_meta.json` is what assemble treats as silent), so this step is a clean skip. Without narration but with `sfx:` cues or host audio the film is not silent: `audio.mjs` writes the audio_meta.json file for `fetch-sfx` to fill. `music: none` with narration keeps TTS and turns only BGM off. Use exactly this spelling — don't improvise other markers. -**Gate:** audio job has started, or the project is marked silent (`music: none` + no `SCRIPT.md`). +**Gate:** audio job has started, or the project is marked silent (`music: none`, no `SCRIPT.md`, no `sfx:` cues or host audio). --- diff --git a/skills/pr-to-video/references/story-design.md b/skills/pr-to-video/references/story-design.md index 95bd6d7922..5aa779af31 100644 --- a/skills/pr-to-video/references/story-design.md +++ b/skills/pr-to-video/references/story-design.md @@ -190,7 +190,7 @@ Estimate while writing: `duration ≈ ceil(word_count / 2.2)`. 29 words → 13s The storyboard's top YAML block carries a `music:` field — the BGM mood the audio step retrieves against (e.g. `music: confident minimal tech underscore`). Omitting it falls back to `message:` → `arc:` → a neutral default, so BGM plays unless turned off explicitly. - **`music: none`** — BGM off (narration, if any, still runs). -- **`music: none` + no `SCRIPT.md`** — the canonical **fully-silent marker**: no narration, no BGM, no SFX. `audio.mjs` generates nothing and the audio step is a clean skip. Use exactly this spelling when the user asks for a silent / music-free video. +- **`music: none` + no `SCRIPT.md` + no `sfx:` cues or host audio** — the canonical **fully-silent marker**: no narration, no BGM, no SFX. `audio.mjs` generates nothing and the audio step is a clean skip. Use exactly this spelling when the user asks for a silent / music-free video. ## Frame template diff --git a/skills/pr-to-video/scripts/audio.mjs b/skills/pr-to-video/scripts/audio.mjs index 30ca2bd838..5e33feb45d 100644 --- a/skills/pr-to-video/scripts/audio.mjs +++ b/skills/pr-to-video/scripts/audio.mjs @@ -26,6 +26,7 @@ import { spawnSync } from "node:child_process"; import { existsSync, readFileSync, rmSync, writeFileSync } from "node:fs"; import { dirname, join, resolve } from "node:path"; import { fileURLToPath } from "node:url"; +import { hostAudio, keepHostAudio } from "./lib/host-audio.mjs"; import { parseStoryboard } from "./lib/storyboard.mjs"; const HERE = dirname(fileURLToPath(import.meta.url)); @@ -150,6 +151,28 @@ function runGenerate(argv) { String(g.extra?.music ?? "") .trim() .toLowerCase() === "none"; + const host = previousHostAudio(outPath, hyperframesDir, die, false); + const scored = sfxCueLines(manifest).length || host.bgm || host.sfx.length; + if (bgmOff && !lines.length && scored) { + // No narration and no looked-up music: the host's entries and any sounds fetch-sfx already found; a stale + // voice or bed in the engine's sidecar goes. + const neutral = neutralPath(outPath); + let looked = []; + try { + looked = JSON.parse(readFileSync(neutral, "utf8")).sfx ?? []; + } catch (err) { + if (err.code !== "ENOENT") die(`${neutral} does not parse: ${err.message}`); + } + const kept = { voices: [], bgm: null, bgm_pending: false, sfx: looked }; + writeFileSync(neutral, JSON.stringify(kept)); + const meta = keepHostAudio( + toProductLaunchMeta(kept), + previousHostAudio(outPath, hyperframesDir, die), + ); + writeFileSync(outPath, JSON.stringify(meta, null, 2)); + console.log(`✓ audio generate: no narration or music to make → ${outPath}`); + return; + } if (bgmOff && !lines.length) { rmSync(outPath, { force: true }); rmSync(neutralPath(outPath), { force: true }); @@ -178,13 +201,46 @@ function runGenerate(argv) { const neutral = neutralPath(outPath); runEngine({ request, hyperframesDir, neutral, only: "tts,bgm" }, die); - const meta = toProductLaunchMeta(JSON.parse(readFileSync(neutral, "utf8"))); + const rebuilt = toProductLaunchMeta(JSON.parse(readFileSync(neutral, "utf8"))); + const meta = keepHostAudio(rebuilt, previousHostAudio(outPath, hyperframesDir, die)); writeFileSync(outPath, JSON.stringify(meta, null, 2)); console.log( `✓ audio generate: ${meta.voices.length} voice + ${meta.bgm ? "1 bgm" : "no bgm"} → ${outPath}`, ); } +// Per-frame `sfx:` cues (comma-separated) → engine lines carrying only sfx. +// `filter(Boolean)` alone is not enough: a storyboard that spells "no SFX here" as +// `sfx: none` reaches the engine as a cue literally NAMED "none", which then fails to +// resolve. The absence sentinels are part of the storyboard vocabulary, so drop them. +const SFX_NONE = new Set(["none", "no", "n/a", "na", "skip", "-", "—", "–"]); +function sfxCueLines(manifest) { + const lines = []; + for (const f of manifest.frames) { + const names = (f.extra?.sfx ?? "") + .split(",") + .map((s) => s.trim()) + .filter((s) => s && !SFX_NONE.has(s.toLowerCase())); + if (names.length && f.number != null) lines.push({ id: pad2(f.number), sfx: names }); + } + return lines; +} + +// The host's own entries in audio_meta.json now (lib/host-audio.mjs). A pass reads them again for its final +// write, since the host may add audio while the engine runs. Only a missing file means none: one that does not +// parse stops the pass. +function previousHostAudio(outPath, hyperframesDir, die, warn = true) { + let previous = null; + try { + previous = JSON.parse(readFileSync(outPath, "utf8")); + } catch (err) { + if (err.code !== "ENOENT") die(`${outPath} does not parse: ${err.message}`); + } + const host = hostAudio(previous, (path) => existsSync(join(hyperframesDir, path))); + if (warn) for (const why of host.dropped) console.warn(`⚠ audio: host audio dropped: ${why}`); + return host; +} + // ── fetch-sfx ──────────────────────────────────────────────────────────────── function runFetchSfx(argv) { const die = (m) => { @@ -198,19 +254,8 @@ function runFetchSfx(argv) { if (!existsSync(storyboardPath)) die(`STORYBOARD.md not found at ${storyboardPath}`); const manifest = parseStoryboard(readFileSync(storyboardPath, "utf8")); - // Per-frame `sfx:` cues (comma-separated) → engine lines carrying only sfx. - // `filter(Boolean)` alone is not enough: a storyboard that spells "no SFX here" as - // `sfx: none` reaches the engine as a cue literally NAMED "none", which then fails to - // resolve. The absence sentinels are part of the storyboard vocabulary, so drop them. - const SFX_NONE = new Set(["none", "no", "n/a", "na", "skip", "-", "—", "–"]); - const lines = []; - for (const f of manifest.frames) { - const names = (f.extra?.sfx ?? "") - .split(",") - .map((s) => s.trim()) - .filter((s) => s && !SFX_NONE.has(s.toLowerCase())); - if (names.length && f.number != null) lines.push({ id: pad2(f.number), sfx: names }); - } + const host = previousHostAudio(outPath, hyperframesDir, die, false); + const lines = sfxCueLines(manifest).filter((line) => !host.frames.has(Number(line.id))); const neutral = neutralPath(outPath); const request = { lines, bgm: { mode: "none" } }; @@ -219,7 +264,8 @@ function runFetchSfx(argv) { // voices/bgm written by the earlier generate (--only tts,bgm) pass are preserved. runEngine({ request, hyperframesDir, neutral, only: "sfx" }, die); - const meta = toProductLaunchMeta(JSON.parse(readFileSync(neutral, "utf8"))); + const rebuilt = toProductLaunchMeta(JSON.parse(readFileSync(neutral, "utf8"))); + const meta = keepHostAudio(rebuilt, previousHostAudio(outPath, hyperframesDir, die)); writeFileSync(outPath, JSON.stringify(meta, null, 2)); console.log(`✓ audio fetch-sfx: ${meta.sfx.length} SFX cue(s) → ${outPath}`); // This pass rewrites audio_meta.json from the neutral sidecar. If a detached BGM generate is diff --git a/skills/pr-to-video/scripts/lib/host-audio.mjs b/skills/pr-to-video/scripts/lib/host-audio.mjs new file mode 100644 index 0000000000..fa45826a82 --- /dev/null +++ b/skills/pr-to-video/scripts/lib/host-audio.mjs @@ -0,0 +1,28 @@ +// Audio the host app supplied survives every pass that rebuilds audio_meta.json from the storyboard: an entry +// marked "source": "host" whose file exists. A frame the host scored takes no looked-up cue sound. +/** The host's own entries in an earlier audio_meta.json, and why any other host-marked entry was dropped; `exists` + * resolves a path from the project root. A frame written as "1" is frame 1. */ +export function hostAudio(previous, exists) { + const marked = [[previous?.bgm, "path"], ...(previous?.sfx ?? []).map((s) => [s, "file"])].filter( + ([entry]) => entry?.source === "host", + ); + const kept = new Set(); + const dropped = []; + for (const [entry, key] of marked) { + const path = entry[key]; + if (typeof path !== "string") dropped.push(`a host entry has no "${key}"`); + else if (!exists(path)) dropped.push(`${path} is not on disk`); + else kept.add(entry); + } + const bgm = kept.has(previous?.bgm) ? previous.bgm : null; + const sfx = (previous?.sfx ?? []) + .filter((s) => kept.has(s)) + .map((s) => ({ ...s, frame: Number(s.frame) })); + return { bgm, sfx, frames: new Set(sfx.map((s) => s.frame)), dropped }; +} + +/** `rebuilt` with the host's bed and sounds put back; the host's bed wins over a looked-up one. */ +export function keepHostAudio(rebuilt, host) { + const looked = (rebuilt.sfx ?? []).filter((s) => !host.frames.has(s.frame)); + return { ...rebuilt, bgm: host.bgm ?? rebuilt.bgm, sfx: [...host.sfx, ...looked] }; +} diff --git a/skills/product-launch-video/SKILL.md b/skills/product-launch-video/SKILL.md index 6b10e9db02..bf763957cb 100644 --- a/skills/product-launch-video/SKILL.md +++ b/skills/product-launch-video/SKILL.md @@ -7,7 +7,7 @@ description: "Turn a product or marketing URL, pasted script, or brief into a pr > **First, keep this skill fresh — confirm with the user before running:** `npx hyperframes skills update product-launch-video`. A fast no-op when everything is current; otherwise it refreshes this skill plus the core domain skills it depends on before you rely on them. -> **media-use**: Before sourcing audio/images/logos, call `/media-use` to resolve BGM/SFX/images from the HeyGen catalog and brand logos from their official sources. Run `--adopt` first to register existing assets. See `/media-use` skill. +> **media-use**: Before sourcing audio/images/logos, call `/media-use` to resolve BGM/SFX/images from the HeyGen catalog and brand logos from their official sources. Run `--adopt` first to register existing assets. See `/media-use` skill. When the host app provides its own music or sound-effect tools, use those for music and sound effects. > **figma source**: If the source is a figma.com URL, run `/figma` first — asset export, brand tokens, and components/storyboard reconstruction if needed — then build this workflow from its output. Don't drive Figma via raw MCP tools directly: that skips SVG sanitization, `.media/manifest.jsonl` provenance, and brand-token `var()` binding, so a later brand change can't propagate without a full re-import. @@ -128,13 +128,13 @@ Start audio after Step 3 approval. Run it in the background, then continue to St `node /scripts/audio.mjs --script ./SCRIPT.md --storyboard ./STORYBOARD.md --hyperframes . --out ./audio_meta.json --provider --voice &` -The audio script handles narration, word timings, BGM lookup from HeyGen's music library, and timing metadata. BGM mood comes from the storyboard's `music:` field; **`music: none` turns BGM off**. This uses the HeyGen Audio API for retrieval, not generation, and uses the same `~/.heygen` credential as TTS. For provider details, read `../media-use/audio/references/tts.md`. +The audio script handles narration, word timings, BGM lookup from HeyGen's music library, and timing metadata. BGM mood comes from the storyboard's `music:` field; **`music: none` turns BGM off**. This uses the HeyGen Audio API for retrieval, not generation, and uses the same `~/.heygen` credential as TTS. For provider details, read `../media-use/audio/references/tts.md`. When the host app's own tools make the music or a sound effect, put the file under assets/ and list it in the audio_meta.json file marked `"source": "host"`: music as `bgm` `{ "path": "assets/…", "source": "host" }` with `music: none` in the storyboard; a sound as an `sfx` entry `{ "frame": , "file": "assets/…", "offset_s": 0, "duration_s": , "source": "host" }`. Every audio pass keeps these, and a frame with a host sound gets no looked-up cue. If there is no narration and no `SCRIPT.md`, skip voice generation. BGM may still run if the storyboard has a music mood. -**The canonical fully-silent marker:** `music: none` in the STORYBOARD.md top YAML block **and** no `SCRIPT.md`. That combination marks the project silent — no narration, no BGM, no SFX. `audio.mjs` recognizes it and generates nothing (it removes any stale `audio_meta.json`; an absent `audio_meta.json` is what assemble treats as silent), so Step 3.1 is a clean skip. Use it when the user asks for a silent / music-free video — don't improvise other spellings. +**The canonical fully-silent marker:** `music: none` in the STORYBOARD.md top YAML block, no `SCRIPT.md`, **and** no `sfx:` cues or host audio. That combination marks the project silent — no narration, no BGM, no SFX. `audio.mjs` recognizes it and generates nothing (it removes any stale `audio_meta.json`; an absent `audio_meta.json` is what assemble treats as silent), so Step 3.1 is a clean skip. Without narration but with `sfx:` cues or host audio the film is not silent: `audio.mjs` writes the audio_meta.json file for `fetch-sfx` to fill. Use it when the user asks for a silent / music-free video — don't improvise other spellings. -**Gate:** audio job has started, or the project is marked silent (`music: none` + no `SCRIPT.md`). +**Gate:** audio job has started, or the project is marked silent (`music: none`, no `SCRIPT.md`, no `sfx:` cues or host audio). --- diff --git a/skills/product-launch-video/references/story-design.md b/skills/product-launch-video/references/story-design.md index e6b68fe3e1..1a5a2b8415 100644 --- a/skills/product-launch-video/references/story-design.md +++ b/skills/product-launch-video/references/story-design.md @@ -465,7 +465,7 @@ Pick a small set and repeat them: default to `crossfade` (or `blur-crossfade` wh The storyboard's top YAML block carries a `music:` field — the BGM mood the audio step retrieves against (e.g. `music: confident minimal tech underscore`). Omitting it falls back to `message:` → `arc:` → a neutral default, so BGM plays unless turned off explicitly. - **`music: none`** — BGM off (narration, if any, still runs). -- **`music: none` + no `SCRIPT.md`** — the canonical **fully-silent marker**: no narration, no BGM, no SFX. `audio.mjs` generates nothing and Step 3.1 is a clean skip. Use exactly this spelling when the user asks for a silent / music-free video. +- **`music: none` + no `SCRIPT.md` + no `sfx:` cues or host audio** — the canonical **fully-silent marker**: no narration, no BGM, no SFX. `audio.mjs` generates nothing and Step 3.1 is a clean skip. Use exactly this spelling when the user asks for a silent / music-free video. ## Frame template diff --git a/skills/product-launch-video/scripts/audio.mjs b/skills/product-launch-video/scripts/audio.mjs index 9236202f78..3fd7dc3a9c 100644 --- a/skills/product-launch-video/scripts/audio.mjs +++ b/skills/product-launch-video/scripts/audio.mjs @@ -24,6 +24,7 @@ import { spawnSync } from "node:child_process"; import { existsSync, readFileSync, rmSync, writeFileSync } from "node:fs"; import { dirname, join, resolve } from "node:path"; import { fileURLToPath } from "node:url"; +import { hostAudio, keepHostAudio } from "./lib/host-audio.mjs"; import { parseStoryboard } from "./lib/storyboard.mjs"; const HERE = dirname(fileURLToPath(import.meta.url)); @@ -149,6 +150,28 @@ function runGenerate(argv) { String(g.extra?.music ?? "") .trim() .toLowerCase() === "none"; + const host = previousHostAudio(outPath, hyperframesDir, die, false); + const scored = sfxCueLines(manifest).length || host.bgm || host.sfx.length; + if (bgmOff && !lines.length && scored) { + // No narration and no looked-up music: the host's entries and any sounds fetch-sfx already found; a stale + // voice or bed in the engine's sidecar goes. + const neutral = neutralPath(outPath); + let looked = []; + try { + looked = JSON.parse(readFileSync(neutral, "utf8")).sfx ?? []; + } catch (err) { + if (err.code !== "ENOENT") die(`${neutral} does not parse: ${err.message}`); + } + const kept = { voices: [], bgm: null, bgm_pending: false, sfx: looked }; + writeFileSync(neutral, JSON.stringify(kept)); + const meta = keepHostAudio( + toProductLaunchMeta(kept), + previousHostAudio(outPath, hyperframesDir, die), + ); + writeFileSync(outPath, JSON.stringify(meta, null, 2)); + console.log(`✓ audio generate: no narration or music to make → ${outPath}`); + return; + } if (bgmOff && !lines.length) { rmSync(outPath, { force: true }); rmSync(neutralPath(outPath), { force: true }); @@ -177,13 +200,46 @@ function runGenerate(argv) { const neutral = neutralPath(outPath); runEngine({ request, hyperframesDir, neutral, only: "tts,bgm" }, die); - const meta = toProductLaunchMeta(JSON.parse(readFileSync(neutral, "utf8"))); + const rebuilt = toProductLaunchMeta(JSON.parse(readFileSync(neutral, "utf8"))); + const meta = keepHostAudio(rebuilt, previousHostAudio(outPath, hyperframesDir, die)); writeFileSync(outPath, JSON.stringify(meta, null, 2)); console.log( `✓ audio generate: ${meta.voices.length} voice + ${meta.bgm ? "1 bgm" : "no bgm"} → ${outPath}`, ); } +// Per-frame `sfx:` cues (comma-separated) → engine lines carrying only sfx. +// `filter(Boolean)` alone is not enough: a storyboard that spells "no SFX here" as +// `sfx: none` reaches the engine as a cue literally NAMED "none", which then fails to +// resolve. The absence sentinels are part of the storyboard vocabulary, so drop them. +const SFX_NONE = new Set(["none", "no", "n/a", "na", "skip", "-", "—", "–"]); +function sfxCueLines(manifest) { + const lines = []; + for (const f of manifest.frames) { + const names = (f.extra?.sfx ?? "") + .split(",") + .map((s) => s.trim()) + .filter((s) => s && !SFX_NONE.has(s.toLowerCase())); + if (names.length && f.number != null) lines.push({ id: pad2(f.number), sfx: names }); + } + return lines; +} + +// The host's own entries in audio_meta.json now (lib/host-audio.mjs). A pass reads them again for its final +// write, since the host may add audio while the engine runs. Only a missing file means none: one that does not +// parse stops the pass. +function previousHostAudio(outPath, hyperframesDir, die, warn = true) { + let previous = null; + try { + previous = JSON.parse(readFileSync(outPath, "utf8")); + } catch (err) { + if (err.code !== "ENOENT") die(`${outPath} does not parse: ${err.message}`); + } + const host = hostAudio(previous, (path) => existsSync(join(hyperframesDir, path))); + if (warn) for (const why of host.dropped) console.warn(`⚠ audio: host audio dropped: ${why}`); + return host; +} + // ── fetch-sfx ──────────────────────────────────────────────────────────────── function runFetchSfx(argv) { const die = (m) => { @@ -197,19 +253,8 @@ function runFetchSfx(argv) { if (!existsSync(storyboardPath)) die(`STORYBOARD.md not found at ${storyboardPath}`); const manifest = parseStoryboard(readFileSync(storyboardPath, "utf8")); - // Per-frame `sfx:` cues (comma-separated) → engine lines carrying only sfx. - // `filter(Boolean)` alone is not enough: a storyboard that spells "no SFX here" as - // `sfx: none` used to reach the engine as a cue literally NAMED "none", which then failed - // to resolve. The absence sentinels are part of the storyboard vocabulary, so drop them. - const SFX_NONE = new Set(["none", "no", "n/a", "na", "skip", "-", "—", "–"]); - const lines = []; - for (const f of manifest.frames) { - const names = (f.extra?.sfx ?? "") - .split(",") - .map((s) => s.trim()) - .filter((s) => s && !SFX_NONE.has(s.toLowerCase())); - if (names.length && f.number != null) lines.push({ id: pad2(f.number), sfx: names }); - } + const host = previousHostAudio(outPath, hyperframesDir, die, false); + const lines = sfxCueLines(manifest).filter((line) => !host.frames.has(Number(line.id))); const neutral = neutralPath(outPath); const request = { lines, bgm: { mode: "none" } }; @@ -218,7 +263,8 @@ function runFetchSfx(argv) { // voices/bgm written by the earlier generate (--only tts,bgm) pass are preserved. runEngine({ request, hyperframesDir, neutral, only: "sfx" }, die); - const meta = toProductLaunchMeta(JSON.parse(readFileSync(neutral, "utf8"))); + const rebuilt = toProductLaunchMeta(JSON.parse(readFileSync(neutral, "utf8"))); + const meta = keepHostAudio(rebuilt, previousHostAudio(outPath, hyperframesDir, die)); writeFileSync(outPath, JSON.stringify(meta, null, 2)); console.log(`✓ audio fetch-sfx: ${meta.sfx.length} SFX cue(s) → ${outPath}`); // This pass rewrites audio_meta.json from the neutral sidecar. If a detached BGM generate is diff --git a/skills/product-launch-video/scripts/audio.test.mjs b/skills/product-launch-video/scripts/audio.test.mjs index de3f419055..89a27a8daa 100644 --- a/skills/product-launch-video/scripts/audio.test.mjs +++ b/skills/product-launch-video/scripts/audio.test.mjs @@ -1,7 +1,7 @@ import assert from "node:assert/strict"; -import { existsSync, mkdtempSync, readFileSync, writeFileSync } from "node:fs"; +import { existsSync, mkdirSync, mkdtempSync, readFileSync, writeFileSync } from "node:fs"; import { tmpdir } from "node:os"; -import { join } from "node:path"; +import { dirname, join } from "node:path"; import { spawnSync } from "node:child_process"; import test from "node:test"; @@ -67,12 +67,22 @@ test("--provider takes precedence over HF_TTS_PROVIDER", () => { // audio_meta.json is what assemble treats as silent) and clear stale meta. /** Like runAudio, but with a caller-controlled storyboard and no request assertion. */ -function runAudioRaw({ storyboard, scriptMd = null, preexistingMeta = null }) { +function runAudioRaw({ + storyboard, + scriptMd = null, + preexistingMeta = null, + files = [], + duringEngine = "", +}) { const dir = mkdtempSync(join(tmpdir(), "product-launch-audio-")); const engine = join(dir, "engine.mjs"); writeFileSync(join(dir, "STORYBOARD.md"), storyboard); if (scriptMd != null) writeFileSync(join(dir, "SCRIPT.md"), scriptMd); if (preexistingMeta != null) writeFileSync(join(dir, "audio_meta.json"), preexistingMeta); + for (const file of files) { + mkdirSync(dirname(join(dir, file)), { recursive: true }); + writeFileSync(join(dir, file), ""); + } writeFileSync( engine, `import { readFileSync, writeFileSync } from "node:fs"; @@ -81,7 +91,7 @@ const flag = (name) => argv[argv.indexOf(name) + 1]; const request = JSON.parse(readFileSync(flag("--request"), "utf8")); writeFileSync(new URL("request.json", import.meta.url), JSON.stringify(request)); writeFileSync(flag("--out"), JSON.stringify({ voices: [], bgm: null, sfx: [] })); -`, +${duringEngine}`, ); const result = spawnSync( process.execPath, @@ -153,10 +163,22 @@ test("a storyboard music mood still retrieves BGM (marker is exact, not fuzzy)", // (linear.app site showcase, 2026-07-30). /** Runs the fetch-sfx subcommand. `neutralOut` is what the stub engine writes to --out. */ -function runFetchSfx({ storyboard, neutralOut = { voices: [], bgm: null, sfx: [] } }) { +function runFetchSfx({ + storyboard, + neutralOut = { voices: [], bgm: null, sfx: [] }, + preexistingMeta = null, + files = [], + duringEngine = "", +}) { const dir = mkdtempSync(join(tmpdir(), "product-launch-sfx-")); const engine = join(dir, "engine.mjs"); writeFileSync(join(dir, "STORYBOARD.md"), storyboard); + if (preexistingMeta != null) + writeFileSync(join(dir, "audio_meta.json"), JSON.stringify(preexistingMeta)); + for (const file of files) { + mkdirSync(dirname(join(dir, file)), { recursive: true }); + writeFileSync(join(dir, file), ""); + } writeFileSync( engine, `import { readFileSync, writeFileSync } from "node:fs"; @@ -165,7 +187,7 @@ const flag = (name) => argv[argv.indexOf(name) + 1]; const request = JSON.parse(readFileSync(flag("--request"), "utf8")); writeFileSync(new URL("request.json", import.meta.url), JSON.stringify(request)); writeFileSync(flag("--out"), ${JSON.stringify(JSON.stringify(neutralOut))}); -`, +${duringEngine}`, ); const result = spawnSync( process.execPath, @@ -232,3 +254,146 @@ test("fetch-sfx: a resolved bed reports bgm_pending false and no warning", () => assert.equal(meta.bgm.path, "assets/bgm/track.mp3"); assert.doesNotMatch(result.stderr, /pending/i); }); + +// ── host audio (lib/host-audio.mjs) ───────────────────────────────────────── +// A host app's own music or sound, listed in audio_meta.json as "source": "host", survives every audio pass. + +const HOST_WHOOSH = { + frame: 1, + file: "assets/sfx/custom-whoosh.mp3", + offset_s: 0, + duration_s: 2, + source: "host", +}; +const HOST_BED = { path: "assets/bgm/host.mp3", source: "host" }; + +test("fetch-sfx keeps a host sound and looks up no cue for its frame", () => { + const { dir, result } = runFetchSfx({ + storyboard: FRAME_WITH_SFX("whoosh"), + preexistingMeta: { bgm: null, voices: [], sfx: [HOST_WHOOSH] }, + files: [HOST_WHOOSH.file], + neutralOut: { voices: [], bgm: null, sfx: [{ id: "01", file: "assets/sfx/whoosh.mp3" }] }, + }); + + assert.equal(result.status, 0, result.stderr); + assert.deepEqual(JSON.parse(readFileSync(join(dir, "request.json"), "utf8")).lines, []); + assert.deepEqual(JSON.parse(readFileSync(join(dir, "audio_meta.json"), "utf8")).sfx, [ + HOST_WHOOSH, + ]); +}); + +test("fetch-sfx keeps a host bed the storyboard does not look up", () => { + const { dir, result } = runFetchSfx({ + storyboard: FRAME_WITH_SFX("whoosh"), + preexistingMeta: { bgm: HOST_BED, voices: [], sfx: [] }, + files: [HOST_BED.path], + }); + + assert.equal(result.status, 0, result.stderr); + assert.deepEqual(JSON.parse(readFileSync(join(dir, "audio_meta.json"), "utf8")).bgm, HOST_BED); +}); + +test("music: none + no SCRIPT.md with a sound cue is not silent: audio_meta.json stays for fetch-sfx", () => { + const { dir, result } = runAudioRaw({ + storyboard: + "---\nmessage: Test\nmusic: none\n---\n\n## Frame 1 — Hook\n- duration: 3s\n- sfx: whoosh\n", + }); + + assert.equal(result.status, 0, result.stderr); + assert.doesNotMatch(result.stdout, /marked silent/); + assert.equal(existsSync(join(dir, "request.json")), false); + assert.deepEqual(JSON.parse(readFileSync(join(dir, "audio_meta.json"), "utf8")).sfx, []); +}); + +test("generate keeps a host bed when there is no narration and music: none", () => { + const { dir, result } = runAudioRaw({ + storyboard: "---\nmessage: Test\nmusic: none\n---\n", + preexistingMeta: JSON.stringify({ bgm: HOST_BED, voices: [], sfx: [] }), + files: [HOST_BED.path], + }); + + assert.equal(result.status, 0, result.stderr); + assert.deepEqual(JSON.parse(readFileSync(join(dir, "audio_meta.json"), "utf8")).bgm, HOST_BED); +}); + +test("fetch-sfx stops on an audio_meta.json that does not parse, so host entries are not lost", () => { + const { dir } = runFetchSfx({ storyboard: FRAME_WITH_SFX("whoosh") }); + writeFileSync( + join(dir, "audio_meta.json"), + '{ "bgm": { "path": "assets/bgm/host.mp3", "source": "host" }, }', + ); + const again = spawnSync( + process.execPath, + [script, "fetch-sfx", "--hyperframes", dir, "--storyboard", join(dir, "STORYBOARD.md")], + { encoding: "utf8", env: { ...process.env, HF_MEDIA_ENGINE: join(dir, "engine.mjs") } }, + ); + + assert.equal(again.status, 1); + assert.match(again.stderr, /does not parse/); + assert.match(readFileSync(join(dir, "audio_meta.json"), "utf8"), /host\.mp3/); +}); + +test("re-running generate with no narration keeps the sounds fetch-sfx found", () => { + const storyboard = + "---\nmessage: Test\nmusic: none\n---\n\n## Frame 1 — Hook\n- duration: 3s\n- sfx: whoosh\n"; + const { dir, result } = runAudioRaw({ storyboard }); + assert.equal(result.status, 0, result.stderr); + writeFileSync( + join(dir, "audio_engine_meta.json"), + JSON.stringify({ + voices: [{ id: "01", path: "old.wav" }], + bgm: { path: "old.mp3" }, + sfx: [{ id: "01", file: "assets/sfx/whoosh.mp3" }], + }), + ); + const again = spawnSync( + process.execPath, + [script, "--hyperframes", dir, "--storyboard", join(dir, "STORYBOARD.md")], + { + encoding: "utf8", + }, + ); + + assert.equal(again.status, 0, again.stderr); + const meta = JSON.parse(readFileSync(join(dir, "audio_meta.json"), "utf8")); + assert.deepEqual( + meta.sfx.map((s) => s.file), + ["assets/sfx/whoosh.mp3"], + ); + assert.deepEqual(meta.voices, []); + assert.equal(meta.bgm, null); +}); + +// What a host app writes into audio_meta.json while a pass's engine is still running. +const HOST_WRITES_MID_PASS = ` +const { mkdirSync } = await import("node:fs"); +for (const dir of ["assets/bgm/", "assets/sfx/"]) mkdirSync(new URL(dir, import.meta.url), { recursive: true }); +writeFileSync(new URL("${HOST_BED.path}", import.meta.url), ""); +writeFileSync(new URL("${HOST_WHOOSH.file}", import.meta.url), ""); +writeFileSync(new URL("audio_meta.json", import.meta.url), ${JSON.stringify(JSON.stringify({ bgm: HOST_BED, voices: [], sfx: [HOST_WHOOSH] }))}); +`; + +test("generate keeps host audio written while its engine runs", () => { + const { dir, result } = runAudioRaw({ + storyboard: "---\nmessage: Test\nmusic: calm piano\n---\n", + duringEngine: HOST_WRITES_MID_PASS, + }); + + assert.equal(result.status, 0, result.stderr); + const meta = JSON.parse(readFileSync(join(dir, "audio_meta.json"), "utf8")); + assert.deepEqual(meta.bgm, HOST_BED); + assert.deepEqual(meta.sfx, [HOST_WHOOSH]); +}); + +test("fetch-sfx keeps host audio written while its engine runs, and the host's sound wins its frame", () => { + const { dir, result } = runFetchSfx({ + storyboard: FRAME_WITH_SFX("whoosh"), + neutralOut: { voices: [], bgm: null, sfx: [{ id: "01", file: "assets/sfx/whoosh.mp3" }] }, + duringEngine: HOST_WRITES_MID_PASS, + }); + + assert.equal(result.status, 0, result.stderr); + const meta = JSON.parse(readFileSync(join(dir, "audio_meta.json"), "utf8")); + assert.deepEqual(meta.bgm, HOST_BED); + assert.deepEqual(meta.sfx, [HOST_WHOOSH]); +}); diff --git a/skills/product-launch-video/scripts/lib/host-audio.mjs b/skills/product-launch-video/scripts/lib/host-audio.mjs new file mode 100644 index 0000000000..fa45826a82 --- /dev/null +++ b/skills/product-launch-video/scripts/lib/host-audio.mjs @@ -0,0 +1,28 @@ +// Audio the host app supplied survives every pass that rebuilds audio_meta.json from the storyboard: an entry +// marked "source": "host" whose file exists. A frame the host scored takes no looked-up cue sound. +/** The host's own entries in an earlier audio_meta.json, and why any other host-marked entry was dropped; `exists` + * resolves a path from the project root. A frame written as "1" is frame 1. */ +export function hostAudio(previous, exists) { + const marked = [[previous?.bgm, "path"], ...(previous?.sfx ?? []).map((s) => [s, "file"])].filter( + ([entry]) => entry?.source === "host", + ); + const kept = new Set(); + const dropped = []; + for (const [entry, key] of marked) { + const path = entry[key]; + if (typeof path !== "string") dropped.push(`a host entry has no "${key}"`); + else if (!exists(path)) dropped.push(`${path} is not on disk`); + else kept.add(entry); + } + const bgm = kept.has(previous?.bgm) ? previous.bgm : null; + const sfx = (previous?.sfx ?? []) + .filter((s) => kept.has(s)) + .map((s) => ({ ...s, frame: Number(s.frame) })); + return { bgm, sfx, frames: new Set(sfx.map((s) => s.frame)), dropped }; +} + +/** `rebuilt` with the host's bed and sounds put back; the host's bed wins over a looked-up one. */ +export function keepHostAudio(rebuilt, host) { + const looked = (rebuilt.sfx ?? []).filter((s) => !host.frames.has(s.frame)); + return { ...rebuilt, bgm: host.bgm ?? rebuilt.bgm, sfx: [...host.sfx, ...looked] }; +}