From 0a60427fdfba35efa5356ca1a423ad5ee5dd7c18 Mon Sep 17 00:00:00 2001
From: ukimsanov <ular.kimsanov@heygen.com>
Date: Thu, 11 Jun 2026 17:35:15 -0700
Subject: [PATCH] fix(skill): move music fetch from Step 5 to Step 3 (beats
 align to track structure)
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

User flag: music selection in Step 5 was the wrong timing. Music has
structural moments (builds, peaks, drops, fills) that the video's
rhythm should honor — a build at 22s wants beat N to land at 22s.
With Step-5 fetch, the storyboard wrote beat timing BLIND to the
actual track, then Step 5 produced a structural mismatch.

SFX timing was already correct (per-beat punctuation, decided in
storyboard, fetched in build with full beat context). Only music
needed the move.

New flow:
  Step 2 (yes/no/auto) → Step 3 Music Fetch (search + pick + add +
  inspect + record sparkline+peaks in STORYBOARD.md) → Step 3
  beat-timing AWARE of track structure → Step 4 (no-op for music)
  → Step 5 (just wire the already-on-disk file)

Changes:

step-3-storyboard.md:
- New ## Music Fetch section between PACING and Technique-pick.
  6 sub-steps: describe mood → search catalog → pick (collaborative
  or auto by score) → music add (downloads + prints LUFS/peaks/
  sparkline) → record in Global Direction → time beats against
  structure (build@22s → schedule reveal at 22s, etc.).
- Global Direction template adds:
    **Music file:** assets/music/<id>.<ext>
    **Music direction:** mood + volume rule
    **Music structure:** sparkline + peak + duration + suggested trim
- Updated the existing "Music direction details" subsection to
  reference Step 2 Q5 + background-music.md (renamed from
  "Underscore/music direction").

step-4-vo.md:
- Replaced "Music (storyboard-driven, decided in Step 2)" section
  with "Music (already fetched in Step 3 — verify timing only)".
  Explicitly: do NOT re-fetch or re-ask here. Step 4 may align VO
  start to music's quiet intro if relevant.

step-5-build.md:
- Background music paragraph: "track is already on disk from Step 3
  Music Fetch; do NOT re-fetch or re-search here. Just wire as a
  BGM <audio> on track lane 11 with data-volume per storyboard."

w2h-prep.mjs:
- New BGM detection block (after voice auto-detect). Parses
  STORYBOARD.md for `**Music file:** <path>` line; falls back to
  auto-detect from `assets/music/` (single-file or alphabetical with
  warning when >1). Emits `bgm_path` field in group_spec.json,
  which assemble-index.mjs already consumes (line 289).
- Smoke-verified end-to-end: storyboard with Music file line →
  bgm_path: "assets/music/<id>.mp3" in group_spec.

This closes the loop that was missing: music decision → fetch →
analysis → beat timing → group_spec → assembler → final video.
---
 .../references/step-3-storyboard.md           | 56 ++++++++++++++++++-
 .../references/step-4-vo.md                   | 10 +++-
 .../references/step-5-build.md                |  2 +-
 .../scripts/w2h-prep.mjs                      | 34 +++++++++++
 4 files changed, 97 insertions(+), 5 deletions(-)

diff --git a/skills/website-to-hyperframes/references/step-3-storyboard.md b/skills/website-to-hyperframes/references/step-3-storyboard.md
index 177d509487..6f829dc882 100644
--- a/skills/website-to-hyperframes/references/step-3-storyboard.md
+++ b/skills/website-to-hyperframes/references/step-3-storyboard.md
@@ -59,6 +59,58 @@ If the user said "dark cinematic feel" — that's SLOW, not fast. If they said "
 
 ---
 
+## Music Fetch (only if Step 2 Q5 said yes — do this BEFORE writing per-beat timing)
+
+If Step 2 Q5 picked "Yes, music bed" (or auto-decide chose yes), **fetch the track now, before timing any beats.** Music has structure (builds, peaks, drops, fills) that the video's rhythm should honor — a build at 22s wants beat N to land at 22s, not at 18s. Search → pick → analyze → THEN write beat timing. Doing this after the storyboard is the wrong order: you'd end up with beats that don't align to the track's structural moments.
+
+Skip this section entirely if Step 2 said no music (the storyboard's Global Direction has no `**Music file:**` line and Step 5 skips the music wire-up cleanly).
+
+**1. Describe the mood in one line.** Informed by Step 2 brief style + video type + Arc choice. Search by feel + function, not by filename:
+
+- e.g. `"moody ambient pad with slow build for a brand reel"`
+- e.g. `"upbeat lo-fi reel bed under voice"`
+- e.g. `"tense synth drone rising for product launch"`
+- e.g. `"warm acoustic guitar cozy"` / `"epic cinematic orchestral swell"` / `"minimal piano introspective"`
+
+See [`background-music.md`](background-music.md) "Search prompts that work" for more shapes.
+
+**2. Search the catalog.** Pull 5 candidates:
+
+```bash
+npx tsx packages/cli/src/cli.ts music search "<your mood line>" --limit 5
+```
+
+Returns `id`, `name`, `description`, `duration`, `score` per candidate.
+
+**3. Pick a track.** In collaborative mode: present the top 3-5 candidates to the user (one line each: `id` + `name` + `duration` + first ~80 chars of description) and ask them to pick. In auto mode: pick the highest-score candidate whose `duration` is ≥ the planned video duration (so the track covers the whole video without looping).
+
+**4. Download + analyze.** This is the load-bearing step:
+
+```bash
+npx tsx packages/cli/src/cli.ts music add <chosen-id>
+```
+
+This writes the track to `assets/music/<id>.<ext>` AND prints the analysis — LUFS / true peak / **peak time** / onset / tail / **loudness sparkline** (e.g. `shape ▁▂▅█▇▃▁`). The sparkline shows energy contour over time — read it to know WHERE the track builds, drops, breaks. Each block roughly maps to `t ≈ (i + 0.5) / length × duration`.
+
+**5. Record in the storyboard's Global Direction** (the template above). Three lines:
+
+```markdown
+**Music file:** assets/music/<id>.<ext>
+**Music direction:** <mood + volume rule, e.g. "moody ambient pad sits at 0.45 under VO, lifts to 0.75 for the 2s pre-CTA stretch">
+**Music structure:** sparkline=▁▂▅█▇▃▁ · peak=Xs · duration=Ds · suggested-trim=<head>s-<tail>s
+```
+
+**6. Time beats against the structure** (when you write the per-beat sequence below):
+
+- If the music has a build at 22s → schedule a hero/reveal/peak beat to land at 22s.
+- If the music drops at 35s → that's a CTA-friendly silence; schedule the CTA there.
+- If the music is even/ambient with no notable peaks → treat it as a bed; time beats against VO + visual rhythm instead.
+- If the music has a clear intro (first N seconds quiet, sparkline starts at `▁▁`), VO start timing can wait for the intro to settle — e.g. VO starts at the first `▂` block.
+
+In Step 5, the music is already on disk — beat workers read `**Music file:**` from STORYBOARD.md Global Direction and the orchestrator embeds it as the BGM lane (track 11) without re-fetching.
+
+---
+
 ## Technique-pick checklist (REQUIRED, do this BEFORE writing beat copy)
 
 For every beat you plan, name **2–4 techniques** it will use. A beat with one technique is a slideshow frame — if you can't name two, redesign that beat.
@@ -135,7 +187,9 @@ Every STORYBOARD.md starts with global settings:
 **Audio:** [TTS provider] voiceover + [music yes/no, from Step 2 Q5] + SFX
 **VO direction:** [voice character — e.g., "mid-age male, calm confident delivery,
 Apple keynote register — economy of words, silence between sentences is a feature"]
-**Music direction:** [only if Step 2 Q5 said yes — one line of mood/feel for `hyperframes music search`, plus volume rule. e.g., "moody ambient pad with slow build, sits at ~0.45 under VO, lifts to ~0.75 in the 2s pre-CTA stretch where there's no narration"]
+**Music file:** [only if music = yes — path to the track fetched in Music Fetch below, e.g., `assets/music/a3f5b2e1.mp3`]
+**Music direction:** [only if music = yes — one line of mood/volume rule, e.g., "moody ambient pad, sits at ~0.45 under VO, lifts to ~0.75 in the 2s pre-CTA stretch"]
+**Music structure:** [only if music = yes — copied verbatim from `music add` analysis: `sparkline=▁▂▅█▇▃▁ · peak=Xs · duration=Ds · suggested-trim=<head>s-<tail>s`. Use this to time beats against the track's builds/drops.]
 **Style basis:** DESIGN.md (brand colors, fonts, components from the captured site)
 ```
 
diff --git a/skills/website-to-hyperframes/references/step-4-vo.md b/skills/website-to-hyperframes/references/step-4-vo.md
index d4c664ee16..c423d9221f 100644
--- a/skills/website-to-hyperframes/references/step-4-vo.md
+++ b/skills/website-to-hyperframes/references/step-4-vo.md
@@ -109,11 +109,15 @@ The final beat timings (real-audio-derived `start` and `duration`) live in STORY
 
 ---
 
-## Music (storyboard-driven, decided in Step 2)
+## Music (already fetched in Step 3 — verify timing only)
 
-Music is a Step-2 user-preference question (see step-2-brief.md Question 5) — **do NOT ask the user about music at this step**, the decision is already locked in the brief. If Step 2 said yes, the storyboard's Global Direction names a mood (`**Music direction:** moody ambient pad, slow build, sits at ~0.45 under VO`); Step 5 fetches it via `hyperframes music search` + `music add` against the HeyGen catalog (free, same key as `sfx`).
+Music is decided in Step 2 (yes/no/auto), fetched and analyzed in Step 3's Music Fetch sub-step (which writes `**Music file:**`, `**Music direction:**`, and `**Music structure:**` into STORYBOARD.md's Global Direction). **Do NOT re-fetch or re-ask here.** By the time you reach Step 4, the track is already on disk at `assets/music/<id>.<ext>` and the storyboard's beat timing already accounts for the track's peaks/drops.
 
-If the user supplies their own track instead of using the catalog, note its file path in STORYBOARD.md's Global Direction as `**Music file:** path/to/track.mp3` and Step 5 wires that directly without a fetch. If Step 2 said no music, no Music line appears in Global Direction and Step 5 skips the fetch cleanly.
+What Step 4 does for music: **align VO timing against music structure if relevant.** If the track has a quiet intro before the first build, VO can start at the first build (typically 1-3s in). Note this in STORYBOARD.md's `**Narration start:**` field so Step 5 wires the VO `<audio>` `data-start` to match.
+
+If the user supplied their own track (rather than the catalog), the Step 3 fetch step recorded that file path instead. Either way, Step 5 wires whatever `**Music file:**` points to as the BGM lane.
+
+If Step 2 said no music, none of this applies — there's no `**Music file:**` line in Global Direction and Step 5 skips the music wire-up cleanly.
 
 See [`background-music.md`](background-music.md) for the full model: when music helps vs hurts, the volume hierarchy under VO (`0.4–0.6` vs `0.7–0.9` pure-music), manual ducking patterns since there's no auto-ducking, and the one-bed-per-video rule.
 
diff --git a/skills/website-to-hyperframes/references/step-5-build.md b/skills/website-to-hyperframes/references/step-5-build.md
index c1713fd482..ea3e881826 100644
--- a/skills/website-to-hyperframes/references/step-5-build.md
+++ b/skills/website-to-hyperframes/references/step-5-build.md
@@ -27,7 +27,7 @@ hyperframes sfx add <id>                                # → assets/sfx/<id>.mp
 
 `sfx add` writes the clip to `assets/sfx/` and prints its analysis (peak time, onset/tail, loudness) — use it to trim and anchor per the global doc. If `HEYGEN_API_KEY` isn't set, `sfx` prints how to get a free key; ask the user for one or build without SFX — don't silently drop them.
 
-**Background music** (optional, storyboard-driven). If `STORYBOARD.md`'s Global Direction names a music bed, search and download it the same way with `hyperframes music search` / `music add` — output lands in `assets/music/<id>.<ext>`. Volume hierarchy is different from SFX (BGM `0.4–0.6` under VO; `0.7–0.9` pure-music) and there's no auto-ducking; see [`background-music.md`](background-music.md) for when music helps vs hurts, manual ducking patterns, and the one-bed-per-video rule.
+**Background music** (already on disk from Step 3 Music Fetch). If `STORYBOARD.md`'s Global Direction has a `**Music file:**` line, the track was fetched and analyzed in Step 3 — it's already at `assets/music/<id>.<ext>`. **Do NOT re-fetch or re-search here.** Just wire the file as a BGM `<audio>` element on track lane 11 with `data-volume` per the storyboard's `**Music direction:**` line. If there's no `**Music file:**` line (Step 2 said no music), skip the music wire-up cleanly — assembler emits no BGM lane. Volume hierarchy: BGM `0.4–0.6` under VO; `0.7–0.9` if pure-music. No auto-ducking; see [`background-music.md`](background-music.md) for manual ducking patterns and the one-bed-per-video rule.
 
 ## Known landmines — read before writing each beat composition
 
diff --git a/skills/website-to-hyperframes/scripts/w2h-prep.mjs b/skills/website-to-hyperframes/scripts/w2h-prep.mjs
index 24de574d0b..0c9d46ec8e 100644
--- a/skills/website-to-hyperframes/scripts/w2h-prep.mjs
+++ b/skills/website-to-hyperframes/scripts/w2h-prep.mjs
@@ -536,6 +536,36 @@ function main() {
       }
     : undefined;
 
+  // Detect BGM: prefer storyboard's `**Music file:** <path>` line (Step 3's
+  // Music Fetch sub-step writes this), fall back to auto-detect from
+  // `assets/music/` (one-bed-per-video rule means a single file is the norm).
+  // assemble-index.mjs reads `bgm_path` and emits the BGM <audio> on track 11.
+  let bgmPath;
+  const storyboardPath = join(projectDir, "STORYBOARD.md");
+  if (existsSync(storyboardPath)) {
+    const sb = readFileSync(storyboardPath, "utf-8");
+    const m = sb.match(/^\s*\*\*Music file:\*\*\s*(\S[^\s\n]*)/m);
+    if (m) bgmPath = m[1];
+  }
+  if (!bgmPath) {
+    const musicDir = join(projectDir, "assets", "music");
+    if (existsSync(musicDir)) {
+      const musicFiles = readdirSync(musicDir)
+        .filter((f) => /\.(mp3|wav|flac|m4a|ogg)$/i.test(f))
+        .sort();
+      if (musicFiles.length === 1) {
+        bgmPath = `assets/music/${musicFiles[0]}`;
+      } else if (musicFiles.length > 1) {
+        // Multiple candidates — pick the first alphabetically but warn so the
+        // orchestrator can disambiguate via STORYBOARD.md's `**Music file:**`.
+        bgmPath = `assets/music/${musicFiles[0]}`;
+        console.warn(
+          `! w2h-prep: ${musicFiles.length} music files in assets/music/ but no \`**Music file:**\` line in STORYBOARD.md. Using ${musicFiles[0]}; add an explicit line to disambiguate.`,
+        );
+      }
+    }
+  }
+
   const spec = {
     total_duration_s: totalDuration,
     total_scenes: flatScenes.length,
@@ -549,6 +579,7 @@ function main() {
     captions_enabled: existsSync(join(projectDir, "compositions/captions.html")),
     sfx: [],
     ...(voice ? { voice } : {}),
+    ...(bgmPath ? { bgm_path: bgmPath } : {}),
     // shader_transitions auto-derived from STORYBOARD.md "Shader Transitions"
     // block (PHASE C(b)). Canonical line format:
     //   - between beat-X and beat-Y: shader=<name>[, duration=<n>]
@@ -595,6 +626,9 @@ function main() {
   if (voice) {
     console.log(`  + voice (${voice.path}, ${voice.duration_s}s)`);
   }
+  if (bgmPath) {
+    console.log(`  + bgm_path (${bgmPath})`);
+  }
   if (shaderTransitions) {
     const namedCount = shaderTransitions.transitions.filter((t) => t.shader).length;
     console.log(