mirror of
https://github.com/heygen-com/hyperframes.git
synced 2026-09-11 14:50:02 +00:00
feat(skills): unify bgm-to-video flows into music-to-video
Replace bgm-to-video, bgm-to-video-new, bgm-to-video-refactor, and the standalone beat-sync/montage skills with a single music-to-video skill. Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
This commit is contained in:
Miao Yang
co-authored by
Claude Opus 4.8
Claude Opus 4.8
parent
0dcf914e14
commit
a34d3dba4d
@@ -0,0 +1,531 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Beat-grid + drum/event analysis engine for music-to-video.
|
||||
|
||||
Turns a BGM track directly into a deterministic `audiomap.json` — the music skeleton
|
||||
that the Director and Builder hang visuals on. It merges:
|
||||
|
||||
- a reliable tempo + beat grid + downbeat (librosa beat tracker),
|
||||
- metrical position per event (strong / weak / syncopated / off-grid) over a 16th-note bar grid,
|
||||
- drum-element classification (kick / snare / hihat / perc) via band-split,
|
||||
- special audio events (riser / glitch / crash-impact / hard-stop silence),
|
||||
- an energy narrative (audio-driven energy phases + builds + key moments; the Music
|
||||
Reader names the sections — no fixed Intro/Build/Drop/Outro template),
|
||||
- a phrase layer + per-section density budgets for visual planning.
|
||||
|
||||
Output is the canonical `audiomap.json` documented in the skill.
|
||||
|
||||
Usage:
|
||||
python3 analyze-beatgrid.py track.mp3 -o audiomap.json
|
||||
python3 analyze-beatgrid.py track.mp3 --print # also print a readable brief
|
||||
|
||||
Deps: ffmpeg/ffprobe on PATH + librosa, numpy, soundfile (band-split heuristics,
|
||||
no learned models / no madmom).
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import json
|
||||
import subprocess
|
||||
import sys
|
||||
import tempfile
|
||||
from pathlib import Path
|
||||
|
||||
import librosa
|
||||
import numpy as np
|
||||
import soundfile as sf
|
||||
|
||||
SR = 22050
|
||||
HOP = 512 # ~23 ms frames
|
||||
AUDIOMAP_VERSION = 2
|
||||
|
||||
|
||||
# ── decode ────────────────────────────────────────────────────────────────
|
||||
def load_audio(path: str) -> tuple[np.ndarray, int, float]:
|
||||
"""Decode any ffmpeg-readable file to mono float32 @ SR via a temp wav."""
|
||||
with tempfile.NamedTemporaryFile(suffix=".wav", delete=False) as tmp:
|
||||
wav = tmp.name
|
||||
subprocess.run(
|
||||
["ffmpeg", "-y", "-i", path, "-ac", "1", "-ar", str(SR), wav],
|
||||
capture_output=True, check=True,
|
||||
)
|
||||
y, sr = sf.read(wav, dtype="float32")
|
||||
Path(wav).unlink(missing_ok=True)
|
||||
if y.ndim > 1:
|
||||
y = y.mean(axis=1)
|
||||
return y, sr, len(y) / sr
|
||||
|
||||
|
||||
# ── tempo + beat grid + downbeat phase ──────────────────────────────────────
|
||||
def beat_grid(y: np.ndarray, sr: int) -> dict:
|
||||
tempo, beat_frames = librosa.beat.beat_track(y=y, sr=sr, hop_length=HOP, units="frames")
|
||||
beats = librosa.frames_to_time(beat_frames, sr=sr, hop_length=HOP)
|
||||
return {"bpm": float(np.atleast_1d(tempo)[0]), "beats": beats, "beat_frames": beat_frames}
|
||||
|
||||
|
||||
def _norm_flux(band: np.ndarray) -> np.ndarray:
|
||||
"""Positive first-difference (onset flux) of a band-energy curve, normalized."""
|
||||
flux = np.maximum(0.0, np.diff(np.sqrt(band), prepend=band[:1]))
|
||||
return flux / (flux.max() + 1e-9)
|
||||
|
||||
|
||||
def band_energy_curves(y: np.ndarray, sr: int) -> dict:
|
||||
"""Per-frame band energy + per-band normalized onset flux (for drum typing)."""
|
||||
S = np.abs(librosa.stft(y, hop_length=HOP)) ** 2
|
||||
freqs = librosa.fft_frequencies(sr=sr)
|
||||
# Tight drum bands: kick fundamental, snare body, hihat sizzle. Narrow bands
|
||||
# keep one drum's transient from leaking into another's flux on a full mix.
|
||||
low = (freqs < 150) # kick
|
||||
mid = (freqs >= 150) & (freqs < 900) # snare body
|
||||
high = (freqs >= 5000) # hihat / cymbal
|
||||
e_low, e_mid, e_high = S[low].sum(axis=0), S[mid].sum(axis=0), S[high].sum(axis=0)
|
||||
return {
|
||||
"S": S,
|
||||
"low": e_low, "mid": e_mid, "high": e_high,
|
||||
"total": S.sum(axis=0) + 1e-9,
|
||||
"flux_low": _norm_flux(e_low), "flux_mid": _norm_flux(e_mid), "flux_high": _norm_flux(e_high),
|
||||
"flatness": librosa.feature.spectral_flatness(S=np.sqrt(S))[0],
|
||||
"centroid": librosa.feature.spectral_centroid(S=np.sqrt(S), sr=sr)[0],
|
||||
"n": S.shape[1],
|
||||
}
|
||||
|
||||
|
||||
def downbeat_phase(beat_frames: np.ndarray, bc: dict, beats_per_bar: int = 4) -> int:
|
||||
"""Pick the bar phase whose beats carry the most KICK (low-band) energy."""
|
||||
kick = bc["low"] / bc["total"]
|
||||
best_p, best_score = 0, -1.0
|
||||
for p in range(beats_per_bar):
|
||||
idx = [bf for i, bf in enumerate(beat_frames) if (i - p) % beats_per_bar == 0]
|
||||
idx = [min(f, bc["n"] - 1) for f in idx]
|
||||
score = float(np.sum([kick[f] for f in idx])) if idx else 0.0
|
||||
if score > best_score:
|
||||
best_p, best_score = p, score
|
||||
return best_p
|
||||
|
||||
|
||||
# ── metrical position: strong / weak / syncopated / off-grid ─────────────────
|
||||
# 16-step bar grid (4 beats x 4 sixteenths). Strength by metrical weight.
|
||||
GRID_CLASS = {0: "strong", 8: "strong", 4: "weak", 12: "weak",
|
||||
2: "weak", 6: "weak", 10: "weak", 14: "weak"} # else (odd 16ths) -> syncopated
|
||||
|
||||
|
||||
def classify_metric(t: float, beats: np.ndarray, phase: int, bpb: int = 4) -> tuple:
|
||||
"""Return (grid_class, bar, beat_in_bar, step16) for a time t."""
|
||||
if len(beats) < 2:
|
||||
return "off-grid", -1, -1, -1
|
||||
i = int(np.searchsorted(beats, t) - 1)
|
||||
i = max(0, min(i, len(beats) - 2))
|
||||
beat_dur = beats[i + 1] - beats[i]
|
||||
frac = (t - beats[i]) / max(beat_dur, 1e-6) # 0..1 within the beat
|
||||
sixteenth = int(round(frac * 4)) % 4 # nearest 16th in beat
|
||||
carry = 1 if round(frac * 4) >= 4 else 0
|
||||
beat_idx = i + carry
|
||||
beat_in_bar = (beat_idx - phase) % bpb # 0..3
|
||||
bar = (beat_idx - phase) // bpb
|
||||
step16 = beat_in_bar * 4 + sixteenth # 0..15
|
||||
# distance to the nearest 16th line (in seconds) → off-grid test
|
||||
nearest = beats[i] + (round(frac * 4) / 4) * beat_dur
|
||||
if abs(t - nearest) > 0.5 * (beat_dur / 4):
|
||||
return "off-grid", bar, beat_in_bar + 1, step16
|
||||
return GRID_CLASS.get(step16, "syncopated"), bar, beat_in_bar + 1, step16
|
||||
|
||||
|
||||
# ── drum classification (band-split heuristic) ──────────────────────────────
|
||||
def frame_at(t: float, sr: int, n: int) -> int:
|
||||
return min(int(round(t * sr / HOP)), n - 1)
|
||||
|
||||
|
||||
def classify_drum(t: float, bc: dict, sr: int) -> tuple:
|
||||
"""(drum_type, energy_norm, feel): which band's onset TRANSIENT dominates.
|
||||
|
||||
Uses per-band normalized flux (relative transient strength), the standard
|
||||
way to separate kick (low) / snare (mid+noise) / hihat (high). Falls back to
|
||||
glitch for noisy non-harmonic bursts and perc when no band clearly leads.
|
||||
"""
|
||||
f = frame_at(t, sr, bc["n"])
|
||||
win = slice(max(0, f - 1), min(bc["n"], f + 2))
|
||||
fl = float(bc["flux_low"][win].max())
|
||||
fm = float(bc["flux_mid"][win].max())
|
||||
fh = float(bc["flux_high"][win].max())
|
||||
lo = float(bc["low"][win].mean()); md = float(bc["mid"][win].mean())
|
||||
hi = float(bc["high"][win].mean()); tot = float(bc["total"][win].mean())
|
||||
flat = float(bc["flatness"][win].mean())
|
||||
lr, mr, hr = lo / tot, md / tot, hi / tot
|
||||
|
||||
fluxes = {"kick": fl, "snare": fm, "hihat": fh}
|
||||
lead = max(fluxes, key=fluxes.get)
|
||||
lead_val = fluxes[lead]
|
||||
if lead_val < 0.06: # no real transient → texture/perc
|
||||
drum = "glitch" if flat > 0.30 else "perc"
|
||||
elif lead == "snare" and flat > 0.30 and mr < 0.30:
|
||||
drum = "glitch" # mid-band but noisy & thin → scratch/glitch
|
||||
else:
|
||||
drum = lead
|
||||
# feel (frequency character)
|
||||
has_bot, has_top, has_mid = lr > 0.30, hr > 0.20, mr > 0.30
|
||||
feel = ("full" if has_bot and has_top and has_mid else
|
||||
"heavy" if has_bot and not has_top else
|
||||
"bright" if has_top and not has_bot else
|
||||
"intimate" if has_mid else "sparse")
|
||||
return drum, tot, feel
|
||||
|
||||
|
||||
# ── energy structure (RMS @1s) + sections + key moments + builds ────────────
|
||||
def energy_structure(y: np.ndarray, sr: int, dur: float, first_onset: float = 0.0) -> dict:
|
||||
rms = librosa.feature.rms(y=y, hop_length=sr)[0] # ~1s frames
|
||||
rms = rms / (rms.max() + 1e-9)
|
||||
norms = rms.tolist()
|
||||
|
||||
def lvl(n):
|
||||
return "VOID" if n < 0.2 else "LOW" if n < 0.4 else "MEDIUM" if n < 0.65 else "HIGH"
|
||||
|
||||
phases, cur, cs = [], None, 0
|
||||
for i, n in enumerate(norms):
|
||||
l = lvl(n)
|
||||
if l != cur:
|
||||
if cur:
|
||||
phases.append({"s": cs, "e": i, "lvl": cur})
|
||||
cur, cs = l, i
|
||||
if cur:
|
||||
phases.append({"s": cs, "e": len(norms), "lvl": cur})
|
||||
|
||||
moments = []
|
||||
for i in range(1, len(norms)):
|
||||
d = norms[i] - norms[i - 1]
|
||||
if abs(d) > 0.12:
|
||||
moments.append({"t": i, "kind": "DROP" if d < 0 else "SURGE", "delta": round(d, 2)})
|
||||
moments.sort(key=lambda m: abs(m["delta"]), reverse=True)
|
||||
|
||||
# hard stop: a HIGH→low cliff (a sudden stop) in the back third
|
||||
hard_stops = [m for m in moments if m["kind"] == "DROP" and m["t"] > dur * 0.6 and m["delta"] < -0.25]
|
||||
|
||||
# NO forced Intro/Build/Drop/Outro template. The energy phases (audio-driven runs of
|
||||
# one energy level, variable count) are the raw structural blocks. The Music Reader
|
||||
# (LLM) decides the actual sections — count, boundaries, and free-form names — from
|
||||
# these phases + key_moments + rolls + hard_stops + phrases. Sections are
|
||||
# interpretation; only the timing they snap to is fact.
|
||||
phases_sec = []
|
||||
for p in phases:
|
||||
seg = norms[p["s"]:max(p["s"] + 1, p["e"])]
|
||||
phases_sec.append({
|
||||
"start": float(p["s"]),
|
||||
"end": float(min(p["e"], round(dur, 1))),
|
||||
"level": p["lvl"],
|
||||
"energy": round(float(np.mean(seg)) if seg else 0.0, 2),
|
||||
})
|
||||
|
||||
return {"norms": [round(n, 2) for n in norms], "phases": phases_sec,
|
||||
"moments": moments[:8], "hard_stops": hard_stops}
|
||||
|
||||
|
||||
# ── rolls / fills (localized rapid-onset runs) ───────────────────────────────
|
||||
# A roll is where choreography should switch from discrete hits to a continuous /
|
||||
# cascading visual (per-letter cascade, stagger). Derived straight from the onset
|
||||
# stream — runs never overlap, so no dedup is needed (unlike a band-energy detector).
|
||||
ROLL_MIN_HITS = 4
|
||||
ROLL_CONT = 0.55 # × beat_dur: gap up to ~half a beat still keeps a run alive
|
||||
ROLL_ACCEPT = 0.42 # × beat_dur: mean spacing denser than an 8th note counts
|
||||
ROLL_DEDUP = 0.08 # seconds: merge onsets closer than a 32nd (double-trigger)
|
||||
|
||||
|
||||
def detect_rolls(events: list, beat_dur: float) -> list:
|
||||
"""Runs of >=4 onsets whose MEAN spacing is denser than an 8th note. Tuned so a
|
||||
full hihat/snare roll is captured as ONE span (not fragmented down to its tail),
|
||||
while a sparse groove stays out — validated against the golden 7.5-9.5s roll.
|
||||
The linear scan means runs never overlap (no LEGACY-style double-counting)."""
|
||||
cont = beat_dur * ROLL_CONT # max gap that keeps a run alive
|
||||
accept = beat_dur * ROLL_ACCEPT # max MEAN gap for a run to count
|
||||
# collapse onset double-triggers (two onsets < a 32nd apart = one hit) so a
|
||||
# held/sparse passage can't masquerade as a roll on a duplicated transient.
|
||||
ev = []
|
||||
for e in events:
|
||||
if ev and e["t"] - ev[-1]["t"] <= ROLL_DEDUP:
|
||||
if e.get("energy", 0) > ev[-1].get("energy", 0):
|
||||
ev[-1] = e
|
||||
continue
|
||||
ev.append(e)
|
||||
times = [e["t"] for e in ev]
|
||||
n = len(times)
|
||||
rolls, i = [], 0
|
||||
while i < n - 1:
|
||||
j = i
|
||||
while j + 1 < n and (times[j + 1] - times[j]) <= cont:
|
||||
j += 1
|
||||
if j - i + 1 >= ROLL_MIN_HITS:
|
||||
gaps = [times[k + 1] - times[k] for k in range(i, j)]
|
||||
if sum(gaps) / len(gaps) <= accept:
|
||||
t0, t1 = times[i], times[j]
|
||||
half = len(gaps) // 2
|
||||
accel = (half >= 1 and
|
||||
sum(gaps[half:]) / (len(gaps) - half) <
|
||||
sum(gaps[:half]) / half * 0.85)
|
||||
dcount: dict[str, int] = {}
|
||||
for e in ev[i:j + 1]:
|
||||
dcount[e["drum"]] = dcount.get(e["drum"], 0) + 1
|
||||
rolls.append({
|
||||
"start": round(t0, 3), "end": round(t1, 3),
|
||||
"dur_sec": round(t1 - t0, 3),
|
||||
"hits": j - i + 1,
|
||||
"rate_per_min": round((j - i) / max(t1 - t0, 1e-6) * 60),
|
||||
"kind": "accel-roll" if accel else ("sustained-fill" if t1 - t0 > 1.2 else "fill"),
|
||||
"drum": max(dcount, key=dcount.get),
|
||||
})
|
||||
i = j + 1
|
||||
return rolls
|
||||
|
||||
|
||||
# ── per-section spectral character (sustained "feel") ────────────────────────
|
||||
# Coarse, reliable bands for how a SECTION sounds (not the noisy per-second dump).
|
||||
# Distinct from a per-event `feel`, which is the transient color of a single hit.
|
||||
FEEL_BANDS = [("sub", 0, 60), ("bass", 60, 250), ("low_mid", 250, 800),
|
||||
("mid", 800, 2500), ("presence", 2500, 6000), ("air", 6000, 1e9)]
|
||||
|
||||
|
||||
def annotate_section_feel(bc: dict, sr: int, sections: list) -> None:
|
||||
"""Attach {character, bands} to each energy phase from its sustained band balance."""
|
||||
S, n = bc["S"], bc["n"]
|
||||
freqs = librosa.fft_frequencies(sr=sr)
|
||||
fps = sr / HOP
|
||||
masks = [(name, (freqs >= lo) & (freqs < hi)) for name, lo, hi in FEEL_BANDS]
|
||||
for s in sections:
|
||||
f0 = int(s["start"] * fps)
|
||||
f1 = min(max(f0 + 1, int(s["end"] * fps)), n)
|
||||
seg = S[:, f0:f1]
|
||||
if seg.shape[1] == 0 or s.get("energy", 0) < 0.15:
|
||||
s["feel"] = {"character": "sparse", "bands": []}
|
||||
continue
|
||||
en = {name: float(seg[m].sum()) for name, m in masks}
|
||||
tot = sum(en.values()) + 1e-9
|
||||
ratios = {name: en[name] / tot for name in en}
|
||||
present = sorted([bn for bn, r in ratios.items() if r > 0.12],
|
||||
key=lambda bn: -ratios[bn])
|
||||
lo = ratios["sub"] + ratios["bass"]
|
||||
hi = ratios["presence"] + ratios["air"]
|
||||
mid = ratios["low_mid"] + ratios["mid"]
|
||||
char = ("heavy" if lo > 0.5 else "bright" if hi > 0.45 else
|
||||
"full" if lo > 0.25 and hi > 0.25 else
|
||||
"warm" if mid > 0.5 else "sparse")
|
||||
s["feel"] = {"character": char, "bands": present}
|
||||
|
||||
|
||||
# ── audiomap enrichment: phrase layer + section density budgets ─────────────
|
||||
def round3(n: float) -> float:
|
||||
return round(float(n), 3)
|
||||
|
||||
|
||||
def derive_phrases(downbeats: list[float], phrase_bars: int, duration_sec: float) -> list:
|
||||
"""Group downbeats into phrase spans of `phrase_bars` bars."""
|
||||
phrases = []
|
||||
if not downbeats:
|
||||
return phrases
|
||||
index = 0
|
||||
for i in range(0, len(downbeats), phrase_bars):
|
||||
start = downbeats[i]
|
||||
next_idx = i + phrase_bars
|
||||
end = downbeats[next_idx] if next_idx < len(downbeats) else duration_sec
|
||||
phrases.append({
|
||||
"index": index,
|
||||
"start": round3(start),
|
||||
"end": round3(end),
|
||||
"bars": min(phrase_bars, len(downbeats) - i),
|
||||
})
|
||||
index += 1
|
||||
return phrases
|
||||
|
||||
|
||||
def count_in(times: list[float], start: float, end: float) -> int:
|
||||
return sum(1 for t in times if t >= start - 1e-6 and t < end - 1e-6)
|
||||
|
||||
|
||||
def derive_phase_budgets(timeline: dict) -> list:
|
||||
"""Attach an objective density read to each energy phase.
|
||||
|
||||
Density is a fact (onsets-per-second + rolls). It is a hint for how much visual
|
||||
content a span can hold; it does not set timing or sections. The Music Reader uses
|
||||
these phases to decide the actual sections.
|
||||
"""
|
||||
phases = timeline.get("energy_phases", [])
|
||||
onset_times = [e["t"] for e in timeline.get("events", [])]
|
||||
rolls = timeline.get("rolls", [])
|
||||
hard_stops = timeline.get("hard_stops", [])
|
||||
|
||||
out = []
|
||||
for s in phases:
|
||||
span = max(1e-6, float(s.get("end", 0)) - float(s.get("start", 0)))
|
||||
onsets = count_in(onset_times, s["start"], s["end"])
|
||||
ph_rolls = [
|
||||
{"start": r["start"], "end": r["end"], "kind": r["kind"], "drum": r["drum"]}
|
||||
for r in rolls
|
||||
if r["start"] < s["end"] - 1e-6 and r["end"] > s["start"] + 1e-6
|
||||
]
|
||||
ph_stops = [
|
||||
h["t"]
|
||||
for h in hard_stops
|
||||
if h["t"] >= s["start"] - 1e-6 and h["t"] < s["end"] + 1e-6
|
||||
]
|
||||
|
||||
if s.get("energy", 0) < 0.2 or onsets < 6:
|
||||
density = "sparse"
|
||||
elif onsets >= 18 or ph_rolls:
|
||||
density = "dense"
|
||||
else:
|
||||
density = "medium"
|
||||
|
||||
enriched = dict(s)
|
||||
enriched["onsets"] = onsets
|
||||
enriched["onsetRate"] = round(onsets / span, 1)
|
||||
enriched["rolls"] = ph_rolls
|
||||
enriched["hardStops"] = ph_stops
|
||||
enriched["density"] = density
|
||||
out.append(enriched)
|
||||
return out
|
||||
|
||||
|
||||
def finalize_audiomap(timeline: dict, phrase_bars: int = 4) -> dict:
|
||||
downbeats = timeline.get("grid", {}).get("downbeats_sec", [])
|
||||
duration_sec = timeline.get("audio", {}).get("duration_sec", 0)
|
||||
energy_phases = derive_phase_budgets(timeline)
|
||||
phrases = derive_phrases(downbeats, phrase_bars, duration_sec)
|
||||
return {
|
||||
"version": AUDIOMAP_VERSION,
|
||||
"phraseBars": phrase_bars,
|
||||
**timeline,
|
||||
"energy_phases": energy_phases,
|
||||
"phrases": phrases,
|
||||
}
|
||||
|
||||
|
||||
# ── main ────────────────────────────────────────────────────────────────────
|
||||
def analyze(path: str, phrase_bars: int = 4) -> dict:
|
||||
y, sr, dur = load_audio(path)
|
||||
bg = beat_grid(y, sr)
|
||||
bc = band_energy_curves(y, sr)
|
||||
phase = downbeat_phase(bg["beat_frames"], bc)
|
||||
beats = bg["beats"]
|
||||
downbeats = [float(beats[i]) for i in range(len(beats)) if (i - phase) % 4 == 0]
|
||||
|
||||
# onsets → events
|
||||
onset_t = librosa.onset.onset_detect(
|
||||
y=y, sr=sr, hop_length=HOP, units="time", backtrack=True
|
||||
)
|
||||
en_at = bc["total"]
|
||||
en_max = float(en_at.max()) + 1e-9
|
||||
events = []
|
||||
for t in onset_t:
|
||||
gclass, bar, bib, step16 = classify_metric(float(t), beats, phase)
|
||||
drum, energy, feel = classify_drum(float(t), bc, sr)
|
||||
f = frame_at(float(t), sr, bc["n"])
|
||||
events.append({
|
||||
"t": round(float(t), 3),
|
||||
"bar": int(bar), "beat_in_bar": int(bib), "step16": int(step16),
|
||||
"grid": gclass, "drum": drum,
|
||||
"energy": round(float(en_at[f]) / en_max, 2), "feel": feel,
|
||||
"special": None,
|
||||
})
|
||||
|
||||
first_onset = next((float(t) for t in onset_t if t >= 2.0), 0.0)
|
||||
es = energy_structure(y, sr, dur, first_onset)
|
||||
|
||||
# tag specials onto nearby events
|
||||
for hs in es["hard_stops"]:
|
||||
for e in events:
|
||||
if abs(e["t"] - hs["t"]) < 0.6:
|
||||
e["special"] = "hard_stop"
|
||||
# riser: events inside a 1.5s+ rising-energy run that precedes a SURGE
|
||||
surges = [m["t"] for m in es["moments"] if m["kind"] == "SURGE"]
|
||||
for st in surges:
|
||||
for e in events:
|
||||
if st - 2.0 <= e["t"] < st and e["special"] is None and e["drum"] in ("perc", "glitch"):
|
||||
e["special"] = "riser"
|
||||
|
||||
# rolls / fills + whether each leads straight into a surge/drop (cascade cue)
|
||||
beat_dur = float(np.median(np.diff(beats))) if len(beats) > 1 else 60.0 / max(bg["bpm"], 1e-6)
|
||||
rolls = detect_rolls(events, beat_dur)
|
||||
for r in rolls:
|
||||
r["leads_to"] = next((m["kind"] for m in es["moments"]
|
||||
if 0 <= m["t"] - r["end"] <= 1.2), None)
|
||||
# near-silent windows (= VOID energy phases) — convenience for "hold / breathe"
|
||||
silences = [{"start": p["start"], "end": p["end"]}
|
||||
for p in es["phases"] if p["level"] == "VOID"]
|
||||
# per-phase sustained spectral character (how each energy block FEELS)
|
||||
annotate_section_feel(bc, sr, es["phases"])
|
||||
|
||||
n_drum = {}
|
||||
for e in events:
|
||||
n_drum[e["drum"]] = n_drum.get(e["drum"], 0) + 1
|
||||
n_grid = {}
|
||||
for e in events:
|
||||
n_grid[e["grid"]] = n_grid.get(e["grid"], 0) + 1
|
||||
|
||||
summary = (f"{bg['bpm']:.0f} BPM · {len(beats)} beats / {len(downbeats)} bars · "
|
||||
f"{len(events)} events ({n_drum}) · {len(rolls)} rolls · "
|
||||
f"{len(es['phases'])} energy phases · {dur:.1f}s")
|
||||
|
||||
timeline = {
|
||||
"summary": summary,
|
||||
"audio": {"path": path, "duration_sec": round(dur, 3), "sr": sr},
|
||||
"tempo": {"bpm": round(bg["bpm"], 1), "beats_per_bar": 4,
|
||||
"downbeat_phase": phase, "n_beats": len(beats), "n_bars": len(downbeats)},
|
||||
"grid": {"beats_sec": [round(float(b), 3) for b in beats],
|
||||
"downbeats_sec": [round(b, 3) for b in downbeats]},
|
||||
"energy_phases": es["phases"],
|
||||
"key_moments": es["moments"],
|
||||
"hard_stops": es["hard_stops"],
|
||||
"rolls": rolls,
|
||||
"silences": silences,
|
||||
"stats": {"drum_counts": n_drum, "grid_counts": n_grid},
|
||||
"events": events,
|
||||
}
|
||||
return finalize_audiomap(timeline, phrase_bars)
|
||||
|
||||
|
||||
def print_brief(d: dict) -> None:
|
||||
print(f"\n{d['summary']}\n{'='*70}")
|
||||
print("ENERGY PHASES (audio-driven blocks; the Music Reader names the sections)")
|
||||
for s in d.get("energy_phases", []):
|
||||
feel = s.get("feel", {})
|
||||
bands = ",".join(feel.get("bands", []))
|
||||
print(f" {s['start']:5.1f}-{s['end']:5.1f}s {s.get('level', ''):6s} energy={s.get('energy')} "
|
||||
f"{feel.get('character', ''):6s} [{bands}] {s.get('density', '?')}")
|
||||
print("PHRASES")
|
||||
for p in d.get("phrases", []):
|
||||
print(f" #{p['index']} {p['start']:5.2f}-{p['end']:5.2f}s bars={p['bars']}")
|
||||
print("KEY MOMENTS")
|
||||
for m in d["key_moments"]:
|
||||
print(f" {m['t']:3d}s {m['kind']:5s} Δ{m['delta']:+.2f}")
|
||||
print(f"HARD STOPS: {[h['t'] for h in d['hard_stops']]}")
|
||||
print("ROLLS / FILLS")
|
||||
for r in d.get("rolls", []):
|
||||
lead = f" → {r['leads_to']}" if r.get("leads_to") else ""
|
||||
print(f" {r['start']:6.2f}-{r['end']:5.2f}s {r['hits']:2d} hits @ {r['rate_per_min']:4d}/min "
|
||||
f"{r['kind']:10s} ({r['drum']}){lead}")
|
||||
print(f"SILENCES: {[(s['start'], s['end']) for s in d.get('silences', [])]}")
|
||||
print(f"DRUM COUNTS: {d['stats']['drum_counts']} GRID: {d['stats']['grid_counts']}")
|
||||
print(f"\nEVENTS ({len(d['events'])}) [t · bar:beat · grid · drum · energy · special]")
|
||||
for e in d["events"]:
|
||||
sp = f" <{e['special']}>" if e["special"] else ""
|
||||
print(f" {e['t']:6.2f}s b{e['bar']}:{e['beat_in_bar']} {e['grid']:3s} "
|
||||
f"{e['drum']:6s} e={e['energy']:.2f} {e['feel']:8s}{sp}")
|
||||
|
||||
|
||||
def main() -> None:
|
||||
ap = argparse.ArgumentParser()
|
||||
ap.add_argument("audio")
|
||||
ap.add_argument("-o", "--out", default=None)
|
||||
ap.add_argument("--phrase-bars", type=int, default=4)
|
||||
ap.add_argument("--print", action="store_true", dest="do_print")
|
||||
a = ap.parse_args()
|
||||
d = analyze(a.audio, phrase_bars=a.phrase_bars)
|
||||
if a.out:
|
||||
Path(a.out).write_text(json.dumps(d, ensure_ascii=False, indent=2))
|
||||
dens = " ".join(f"{s.get('level', '?')}:{s.get('density', '?')}" for s in d.get("energy_phases", []))
|
||||
print(
|
||||
f"[analyze-beatgrid] wrote audiomap {a.out} · {len(d.get('energy_phases', []))} phases · density [{dens}]",
|
||||
file=sys.stderr,
|
||||
)
|
||||
if a.do_print or not a.out:
|
||||
print_brief(d)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,214 @@
|
||||
#!/usr/bin/env node
|
||||
// assemble-index.mjs — deterministic top-level index.html assembly for a
|
||||
// music-to-video project. No subagent, no judgment: turns STORYBOARD.md + the
|
||||
// built per-frame composition files (+ assets/bgm.mp3) into the standalone
|
||||
// index.html the renderer consumes.
|
||||
//
|
||||
// index.html is a *standalone* composition (root <div id="root"> directly in
|
||||
// <body>, no <template> wrapper — template is for the frame sub-comps). Each
|
||||
// frame is referenced (not inlined) as a <div class="frame"> with
|
||||
// data-composition-src pointing at its file; the renderer seeks each at its
|
||||
// absolute data-start. Frames tile the track gap-free, so frame→frame is a
|
||||
// plain back-to-back HARD CUT — there is NO transition injector (unlike
|
||||
// product-launch, whose transitions.mjs exists only for soft transitions).
|
||||
//
|
||||
// Track lanes:
|
||||
// 1 frame clips (sequential, gap-free, hard cut between)
|
||||
// 10 optional per-frame VO <audio> (deferred; mounted only if audio_meta has it)
|
||||
// 11 BGM <audio> (full duration)
|
||||
//
|
||||
// Reads: --storyboard STORYBOARD.md, --hyperframes <root>, [--audiomap audiomap.json],
|
||||
// [--bgm assets/bgm.mp3], [--audio-meta audio_meta.json]. On disk: each frame's src html.
|
||||
// Writes: <project>/index.html
|
||||
//
|
||||
// Exit 0 = index.html written + summary. Exit 1 = fatal contract break (no
|
||||
// frames, a frame missing/empty/with-no-duration, an inner id mismatch).
|
||||
|
||||
import { existsSync, readFileSync, writeFileSync } from "node:fs";
|
||||
import { basename, join, resolve } from "node:path";
|
||||
import { parseStoryboard } from "./lib/storyboard.mjs";
|
||||
|
||||
const argv = process.argv.slice(2);
|
||||
const flag = (name, def) => {
|
||||
const i = argv.indexOf(`--${name}`);
|
||||
return i >= 0 && i + 1 < argv.length ? argv[i + 1] : def;
|
||||
};
|
||||
function die(msg) {
|
||||
console.error(`✗ assemble-index.mjs: ${msg}`);
|
||||
process.exit(1);
|
||||
}
|
||||
const r3 = (x) => Math.round(x * 1000) / 1000;
|
||||
const anomalies = [];
|
||||
|
||||
const hyperframesDir = resolve(flag("hyperframes", "."));
|
||||
const storyboardPath = resolve(flag("storyboard", join(hyperframesDir, "STORYBOARD.md")));
|
||||
const audiomapPath = resolve(flag("audiomap", join(hyperframesDir, "audiomap.json")));
|
||||
const audioMetaPath = resolve(flag("audio-meta", join(hyperframesDir, "audio_meta.json")));
|
||||
const bgmRel = flag("bgm", "assets/bgm.mp3");
|
||||
const outPath = resolve(flag("out", join(hyperframesDir, "index.html")));
|
||||
|
||||
// ---------- parse storyboard ----------
|
||||
if (!existsSync(storyboardPath)) die(`STORYBOARD.md not found at ${storyboardPath}`);
|
||||
const manifest = parseStoryboard(readFileSync(storyboardPath, "utf8"));
|
||||
const G = manifest.globals.extra ?? {};
|
||||
|
||||
// canvas from frontmatter `canvas: {"w":1920,"h":1080,"fps":30}` (key lowercased by parser)
|
||||
let WIDTH = 1920,
|
||||
HEIGHT = 1080;
|
||||
if (G.canvas) {
|
||||
try {
|
||||
const c = JSON.parse(G.canvas);
|
||||
if (Number.isFinite(c.w)) WIDTH = c.w;
|
||||
if (Number.isFinite(c.h)) HEIGHT = c.h;
|
||||
} catch {
|
||||
anomalies.push(`could not JSON.parse canvas frontmatter: ${G.canvas} — using ${WIDTH}×${HEIGHT}`);
|
||||
}
|
||||
}
|
||||
|
||||
// audio duration is the spine truth
|
||||
let audioDur = null;
|
||||
if (existsSync(audiomapPath)) {
|
||||
try {
|
||||
audioDur = JSON.parse(readFileSync(audiomapPath, "utf8"))?.audio?.duration_sec ?? null;
|
||||
} catch (e) {
|
||||
anomalies.push(`audiomap.json parse failed (${e.message}) — using frame sum for duration`);
|
||||
}
|
||||
}
|
||||
|
||||
// ---------- resolve mountable frames in document order ----------
|
||||
const mounted = [];
|
||||
for (const f of manifest.frames) {
|
||||
const label = `frame ${f.number ?? f.index}${f.title ? ` (${f.title})` : ""}`;
|
||||
if (!f.src) die(`${label} has no \`src\` — the planner must write it in STORYBOARD.md`);
|
||||
const compAbs = join(hyperframesDir, f.src);
|
||||
if (!existsSync(compAbs))
|
||||
die(`${label}: src ${f.src} is not on disk — re-dispatch its frame-worker before assembling`);
|
||||
if (!Number.isFinite(f.durationSeconds) || f.durationSeconds <= 0)
|
||||
die(`${label}: no positive \`duration\` (got ${JSON.stringify(f.duration)})`);
|
||||
const compId = basename(f.src).replace(/\.html?$/i, "");
|
||||
const inner = readFileSync(compAbs, "utf8");
|
||||
if (!inner.trim() || !/<\w/.test(inner))
|
||||
die(`${label}: ${f.src} is empty/blank — the frame-worker wrote a partial file. Re-dispatch it.`);
|
||||
if (
|
||||
!inner.includes(`data-composition-id="${compId}"`) &&
|
||||
!inner.includes(`data-composition-id='${compId}'`)
|
||||
)
|
||||
die(`${label}: ${f.src} has no data-composition-id="${compId}" (host/inner id must match)`);
|
||||
mounted.push({ frame: f, compId, durationSeconds: r3(f.durationSeconds) });
|
||||
}
|
||||
if (mounted.length === 0) die("no mountable frames (none with an on-disk src)");
|
||||
|
||||
// cumulative starts — start[i] + duration[i] == start[i+1] exactly (gap-free hard cuts)
|
||||
let acc = 0;
|
||||
for (const m of mounted) {
|
||||
m.start = r3(acc);
|
||||
acc += m.durationSeconds;
|
||||
}
|
||||
const FRAME_SUM = r3(acc);
|
||||
const TOTAL = r3(audioDur ?? FRAME_SUM);
|
||||
if (audioDur != null && Math.abs(FRAME_SUM - audioDur) > 0.1)
|
||||
anomalies.push(
|
||||
`frames sum to ${FRAME_SUM}s but audio is ${audioDur}s (Δ${r3(FRAME_SUM - audioDur)}s) — frames should tile the track; check the plan`,
|
||||
);
|
||||
|
||||
// ---------- optional VO (deferred hook) ----------
|
||||
let audio = { voices: [] };
|
||||
if (existsSync(audioMetaPath)) {
|
||||
try {
|
||||
audio = JSON.parse(readFileSync(audioMetaPath, "utf8"));
|
||||
} catch (e) {
|
||||
anomalies.push(`audio_meta.json parse: ${e.message}`);
|
||||
}
|
||||
}
|
||||
const voiceByNum = new Map();
|
||||
for (const v of audio.voices ?? []) if (v.frame != null) voiceByNum.set(v.frame, v);
|
||||
|
||||
// ---------- build <body> ----------
|
||||
const body = [];
|
||||
let voiceCount = 0;
|
||||
for (const m of mounted) {
|
||||
body.push(
|
||||
` <div`,
|
||||
` id="el-${m.compId}"`,
|
||||
` class="frame"`,
|
||||
` data-composition-id="${m.compId}"`,
|
||||
` data-composition-src="${m.frame.src}"`,
|
||||
` data-start="${m.start}"`,
|
||||
` data-duration="${m.durationSeconds}"`,
|
||||
` data-track-index="1"`,
|
||||
` ></div>`,
|
||||
);
|
||||
const v = m.frame.number != null ? voiceByNum.get(m.frame.number) : undefined;
|
||||
if (v?.path && existsSync(join(hyperframesDir, v.path))) {
|
||||
body.push(
|
||||
` <audio id="el-${m.compId}-voice" src="${v.path}" data-start="${m.start}"`,
|
||||
` data-duration="${m.durationSeconds}" data-track-index="10" data-volume="1"></audio>`,
|
||||
);
|
||||
voiceCount++;
|
||||
}
|
||||
body.push("");
|
||||
}
|
||||
|
||||
// BGM (track 11) — full duration; duck slightly when VO present
|
||||
let bgmEmitted = false;
|
||||
if (existsSync(join(hyperframesDir, bgmRel))) {
|
||||
const vol = voiceCount > 0 ? 0.8 : 0.9;
|
||||
body.push(
|
||||
` <!-- BGM -->`,
|
||||
` <audio id="el-bgm" src="${bgmRel}" data-start="0" data-duration="${TOTAL}"`,
|
||||
` data-track-index="11" data-volume="${vol}"></audio>`,
|
||||
);
|
||||
bgmEmitted = true;
|
||||
} else {
|
||||
anomalies.push(`BGM not found at ${bgmRel} — index has no music track`);
|
||||
}
|
||||
|
||||
// ---------- head + emit ----------
|
||||
const headStyle = [
|
||||
" * { margin: 0; padding: 0; box-sizing: border-box; }",
|
||||
` html, body { width: ${WIDTH}px; height: ${HEIGHT}px; overflow: hidden; background: #000; }`,
|
||||
` #root { position: relative; width: ${WIDTH}px; height: ${HEIGHT}px; overflow: hidden; }`,
|
||||
" .frame { position: absolute; inset: 0; width: 100%; height: 100%; }",
|
||||
].join("\n");
|
||||
|
||||
const html = `<!doctype html>
|
||||
<html lang="en">
|
||||
<head>
|
||||
<meta charset="UTF-8" />
|
||||
<meta name="viewport" content="width=${WIDTH}, height=${HEIGHT}" />
|
||||
<script src="https://cdn.jsdelivr.net/npm/gsap@3.14.2/dist/gsap.min.js"></script>
|
||||
<style>
|
||||
${headStyle}
|
||||
</style>
|
||||
</head>
|
||||
<body>
|
||||
<div
|
||||
id="root"
|
||||
data-composition-id="main"
|
||||
data-start="0"
|
||||
data-duration="${TOTAL}"
|
||||
data-width="${WIDTH}"
|
||||
data-height="${HEIGHT}"
|
||||
>
|
||||
${body.join("\n")}
|
||||
</div>
|
||||
|
||||
<script>
|
||||
window.__timelines = window.__timelines || {};
|
||||
window.__timelines["main"] = gsap.timeline({ paused: true });
|
||||
</script>
|
||||
</body>
|
||||
</html>
|
||||
`;
|
||||
writeFileSync(outPath, html);
|
||||
|
||||
console.log(`✓ wrote ${outPath}`);
|
||||
console.log(` canvas: ${WIDTH}×${HEIGHT}`);
|
||||
console.log(` frames (track 1): ${mounted.length}`);
|
||||
console.log(` bgm (track 11): ${bgmEmitted ? bgmRel : "MISSING"}`);
|
||||
console.log(` vo (track 10): ${voiceCount}`);
|
||||
console.log(` total duration: ${TOTAL}s` + (audioDur != null ? ` (audio ${audioDur}s)` : ""));
|
||||
if (anomalies.length) {
|
||||
console.log(`\nanomalies (non-fatal):`);
|
||||
for (const a of anomalies) console.log(` - ${a}`);
|
||||
}
|
||||
@@ -0,0 +1,249 @@
|
||||
// storyboard.mjs — vendored lenient parser for STORYBOARD.md.
|
||||
//
|
||||
// Faithful plain-JS port of @hyperframes/core/storyboard
|
||||
// (packages/core/src/storyboard/parseStoryboard.ts). Vendored because skills
|
||||
// ship standalone: installed via `npx skills add`, a skill's scripts can't reach
|
||||
// the monorepo's core package, and the core export points at .ts source that
|
||||
// `node` (which runs these scripts) can't load. CANONICAL contract = the core
|
||||
// parser + skills/hyperframes-core/references/storyboard-format.md; keep this in
|
||||
// lockstep. Behavior: never throws, accepts freeform narrative, recognizes
|
||||
// Frame/Beat/Scene headings at H2/H3, preserves unknown keys verbatim under
|
||||
// `extra` (keys lowercased). Pure node — no deps.
|
||||
|
||||
export const FRAME_STATUSES = ["outline", "built", "animated"];
|
||||
export const DEFAULT_FRAME_STATUS = "outline";
|
||||
|
||||
// Detection-only frame heading (ends at the keyword); ReDoS-hardened — keep as-is.
|
||||
const FRAME_HEADING_RE = /^(#{2,3})[ \t]+(?:frame|beat|scene)\b/i;
|
||||
const FRAME_TITLE_SEP_RE = /^[\s.:—-]+/;
|
||||
const HEADING_LEVEL_RE = /^(#{1,6})\s+/;
|
||||
const META_RE = /^\s*[-*]\s+([A-Za-z_][\w-]*)\s*:\s*(.+?)\s*$/;
|
||||
const LEADING_INT_RE = /^(\d+)/;
|
||||
const DURATION_NUM_RE = /(\d+(?:\.\d+)?)/;
|
||||
const TRANSITION_KEYS = new Set(["transition_in", "transitionin", "transition"]);
|
||||
const SCENE_KEYS = new Set(["scene", "description", "summary", "caption"]);
|
||||
export const VOICEOVER_ALIASES = ["voiceover", "vo", "voice_over", "narration"];
|
||||
const VOICEOVER_KEYS = new Set(VOICEOVER_ALIASES);
|
||||
|
||||
export function parseStoryboard(source) {
|
||||
const warnings = [];
|
||||
const { globals, bodyStartLine, body } = parseFrontmatter(source, warnings);
|
||||
const frames = parseFrames(body, bodyStartLine, warnings);
|
||||
return { globals, frames, warnings };
|
||||
}
|
||||
|
||||
function emptyGlobals() {
|
||||
return { extra: {} };
|
||||
}
|
||||
|
||||
function isFrameStatus(value) {
|
||||
return FRAME_STATUSES.includes(value);
|
||||
}
|
||||
|
||||
// ── Frontmatter ─────────────────────────────────────────────────────────────
|
||||
function findFrontmatterRange(lines, warnings) {
|
||||
let start = 0;
|
||||
while (start < lines.length && (lines[start] ?? "").trim() === "") start++;
|
||||
if ((lines[start] ?? "").trim() !== "---") return null;
|
||||
for (let i = start + 1; i < lines.length; i++) {
|
||||
if ((lines[i] ?? "").trim() === "---") return { start, end: i };
|
||||
}
|
||||
warnings.push({
|
||||
message: "Frontmatter opening '---' has no closing '---'; treating whole file as body.",
|
||||
line: start + 1,
|
||||
});
|
||||
return null;
|
||||
}
|
||||
|
||||
function parseFrontmatterEntries(lines, start, end, warnings) {
|
||||
const globals = emptyGlobals();
|
||||
for (let i = start + 1; i < end; i++) {
|
||||
const raw = lines[i] ?? "";
|
||||
if (raw.trim() === "") continue;
|
||||
const colon = raw.indexOf(":");
|
||||
if (colon === -1) {
|
||||
warnings.push({
|
||||
message: `Ignored non key:value frontmatter line: "${raw.trim()}"`,
|
||||
line: i + 1,
|
||||
});
|
||||
continue;
|
||||
}
|
||||
const key = raw.slice(0, colon).trim().toLowerCase();
|
||||
assignGlobal(globals, key, stripQuotes(raw.slice(colon + 1).trim()));
|
||||
}
|
||||
return globals;
|
||||
}
|
||||
|
||||
function parseFrontmatter(source, warnings) {
|
||||
const lines = source.split(/\r?\n/);
|
||||
const range = findFrontmatterRange(lines, warnings);
|
||||
if (!range) return { globals: emptyGlobals(), bodyStartLine: 1, body: source };
|
||||
const globals = parseFrontmatterEntries(lines, range.start, range.end, warnings);
|
||||
const body = lines.slice(range.end + 1).join("\n");
|
||||
return { globals, bodyStartLine: range.end + 2, body };
|
||||
}
|
||||
|
||||
function assignGlobal(globals, key, value) {
|
||||
switch (key) {
|
||||
case "format":
|
||||
globals.format = value;
|
||||
break;
|
||||
case "message":
|
||||
globals.message = value;
|
||||
break;
|
||||
case "arc":
|
||||
globals.arc = value;
|
||||
break;
|
||||
case "audience":
|
||||
globals.audience = value;
|
||||
break;
|
||||
default:
|
||||
globals.extra[key] = value;
|
||||
}
|
||||
}
|
||||
|
||||
// ── Frames ──────────────────────────────────────────────────────────────────
|
||||
function openFrameSection(line, headingLine) {
|
||||
const match = FRAME_HEADING_RE.exec(line);
|
||||
if (!match) return null;
|
||||
const headingText = line.slice(match[0].length).replace(FRAME_TITLE_SEP_RE, "").trim();
|
||||
return { headingText, headingLine, level: (match[1] ?? "##").length, lines: [] };
|
||||
}
|
||||
|
||||
function endsFrameSection(line, current) {
|
||||
if (!current) return false;
|
||||
const heading = HEADING_LEVEL_RE.exec(line);
|
||||
return heading !== null && (heading[1] ?? "").length <= current.level;
|
||||
}
|
||||
|
||||
function parseFrames(body, bodyStartLine, warnings) {
|
||||
const lines = body.split(/\r?\n/);
|
||||
const sections = [];
|
||||
let current = null;
|
||||
for (let i = 0; i < lines.length; i++) {
|
||||
const line = lines[i] ?? "";
|
||||
const opened = openFrameSection(line, bodyStartLine + i);
|
||||
if (opened) {
|
||||
sections.push(opened);
|
||||
current = opened;
|
||||
} else if (endsFrameSection(line, current)) {
|
||||
current = null;
|
||||
} else if (current) {
|
||||
current.lines.push(line);
|
||||
}
|
||||
}
|
||||
return sections.map((section, idx) => buildFrame(section, idx + 1, warnings));
|
||||
}
|
||||
|
||||
function buildFrame(section, index, warnings) {
|
||||
const frame = { index, status: DEFAULT_FRAME_STATUS, narrative: "", extra: {} };
|
||||
const { number, title } = parseHeading(section.headingText);
|
||||
if (number !== undefined) frame.number = number;
|
||||
if (title) frame.title = title;
|
||||
|
||||
const narrativeLines = [];
|
||||
for (const line of section.lines) {
|
||||
const meta = META_RE.exec(line);
|
||||
if (meta) {
|
||||
applyMeta(
|
||||
frame,
|
||||
(meta[1] ?? "").toLowerCase(),
|
||||
(meta[2] ?? "").trim(),
|
||||
section.headingLine,
|
||||
warnings,
|
||||
);
|
||||
} else {
|
||||
narrativeLines.push(line);
|
||||
}
|
||||
}
|
||||
frame.narrative = narrativeLines.join("\n").trim();
|
||||
return frame;
|
||||
}
|
||||
|
||||
function parseHeading(text) {
|
||||
if (!text) return {};
|
||||
const intMatch = LEADING_INT_RE.exec(text);
|
||||
if (!intMatch) return { title: text };
|
||||
const number = Number.parseInt(intMatch[1] ?? "", 10);
|
||||
const rest = text
|
||||
.slice((intMatch[0] ?? "").length)
|
||||
.replace(/^[\s.:—-]+/, "")
|
||||
.trim();
|
||||
return { number, title: rest || undefined };
|
||||
}
|
||||
|
||||
// Dispatch a recognized metadata key to its field, else stash under `extra`.
|
||||
// Mirrors core's META_SETTERS map exactly (direct keys + alias sets).
|
||||
function applyMeta(frame, key, value, headingLine, warnings) {
|
||||
switch (key) {
|
||||
case "duration":
|
||||
applyDuration(frame, value, headingLine, warnings);
|
||||
return;
|
||||
case "status":
|
||||
applyStatus(frame, value, headingLine, warnings);
|
||||
return;
|
||||
case "poster":
|
||||
applyPoster(frame, value);
|
||||
return;
|
||||
case "src":
|
||||
frame.src = value;
|
||||
return;
|
||||
}
|
||||
if (TRANSITION_KEYS.has(key)) {
|
||||
frame.transitionIn = value;
|
||||
return;
|
||||
}
|
||||
if (SCENE_KEYS.has(key)) {
|
||||
frame.scene = value;
|
||||
return;
|
||||
}
|
||||
if (VOICEOVER_KEYS.has(key)) {
|
||||
frame.voiceover = stripQuotes(value);
|
||||
return;
|
||||
}
|
||||
frame.extra[key] = value;
|
||||
}
|
||||
|
||||
function applyPoster(frame, value) {
|
||||
const num = DURATION_NUM_RE.exec(value);
|
||||
if (num) frame.poster = Number.parseFloat(num[1] ?? "");
|
||||
}
|
||||
|
||||
function applyDuration(frame, value, headingLine, warnings) {
|
||||
frame.duration = value;
|
||||
const num = DURATION_NUM_RE.exec(value);
|
||||
if (num) {
|
||||
frame.durationSeconds = Number.parseFloat(num[1] ?? "");
|
||||
return;
|
||||
}
|
||||
warnings.push({
|
||||
message: `Frame ${frame.index}: could not parse duration "${value}".`,
|
||||
line: headingLine,
|
||||
frameIndex: frame.index,
|
||||
});
|
||||
}
|
||||
|
||||
function applyStatus(frame, value, headingLine, warnings) {
|
||||
const normalized = value.toLowerCase();
|
||||
if (isFrameStatus(normalized)) {
|
||||
frame.status = normalized;
|
||||
return;
|
||||
}
|
||||
frame.extra.status = value;
|
||||
warnings.push({
|
||||
message: `Frame ${frame.index}: unknown status "${value}"; defaulting to "${DEFAULT_FRAME_STATUS}".`,
|
||||
line: headingLine,
|
||||
frameIndex: frame.index,
|
||||
});
|
||||
}
|
||||
|
||||
function stripQuotes(value) {
|
||||
if (value.length >= 2) {
|
||||
const first = value[0];
|
||||
const last = value[value.length - 1];
|
||||
if ((first === '"' && last === '"') || (first === "'" && last === "'")) {
|
||||
return value.slice(1, -1);
|
||||
}
|
||||
}
|
||||
return value;
|
||||
}
|
||||
@@ -0,0 +1,55 @@
|
||||
#!/usr/bin/env node
|
||||
// stage-assets.mjs — copy user-supplied media into the project's assets/ so scene
|
||||
// files (and lint/validate/render) can reference them locally. Only needed when
|
||||
// the user provides images/videos for asset treatments (montage.md). First-wins,
|
||||
// idempotent, safe to run twice. Never fetches remote URLs.
|
||||
//
|
||||
// Usage: node stage-assets.mjs --from <srcDir> --hyperframes <projectRoot>
|
||||
// [--into public] (subdir under assets/; default copies flat into assets/)
|
||||
//
|
||||
// Copies common media extensions only; reports what landed.
|
||||
|
||||
import { existsSync, mkdirSync, readdirSync, copyFileSync, statSync } from "node:fs";
|
||||
import { extname, join, resolve, basename } from "node:path";
|
||||
|
||||
const argv = process.argv.slice(2);
|
||||
const flag = (n, d) => {
|
||||
const i = argv.indexOf(`--${n}`);
|
||||
return i >= 0 && i + 1 < argv.length ? argv[i + 1] : d;
|
||||
};
|
||||
function die(m) {
|
||||
console.error(`✗ stage-assets.mjs: ${m}`);
|
||||
process.exit(1);
|
||||
}
|
||||
|
||||
const fromDir = flag("from", null);
|
||||
if (!fromDir) die("missing --from <srcDir>");
|
||||
const fromAbs = resolve(fromDir);
|
||||
if (!existsSync(fromAbs) || !statSync(fromAbs).isDirectory()) die(`--from is not a directory: ${fromAbs}`);
|
||||
const hyperframesDir = resolve(flag("hyperframes", "."));
|
||||
const into = flag("into", "");
|
||||
const destDir = join(hyperframesDir, "assets", into);
|
||||
|
||||
const MEDIA = new Set([".jpg", ".jpeg", ".png", ".webp", ".gif", ".mp4", ".mov", ".webm", ".m4v"]);
|
||||
|
||||
mkdirSync(destDir, { recursive: true });
|
||||
let staged = 0,
|
||||
skipped = 0;
|
||||
const landed = [];
|
||||
for (const name of readdirSync(fromAbs)) {
|
||||
const src = join(fromAbs, name);
|
||||
if (!statSync(src).isFile()) continue;
|
||||
if (!MEDIA.has(extname(name).toLowerCase())) continue;
|
||||
const dest = join(destDir, basename(name));
|
||||
if (existsSync(dest)) {
|
||||
skipped++;
|
||||
continue;
|
||||
} // first-wins
|
||||
copyFileSync(src, dest);
|
||||
staged++;
|
||||
landed.push(join("assets", into, basename(name)));
|
||||
}
|
||||
|
||||
console.log(`✓ stage-assets: ${staged} copied, ${skipped} already present → ${join("assets", into)}/`);
|
||||
for (const l of landed) console.log(` ${l}`);
|
||||
if (staged === 0 && skipped === 0) console.log(` (no media files found in ${fromAbs})`);
|
||||
@@ -0,0 +1,144 @@
|
||||
#!/usr/bin/env node
|
||||
// validate-plan.mjs — machine-check STORYBOARD.md against the audiomap + template
|
||||
// catalog at Step 3, before any frame is built. Runs on the PLAN (frame files do
|
||||
// not exist yet), so it checks fields, not on-disk html.
|
||||
//
|
||||
// HARD (exit 1): frontmatter duration_s == audiomap duration; >=1 frame; each frame
|
||||
// has src + positive duration; frames tile the track gap-free (sum == duration_s).
|
||||
// WARN (exit 0): best-effort group checks — each group exactly one of
|
||||
// template/free_design/asset; a template id exists under the --templates dir;
|
||||
// phrase_flow frame has no beat_cut asset treatment.
|
||||
//
|
||||
// Frame-level checks use the vendored storyboard parser. Group-level checks re-scan
|
||||
// the RAW source (the parser's META_RE consumes indented `- params:`/`- asset:` lines).
|
||||
//
|
||||
// Reads: --storyboard, --audiomap, --hyperframes <root> (for templates/).
|
||||
|
||||
import { existsSync, readFileSync } from "node:fs";
|
||||
import { join, resolve } from "node:path";
|
||||
import { parseStoryboard } from "./lib/storyboard.mjs";
|
||||
|
||||
const argv = process.argv.slice(2);
|
||||
const flag = (n, d) => {
|
||||
const i = argv.indexOf(`--${n}`);
|
||||
return i >= 0 && i + 1 < argv.length ? argv[i + 1] : d;
|
||||
};
|
||||
const hyperframesDir = resolve(flag("hyperframes", "."));
|
||||
const storyboardPath = resolve(flag("storyboard", join(hyperframesDir, "STORYBOARD.md")));
|
||||
const audiomapPath = resolve(flag("audiomap", join(hyperframesDir, "audiomap.json")));
|
||||
const templatesDir = resolve(flag("templates", join(hyperframesDir, "templates")));
|
||||
|
||||
const errors = [];
|
||||
const warns = [];
|
||||
const r3 = (x) => Math.round(x * 1000) / 1000;
|
||||
|
||||
if (!existsSync(storyboardPath)) {
|
||||
console.error(`✗ STORYBOARD.md not found at ${storyboardPath}`);
|
||||
process.exit(1);
|
||||
}
|
||||
const raw = readFileSync(storyboardPath, "utf8");
|
||||
const manifest = parseStoryboard(raw);
|
||||
const G = manifest.globals.extra ?? {};
|
||||
|
||||
// ---------- audio duration ----------
|
||||
let audioDur = null;
|
||||
if (existsSync(audiomapPath)) {
|
||||
try {
|
||||
audioDur = JSON.parse(readFileSync(audiomapPath, "utf8"))?.audio?.duration_sec ?? null;
|
||||
} catch (e) {
|
||||
warns.push(`audiomap parse failed: ${e.message}`);
|
||||
}
|
||||
} else {
|
||||
warns.push(`audiomap not found at ${audiomapPath} — skipping duration cross-check`);
|
||||
}
|
||||
|
||||
// ---------- frontmatter duration_s ----------
|
||||
const declaredDur = G.duration_s != null ? Number.parseFloat(G.duration_s) : NaN;
|
||||
if (!Number.isFinite(declaredDur)) errors.push(`frontmatter \`duration_s\` missing or unparseable`);
|
||||
else if (audioDur != null && Math.abs(declaredDur - audioDur) > 0.05)
|
||||
errors.push(`frontmatter duration_s (${declaredDur}) != audiomap duration (${audioDur})`);
|
||||
|
||||
// ---------- frames (hard) ----------
|
||||
const frames = manifest.frames;
|
||||
if (frames.length === 0) errors.push(`no frames (no \`## Frame N — <id>\` headings found)`);
|
||||
|
||||
let sum = 0;
|
||||
for (const f of frames) {
|
||||
const label = `frame ${f.number ?? f.index}${f.title ? ` (${f.title})` : ""}`;
|
||||
if (!f.src) errors.push(`${label}: missing \`- src:\``);
|
||||
if (!Number.isFinite(f.durationSeconds) || f.durationSeconds <= 0)
|
||||
errors.push(`${label}: missing/!positive \`- duration:\` (got ${JSON.stringify(f.duration)})`);
|
||||
else sum += f.durationSeconds;
|
||||
}
|
||||
sum = r3(sum);
|
||||
const tileTarget = Number.isFinite(declaredDur) ? declaredDur : audioDur;
|
||||
if (tileTarget != null && Math.abs(sum - tileTarget) > 0.1)
|
||||
errors.push(`frame durations sum to ${sum}s but the track is ${tileTarget}s — frames must tile it gap-free`);
|
||||
|
||||
// ---------- group checks (warns) — parse RAW text ----------
|
||||
const FRAME_HEAD = /^##\s+(?:frame|scene|section)\b/i;
|
||||
const GROUP_HEAD = /^\s*[-*]\s*\*\*\s*(\w+)\s*\*\*\s*[—:-]\s*(template|free_design|asset)\b(.*)$/i;
|
||||
const templateExistsCache = new Map();
|
||||
function templateExists(id) {
|
||||
if (templateExistsCache.has(id)) return templateExistsCache.get(id);
|
||||
const ok = existsSync(join(templatesDir, id, "index.html"));
|
||||
templateExistsCache.set(id, ok);
|
||||
return ok;
|
||||
}
|
||||
|
||||
// walk raw lines → group blocks tagged with their frame label + pacing
|
||||
const blocks = [];
|
||||
let frameLabel = "?";
|
||||
let pacing = "";
|
||||
let cur = null;
|
||||
for (const ln of raw.split(/\r?\n/)) {
|
||||
if (FRAME_HEAD.test(ln)) {
|
||||
frameLabel = ln.replace(/^#+\s+/, "").trim();
|
||||
pacing = "";
|
||||
cur = null;
|
||||
continue;
|
||||
}
|
||||
const pm = ln.match(/^\s*[-*]\s*pacing\s*:\s*([A-Za-z_]+)/i);
|
||||
if (pm && !cur) {
|
||||
pacing = pm[1].toLowerCase();
|
||||
continue;
|
||||
}
|
||||
const h = GROUP_HEAD.exec(ln);
|
||||
if (h) {
|
||||
cur = { frameLabel, pacing, name: h[1], kind: h[2].toLowerCase(), rest: h[3] ?? "", lines: [] };
|
||||
blocks.push(cur);
|
||||
continue;
|
||||
}
|
||||
if (cur) cur.lines.push(ln);
|
||||
}
|
||||
|
||||
const framesWithGroups = new Set(blocks.map((b) => b.frameLabel));
|
||||
for (const f of frames) {
|
||||
const lbl = `Frame ${f.number ?? ""} — ${f.title ?? f.index}`.replace(/\s+—\s+$/, "");
|
||||
if (![...framesWithGroups].some((s) => s.includes(String(f.title ?? "")))) {
|
||||
warns.push(`${lbl}: no parseable groups (expected \`- **gN** — template|free_design|asset …\`)`);
|
||||
}
|
||||
}
|
||||
|
||||
for (const b of blocks) {
|
||||
const gid = `${b.frameLabel} / ${b.name}`;
|
||||
const blockText = b.rest + " " + b.lines.join(" ");
|
||||
if (b.kind === "template") {
|
||||
const m = b.rest.match(/`([^`]+)`/) || b.rest.match(/:\s*([\w-]+)/);
|
||||
const id = m ? m[1].trim() : null;
|
||||
if (!id) warns.push(`${gid}: template kind but no template id on the head line`);
|
||||
else if (!templateExists(id))
|
||||
warns.push(`${gid}: template \`${id}\` not found at templates/${id}/index.html`);
|
||||
}
|
||||
if (b.kind === "asset" && b.pacing === "phrase_flow" && /beat_cut/.test(blockText))
|
||||
warns.push(`${gid}: beat_cut asset treatment on a phrase_flow frame — use ken_burns/crossfade instead`);
|
||||
}
|
||||
|
||||
// ---------- report ----------
|
||||
for (const w of warns) console.log(`⚠ ${w}`);
|
||||
if (errors.length) {
|
||||
for (const e of errors) console.error(`✗ ${e}`);
|
||||
console.error(`\nvalidate-plan: ${errors.length} error(s), ${warns.length} warning(s)`);
|
||||
process.exit(1);
|
||||
}
|
||||
console.log(`✓ validate-plan: ${frames.length} frames tile ${sum}s; ${blocks.length} groups; ${warns.length} warning(s)`);
|
||||
Reference in New Issue
Block a user