diff --git a/skills/gsap-effects/audio-visualizer.md b/skills/gsap-effects/audio-visualizer.md index 0526d1e47..2d172320a 100644 --- a/skills/gsap-effects/audio-visualizer.md +++ b/skills/gsap-effects/audio-visualizer.md @@ -10,7 +10,7 @@ HyperFrames renders frame-by-frame in headless Chrome — there's no audio playi ```bash python skills/gsap-effects/scripts/extract-audio-data.py audio.mp3 -o audio-data.json -python skills/gsap-effects/scripts/extract-audio-data.py video.mp4 --fps 30 --bands 8 -o audio-data.json +python skills/gsap-effects/scripts/extract-audio-data.py video.mp4 --fps 30 --bands 16 -o audio-data.json ``` Requires ffmpeg. Optional: numpy (faster FFT, falls back to pure Python). @@ -18,26 +18,36 @@ Requires ffmpeg. Optional: numpy (faster FFT, falls back to pure Python). | Flag | Default | Description | | --------- | --------------- | -------------------------------------------------------- | | `--fps` | 30 | Must match the composition/render FPS | -| `--bands` | 8 | Number of frequency bands (more = finer spectrum detail) | +| `--bands` | 16 | Number of frequency bands (more = finer spectrum detail) | | `-o` | audio-data.json | Output path | +The script uses a 4096-sample FFT window (not the per-frame sample count) to ensure each frequency band maps to distinct FFT bins. Bands are logarithmically spaced from 30Hz to 16kHz — the useful range for music. Each band is normalized independently across the full track so treble activity is visible even when bass is louder in absolute terms. + Output structure: ```json { "duration": 180.5, "fps": 30, - "bands": 8, + "bands": 16, "totalFrames": 5415, "frames": [ - { "time": 0.0, "rms": 0.0, "bands": [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0] }, - { "time": 0.0333, "rms": 0.42, "bands": [0.8, 0.6, 0.3, 0.2, 0.1, 0.1, 0.05, 0.02] } + { "time": 0.0, "rms": 0.0, "bands": [0.0, 0.0, 0.0, ...] }, + { "time": 0.0333, "rms": 0.42, "bands": [0.8, 0.6, 0.3, ...] } ] } ``` - `rms` — overall amplitude, normalized 0-1 across the track. Drives pulsing, bouncing, glow. -- `bands` — frequency magnitudes, normalized 0-1 per frame. Index 0 = bass, last = treble. Drives spectrum bars, EQ displays. +- `bands` — frequency magnitudes per band, each normalized 0-1 independently across the track. Index 0 = lowest bass (30Hz), last index = highest treble (16kHz). Drives spectrum bars, EQ displays. + +## Band Ordering + +Bands are always ordered low-to-high frequency: index 0 is bass, last index is treble. When drawing visualizations: + +- **Horizontal layouts** (spectrum bars, EQ): low frequencies on the left, high frequencies on the right. Iterate bands left-to-right as index 0, 1, 2, ... +- **Vertical layouts**: low frequencies at the bottom, high frequencies at the top. Iterate bands bottom-to-top. +- **Circular layouts**: bass starts at the top (12 o'clock) and wraps clockwise. ## Step 2: Embed Data in the Composition diff --git a/skills/gsap-effects/scripts/extract-audio-data.py b/skills/gsap-effects/scripts/extract-audio-data.py index 2cc77b196..bc57e1ac6 100644 --- a/skills/gsap-effects/scripts/extract-audio-data.py +++ b/skills/gsap-effects/scripts/extract-audio-data.py @@ -7,12 +7,12 @@ ready to embed in a HyperFrames composition. Usage: python extract-audio-data.py input.mp3 -o audio-data.json - python extract-audio-data.py input.mp4 --fps 30 --bands 8 -o audio-data.json + python extract-audio-data.py input.mp4 --fps 30 --bands 16 -o audio-data.json Requirements: - Python 3.9+ - ffmpeg (for decoding audio) - - numpy (pip install numpy) + - numpy (pip install numpy — optional but 100x faster) """ import argparse @@ -22,15 +22,34 @@ import struct import sys import math +# --------------------------------------------------------------------------- +# FFT parameters +# +# The FFT window must be large enough to resolve low-frequency bands cleanly. +# At 44100Hz, a 4096-sample window gives ~10.8 Hz per bin — enough to +# distinguish 30Hz bass from 45Hz sub-bass. The per-frame audio slice +# (44100/30 = 1470 samples at 30fps) is far too small and causes the lowest +# bands to map to the same FFT bins, producing duplicate values. +# +# The window is centered on each frame's timestamp and zero-padded if it +# extends beyond the audio boundaries. +# --------------------------------------------------------------------------- + +FFT_SIZE = 4096 + +# Frequency range for music: 30Hz–16kHz. Below 30Hz is sub-bass rumble that +# most speakers can't reproduce. Above 16kHz is noise/harmonics that don't +# contribute to perceived rhythm or melody. +MIN_FREQ = 30.0 +MAX_FREQ = 16000.0 + + def decode_audio(path: str, sample_rate: int = 44100) -> tuple[bytes, int]: """Decode audio to raw PCM s16le mono via ffmpeg.""" cmd = [ "ffmpeg", "-i", path, - "-vn", # no video - "-ac", "1", # mono - "-ar", str(sample_rate), # resample - "-f", "s16le", # raw 16-bit signed little-endian - "-acodec", "pcm_s16le", + "-vn", "-ac", "1", "-ar", str(sample_rate), + "-f", "s16le", "-acodec", "pcm_s16le", "-loglevel", "error", "pipe:1", ] @@ -55,22 +74,38 @@ def compute_rms(samples: list[float]) -> float: return math.sqrt(sum(s * s for s in samples) / len(samples)) -def compute_fft_bands(samples: list[float], sample_rate: int, n_bands: int) -> list[float]: - """Compute magnitude in frequency bands via FFT (no numpy needed).""" +def get_fft_window(samples: list[float], center: int, fft_size: int) -> list[float]: + """Extract a window of samples centered on `center`, zero-padded at edges.""" + half = fft_size // 2 + start = center - half + end = center + half n = len(samples) + + window = [] + for i in range(start, end): + if 0 <= i < n: + window.append(samples[i]) + else: + window.append(0.0) + + # Apply Hann window + for i in range(len(window)): + window[i] *= 0.5 - 0.5 * math.cos(2 * math.pi * i / len(window)) + + return window + + +def compute_fft_bands(windowed: list[float], sample_rate: int, n_bands: int) -> list[float]: + """Compute magnitude in logarithmically-spaced frequency bands via FFT.""" + n = len(windowed) if n == 0: return [0.0] * n_bands - # Apply Hann window - windowed = [samples[i] * (0.5 - 0.5 * math.cos(2 * math.pi * i / n)) for i in range(n)] - - # Use numpy if available for speed, fall back to pure Python try: import numpy as np fft = np.fft.rfft(windowed) - magnitudes = list(np.abs(fft)) + magnitudes = np.abs(fft).tolist() except ImportError: - # Pure Python DFT (slow but works without numpy) half = n // 2 + 1 magnitudes = [] for k in range(half): @@ -78,14 +113,11 @@ def compute_fft_bands(samples: list[float], sample_rate: int, n_bands: int) -> l im = sum(windowed[i] * math.sin(2 * math.pi * k * i / n) for i in range(n)) magnitudes.append(math.sqrt(re * re + im * im)) - # Frequency resolution freq_per_bin = sample_rate / n n_bins = len(magnitudes) - # Split bins into bands using logarithmic spacing (20Hz to Nyquist) - min_freq = 20.0 - max_freq = sample_rate / 2.0 - band_edges = [min_freq * (max_freq / min_freq) ** (i / n_bands) for i in range(n_bands + 1)] + # Logarithmic band edges from MIN_FREQ to MAX_FREQ + band_edges = [MIN_FREQ * (MAX_FREQ / MIN_FREQ) ** (i / n_bands) for i in range(n_bands + 1)] bands = [] for b in range(n_bands): @@ -93,14 +125,11 @@ def compute_fft_bands(samples: list[float], sample_rate: int, n_bands: int) -> l high_bin = min(n_bins - 1, int(band_edges[b + 1] / freq_per_bin)) if high_bin <= low_bin: high_bin = low_bin + 1 - band_mag = sum(magnitudes[low_bin:high_bin]) / max(1, high_bin - low_bin) + # Use max magnitude in the band (peak), not average — peaks are more + # perceptually relevant and make the visualization more responsive. + band_mag = max(magnitudes[low_bin:high_bin]) bands.append(band_mag) - # Normalize to 0-1 range - peak = max(bands) if bands else 1.0 - if peak > 0: - bands = [b / peak for b in bands] - return bands @@ -110,33 +139,57 @@ def extract(path: str, fps: int, n_bands: int) -> dict: pcm, sample_rate = decode_audio(path) samples = pcm_to_floats(pcm) duration = len(samples) / sample_rate - frame_size = sample_rate // fps + frame_step = sample_rate // fps total_frames = int(duration * fps) print(f"Duration: {duration:.1f}s, {total_frames} frames at {fps}fps", file=sys.stderr) - print(f"Extracting RMS + {n_bands} frequency bands per frame...", file=sys.stderr) + print(f"FFT window: {FFT_SIZE} samples ({sample_rate/FFT_SIZE:.1f} Hz/bin)", file=sys.stderr) + print(f"Frequency range: {MIN_FREQ:.0f}-{MAX_FREQ:.0f} Hz, {n_bands} bands", file=sys.stderr) - frames = [] + # Pass 1: extract raw values + raw_frames = [] for f in range(total_frames): - start = f * frame_size - end = start + frame_size - frame_samples = samples[start:end] + center = f * frame_step + frame_step // 2 + rms_start = f * frame_step + rms_end = rms_start + frame_step + frame_samples = samples[rms_start:rms_end] rms = compute_rms(frame_samples) - bands = compute_fft_bands(frame_samples, sample_rate, n_bands) + window = get_fft_window(samples, center, FFT_SIZE) + bands = compute_fft_bands(window, sample_rate, n_bands) + + raw_frames.append({"rms": rms, "bands": bands}) + + # Pass 2: normalize RMS to 0-1 across the whole track + peak_rms = max(f["rms"] for f in raw_frames) if raw_frames else 1.0 + + # Pass 2b: normalize each band independently across the whole track. + # This ensures that treble activity shows up even when bass is louder + # in absolute terms. Without this, high bands look dead because their + # absolute magnitudes are much smaller than bass/mid. + band_peaks = [0.0] * n_bands + for f in raw_frames: + for i, b in enumerate(f["bands"]): + if b > band_peaks[i]: + band_peaks[i] = b + + # Build output + frames = [] + for f_idx, raw in enumerate(raw_frames): + rms = raw["rms"] / peak_rms if peak_rms > 0 else 0.0 + bands = [] + for i, b in enumerate(raw["bands"]): + if band_peaks[i] > 0: + bands.append(round(b / band_peaks[i], 4)) + else: + bands.append(0.0) frames.append({ - "time": round(f / fps, 4), + "time": round(f_idx / fps, 4), "rms": round(rms, 4), - "bands": [round(b, 4) for b in bands], + "bands": bands, }) - # Normalize RMS to 0-1 across the whole track - peak_rms = max(f["rms"] for f in frames) if frames else 1.0 - if peak_rms > 0: - for f in frames: - f["rms"] = round(f["rms"] / peak_rms, 4) - return { "duration": round(duration, 4), "fps": fps, @@ -151,7 +204,7 @@ def main(): parser.add_argument("input", help="Audio or video file") parser.add_argument("-o", "--output", default="audio-data.json", help="Output JSON path") parser.add_argument("--fps", type=int, default=30, help="Frames per second (default: 30)") - parser.add_argument("--bands", type=int, default=8, help="Number of frequency bands (default: 8)") + parser.add_argument("--bands", type=int, default=16, help="Number of frequency bands (default: 16)") args = parser.parse_args() data = extract(args.input, args.fps, args.bands)