diff --git a/packages/cli/src/cli.ts b/packages/cli/src/cli.ts index 298ef88fa5..7a268670a9 100644 --- a/packages/cli/src/cli.ts +++ b/packages/cli/src/cli.ts @@ -149,6 +149,7 @@ const subCommands = { cloudrun: () => import("./commands/cloudrun.js").then((m) => m.default), cloud: () => import("./commands/cloud.js").then((m) => m.default), auth: () => import("./commands/auth.js").then((m) => m.default), + sfx: () => import("./commands/sfx.js").then((m) => m.default), }; const main = defineCommand({ diff --git a/packages/cli/src/cloud/_gen/client.ts b/packages/cli/src/cloud/_gen/client.ts index 42a0f5764e..f25530e523 100644 --- a/packages/cli/src/cloud/_gen/client.ts +++ b/packages/cli/src/cloud/_gen/client.ts @@ -5,6 +5,7 @@ * experiment-framework to regenerate. */ import type { + AudioSoundItem, CreateHyperframesRenderRequest, CreateHyperframesRenderResponse, DeleteHyperframesRenderResponse, @@ -302,4 +303,32 @@ export class HyperframesCloudClient { signal: args.signal, }); } + + async searchSounds(args: { + query: string; + type?: "music" | "sound_effects"; + limit?: number; + min_score?: number; + token?: string; + signal?: AbortSignal; + }): Promise<{ data?: Array; has_more?: boolean; next_token?: string | null }> { + const query: Record = { + query: args.query, + type: args.type, + limit: args.limit, + min_score: args.min_score, + token: args.token, + }; + return await this.request<{ + data?: Array; + has_more?: boolean; + next_token?: string | null; + }>({ + method: "GET", + path: "/v3/audio/sounds", + query, + unwrapData: false, + signal: args.signal, + }); + } } diff --git a/packages/cli/src/cloud/_gen/types.ts b/packages/cli/src/cloud/_gen/types.ts index a56c511202..da4154527b 100644 --- a/packages/cli/src/cloud/_gen/types.ts +++ b/packages/cli/src/cloud/_gen/types.ts @@ -267,3 +267,36 @@ export interface UploadAssetV3Response { */ size_bytes: number; } + +export type AudioContentType = "music" | "sound_effects"; +export interface AudioSoundItem { + /** + * Unique identifier for the audio track. + */ + id: string; + /** + * Display name of the audio track. + */ + name: string; + /** + * Semantic description the track was indexed under. + */ + description: string; + /** + * Temporary pre-signed URL to download the audio file. Expires; do not cache + * long-term. + */ + audio_url: string; + /** + * Duration of the audio in seconds, when known. + */ + duration?: number | null; + /** + * Semantic similarity score (0-1); higher means a closer match. + */ + score: number; + /** + * Audio content type of this track. + */ + type: AudioContentType; +} diff --git a/packages/cli/src/commands/sfx.ts b/packages/cli/src/commands/sfx.ts new file mode 100644 index 0000000000..6a28415550 --- /dev/null +++ b/packages/cli/src/commands/sfx.ts @@ -0,0 +1,52 @@ +/** + * `hyperframes sfx` — search and download sound effects from the HeyGen + * catalog. Subverbs live in `./sfx/.ts`, loaded dynamically so this + * surface doesn't affect CLI cold-start. + * + * Auth reuses the existing cloud credential chain (`createCloudClient` → + * `resolveCloudAuthHeaders`): `HEYGEN_API_KEY` or `hyperframes auth login`. + * No new credential store, no new env var. + */ + +import { defineCommand } from "citty"; +import type { Example } from "./_examples.js"; +import { c } from "../ui/colors.js"; + +export const examples: Example[] = [ + ["See what families are in the catalog", "hyperframes sfx list"], + ["Search the SFX catalog by meaning", 'hyperframes sfx search "whoosh for a scene change"'], + ["Download a clip into the project", "hyperframes sfx add 8b1ac2f2e69edd9b"], + ["Measure a clip (loudness, peak time)", "hyperframes sfx inspect 8b1ac2f2e69edd9b"], +]; + +const HELP = ` +${c.bold("hyperframes sfx")} ${c.dim("")} + +Find and use sound effects from the HeyGen catalog — AI-generated whooshes, +impacts, UI tones, risers, foley, and more — searchable by description. + +${c.bold("SUBCOMMANDS")} + ${c.accent("list")} List the families in the catalog (start here) + ${c.accent("search")} ${c.dim('""')} Search the catalog by meaning + ${c.accent("add")} ${c.dim("")} Download a clip into ./assets/sfx/ + ${c.accent("inspect")} ${c.dim("")} Measure loudness, peak time, onset/tail + +${c.bold("EXAMPLE")} + hyperframes sfx list + hyperframes sfx search "whoosh for a scene change" + hyperframes sfx add + hyperframes sfx inspect +`; + +export default defineCommand({ + meta: { name: "sfx", description: "Search and download sound effects from the HeyGen catalog" }, + subCommands: { + list: () => import("./sfx/list.js").then((m) => m.default), + search: () => import("./sfx/search.js").then((m) => m.default), + add: () => import("./sfx/add.js").then((m) => m.default), + inspect: () => import("./sfx/inspect.js").then((m) => m.default), + }, + run({ args }) { + if (!args._?.[0]) console.log(HELP); + }, +}); diff --git a/packages/cli/src/commands/sfx/add.ts b/packages/cli/src/commands/sfx/add.ts new file mode 100644 index 0000000000..66f2895b83 --- /dev/null +++ b/packages/cli/src/commands/sfx/add.ts @@ -0,0 +1,130 @@ +import { defineCommand } from "citty"; +import { join, relative, resolve } from "node:path"; +import * as clack from "@clack/prompts"; +import type { Example } from "../_examples.js"; +import { c } from "../../ui/colors.js"; +import { errorBox } from "../../ui/format.js"; +import { createCloudClient, downloadToFile } from "../../cloud/index.js"; +import { reportApiError } from "../../cloud/errors.js"; +import { readCachedSfx, sfxDir, writeSearchCache } from "./state.js"; +import { analyzeAudio, formatAnalysisLines, type AudioAnalysis } from "./analyze.js"; +import { requireSfxKey } from "./auth.js"; + +export const examples: Example[] = [ + ["Download a clip from a prior search", "hyperframes sfx add 8b1ac2f2e69edd9b"], + ["Add into a specific project", "hyperframes sfx add 8b1ac2f2e69edd9b --dir ./my-video"], +]; + +// Presigned URLs are short-lived; refresh anything older than this. +const URL_TTL_MS = 10 * 60 * 1000; + +export default defineCommand({ + meta: { + name: "add", + description: "Download a sound effect into the project's assets/sfx/ directory", + }, + args: { + id: { + type: "positional", + description: "The SFX id from `hyperframes sfx search`", + required: true, + }, + dir: { type: "string", description: "Project directory (default: current directory)" }, + json: { type: "boolean", description: "Output result as JSON", default: false }, + }, + // CLI entrypoint: arg parsing + URL refresh + analysis + json/human + error paths. + // fallow-ignore-next-line complexity + async run({ args }) { + const projectDir = resolve(args.dir ?? "."); + if (!(await requireSfxKey(args.json))) return; + const entry = readCachedSfx(projectDir, args.id); + if (!entry) { + errorBox( + "Unknown SFX id", + `No cached result for "${args.id}".`, + 'Run `hyperframes sfx search ""` first, then add by id.', + ); + process.exit(1); + } + + const spin = args.json ? null : clack.spinner(); + spin?.start(`Adding ${entry.name}...`); + try { + // Refresh the (expiring) presigned URL by re-running the cached query. + let url = entry.audio_url; + if (Date.now() - entry.fetchedAt > URL_TTL_MS) { + const client = await createCloudClient(); + const res = await client.searchSounds({ + query: entry.query, + type: "sound_effects", + limit: 50, + }); + const fresh = (res.data ?? []).find((it) => it.id === args.id); + if (fresh) { + url = fresh.audio_url; + writeSearchCache(projectDir, [ + { ...entry, audio_url: fresh.audio_url, fetchedAt: Date.now() }, + ]); + } + } + + const dest = join(sfxDir(projectDir), `${args.id}${extFromUrl(url)}`); + const result = await downloadToFile( + url, + dest, + spin + ? { + onProgress: (b, t) => + spin.message( + t + ? `Downloading ${Math.round((b / t) * 100)}%` + : `Downloaded ${(b / 1024).toFixed(0)} KB`, + ), + } + : {}, + ); + const rel = relative(projectDir, result.path) || result.path; + const dur = (entry.duration ?? 1).toFixed(2); + + // Measure the clip so the agent can place + balance it without hearing it. + // Best-effort: ffmpeg is optional, so a missing binary just omits the analysis. + let analysis: AudioAnalysis | null = null; + try { + analysis = await analyzeAudio(result.path); + } catch { + /* ffmpeg not installed or unreadable clip — skip the analysis */ + } + + if (args.json) { + console.log( + JSON.stringify({ + ok: true, + id: args.id, + name: entry.name, + path: rel, + duration: entry.duration, + analysis, + }), + ); + return; + } + spin?.stop(c.success(`Added ${c.accent(entry.name)} → ${c.accent(rel)}`)); + console.log(c.dim(" Embed it as a clip (volume low under voiceover):")); + console.log( + ` ${c.dim(``)}`, + ); + if (analysis) { + for (const line of formatAnalysisLines(analysis)) console.log(` ${c.dim(line)}`); + } + } catch (err) { + spin?.stop(c.error("Add failed")); + reportApiError("SFX download failed", err); + } + }, +}); + +/** Pick the file extension from a (possibly query-stringed) URL; default .flac. */ +function extFromUrl(url: string): string { + const match = (url.split("?")[0] ?? url).match(/\.(flac|mp3|wav|m4a|ogg)$/i); + return match ? match[0].toLowerCase() : ".flac"; +} diff --git a/packages/cli/src/commands/sfx/analyze.test.ts b/packages/cli/src/commands/sfx/analyze.test.ts new file mode 100644 index 0000000000..8555fe7d37 --- /dev/null +++ b/packages/cli/src/commands/sfx/analyze.test.ts @@ -0,0 +1,90 @@ +import { describe, expect, it } from "vitest"; +import { envelopeSparkline, formatAnalysisLines, parseFfmpegAnalysis } from "./analyze.js"; + +// Shaped after real `ffmpeg -af ebur128=peak=true,silencedetect=...` stderr: +// 0.5s lead silence, a tone that peaks ~0.9s, 0.5s trailing silence to EOF (2.0s). +const FULL = ` Duration: 00:00:02.00, bitrate: 705 kb/s +[Parsed_silencedetect_1 @ 0x0] silence_start: 0 +[Parsed_ebur128_0 @ 0x0] t: 0.0999773 TARGET:-23 LUFS M:-120.7 S:-120.7 I: -70.0 LUFS LRA: 0.0 LU FTPK: -inf dBFS TPK: -inf dBFS +[Parsed_silencedetect_1 @ 0x0] silence_end: 0.5 | silence_duration: 0.5 +[Parsed_ebur128_0 @ 0x0] t: 0.899977 TARGET:-23 LUFS M: -20.0 S:-120.7 I: -22.0 LUFS LRA: 0.0 LU FTPK: -inf dBFS TPK: -18.1 dBFS +[Parsed_ebur128_0 @ 0x0] t: 1.399977 TARGET:-23 LUFS M: -24.0 S:-120.7 I: -22.5 LUFS LRA: 0.0 LU FTPK: -inf dBFS TPK: -18.1 dBFS +[Parsed_silencedetect_1 @ 0x0] silence_start: 1.5 + Integrated loudness: + I: -22.9 LUFS + True peak: + Peak: -18.1 dBFS +`; + +describe("parseFfmpegAnalysis", () => { + it("extracts duration, loudness, true peak, peak time, onset, and tail", () => { + const a = parseFfmpegAnalysis(FULL); + expect(a.durationSec).toBe(2); + expect(a.integratedLufs).toBe(-22.9); // the summary value, not the per-frame running I: + expect(a.truePeakDb).toBe(-18.1); + expect(a.peakTimeSec).toBeCloseTo(0.9, 1); // loudest momentary M (-20.0), not the silent frames + expect(a.onsetSec).toBeCloseTo(0.5, 2); // end of the leading silence + expect(a.tailStartSec).toBeCloseTo(1.5, 2); // trailing silence runs to EOF without a silence_end + // contour: silent frame floored to -70, then the two audible frames + expect(a.envelope).toEqual([-70, -20, -24]); + }); + + it("builds an empty envelope and sparkline when there are no momentary lines", () => { + expect(parseFfmpegAnalysis("").envelope).toEqual([]); + expect(envelopeSparkline([])).toBe(""); + }); + + it("renders a sparkline spanning low→high blocks across the contour", () => { + const spark = envelopeSparkline([-70, -20, -24]); + expect(spark).toHaveLength(3); + expect(spark[0]).toBe("▁"); // quietest → lowest block + expect(spark[1]).toBe("█"); // loudest → highest block + }); + + it("ignores silent frames (M ≈ -120) when finding the peak", () => { + // Only silence-level momentary lines → no defensible peak. + const silentish = ` Duration: 00:00:01.00, bitrate: 705 kb/s +[Parsed_ebur128_0 @ 0x0] t: 0.5 TARGET:-23 LUFS M:-120.7 S:-120.7 I: -70.0 LUFS LRA: 0.0 LU +`; + expect(parseFfmpegAnalysis(silentish).peakTimeSec).toBeNull(); + }); + + it("reports onset 0 and tail null for a clip with no detected silence", () => { + const hot = ` Duration: 00:00:01.00, bitrate: 705 kb/s +[Parsed_ebur128_0 @ 0x0] t: 0.3 TARGET:-23 LUFS M: -12.0 S:-15.0 I: -12.0 LUFS LRA: 0.0 LU + Integrated loudness: + I: -12.0 LUFS + True peak: + Peak: -1.0 dBFS +`; + const a = parseFfmpegAnalysis(hot); + expect(a.onsetSec).toBe(0); + expect(a.tailStartSec).toBeNull(); + expect(a.peakTimeSec).toBeCloseTo(0.3, 2); + expect(a.truePeakDb).toBe(-1); + }); + + it("emits a trim line with the effective window when the clip has dead air", () => { + const a = parseFfmpegAnalysis(FULL); // onset 0.5, tail 1.5, dur 2.0 + const trim = formatAnalysisLines(a).find((l) => l.startsWith("trim")); + expect(trim).toBeDefined(); + expect(trim).toContain("data-media-start=0.5"); + expect(trim).toContain("data-duration=1"); // tail(1.5) − onset(0.5) + }); + + it("omits the trim line when the clip starts hot and has no trailing silence", () => { + const hot = parseFfmpegAnalysis( + " Duration: 00:00:01.00, bitrate: 705 kb/s\n[Parsed_ebur128_0 @ 0x0] t: 0.3 M: -12.0 I: -12.0 LUFS\n", + ); + expect(formatAnalysisLines(hot).some((l) => l.startsWith("trim"))).toBe(false); + }); + + it("returns nulls gracefully on empty input", () => { + const a = parseFfmpegAnalysis(""); + expect(a.durationSec).toBeNull(); + expect(a.integratedLufs).toBeNull(); + expect(a.peakTimeSec).toBeNull(); + expect(a.onsetSec).toBe(0); + expect(a.tailStartSec).toBeNull(); + }); +}); diff --git a/packages/cli/src/commands/sfx/analyze.ts b/packages/cli/src/commands/sfx/analyze.ts new file mode 100644 index 0000000000..8281d6dced --- /dev/null +++ b/packages/cli/src/commands/sfx/analyze.ts @@ -0,0 +1,243 @@ +/** + * Audio analysis for sound-effect clips. + * + * Agents can't *hear* a clip, so they can't guess where a riser peaks or how + * loud one sound is next to another. This runs ffmpeg once over a local file + * and extracts the few facts that drive placement decisions: + * + * - integratedLufs / truePeakDb → set `data-volume` by measured level + * - peakTimeSec → where to fire the payoff in a chain + * - onsetSec / tailStartSec → trim dead air (`data-media-start`) / real end + * + * Everything comes from a single `ebur128 + silencedetect` pass (both filters + * pass audio through and report to stderr), so there's no new dependency — + * ffmpeg is already the local-render soft-dep. The parser is split out as a + * pure function so it can be unit-tested without invoking ffmpeg. + */ + +import { execFile } from "node:child_process"; +import { promisify } from "node:util"; +import { findFFmpeg, getFFmpegInstallHint } from "../../browser/ffmpeg.js"; + +const execFileAsync = promisify(execFile); + +export interface AudioAnalysis { + /** Clip length in seconds (from the container), or null if unknown. */ + durationSec: number | null; + /** EBU R128 integrated loudness in LUFS (perceived level), or null. */ + integratedLufs: number | null; + /** True peak in dBFS (clipping headroom; 0 = full scale), or null. */ + truePeakDb: number | null; + /** Time of the loudest moment (max momentary loudness) — a chain's resolve point. */ + peakTimeSec: number | null; + /** First audible moment: end of any leading silence (0 if it starts hot). */ + onsetSec: number; + /** Start of trailing silence — the clip's effective end — or null if it plays to the end. */ + tailStartSec: number | null; + /** + * Loudness contour over the clip — momentary LUFS sampled across time (silence + * floored at -70), downsampled to at most ENVELOPE_BINS values. This is the + * *shape* (build / drop / break), not just the peak: bin i is centered at + * t ≈ (i + 0.5) / envelope.length × durationSec. + */ + envelope: number[]; +} + +/** Max points in the returned envelope — bounded regardless of clip length. */ +const ENVELOPE_BINS = 64; +/** Momentary LUFS below this counts as silence for the contour. */ +const SILENCE_FLOOR_LUFS = -70; + +export class FFmpegNotFoundError extends Error { + readonly hint: string; + constructor() { + super("ffmpeg not found"); + this.name = "FFmpegNotFoundError"; + this.hint = getFFmpegInstallHint(); + } +} + +/** Run ffmpeg's ebur128 + silencedetect over `filePath` and parse the result. */ +export async function analyzeAudio(filePath: string): Promise { + const ffmpeg = findFFmpeg(); + if (!ffmpeg) throw new FFmpegNotFoundError(); + + const args = [ + "-hide_banner", + "-nostats", + "-i", + filePath, + "-af", + "ebur128=peak=true,silencedetect=noise=-50dB:d=0.1", + "-f", + "null", + "-", + ]; + + let stderr = ""; + try { + const res = await execFileAsync(ffmpeg, args, { maxBuffer: 64 * 1024 * 1024 }); + stderr = res.stderr; + } catch (err) { + // ffmpeg can exit non-zero yet still emit the full analysis on stderr. + stderr = (err as { stderr?: string })?.stderr ?? ""; + if (!stderr) throw err; + } + return parseFfmpegAnalysis(stderr); +} + +/** + * Pure parser for the stderr of `ffmpeg -af ebur128=peak=true,silencedetect=...`. + * Exported for unit tests — no I/O. + */ +export function parseFfmpegAnalysis(stderr: string): AudioAnalysis { + const lines = stderr.split(/\r?\n/); + const durationSec = parseDuration(stderr); + const { peakTimeSec, envelope } = parseMomentary(lines); + return { + durationSec, + // Both the per-frame lines and the final summary print "I: ... LUFS"; the last + // occurrence is the integrated value. "Peak:" only appears in the True-peak summary. + integratedLufs: lastMatchedNumber(stderr, /\bI:\s*(-?\d+(?:\.\d+)?)\s*LUFS/g), + truePeakDb: lastMatchedNumber(stderr, /\bPeak:\s*(-?\d+(?:\.\d+)?)\s*dBFS/g), + peakTimeSec, + ...parseSilence(lines, durationSec), + envelope, + }; +} + +/** "Duration: 00:00:08.01, ..." → seconds. */ +function parseDuration(stderr: string): number | null { + const dur = stderr.match(/Duration:\s*(\d+):(\d+):(\d+(?:\.\d+)?)/); + if (dur?.[1] && dur[2] && dur[3]) + return Number(dur[1]) * 3600 + Number(dur[2]) * 60 + Number.parseFloat(dur[3]); + return null; +} + +/** Last capture-group-1 match of `re` across the whole stderr, as a number. */ +function lastMatchedNumber(stderr: string, re: RegExp): number | null { + const last = [...stderr.matchAll(re)].at(-1); + return last?.[1] ? Number.parseFloat(last[1]) : null; +} + +/** + * Momentary loudness over time → the loudest moment and the full contour. + * ebur128 prints e.g. "t: 1.20 TARGET:-23 LUFS M: -22.3 S: ... I: ... LRA: ...". + */ +function parseMomentary(lines: string[]): { peakTimeSec: number | null; envelope: number[] } { + const mom: Array<{ t: number; m: number }> = []; + const tRe = /\bt:\s*(-?\d+(?:\.\d+)?)\b.*?\bM:\s*(-?\d+(?:\.\d+)?)/; + for (const ln of lines) { + const m = ln.match(tRe); + if (m?.[1] && m[2]) mom.push({ t: Number.parseFloat(m[1]), m: Number.parseFloat(m[2]) }); + } + let peakTimeSec: number | null = null; + let maxM = Number.NEGATIVE_INFINITY; + for (const { t, m } of mom) { + // M of about -120 means silence/-inf; ignore it when finding the peak. + if (m > SILENCE_FLOOR_LUFS && m > maxM) { + maxM = m; + peakTimeSec = t; + } + } + return { peakTimeSec, envelope: buildEnvelope(mom.map((p) => p.m)) }; +} + +/** Pair up silencedetect start/end lines into [start, end] intervals (trailing runs to EOF). */ +function collectSilenceIntervals( + lines: string[], + durationSec: number | null, +): Array<[number, number]> { + const intervals: Array<[number, number]> = []; + let pendingStart: number | null = null; + for (const ln of lines) { + const s = ln.match(/silence_start:\s*(-?\d+(?:\.\d+)?)/); + if (s?.[1]) { + pendingStart = Number.parseFloat(s[1]); + continue; + } + const e = ln.match(/silence_end:\s*(-?\d+(?:\.\d+)?)/); + if (e?.[1] && pendingStart !== null) { + intervals.push([pendingStart, Number.parseFloat(e[1])]); + pendingStart = null; + } + } + if (pendingStart !== null && durationSec !== null) intervals.push([pendingStart, durationSec]); + return intervals; +} + +/** Leading silence → onset; trailing silence (reaching EOF) → tail start. */ +function parseSilence( + lines: string[], + durationSec: number | null, +): { onsetSec: number; tailStartSec: number | null } { + let onsetSec = 0; + let tailStartSec: number | null = null; + for (const [start, end] of collectSilenceIntervals(lines, durationSec)) { + if (start <= 0.05) onsetSec = Math.max(onsetSec, end); + if (durationSec !== null && end >= durationSec - 0.2) tailStartSec = start; + } + return { onsetSec, tailStartSec }; +} + +/** Floor silence, then downsample the momentary-LUFS series to at most ENVELOPE_BINS bins. */ +function buildEnvelope(raw: number[]): number[] { + if (!raw.length) return []; + const vals = raw.map((m) => (m < SILENCE_FLOOR_LUFS ? SILENCE_FLOOR_LUFS : m)); + const round1 = (v: number) => Math.round(v * 10) / 10; + if (vals.length <= ENVELOPE_BINS) return vals.map(round1); + const out: number[] = []; + for (let i = 0; i < ENVELOPE_BINS; i++) { + const start = Math.floor((i * vals.length) / ENVELOPE_BINS); + const end = Math.max(start + 1, Math.floor(((i + 1) * vals.length) / ENVELOPE_BINS)); + let sum = 0; + let n = 0; + for (let j = start; j < end && j < vals.length; j++) { + sum += vals[j] as number; + n++; + } + out.push(round1(sum / n)); + } + return out; +} + +/** Unicode sparkline of the loudness contour (build/drop/break at a glance). */ +export function envelopeSparkline(envelope: number[]): string { + if (!envelope.length) return ""; + const blocks = "▁▂▃▄▅▆▇█"; + const max = Math.max(...envelope); + const min = Math.min(...envelope); + const span = Math.max(1e-6, max - min); + return envelope + .map((v) => { + const idx = Math.min(7, Math.max(0, Math.round(((v - min) / span) * 7))); + return blocks[idx]; + }) + .join(""); +} + +/** Human-readable lines for the terminal (no color — caller wraps as needed). */ +export function formatAnalysisLines(a: AudioAnalysis): string[] { + const s = (n: number | null, unit: string, digits = 2) => + n === null ? "—" : `${n.toFixed(digits)}${unit}`; + const loud = a.integratedLufs === null ? "—" : `${a.integratedLufs.toFixed(1)} LUFS`; + const peak = a.truePeakDb === null ? "—" : `${a.truePeakDb.toFixed(1)} dBFS`; + const out = [ + `loudness ${loud} true-peak ${peak}`, + `peak at ${s(a.peakTimeSec, "s")} onset ${s(a.onsetSec, "s")} tail ${s(a.tailStartSec, "s")}`, + ]; + // Suggested trim window when the clip carries silence padding — so a padded + // riser/whoosh plays only its real content instead of firing late on dead air. + const end = a.tailStartSec ?? a.durationSec; + const hasDeadAir = a.onsetSec > 0.05 || a.tailStartSec !== null; + if (hasDeadAir && end !== null && end > a.onsetSec) { + const r2 = (n: number) => Math.round(n * 100) / 100; + const parts: string[] = []; + if (a.onsetSec > 0.05) parts.push(`data-media-start=${r2(a.onsetSec)}`); + parts.push(`data-duration=${r2(end - a.onsetSec)}`); + out.push(`trim ${parts.join(" ")}`); + } + const spark = envelopeSparkline(a.envelope); + if (spark) out.push(`shape ${spark}`); + return out; +} diff --git a/packages/cli/src/commands/sfx/auth.ts b/packages/cli/src/commands/sfx/auth.ts new file mode 100644 index 0000000000..d742f75957 --- /dev/null +++ b/packages/cli/src/commands/sfx/auth.ts @@ -0,0 +1,44 @@ +/** + * SFX access gate. The catalog is free, but searching/downloading it needs a + * HeyGen API key. Rather than letting the API call fail with a raw 401, the + * network-touching subcommands (`search`, `add`) call this first: if no + * credential resolves, it prints a friendly opt-in (get a free key, set it, or + * skip SFX) — mirroring how the VO/Gemini steps ask — and the caller bails. + * + * `list` and `inspect` are local and don't gate on this. + */ + +import { tryResolveCredential } from "../../auth/resolver.js"; +import { c } from "../../ui/colors.js"; + +const SFX_SIGNUP_URL = "https://app.heygen.com/developers/api"; + +/** + * Returns true if a HeyGen credential is available. If not, prints guidance + * (human or JSON) and returns false — the caller should stop. + */ +export async function requireSfxKey(json = false): Promise { + const cred = await tryResolveCredential(); + if (cred) return true; + + if (json) { + console.log( + JSON.stringify({ + error: "missing_api_key", + message: "Sound effects need a free HeyGen API key.", + get_key_url: SFX_SIGNUP_URL, + set_via: ["HEYGEN_API_KEY in a .env file or environment", "hyperframes auth login"], + }), + ); + return false; + } + + console.log(`\n ${c.bold("Sound effects need a free HeyGen API key.")}`); + console.log(` ${c.dim("HeyGen's SFX catalog is free — you just need a key to search it.")}`); + console.log(` ${c.dim("Get one at")} ${c.accent(SFX_SIGNUP_URL)}`); + console.log( + ` ${c.dim("then add")} ${c.accent("HEYGEN_API_KEY=your-key")} ${c.dim("to a .env file, or run")} ${c.accent("hyperframes auth login")}${c.dim(".")}`, + ); + console.log(` ${c.dim("Or skip SFX — the video just won't have them.")}\n`); + return false; +} diff --git a/packages/cli/src/commands/sfx/catalog-manifest.test.ts b/packages/cli/src/commands/sfx/catalog-manifest.test.ts new file mode 100644 index 0000000000..0641ad0ade --- /dev/null +++ b/packages/cli/src/commands/sfx/catalog-manifest.test.ts @@ -0,0 +1,34 @@ +import { describe, expect, it } from "vitest"; +import { CATALOG_MANIFEST, type SfxMode } from "./catalog-manifest.js"; + +const MODES: SfxMode[] = ["hit", "build", "bed", "sting", "meme"]; + +describe("CATALOG_MANIFEST", () => { + it("is non-empty and self-consistent", () => { + expect(CATALOG_MANIFEST.families.length).toBeGreaterThan(0); + for (const f of CATALOG_MANIFEST.families) { + expect(f.category).toMatch(/^[a-z]+$/); + expect(f.count).toBeGreaterThan(0); + expect(MODES).toContain(f.mode); + expect(f.use.length).toBeGreaterThan(0); + expect(f.examples.length).toBeGreaterThan(0); + expect(f.examples.length).toBeLessThanOrEqual(3); + } + }); + + it("totalClips equals the sum of family counts", () => { + const sum = CATALOG_MANIFEST.families.reduce((n, f) => n + f.count, 0); + expect(CATALOG_MANIFEST.totalClips).toBe(sum); + }); + + it("lists families largest-first so the common kinds surface first", () => { + const counts = CATALOG_MANIFEST.families.map((f) => f.count); + expect(counts).toEqual([...counts].sort((a, b) => b - a)); + }); + + it("covers the non-obvious families an agent tends to forget", () => { + const cats = new Set(CATALOG_MANIFEST.families.map((f) => f.category)); + for (const c of ["meme", "ambience", "downlifter", "glitch", "stinger"]) + expect(cats).toContain(c); + }); +}); diff --git a/packages/cli/src/commands/sfx/catalog-manifest.ts b/packages/cli/src/commands/sfx/catalog-manifest.ts new file mode 100644 index 0000000000..cb9ed2785e --- /dev/null +++ b/packages/cli/src/commands/sfx/catalog-manifest.ts @@ -0,0 +1,180 @@ +/** + * Catalog map for the SFX library — families, counts, and what each is for. + * + * AUTO-GENERATED by scripts/sfx_rag/build_catalog_manifest.py — do not hand-edit. + */ + +export type SfxMode = "hit" | "build" | "bed" | "sting" | "meme"; + +export interface SfxFamily { + category: string; + count: number; + mode: SfxMode; + use: string; + examples: string[]; +} + +export interface SfxCatalogManifest { + totalClips: number; + families: SfxFamily[]; +} + +export const CATALOG_MANIFEST: SfxCatalogManifest = { + totalClips: 180, + families: [ + { + category: "impact", + count: 29, + mode: "hit", + use: "Slams, booms, sub-drops — punctuate a hit or number landing on the impact frame.", + examples: ["Sub Drop", "Slam 1", "Slam 2"], + }, + { + category: "foley", + count: 27, + mode: "hit", + use: "Real-world textures — footsteps, taps, rips, handling; realism and walk cycles.", + examples: ["Single Footstep", "Heavy Footstep", "Tape Rip 1"], + }, + { + category: "whoosh", + count: 22, + mode: "hit", + use: "Transitions and scene changes — fire on the cut.", + examples: ["Whoosh", "Whoosh effect", "Whoosh bamboo"], + }, + { + category: "meme", + count: 14, + mode: "meme", + use: "Cultural punchlines — vine boom, record scratch, air horn, sad trombone; sparingly, on a comedic turn.", + examples: ["Vine Boom", "Heavy Record Scratch", "Air Horn"], + }, + { + category: "stinger", + count: 13, + mode: "sting", + use: "Logo reveals, brand intros, suspense stings — on a reveal or lock.", + examples: ["Suspense Sting 1", "Calm Elegant Logo", "Epic Logo Reveal"], + }, + { + category: "bell", + count: 9, + mode: "hit", + use: "Bells, chimes, doorbells, alarms — alerts and gentle accents.", + examples: ["Doorbell", "Chime", "Old Phone Ring"], + }, + { + category: "ui", + count: 9, + mode: "hit", + use: "Clicks, taps, sparkles, beeps — on an interaction.", + examples: ["Generic Click", "Sparkle", "Censor Beep"], + }, + { + category: "human", + count: 6, + mode: "bed", + use: "Crowd cheer, applause, heartbeat — reactions and tension (cheers/applause can run as beds).", + examples: ["General Crowd Cheer", "Polite Audience Applause", "Slow Human Heartbeat"], + }, + { + category: "ambience", + count: 6, + mode: "bed", + use: "Drones, tension beds, sci-fi hum, stadium roar — run UNDER a scene for mood; not on a beat.", + examples: ["Eerie Deep Tension", "Slow Resonant Tension", "Sci Fi Hum"], + }, + { + category: "downlifter", + count: 5, + mode: "build", + use: "Descending sweeps and sub-drops — out of a transition or to deflate energy.", + examples: ["Downlifter deep", "Zoom out pull effect", "Sfx medium subdown sound effect"], + }, + { + category: "riser", + count: 5, + mode: "build", + use: "Ascending builds into a climax — place so the peak lands on the beat.", + examples: ["Riser", "Heavy Metallic Riser", "Soft Dreamy Riser"], + }, + { + category: "notification", + count: 5, + mode: "hit", + use: "Pings, alerts, 'correct' tones — confirmations.", + examples: ["Ping", "Correct", "SMS Alert"], + }, + { + category: "glitch", + count: 4, + mode: "hit", + use: "Digital errors and stutters — tech and hard-cut accents.", + examples: ["Digital Error 1", "Digital Error 2", "Glitch Effect 1"], + }, + { + category: "nature", + count: 4, + mode: "bed", + use: "Wind, rain, thunder — outdoor ambience or a weather hit.", + examples: ["Wind", "Rain", "Sharp Thunder Clap"], + }, + { + category: "vehicle", + count: 4, + mode: "hit", + use: "Sirens, engines, crashes — scene and action sounds.", + examples: ["Police Siren", "Car engine", "Car crash sound effect"], + }, + { + category: "weapon", + count: 4, + mode: "hit", + use: "Sword slices and swings, shots — action accents.", + examples: ["Sword slice", "Sword slash and swing", "Pistol Shot"], + }, + { + category: "money", + count: 3, + mode: "hit", + use: "Coin jingles, cash-register ka-ching — reward or purchase.", + examples: ["Multiple Coins Jingle", "Cash Register 'Kaching'", "Whoosh and Cash Register"], + }, + { + category: "animal", + count: 3, + mode: "hit", + use: "Crickets, duck quack — nature beds or comedic accents.", + examples: ["Cricket Chirp", "Duck Quack"], + }, + { + category: "success", + count: 2, + mode: "hit", + use: "Coin-collect chime, achievement — completion and reward.", + examples: ["Coin Collect Chime", "Achievement"], + }, + { + category: "pop", + count: 2, + mode: "hit", + use: "Pops — element pop-ins.", + examples: ["Heart Pop", "Standard Pop"], + }, + { + category: "water", + count: 2, + mode: "hit", + use: "Splash and drop — water hits.", + examples: ["Water Splash", "Water Drop"], + }, + { + category: "tech", + count: 2, + mode: "sting", + use: "Sci-fi logo reveals — futuristic intros.", + examples: ["Sci fi intro logo reveal"], + }, + ], +}; diff --git a/packages/cli/src/commands/sfx/inspect.ts b/packages/cli/src/commands/sfx/inspect.ts new file mode 100644 index 0000000000..a4feb69d7e --- /dev/null +++ b/packages/cli/src/commands/sfx/inspect.ts @@ -0,0 +1,75 @@ +import { defineCommand } from "citty"; +import { existsSync, statSync } from "node:fs"; +import { join, resolve } from "node:path"; +import type { Example } from "../_examples.js"; +import { c } from "../../ui/colors.js"; +import { errorBox } from "../../ui/format.js"; +import { sfxDir } from "./state.js"; +import { analyzeAudio, FFmpegNotFoundError, formatAnalysisLines } from "./analyze.js"; + +export const examples: Example[] = [ + ["Analyze a downloaded clip by id", "hyperframes sfx inspect 8b1ac2f2e69edd9b"], + ["Analyze a file by path (JSON)", "hyperframes sfx inspect ./assets/sfx/clip.mp3 --json"], +]; + +const AUDIO_EXTS = [".mp3", ".flac", ".wav", ".m4a", ".ogg"]; + +/** Resolve an id (downloaded into assets/sfx/) or a direct file path to a clip on disk. */ +// fallow-ignore-next-line complexity +function resolveClipPath(projectDir: string, idOrPath: string): string | undefined { + if (existsSync(idOrPath) && statSync(idOrPath).isFile()) return resolve(idOrPath); + const dir = sfxDir(projectDir); + for (const ext of AUDIO_EXTS) { + const p = join(dir, `${idOrPath}${ext}`); + if (existsSync(p)) return p; + } + return undefined; +} + +export default defineCommand({ + meta: { + name: "inspect", + description: + "Measure a downloaded sound effect: loudness, true peak, peak time, onset/tail silence", + }, + args: { + id: { + type: "positional", + description: "SFX id (downloaded via `sfx add`) or a path to an audio file", + required: true, + }, + dir: { type: "string", description: "Project directory (default: current directory)" }, + json: { type: "boolean", description: "Output the analysis as JSON", default: false }, + }, + // CLI entrypoint: resolve + json/human + ffmpeg error paths. + // fallow-ignore-next-line complexity + async run({ args }) { + const projectDir = resolve(args.dir ?? "."); + const clip = resolveClipPath(projectDir, args.id); + if (!clip) { + errorBox( + "Clip not found", + `No downloaded clip "${args.id}" in ${sfxDir(projectDir)} and no file at that path.`, + "Run `hyperframes sfx add ` first, or pass a path to an audio file.", + ); + process.exit(1); + } + + try { + const analysis = await analyzeAudio(clip); + if (args.json) { + console.log(JSON.stringify(analysis)); + return; + } + console.log(` ${c.bold(args.id)}`); + for (const line of formatAnalysisLines(analysis)) console.log(` ${c.dim(line)}`); + } catch (err) { + if (err instanceof FFmpegNotFoundError) { + errorBox("ffmpeg not found", "Analysis needs ffmpeg.", `Install it: ${err.hint}`); + process.exit(1); + } + errorBox("Analysis failed", err instanceof Error ? err.message : String(err)); + process.exit(1); + } + }, +}); diff --git a/packages/cli/src/commands/sfx/list.ts b/packages/cli/src/commands/sfx/list.ts new file mode 100644 index 0000000000..6c13f4b008 --- /dev/null +++ b/packages/cli/src/commands/sfx/list.ts @@ -0,0 +1,45 @@ +import { defineCommand } from "citty"; +import type { Example } from "../_examples.js"; +import { c } from "../../ui/colors.js"; +import { CATALOG_MANIFEST } from "./catalog-manifest.js"; + +export const examples: Example[] = [ + ["See what's in the catalog before searching", "hyperframes sfx list"], + ["Machine-readable map (for agents)", "hyperframes sfx list --json"], +]; + +export default defineCommand({ + meta: { + name: "list", + description: + "List the sound-effect families in the catalog — see what's available before searching", + }, + args: { + json: { + type: "boolean", + description: "Output the catalog map as JSON (for agents)", + default: false, + }, + }, + run({ args }) { + if (args.json) { + console.log(JSON.stringify(CATALOG_MANIFEST)); + return; + } + const { totalClips, families } = CATALOG_MANIFEST; + console.log(` ${c.bold(`${totalClips} sound effects across ${families.length} families`)}`); + console.log( + c.dim( + " Pick the family that fits the moment, then search within it — don't default to the same whoosh.\n", + ), + ); + const pad = Math.max(...families.map((f) => f.category.length)); + for (const f of families) { + console.log(` ${c.accent(f.category.padEnd(pad))} ${c.dim(`(${f.count})`)} ${f.use}`); + console.log(` ${" ".repeat(pad)} ${c.dim(`e.g. ${f.examples.join(", ")}`)}`); + } + console.log( + `\n ${c.dim("Then:")} ${c.accent('hyperframes sfx search ""')}`, + ); + }, +}); diff --git a/packages/cli/src/commands/sfx/search.ts b/packages/cli/src/commands/sfx/search.ts new file mode 100644 index 0000000000..b8998cdeca --- /dev/null +++ b/packages/cli/src/commands/sfx/search.ts @@ -0,0 +1,111 @@ +import { defineCommand } from "citty"; +import { resolve } from "node:path"; +import * as clack from "@clack/prompts"; +import type { Example } from "../_examples.js"; +import { c } from "../../ui/colors.js"; +import { createCloudClient } from "../../cloud/index.js"; +import { reportApiError } from "../../cloud/errors.js"; +import { writeSearchCache, type CachedSfx } from "./state.js"; +import { requireSfxKey } from "./auth.js"; + +export const examples: Example[] = [ + ["Search the catalog by meaning", 'hyperframes sfx search "whoosh for a scene change"'], + ["Punchy UI sound, fewer results", 'hyperframes sfx search "punchy ui tap" --limit 5'], + ["JSON output (for agents)", 'hyperframes sfx search "cinematic impact boom" --json'], +]; + +export default defineCommand({ + meta: { + name: "search", + description: "Semantic search over the HeyGen sound-effects catalog", + }, + args: { + query: { + type: "positional", + description: "Natural-language description of the sound you want (function + character)", + required: true, + }, + limit: { type: "string", description: "Max results (default 10)", alias: "n" }, + minScore: { + type: "string", + description: "Minimum similarity score 0–1 (default 0.2, exploratory)", + }, + dir: { type: "string", description: "Project directory (default: current directory)" }, + json: { type: "boolean", description: "Output results as JSON (for agents)", default: false }, + }, + // CLI entrypoint: arg parsing + spinner + json/human + error paths. + // fallow-ignore-next-line complexity + async run({ args }) { + const projectDir = resolve(args.dir ?? "."); + const limit = args.limit ? Number.parseInt(args.limit, 10) : 10; + const minScore = args.minScore ? Number.parseFloat(args.minScore) : 0.2; + + if (!(await requireSfxKey(args.json))) return; + + const spin = args.json ? null : clack.spinner(); + spin?.start(`Searching sound effects for "${args.query}"...`); + try { + const client = await createCloudClient(); + const res = await client.searchSounds({ + query: args.query, + type: "sound_effects", + limit, + min_score: minScore, + }); + const now = Date.now(); + const items: CachedSfx[] = (res.data ?? []).map((it) => ({ + id: it.id, + name: it.name, + description: it.description, + duration: it.duration ?? null, + score: it.score, + audio_url: it.audio_url, + query: args.query, + fetchedAt: now, + })); + + // Cache so `sfx add ` can resolve the (expiring) download URL. + try { + writeSearchCache(projectDir, items); + } catch { + /* cache is best-effort; add can still refresh by re-searching */ + } + + if (args.json) { + console.log( + JSON.stringify({ + items: items.map(({ query: _q, fetchedAt: _f, ...rest }) => rest), + has_more: res.has_more ?? false, + next_token: res.next_token ?? null, + }), + ); + return; + } + + spin?.stop( + c.success( + `Found ${items.length} sound${items.length === 1 ? "" : "s"} for "${args.query}"`, + ), + ); + if (items.length === 0) { + console.log(c.dim(" No matches — try a broader description, or lower --minScore.")); + return; + } + for (const it of items) { + const dur = it.duration != null ? `${it.duration.toFixed(2)}s` : "?"; + console.log( + ` ${c.accent(it.id)} ${c.bold(it.name)} ${c.dim(`${dur} · score ${it.score.toFixed(3)}`)}`, + ); + const desc = + it.description.length > 96 ? `${it.description.slice(0, 96)}…` : it.description; + console.log(` ${c.dim(desc)}`); + } + console.log( + `\n ${c.dim("Add one to your project:")} ${c.accent("hyperframes sfx add ")}`, + ); + } catch (err) { + spin?.stop(c.error("Search failed")); + reportApiError("SFX search failed", err); + } + }, +}); diff --git a/packages/cli/src/commands/sfx/state.ts b/packages/cli/src/commands/sfx/state.ts new file mode 100644 index 0000000000..9e70fc1dd0 --- /dev/null +++ b/packages/cli/src/commands/sfx/state.ts @@ -0,0 +1,65 @@ +/** + * Local state for the `sfx` commands. + * + * The public catalog API is search-only (no fetch-by-id), and presigned + * `audio_url`s expire (~15 min). So `sfx search` caches each result's + * download URL + the query that produced it, and `sfx add ` reads that + * cache — refreshing a stale URL by re-running the cached query. The cache + * is co-located with the downloaded clips under `/sfx/`. + */ + +import { mkdirSync, readFileSync, writeFileSync } from "node:fs"; +import { join, resolve } from "node:path"; +import { loadProjectConfig } from "../../utils/projectConfig.js"; + +export interface CachedSfx { + id: string; + name: string; + description: string; + duration: number | null; + score: number; + audio_url: string; + /** The search query that surfaced this clip — used to refresh an expired URL. */ + query: string; + /** Epoch ms when audio_url was fetched (presigned URLs expire). */ + fetchedAt: number; +} + +const CACHE_FILE = ".catalog-cache.json"; + +/** Directory SFX clips download into: `//sfx`. */ +export function sfxDir(projectDir: string): string { + const config = loadProjectConfig(projectDir); + return join(resolve(projectDir), config.paths.assets, "sfx"); +} + +function cachePath(projectDir: string): string { + return join(sfxDir(projectDir), CACHE_FILE); +} + +/** Merge `items` into the per-project search cache (keyed by id). */ +export function writeSearchCache(projectDir: string, items: CachedSfx[]): void { + const dir = sfxDir(projectDir); + mkdirSync(dir, { recursive: true }); + let all: Record = {}; + try { + all = JSON.parse(readFileSync(cachePath(projectDir), "utf-8")) as Record; + } catch { + /* no cache yet */ + } + for (const it of items) all[it.id] = it; + writeFileSync(cachePath(projectDir), JSON.stringify(all, null, 2) + "\n", "utf-8"); +} + +/** Look up a cached clip by id, or undefined if not in the cache. */ +export function readCachedSfx(projectDir: string, id: string): CachedSfx | undefined { + try { + const all = JSON.parse(readFileSync(cachePath(projectDir), "utf-8")) as Record< + string, + CachedSfx + >; + return all[id]; + } catch { + return undefined; + } +} diff --git a/packages/cli/src/help.ts b/packages/cli/src/help.ts index 44acb3737d..14acb5ee51 100644 --- a/packages/cli/src/help.ts +++ b/packages/cli/src/help.ts @@ -68,6 +68,7 @@ const GROUPS: Group[] = [ "Transcribe audio/video to word-level timestamps, or import an existing transcript", ], ["tts", "Generate speech audio from text using a local AI model (Kokoro-82M)"], + ["sfx", "Search and download sound effects from the HeyGen catalog"], ["remove-background", "Remove background from a video or image to produce transparent media"], ], }, diff --git a/skills/hyperframes/SKILL.md b/skills/hyperframes/SKILL.md index 61951dd2ec..4fb6a3d6c8 100644 --- a/skills/hyperframes/SKILL.md +++ b/skills/hyperframes/SKILL.md @@ -1,6 +1,6 @@ --- name: hyperframes -description: Create video compositions, animations, title cards, overlays, captions, voiceovers, audio-reactive visuals, and scene transitions in HyperFrames HTML. Use when asked to build any HTML-based video content, add captions or subtitles synced to audio, generate text-to-speech narration, create audio-reactive animation (beat sync, glow, pulse driven by music), add animated text highlighting (marker sweeps, hand-drawn circles, burst lines, scribble, sketchout), or add transitions between scenes (crossfades, wipes, reveals, shader transitions). Covers composition authoring, timing, media, and the full video production workflow. For dev-loop CLI commands (init, lint, inspect, preview, render) see the hyperframes-cli skill; for asset preprocessing commands (tts, transcribe, remove-background) see the hyperframes-media skill. +description: Create video compositions, animations, title cards, overlays, captions, voiceovers, audio-reactive visuals, sound effects, and scene transitions in HyperFrames HTML. Use when asked to build any HTML-based video content, add captions or subtitles synced to audio, generate text-to-speech narration, create audio-reactive animation (beat sync, glow, pulse driven by music), add animated text highlighting (marker sweeps, hand-drawn circles, burst lines, scribble, sketchout), add sound effects (whooshes, impacts, risers, UI taps) searched from the HeyGen catalog, or add transitions between scenes (crossfades, wipes, reveals, shader transitions). Covers composition authoring, timing, media, and the full video production workflow. For dev-loop CLI commands (init, lint, inspect, preview, render) see the hyperframes-cli skill; for asset preprocessing commands (tts, transcribe, remove-background) see the hyperframes-media skill. --- # HyperFrames @@ -468,6 +468,7 @@ Skip on small edits (fixing a color, adjusting one duration). Run on new composi - **[references/captions.md](references/captions.md)** — Captions, subtitles, lyrics, karaoke synced to audio. Tone-adaptive style detection, per-word styling, text overflow prevention, caption exit guarantees, word grouping. Read when adding any text synced to audio timing. - **[references/audio-reactive.md](references/audio-reactive.md)** — Audio-reactive animation: map frequency bands and amplitude to GSAP properties. Read when visuals should respond to music, voice, or sound. +- **[references/sound-effects.md](references/sound-effects.md)** — Sound effects: search the HeyGen catalog, download a clip, embed as an audio element. When to punctuate, choosing by function + character, the volume hierarchy, and trigger timing. Read when adding SFX (whooshes, impacts, risers, UI taps) to moments. - **[references/css-patterns.md](references/css-patterns.md)** — CSS+GSAP marker highlighting: highlight, circle, burst, scribble, sketchout. Deterministic, fully seekable. Read when adding visual emphasis to text. - **[references/video-composition.md](references/video-composition.md)** — Video-medium rules: density, color presence, scale, frame composition, the design spec as brand not layout. **Always read** — these override web instincts. - **[references/beat-direction.md](references/beat-direction.md)** — Beat planning: concept, mood, choreography verbs, rhythm templates, transition decisions, depth layers. **Always read for multi-scene compositions.** diff --git a/skills/hyperframes/references/beat-direction.md b/skills/hyperframes/references/beat-direction.md index 2647290b44..f12ec65c01 100644 --- a/skills/hyperframes/references/beat-direction.md +++ b/skills/hyperframes/references/beat-direction.md @@ -142,7 +142,7 @@ What's in foreground, midground, and background. Every beat should have at least ### SFX cues -What sounds at what moment: +What sounds at what moment (see [sound-effects.md](sound-effects.md) to search the catalog, choose by function + character, and time them): - "On the capture pulse — a soft, warm analog shutter click." - "Left side carries a faint low drone. On fold: drone cuts. Silence. Then a single clean chime." diff --git a/skills/hyperframes/references/sound-effects.md b/skills/hyperframes/references/sound-effects.md new file mode 100644 index 0000000000..42b5f6a778 --- /dev/null +++ b/skills/hyperframes/references/sound-effects.md @@ -0,0 +1,183 @@ +# Sound Effects + +Sound effects are searched from HeyGen's catalog by meaning, downloaded locally, and embedded as ordinary audio clips. What a sound _does_ depends on its shape in time — some punctuate a single frame, some build across a stretch, some run quietly under a whole scene — and the real craft is combining them: sequencing, layering, and chaining clips into something that feels designed rather than dropped in. + +## The five things a sound can do + +A clip's **duration tells you its job.** Don't treat them all as one-frame stingers. + +| Mode | What it needs | Examples | Typical len | How it's placed | +| --------- | ----------------------------------- | --------------------------------------------------------------- | ----------- | --------------------------------------- | +| **Hit** | a beat (one frame) | whoosh, impact, pop, ui click, bell, glitch, notification, coin | 0.1–2s | `data-start` = the exact event frame | +| **Build** | a span _into_ a beat | riser (ascends), downlifter (descends) | 3–12s | so it resolves on the beat (see Timing) | +| **Bed** | a scene + a mood — **not** an event | ambience, drone, wind, rain, sci-fi hum, crowd roar | 8–20s | runs _under_ the whole scene, very low | +| **Sting** | a reveal / lock moment | logo reveal, suspense sting, brand intro | 1.5–8s | `data-start` = the reveal frame | +| **Meme** | a comedic beat | vine boom, record scratch, air horn, sad trombone, "sus", boing | 0.4–3s | on the comedic moment — sparingly | + +The key thing: **not every sound is tied to an event.** A _bed_ needs a scene with a mood, not a beat — it's a texture you lay underneath. Hits, builds, stings, and memes are punctuation and _do_ land on beats. + +## Two layers: a bed underneath, punctuation on top + +Think of a clip's sound as (at most) two layers: + +1. **A bed (optional, continuous).** If a scene has an _environment or mood_ worth establishing — an outdoor shot, a tense reveal, a futuristic UI, a stadium moment — lay **one** bed under it at very low volume. It isn't tied to any frame; it sets the floor and gets out of the way. Most plain talking-head / UGC clips don't need one; atmospheric ones come alive with one. Never stack two beds — they muddy each other. + +2. **Punctuation (on beats).** On top of the bed — or with no bed at all — punctuate the real beats with hits, builds, stings, or the occasional meme. + +## Punctuation follows beats, not motion + +This rule governs the _punctuation_ layer (hits / builds / stings / memes), not beds. + +A **beat** is a moment the viewer is meant to _register_ — the edit snaps, lands, or turns: a hard cut, a key element popping in, a number hitting, a build peaking, a CTA, a comedic turn. These earn a punctuation sound. + +**Motion is not a beat.** A card drifting across the frame, a slow zoom, a parallax pan, a looping idle animation, decorative movement — these carry themselves visually. A whoosh on every one of them turns polish into noise. When unsure, ask: _would an editor cut, flash, or snap something here?_ If not, no hit. + +The rule cuts both ways: + +- **No beat left silent.** A reveal montage is a _series_ of beats — skipping SFX there entirely is a miss, not restraint. +- **No hit on mere motion.** Don't punctuate drifts, ambient loops, or decorative animation. If a scene needs _something_ under that motion, that's a job for a quiet bed, not a hit. + +## Same beat, same sound (the motif rule) + +Repeated beats of the **same kind share one sound.** Six cards entering one after another are six instances of _one_ beat → _one_ whoosh, reused: embed the same file in multiple `