mirror of
https://github.com/heygen-com/hyperframes.git
synced 2026-11-12 20:47:30 +00:00
feat(sfx): catalog discovery, analysis, and free-key opt-in (#1138)
Cherry-pick of 995dbb4f (originally on feat/sfx-audio-search) into
feat/w2h-restructure-v3, now mergeable because main was just folded
into this branch and brought cloud/auth.
CLI (packages/cli/src/commands/sfx/):
- list / search / add / inspect
- analyze.ts (ffmpeg ebur128 + silencedetect, LUFS/peak/onset/tail
+ loudness sparkline + suggested trim)
- auth.ts (free HeyGen key opt-in instead of raw 401)
- catalog-manifest.ts (180 clips across 22 families, generated)
Skill docs:
- skills/hyperframes/references/sound-effects.md (the global model)
- skills/hyperframes/SKILL.md + beat-direction.md link sound-effects.md
- skills/website-to-hyperframes step-3/step-5 migrate off bundled SFX
onto catalog flow (incorporates session-13/16 prose: orchestrator
authoring stays out, landmines stay in)
- assets/sfx/ deleted (20 mp3 + manifest.json + CREDITS.md)
Cherry-pick conflicts resolved manually:
- step-3-storyboard.md SFX section: took catalog version
- step-5-build.md: kept our session-13/16 structure (landmines,
assembler-owned index.html), folded in commit's catalog-analysis
paragraph
- manifest.json: deleted per the commit's intent
This commit is contained in:
ukimsanov
@@ -149,6 +149,7 @@ const subCommands = {
|
||||
cloudrun: () => import("./commands/cloudrun.js").then((m) => m.default),
|
||||
cloud: () => import("./commands/cloud.js").then((m) => m.default),
|
||||
auth: () => import("./commands/auth.js").then((m) => m.default),
|
||||
sfx: () => import("./commands/sfx.js").then((m) => m.default),
|
||||
};
|
||||
|
||||
const main = defineCommand({
|
||||
|
||||
@@ -5,6 +5,7 @@
|
||||
* experiment-framework to regenerate.
|
||||
*/
|
||||
import type {
|
||||
AudioSoundItem,
|
||||
CreateHyperframesRenderRequest,
|
||||
CreateHyperframesRenderResponse,
|
||||
DeleteHyperframesRenderResponse,
|
||||
@@ -302,4 +303,32 @@ export class HyperframesCloudClient {
|
||||
signal: args.signal,
|
||||
});
|
||||
}
|
||||
|
||||
async searchSounds(args: {
|
||||
query: string;
|
||||
type?: "music" | "sound_effects";
|
||||
limit?: number;
|
||||
min_score?: number;
|
||||
token?: string;
|
||||
signal?: AbortSignal;
|
||||
}): Promise<{ data?: Array<AudioSoundItem>; has_more?: boolean; next_token?: string | null }> {
|
||||
const query: Record<string, string | number | undefined> = {
|
||||
query: args.query,
|
||||
type: args.type,
|
||||
limit: args.limit,
|
||||
min_score: args.min_score,
|
||||
token: args.token,
|
||||
};
|
||||
return await this.request<{
|
||||
data?: Array<AudioSoundItem>;
|
||||
has_more?: boolean;
|
||||
next_token?: string | null;
|
||||
}>({
|
||||
method: "GET",
|
||||
path: "/v3/audio/sounds",
|
||||
query,
|
||||
unwrapData: false,
|
||||
signal: args.signal,
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
@@ -267,3 +267,36 @@ export interface UploadAssetV3Response {
|
||||
*/
|
||||
size_bytes: number;
|
||||
}
|
||||
|
||||
export type AudioContentType = "music" | "sound_effects";
|
||||
export interface AudioSoundItem {
|
||||
/**
|
||||
* Unique identifier for the audio track.
|
||||
*/
|
||||
id: string;
|
||||
/**
|
||||
* Display name of the audio track.
|
||||
*/
|
||||
name: string;
|
||||
/**
|
||||
* Semantic description the track was indexed under.
|
||||
*/
|
||||
description: string;
|
||||
/**
|
||||
* Temporary pre-signed URL to download the audio file. Expires; do not cache
|
||||
* long-term.
|
||||
*/
|
||||
audio_url: string;
|
||||
/**
|
||||
* Duration of the audio in seconds, when known.
|
||||
*/
|
||||
duration?: number | null;
|
||||
/**
|
||||
* Semantic similarity score (0-1); higher means a closer match.
|
||||
*/
|
||||
score: number;
|
||||
/**
|
||||
* Audio content type of this track.
|
||||
*/
|
||||
type: AudioContentType;
|
||||
}
|
||||
|
||||
@@ -0,0 +1,52 @@
|
||||
/**
|
||||
* `hyperframes sfx` — search and download sound effects from the HeyGen
|
||||
* catalog. Subverbs live in `./sfx/<name>.ts`, loaded dynamically so this
|
||||
* surface doesn't affect CLI cold-start.
|
||||
*
|
||||
* Auth reuses the existing cloud credential chain (`createCloudClient` →
|
||||
* `resolveCloudAuthHeaders`): `HEYGEN_API_KEY` or `hyperframes auth login`.
|
||||
* No new credential store, no new env var.
|
||||
*/
|
||||
|
||||
import { defineCommand } from "citty";
|
||||
import type { Example } from "./_examples.js";
|
||||
import { c } from "../ui/colors.js";
|
||||
|
||||
export const examples: Example[] = [
|
||||
["See what families are in the catalog", "hyperframes sfx list"],
|
||||
["Search the SFX catalog by meaning", 'hyperframes sfx search "whoosh for a scene change"'],
|
||||
["Download a clip into the project", "hyperframes sfx add 8b1ac2f2e69edd9b"],
|
||||
["Measure a clip (loudness, peak time)", "hyperframes sfx inspect 8b1ac2f2e69edd9b"],
|
||||
];
|
||||
|
||||
const HELP = `
|
||||
${c.bold("hyperframes sfx")} ${c.dim("<subcommand>")}
|
||||
|
||||
Find and use sound effects from the HeyGen catalog — AI-generated whooshes,
|
||||
impacts, UI tones, risers, foley, and more — searchable by description.
|
||||
|
||||
${c.bold("SUBCOMMANDS")}
|
||||
${c.accent("list")} List the families in the catalog (start here)
|
||||
${c.accent("search")} ${c.dim('"<description>"')} Search the catalog by meaning
|
||||
${c.accent("add")} ${c.dim("<id>")} Download a clip into ./assets/sfx/
|
||||
${c.accent("inspect")} ${c.dim("<id>")} Measure loudness, peak time, onset/tail
|
||||
|
||||
${c.bold("EXAMPLE")}
|
||||
hyperframes sfx list
|
||||
hyperframes sfx search "whoosh for a scene change"
|
||||
hyperframes sfx add <id>
|
||||
hyperframes sfx inspect <id>
|
||||
`;
|
||||
|
||||
export default defineCommand({
|
||||
meta: { name: "sfx", description: "Search and download sound effects from the HeyGen catalog" },
|
||||
subCommands: {
|
||||
list: () => import("./sfx/list.js").then((m) => m.default),
|
||||
search: () => import("./sfx/search.js").then((m) => m.default),
|
||||
add: () => import("./sfx/add.js").then((m) => m.default),
|
||||
inspect: () => import("./sfx/inspect.js").then((m) => m.default),
|
||||
},
|
||||
run({ args }) {
|
||||
if (!args._?.[0]) console.log(HELP);
|
||||
},
|
||||
});
|
||||
@@ -0,0 +1,130 @@
|
||||
import { defineCommand } from "citty";
|
||||
import { join, relative, resolve } from "node:path";
|
||||
import * as clack from "@clack/prompts";
|
||||
import type { Example } from "../_examples.js";
|
||||
import { c } from "../../ui/colors.js";
|
||||
import { errorBox } from "../../ui/format.js";
|
||||
import { createCloudClient, downloadToFile } from "../../cloud/index.js";
|
||||
import { reportApiError } from "../../cloud/errors.js";
|
||||
import { readCachedSfx, sfxDir, writeSearchCache } from "./state.js";
|
||||
import { analyzeAudio, formatAnalysisLines, type AudioAnalysis } from "./analyze.js";
|
||||
import { requireSfxKey } from "./auth.js";
|
||||
|
||||
export const examples: Example[] = [
|
||||
["Download a clip from a prior search", "hyperframes sfx add 8b1ac2f2e69edd9b"],
|
||||
["Add into a specific project", "hyperframes sfx add 8b1ac2f2e69edd9b --dir ./my-video"],
|
||||
];
|
||||
|
||||
// Presigned URLs are short-lived; refresh anything older than this.
|
||||
const URL_TTL_MS = 10 * 60 * 1000;
|
||||
|
||||
export default defineCommand({
|
||||
meta: {
|
||||
name: "add",
|
||||
description: "Download a sound effect into the project's assets/sfx/ directory",
|
||||
},
|
||||
args: {
|
||||
id: {
|
||||
type: "positional",
|
||||
description: "The SFX id from `hyperframes sfx search`",
|
||||
required: true,
|
||||
},
|
||||
dir: { type: "string", description: "Project directory (default: current directory)" },
|
||||
json: { type: "boolean", description: "Output result as JSON", default: false },
|
||||
},
|
||||
// CLI entrypoint: arg parsing + URL refresh + analysis + json/human + error paths.
|
||||
// fallow-ignore-next-line complexity
|
||||
async run({ args }) {
|
||||
const projectDir = resolve(args.dir ?? ".");
|
||||
if (!(await requireSfxKey(args.json))) return;
|
||||
const entry = readCachedSfx(projectDir, args.id);
|
||||
if (!entry) {
|
||||
errorBox(
|
||||
"Unknown SFX id",
|
||||
`No cached result for "${args.id}".`,
|
||||
'Run `hyperframes sfx search "<description>"` first, then add by id.',
|
||||
);
|
||||
process.exit(1);
|
||||
}
|
||||
|
||||
const spin = args.json ? null : clack.spinner();
|
||||
spin?.start(`Adding ${entry.name}...`);
|
||||
try {
|
||||
// Refresh the (expiring) presigned URL by re-running the cached query.
|
||||
let url = entry.audio_url;
|
||||
if (Date.now() - entry.fetchedAt > URL_TTL_MS) {
|
||||
const client = await createCloudClient();
|
||||
const res = await client.searchSounds({
|
||||
query: entry.query,
|
||||
type: "sound_effects",
|
||||
limit: 50,
|
||||
});
|
||||
const fresh = (res.data ?? []).find((it) => it.id === args.id);
|
||||
if (fresh) {
|
||||
url = fresh.audio_url;
|
||||
writeSearchCache(projectDir, [
|
||||
{ ...entry, audio_url: fresh.audio_url, fetchedAt: Date.now() },
|
||||
]);
|
||||
}
|
||||
}
|
||||
|
||||
const dest = join(sfxDir(projectDir), `${args.id}${extFromUrl(url)}`);
|
||||
const result = await downloadToFile(
|
||||
url,
|
||||
dest,
|
||||
spin
|
||||
? {
|
||||
onProgress: (b, t) =>
|
||||
spin.message(
|
||||
t
|
||||
? `Downloading ${Math.round((b / t) * 100)}%`
|
||||
: `Downloaded ${(b / 1024).toFixed(0)} KB`,
|
||||
),
|
||||
}
|
||||
: {},
|
||||
);
|
||||
const rel = relative(projectDir, result.path) || result.path;
|
||||
const dur = (entry.duration ?? 1).toFixed(2);
|
||||
|
||||
// Measure the clip so the agent can place + balance it without hearing it.
|
||||
// Best-effort: ffmpeg is optional, so a missing binary just omits the analysis.
|
||||
let analysis: AudioAnalysis | null = null;
|
||||
try {
|
||||
analysis = await analyzeAudio(result.path);
|
||||
} catch {
|
||||
/* ffmpeg not installed or unreadable clip — skip the analysis */
|
||||
}
|
||||
|
||||
if (args.json) {
|
||||
console.log(
|
||||
JSON.stringify({
|
||||
ok: true,
|
||||
id: args.id,
|
||||
name: entry.name,
|
||||
path: rel,
|
||||
duration: entry.duration,
|
||||
analysis,
|
||||
}),
|
||||
);
|
||||
return;
|
||||
}
|
||||
spin?.stop(c.success(`Added ${c.accent(entry.name)} → ${c.accent(rel)}`));
|
||||
console.log(c.dim(" Embed it as a clip (volume low under voiceover):"));
|
||||
console.log(
|
||||
` ${c.dim(`<audio src="${rel}" data-start="0" data-duration="${dur}" data-volume="0.3"></audio>`)}`,
|
||||
);
|
||||
if (analysis) {
|
||||
for (const line of formatAnalysisLines(analysis)) console.log(` ${c.dim(line)}`);
|
||||
}
|
||||
} catch (err) {
|
||||
spin?.stop(c.error("Add failed"));
|
||||
reportApiError("SFX download failed", err);
|
||||
}
|
||||
},
|
||||
});
|
||||
|
||||
/** Pick the file extension from a (possibly query-stringed) URL; default .flac. */
|
||||
function extFromUrl(url: string): string {
|
||||
const match = (url.split("?")[0] ?? url).match(/\.(flac|mp3|wav|m4a|ogg)$/i);
|
||||
return match ? match[0].toLowerCase() : ".flac";
|
||||
}
|
||||
@@ -0,0 +1,90 @@
|
||||
import { describe, expect, it } from "vitest";
|
||||
import { envelopeSparkline, formatAnalysisLines, parseFfmpegAnalysis } from "./analyze.js";
|
||||
|
||||
// Shaped after real `ffmpeg -af ebur128=peak=true,silencedetect=...` stderr:
|
||||
// 0.5s lead silence, a tone that peaks ~0.9s, 0.5s trailing silence to EOF (2.0s).
|
||||
const FULL = ` Duration: 00:00:02.00, bitrate: 705 kb/s
|
||||
[Parsed_silencedetect_1 @ 0x0] silence_start: 0
|
||||
[Parsed_ebur128_0 @ 0x0] t: 0.0999773 TARGET:-23 LUFS M:-120.7 S:-120.7 I: -70.0 LUFS LRA: 0.0 LU FTPK: -inf dBFS TPK: -inf dBFS
|
||||
[Parsed_silencedetect_1 @ 0x0] silence_end: 0.5 | silence_duration: 0.5
|
||||
[Parsed_ebur128_0 @ 0x0] t: 0.899977 TARGET:-23 LUFS M: -20.0 S:-120.7 I: -22.0 LUFS LRA: 0.0 LU FTPK: -inf dBFS TPK: -18.1 dBFS
|
||||
[Parsed_ebur128_0 @ 0x0] t: 1.399977 TARGET:-23 LUFS M: -24.0 S:-120.7 I: -22.5 LUFS LRA: 0.0 LU FTPK: -inf dBFS TPK: -18.1 dBFS
|
||||
[Parsed_silencedetect_1 @ 0x0] silence_start: 1.5
|
||||
Integrated loudness:
|
||||
I: -22.9 LUFS
|
||||
True peak:
|
||||
Peak: -18.1 dBFS
|
||||
`;
|
||||
|
||||
describe("parseFfmpegAnalysis", () => {
|
||||
it("extracts duration, loudness, true peak, peak time, onset, and tail", () => {
|
||||
const a = parseFfmpegAnalysis(FULL);
|
||||
expect(a.durationSec).toBe(2);
|
||||
expect(a.integratedLufs).toBe(-22.9); // the summary value, not the per-frame running I:
|
||||
expect(a.truePeakDb).toBe(-18.1);
|
||||
expect(a.peakTimeSec).toBeCloseTo(0.9, 1); // loudest momentary M (-20.0), not the silent frames
|
||||
expect(a.onsetSec).toBeCloseTo(0.5, 2); // end of the leading silence
|
||||
expect(a.tailStartSec).toBeCloseTo(1.5, 2); // trailing silence runs to EOF without a silence_end
|
||||
// contour: silent frame floored to -70, then the two audible frames
|
||||
expect(a.envelope).toEqual([-70, -20, -24]);
|
||||
});
|
||||
|
||||
it("builds an empty envelope and sparkline when there are no momentary lines", () => {
|
||||
expect(parseFfmpegAnalysis("").envelope).toEqual([]);
|
||||
expect(envelopeSparkline([])).toBe("");
|
||||
});
|
||||
|
||||
it("renders a sparkline spanning low→high blocks across the contour", () => {
|
||||
const spark = envelopeSparkline([-70, -20, -24]);
|
||||
expect(spark).toHaveLength(3);
|
||||
expect(spark[0]).toBe("▁"); // quietest → lowest block
|
||||
expect(spark[1]).toBe("█"); // loudest → highest block
|
||||
});
|
||||
|
||||
it("ignores silent frames (M ≈ -120) when finding the peak", () => {
|
||||
// Only silence-level momentary lines → no defensible peak.
|
||||
const silentish = ` Duration: 00:00:01.00, bitrate: 705 kb/s
|
||||
[Parsed_ebur128_0 @ 0x0] t: 0.5 TARGET:-23 LUFS M:-120.7 S:-120.7 I: -70.0 LUFS LRA: 0.0 LU
|
||||
`;
|
||||
expect(parseFfmpegAnalysis(silentish).peakTimeSec).toBeNull();
|
||||
});
|
||||
|
||||
it("reports onset 0 and tail null for a clip with no detected silence", () => {
|
||||
const hot = ` Duration: 00:00:01.00, bitrate: 705 kb/s
|
||||
[Parsed_ebur128_0 @ 0x0] t: 0.3 TARGET:-23 LUFS M: -12.0 S:-15.0 I: -12.0 LUFS LRA: 0.0 LU
|
||||
Integrated loudness:
|
||||
I: -12.0 LUFS
|
||||
True peak:
|
||||
Peak: -1.0 dBFS
|
||||
`;
|
||||
const a = parseFfmpegAnalysis(hot);
|
||||
expect(a.onsetSec).toBe(0);
|
||||
expect(a.tailStartSec).toBeNull();
|
||||
expect(a.peakTimeSec).toBeCloseTo(0.3, 2);
|
||||
expect(a.truePeakDb).toBe(-1);
|
||||
});
|
||||
|
||||
it("emits a trim line with the effective window when the clip has dead air", () => {
|
||||
const a = parseFfmpegAnalysis(FULL); // onset 0.5, tail 1.5, dur 2.0
|
||||
const trim = formatAnalysisLines(a).find((l) => l.startsWith("trim"));
|
||||
expect(trim).toBeDefined();
|
||||
expect(trim).toContain("data-media-start=0.5");
|
||||
expect(trim).toContain("data-duration=1"); // tail(1.5) − onset(0.5)
|
||||
});
|
||||
|
||||
it("omits the trim line when the clip starts hot and has no trailing silence", () => {
|
||||
const hot = parseFfmpegAnalysis(
|
||||
" Duration: 00:00:01.00, bitrate: 705 kb/s\n[Parsed_ebur128_0 @ 0x0] t: 0.3 M: -12.0 I: -12.0 LUFS\n",
|
||||
);
|
||||
expect(formatAnalysisLines(hot).some((l) => l.startsWith("trim"))).toBe(false);
|
||||
});
|
||||
|
||||
it("returns nulls gracefully on empty input", () => {
|
||||
const a = parseFfmpegAnalysis("");
|
||||
expect(a.durationSec).toBeNull();
|
||||
expect(a.integratedLufs).toBeNull();
|
||||
expect(a.peakTimeSec).toBeNull();
|
||||
expect(a.onsetSec).toBe(0);
|
||||
expect(a.tailStartSec).toBeNull();
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,243 @@
|
||||
/**
|
||||
* Audio analysis for sound-effect clips.
|
||||
*
|
||||
* Agents can't *hear* a clip, so they can't guess where a riser peaks or how
|
||||
* loud one sound is next to another. This runs ffmpeg once over a local file
|
||||
* and extracts the few facts that drive placement decisions:
|
||||
*
|
||||
* - integratedLufs / truePeakDb → set `data-volume` by measured level
|
||||
* - peakTimeSec → where to fire the payoff in a chain
|
||||
* - onsetSec / tailStartSec → trim dead air (`data-media-start`) / real end
|
||||
*
|
||||
* Everything comes from a single `ebur128 + silencedetect` pass (both filters
|
||||
* pass audio through and report to stderr), so there's no new dependency —
|
||||
* ffmpeg is already the local-render soft-dep. The parser is split out as a
|
||||
* pure function so it can be unit-tested without invoking ffmpeg.
|
||||
*/
|
||||
|
||||
import { execFile } from "node:child_process";
|
||||
import { promisify } from "node:util";
|
||||
import { findFFmpeg, getFFmpegInstallHint } from "../../browser/ffmpeg.js";
|
||||
|
||||
const execFileAsync = promisify(execFile);
|
||||
|
||||
export interface AudioAnalysis {
|
||||
/** Clip length in seconds (from the container), or null if unknown. */
|
||||
durationSec: number | null;
|
||||
/** EBU R128 integrated loudness in LUFS (perceived level), or null. */
|
||||
integratedLufs: number | null;
|
||||
/** True peak in dBFS (clipping headroom; 0 = full scale), or null. */
|
||||
truePeakDb: number | null;
|
||||
/** Time of the loudest moment (max momentary loudness) — a chain's resolve point. */
|
||||
peakTimeSec: number | null;
|
||||
/** First audible moment: end of any leading silence (0 if it starts hot). */
|
||||
onsetSec: number;
|
||||
/** Start of trailing silence — the clip's effective end — or null if it plays to the end. */
|
||||
tailStartSec: number | null;
|
||||
/**
|
||||
* Loudness contour over the clip — momentary LUFS sampled across time (silence
|
||||
* floored at -70), downsampled to at most ENVELOPE_BINS values. This is the
|
||||
* *shape* (build / drop / break), not just the peak: bin i is centered at
|
||||
* t ≈ (i + 0.5) / envelope.length × durationSec.
|
||||
*/
|
||||
envelope: number[];
|
||||
}
|
||||
|
||||
/** Max points in the returned envelope — bounded regardless of clip length. */
|
||||
const ENVELOPE_BINS = 64;
|
||||
/** Momentary LUFS below this counts as silence for the contour. */
|
||||
const SILENCE_FLOOR_LUFS = -70;
|
||||
|
||||
export class FFmpegNotFoundError extends Error {
|
||||
readonly hint: string;
|
||||
constructor() {
|
||||
super("ffmpeg not found");
|
||||
this.name = "FFmpegNotFoundError";
|
||||
this.hint = getFFmpegInstallHint();
|
||||
}
|
||||
}
|
||||
|
||||
/** Run ffmpeg's ebur128 + silencedetect over `filePath` and parse the result. */
|
||||
export async function analyzeAudio(filePath: string): Promise<AudioAnalysis> {
|
||||
const ffmpeg = findFFmpeg();
|
||||
if (!ffmpeg) throw new FFmpegNotFoundError();
|
||||
|
||||
const args = [
|
||||
"-hide_banner",
|
||||
"-nostats",
|
||||
"-i",
|
||||
filePath,
|
||||
"-af",
|
||||
"ebur128=peak=true,silencedetect=noise=-50dB:d=0.1",
|
||||
"-f",
|
||||
"null",
|
||||
"-",
|
||||
];
|
||||
|
||||
let stderr = "";
|
||||
try {
|
||||
const res = await execFileAsync(ffmpeg, args, { maxBuffer: 64 * 1024 * 1024 });
|
||||
stderr = res.stderr;
|
||||
} catch (err) {
|
||||
// ffmpeg can exit non-zero yet still emit the full analysis on stderr.
|
||||
stderr = (err as { stderr?: string })?.stderr ?? "";
|
||||
if (!stderr) throw err;
|
||||
}
|
||||
return parseFfmpegAnalysis(stderr);
|
||||
}
|
||||
|
||||
/**
|
||||
* Pure parser for the stderr of `ffmpeg -af ebur128=peak=true,silencedetect=...`.
|
||||
* Exported for unit tests — no I/O.
|
||||
*/
|
||||
export function parseFfmpegAnalysis(stderr: string): AudioAnalysis {
|
||||
const lines = stderr.split(/\r?\n/);
|
||||
const durationSec = parseDuration(stderr);
|
||||
const { peakTimeSec, envelope } = parseMomentary(lines);
|
||||
return {
|
||||
durationSec,
|
||||
// Both the per-frame lines and the final summary print "I: ... LUFS"; the last
|
||||
// occurrence is the integrated value. "Peak:" only appears in the True-peak summary.
|
||||
integratedLufs: lastMatchedNumber(stderr, /\bI:\s*(-?\d+(?:\.\d+)?)\s*LUFS/g),
|
||||
truePeakDb: lastMatchedNumber(stderr, /\bPeak:\s*(-?\d+(?:\.\d+)?)\s*dBFS/g),
|
||||
peakTimeSec,
|
||||
...parseSilence(lines, durationSec),
|
||||
envelope,
|
||||
};
|
||||
}
|
||||
|
||||
/** "Duration: 00:00:08.01, ..." → seconds. */
|
||||
function parseDuration(stderr: string): number | null {
|
||||
const dur = stderr.match(/Duration:\s*(\d+):(\d+):(\d+(?:\.\d+)?)/);
|
||||
if (dur?.[1] && dur[2] && dur[3])
|
||||
return Number(dur[1]) * 3600 + Number(dur[2]) * 60 + Number.parseFloat(dur[3]);
|
||||
return null;
|
||||
}
|
||||
|
||||
/** Last capture-group-1 match of `re` across the whole stderr, as a number. */
|
||||
function lastMatchedNumber(stderr: string, re: RegExp): number | null {
|
||||
const last = [...stderr.matchAll(re)].at(-1);
|
||||
return last?.[1] ? Number.parseFloat(last[1]) : null;
|
||||
}
|
||||
|
||||
/**
|
||||
* Momentary loudness over time → the loudest moment and the full contour.
|
||||
* ebur128 prints e.g. "t: 1.20 TARGET:-23 LUFS M: -22.3 S: ... I: ... LRA: ...".
|
||||
*/
|
||||
function parseMomentary(lines: string[]): { peakTimeSec: number | null; envelope: number[] } {
|
||||
const mom: Array<{ t: number; m: number }> = [];
|
||||
const tRe = /\bt:\s*(-?\d+(?:\.\d+)?)\b.*?\bM:\s*(-?\d+(?:\.\d+)?)/;
|
||||
for (const ln of lines) {
|
||||
const m = ln.match(tRe);
|
||||
if (m?.[1] && m[2]) mom.push({ t: Number.parseFloat(m[1]), m: Number.parseFloat(m[2]) });
|
||||
}
|
||||
let peakTimeSec: number | null = null;
|
||||
let maxM = Number.NEGATIVE_INFINITY;
|
||||
for (const { t, m } of mom) {
|
||||
// M of about -120 means silence/-inf; ignore it when finding the peak.
|
||||
if (m > SILENCE_FLOOR_LUFS && m > maxM) {
|
||||
maxM = m;
|
||||
peakTimeSec = t;
|
||||
}
|
||||
}
|
||||
return { peakTimeSec, envelope: buildEnvelope(mom.map((p) => p.m)) };
|
||||
}
|
||||
|
||||
/** Pair up silencedetect start/end lines into [start, end] intervals (trailing runs to EOF). */
|
||||
function collectSilenceIntervals(
|
||||
lines: string[],
|
||||
durationSec: number | null,
|
||||
): Array<[number, number]> {
|
||||
const intervals: Array<[number, number]> = [];
|
||||
let pendingStart: number | null = null;
|
||||
for (const ln of lines) {
|
||||
const s = ln.match(/silence_start:\s*(-?\d+(?:\.\d+)?)/);
|
||||
if (s?.[1]) {
|
||||
pendingStart = Number.parseFloat(s[1]);
|
||||
continue;
|
||||
}
|
||||
const e = ln.match(/silence_end:\s*(-?\d+(?:\.\d+)?)/);
|
||||
if (e?.[1] && pendingStart !== null) {
|
||||
intervals.push([pendingStart, Number.parseFloat(e[1])]);
|
||||
pendingStart = null;
|
||||
}
|
||||
}
|
||||
if (pendingStart !== null && durationSec !== null) intervals.push([pendingStart, durationSec]);
|
||||
return intervals;
|
||||
}
|
||||
|
||||
/** Leading silence → onset; trailing silence (reaching EOF) → tail start. */
|
||||
function parseSilence(
|
||||
lines: string[],
|
||||
durationSec: number | null,
|
||||
): { onsetSec: number; tailStartSec: number | null } {
|
||||
let onsetSec = 0;
|
||||
let tailStartSec: number | null = null;
|
||||
for (const [start, end] of collectSilenceIntervals(lines, durationSec)) {
|
||||
if (start <= 0.05) onsetSec = Math.max(onsetSec, end);
|
||||
if (durationSec !== null && end >= durationSec - 0.2) tailStartSec = start;
|
||||
}
|
||||
return { onsetSec, tailStartSec };
|
||||
}
|
||||
|
||||
/** Floor silence, then downsample the momentary-LUFS series to at most ENVELOPE_BINS bins. */
|
||||
function buildEnvelope(raw: number[]): number[] {
|
||||
if (!raw.length) return [];
|
||||
const vals = raw.map((m) => (m < SILENCE_FLOOR_LUFS ? SILENCE_FLOOR_LUFS : m));
|
||||
const round1 = (v: number) => Math.round(v * 10) / 10;
|
||||
if (vals.length <= ENVELOPE_BINS) return vals.map(round1);
|
||||
const out: number[] = [];
|
||||
for (let i = 0; i < ENVELOPE_BINS; i++) {
|
||||
const start = Math.floor((i * vals.length) / ENVELOPE_BINS);
|
||||
const end = Math.max(start + 1, Math.floor(((i + 1) * vals.length) / ENVELOPE_BINS));
|
||||
let sum = 0;
|
||||
let n = 0;
|
||||
for (let j = start; j < end && j < vals.length; j++) {
|
||||
sum += vals[j] as number;
|
||||
n++;
|
||||
}
|
||||
out.push(round1(sum / n));
|
||||
}
|
||||
return out;
|
||||
}
|
||||
|
||||
/** Unicode sparkline of the loudness contour (build/drop/break at a glance). */
|
||||
export function envelopeSparkline(envelope: number[]): string {
|
||||
if (!envelope.length) return "";
|
||||
const blocks = "▁▂▃▄▅▆▇█";
|
||||
const max = Math.max(...envelope);
|
||||
const min = Math.min(...envelope);
|
||||
const span = Math.max(1e-6, max - min);
|
||||
return envelope
|
||||
.map((v) => {
|
||||
const idx = Math.min(7, Math.max(0, Math.round(((v - min) / span) * 7)));
|
||||
return blocks[idx];
|
||||
})
|
||||
.join("");
|
||||
}
|
||||
|
||||
/** Human-readable lines for the terminal (no color — caller wraps as needed). */
|
||||
export function formatAnalysisLines(a: AudioAnalysis): string[] {
|
||||
const s = (n: number | null, unit: string, digits = 2) =>
|
||||
n === null ? "—" : `${n.toFixed(digits)}${unit}`;
|
||||
const loud = a.integratedLufs === null ? "—" : `${a.integratedLufs.toFixed(1)} LUFS`;
|
||||
const peak = a.truePeakDb === null ? "—" : `${a.truePeakDb.toFixed(1)} dBFS`;
|
||||
const out = [
|
||||
`loudness ${loud} true-peak ${peak}`,
|
||||
`peak at ${s(a.peakTimeSec, "s")} onset ${s(a.onsetSec, "s")} tail ${s(a.tailStartSec, "s")}`,
|
||||
];
|
||||
// Suggested trim window when the clip carries silence padding — so a padded
|
||||
// riser/whoosh plays only its real content instead of firing late on dead air.
|
||||
const end = a.tailStartSec ?? a.durationSec;
|
||||
const hasDeadAir = a.onsetSec > 0.05 || a.tailStartSec !== null;
|
||||
if (hasDeadAir && end !== null && end > a.onsetSec) {
|
||||
const r2 = (n: number) => Math.round(n * 100) / 100;
|
||||
const parts: string[] = [];
|
||||
if (a.onsetSec > 0.05) parts.push(`data-media-start=${r2(a.onsetSec)}`);
|
||||
parts.push(`data-duration=${r2(end - a.onsetSec)}`);
|
||||
out.push(`trim ${parts.join(" ")}`);
|
||||
}
|
||||
const spark = envelopeSparkline(a.envelope);
|
||||
if (spark) out.push(`shape ${spark}`);
|
||||
return out;
|
||||
}
|
||||
@@ -0,0 +1,44 @@
|
||||
/**
|
||||
* SFX access gate. The catalog is free, but searching/downloading it needs a
|
||||
* HeyGen API key. Rather than letting the API call fail with a raw 401, the
|
||||
* network-touching subcommands (`search`, `add`) call this first: if no
|
||||
* credential resolves, it prints a friendly opt-in (get a free key, set it, or
|
||||
* skip SFX) — mirroring how the VO/Gemini steps ask — and the caller bails.
|
||||
*
|
||||
* `list` and `inspect` are local and don't gate on this.
|
||||
*/
|
||||
|
||||
import { tryResolveCredential } from "../../auth/resolver.js";
|
||||
import { c } from "../../ui/colors.js";
|
||||
|
||||
const SFX_SIGNUP_URL = "https://app.heygen.com/developers/api";
|
||||
|
||||
/**
|
||||
* Returns true if a HeyGen credential is available. If not, prints guidance
|
||||
* (human or JSON) and returns false — the caller should stop.
|
||||
*/
|
||||
export async function requireSfxKey(json = false): Promise<boolean> {
|
||||
const cred = await tryResolveCredential();
|
||||
if (cred) return true;
|
||||
|
||||
if (json) {
|
||||
console.log(
|
||||
JSON.stringify({
|
||||
error: "missing_api_key",
|
||||
message: "Sound effects need a free HeyGen API key.",
|
||||
get_key_url: SFX_SIGNUP_URL,
|
||||
set_via: ["HEYGEN_API_KEY in a .env file or environment", "hyperframes auth login"],
|
||||
}),
|
||||
);
|
||||
return false;
|
||||
}
|
||||
|
||||
console.log(`\n ${c.bold("Sound effects need a free HeyGen API key.")}`);
|
||||
console.log(` ${c.dim("HeyGen's SFX catalog is free — you just need a key to search it.")}`);
|
||||
console.log(` ${c.dim("Get one at")} ${c.accent(SFX_SIGNUP_URL)}`);
|
||||
console.log(
|
||||
` ${c.dim("then add")} ${c.accent("HEYGEN_API_KEY=your-key")} ${c.dim("to a .env file, or run")} ${c.accent("hyperframes auth login")}${c.dim(".")}`,
|
||||
);
|
||||
console.log(` ${c.dim("Or skip SFX — the video just won't have them.")}\n`);
|
||||
return false;
|
||||
}
|
||||
@@ -0,0 +1,34 @@
|
||||
import { describe, expect, it } from "vitest";
|
||||
import { CATALOG_MANIFEST, type SfxMode } from "./catalog-manifest.js";
|
||||
|
||||
const MODES: SfxMode[] = ["hit", "build", "bed", "sting", "meme"];
|
||||
|
||||
describe("CATALOG_MANIFEST", () => {
|
||||
it("is non-empty and self-consistent", () => {
|
||||
expect(CATALOG_MANIFEST.families.length).toBeGreaterThan(0);
|
||||
for (const f of CATALOG_MANIFEST.families) {
|
||||
expect(f.category).toMatch(/^[a-z]+$/);
|
||||
expect(f.count).toBeGreaterThan(0);
|
||||
expect(MODES).toContain(f.mode);
|
||||
expect(f.use.length).toBeGreaterThan(0);
|
||||
expect(f.examples.length).toBeGreaterThan(0);
|
||||
expect(f.examples.length).toBeLessThanOrEqual(3);
|
||||
}
|
||||
});
|
||||
|
||||
it("totalClips equals the sum of family counts", () => {
|
||||
const sum = CATALOG_MANIFEST.families.reduce((n, f) => n + f.count, 0);
|
||||
expect(CATALOG_MANIFEST.totalClips).toBe(sum);
|
||||
});
|
||||
|
||||
it("lists families largest-first so the common kinds surface first", () => {
|
||||
const counts = CATALOG_MANIFEST.families.map((f) => f.count);
|
||||
expect(counts).toEqual([...counts].sort((a, b) => b - a));
|
||||
});
|
||||
|
||||
it("covers the non-obvious families an agent tends to forget", () => {
|
||||
const cats = new Set(CATALOG_MANIFEST.families.map((f) => f.category));
|
||||
for (const c of ["meme", "ambience", "downlifter", "glitch", "stinger"])
|
||||
expect(cats).toContain(c);
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,180 @@
|
||||
/**
|
||||
* Catalog map for the SFX library — families, counts, and what each is for.
|
||||
*
|
||||
* AUTO-GENERATED by scripts/sfx_rag/build_catalog_manifest.py — do not hand-edit.
|
||||
*/
|
||||
|
||||
export type SfxMode = "hit" | "build" | "bed" | "sting" | "meme";
|
||||
|
||||
export interface SfxFamily {
|
||||
category: string;
|
||||
count: number;
|
||||
mode: SfxMode;
|
||||
use: string;
|
||||
examples: string[];
|
||||
}
|
||||
|
||||
export interface SfxCatalogManifest {
|
||||
totalClips: number;
|
||||
families: SfxFamily[];
|
||||
}
|
||||
|
||||
export const CATALOG_MANIFEST: SfxCatalogManifest = {
|
||||
totalClips: 180,
|
||||
families: [
|
||||
{
|
||||
category: "impact",
|
||||
count: 29,
|
||||
mode: "hit",
|
||||
use: "Slams, booms, sub-drops — punctuate a hit or number landing on the impact frame.",
|
||||
examples: ["Sub Drop", "Slam 1", "Slam 2"],
|
||||
},
|
||||
{
|
||||
category: "foley",
|
||||
count: 27,
|
||||
mode: "hit",
|
||||
use: "Real-world textures — footsteps, taps, rips, handling; realism and walk cycles.",
|
||||
examples: ["Single Footstep", "Heavy Footstep", "Tape Rip 1"],
|
||||
},
|
||||
{
|
||||
category: "whoosh",
|
||||
count: 22,
|
||||
mode: "hit",
|
||||
use: "Transitions and scene changes — fire on the cut.",
|
||||
examples: ["Whoosh", "Whoosh effect", "Whoosh bamboo"],
|
||||
},
|
||||
{
|
||||
category: "meme",
|
||||
count: 14,
|
||||
mode: "meme",
|
||||
use: "Cultural punchlines — vine boom, record scratch, air horn, sad trombone; sparingly, on a comedic turn.",
|
||||
examples: ["Vine Boom", "Heavy Record Scratch", "Air Horn"],
|
||||
},
|
||||
{
|
||||
category: "stinger",
|
||||
count: 13,
|
||||
mode: "sting",
|
||||
use: "Logo reveals, brand intros, suspense stings — on a reveal or lock.",
|
||||
examples: ["Suspense Sting 1", "Calm Elegant Logo", "Epic Logo Reveal"],
|
||||
},
|
||||
{
|
||||
category: "bell",
|
||||
count: 9,
|
||||
mode: "hit",
|
||||
use: "Bells, chimes, doorbells, alarms — alerts and gentle accents.",
|
||||
examples: ["Doorbell", "Chime", "Old Phone Ring"],
|
||||
},
|
||||
{
|
||||
category: "ui",
|
||||
count: 9,
|
||||
mode: "hit",
|
||||
use: "Clicks, taps, sparkles, beeps — on an interaction.",
|
||||
examples: ["Generic Click", "Sparkle", "Censor Beep"],
|
||||
},
|
||||
{
|
||||
category: "human",
|
||||
count: 6,
|
||||
mode: "bed",
|
||||
use: "Crowd cheer, applause, heartbeat — reactions and tension (cheers/applause can run as beds).",
|
||||
examples: ["General Crowd Cheer", "Polite Audience Applause", "Slow Human Heartbeat"],
|
||||
},
|
||||
{
|
||||
category: "ambience",
|
||||
count: 6,
|
||||
mode: "bed",
|
||||
use: "Drones, tension beds, sci-fi hum, stadium roar — run UNDER a scene for mood; not on a beat.",
|
||||
examples: ["Eerie Deep Tension", "Slow Resonant Tension", "Sci Fi Hum"],
|
||||
},
|
||||
{
|
||||
category: "downlifter",
|
||||
count: 5,
|
||||
mode: "build",
|
||||
use: "Descending sweeps and sub-drops — out of a transition or to deflate energy.",
|
||||
examples: ["Downlifter deep", "Zoom out pull effect", "Sfx medium subdown sound effect"],
|
||||
},
|
||||
{
|
||||
category: "riser",
|
||||
count: 5,
|
||||
mode: "build",
|
||||
use: "Ascending builds into a climax — place so the peak lands on the beat.",
|
||||
examples: ["Riser", "Heavy Metallic Riser", "Soft Dreamy Riser"],
|
||||
},
|
||||
{
|
||||
category: "notification",
|
||||
count: 5,
|
||||
mode: "hit",
|
||||
use: "Pings, alerts, 'correct' tones — confirmations.",
|
||||
examples: ["Ping", "Correct", "SMS Alert"],
|
||||
},
|
||||
{
|
||||
category: "glitch",
|
||||
count: 4,
|
||||
mode: "hit",
|
||||
use: "Digital errors and stutters — tech and hard-cut accents.",
|
||||
examples: ["Digital Error 1", "Digital Error 2", "Glitch Effect 1"],
|
||||
},
|
||||
{
|
||||
category: "nature",
|
||||
count: 4,
|
||||
mode: "bed",
|
||||
use: "Wind, rain, thunder — outdoor ambience or a weather hit.",
|
||||
examples: ["Wind", "Rain", "Sharp Thunder Clap"],
|
||||
},
|
||||
{
|
||||
category: "vehicle",
|
||||
count: 4,
|
||||
mode: "hit",
|
||||
use: "Sirens, engines, crashes — scene and action sounds.",
|
||||
examples: ["Police Siren", "Car engine", "Car crash sound effect"],
|
||||
},
|
||||
{
|
||||
category: "weapon",
|
||||
count: 4,
|
||||
mode: "hit",
|
||||
use: "Sword slices and swings, shots — action accents.",
|
||||
examples: ["Sword slice", "Sword slash and swing", "Pistol Shot"],
|
||||
},
|
||||
{
|
||||
category: "money",
|
||||
count: 3,
|
||||
mode: "hit",
|
||||
use: "Coin jingles, cash-register ka-ching — reward or purchase.",
|
||||
examples: ["Multiple Coins Jingle", "Cash Register 'Kaching'", "Whoosh and Cash Register"],
|
||||
},
|
||||
{
|
||||
category: "animal",
|
||||
count: 3,
|
||||
mode: "hit",
|
||||
use: "Crickets, duck quack — nature beds or comedic accents.",
|
||||
examples: ["Cricket Chirp", "Duck Quack"],
|
||||
},
|
||||
{
|
||||
category: "success",
|
||||
count: 2,
|
||||
mode: "hit",
|
||||
use: "Coin-collect chime, achievement — completion and reward.",
|
||||
examples: ["Coin Collect Chime", "Achievement"],
|
||||
},
|
||||
{
|
||||
category: "pop",
|
||||
count: 2,
|
||||
mode: "hit",
|
||||
use: "Pops — element pop-ins.",
|
||||
examples: ["Heart Pop", "Standard Pop"],
|
||||
},
|
||||
{
|
||||
category: "water",
|
||||
count: 2,
|
||||
mode: "hit",
|
||||
use: "Splash and drop — water hits.",
|
||||
examples: ["Water Splash", "Water Drop"],
|
||||
},
|
||||
{
|
||||
category: "tech",
|
||||
count: 2,
|
||||
mode: "sting",
|
||||
use: "Sci-fi logo reveals — futuristic intros.",
|
||||
examples: ["Sci fi intro logo reveal"],
|
||||
},
|
||||
],
|
||||
};
|
||||
@@ -0,0 +1,75 @@
|
||||
import { defineCommand } from "citty";
|
||||
import { existsSync, statSync } from "node:fs";
|
||||
import { join, resolve } from "node:path";
|
||||
import type { Example } from "../_examples.js";
|
||||
import { c } from "../../ui/colors.js";
|
||||
import { errorBox } from "../../ui/format.js";
|
||||
import { sfxDir } from "./state.js";
|
||||
import { analyzeAudio, FFmpegNotFoundError, formatAnalysisLines } from "./analyze.js";
|
||||
|
||||
export const examples: Example[] = [
|
||||
["Analyze a downloaded clip by id", "hyperframes sfx inspect 8b1ac2f2e69edd9b"],
|
||||
["Analyze a file by path (JSON)", "hyperframes sfx inspect ./assets/sfx/clip.mp3 --json"],
|
||||
];
|
||||
|
||||
const AUDIO_EXTS = [".mp3", ".flac", ".wav", ".m4a", ".ogg"];
|
||||
|
||||
/** Resolve an id (downloaded into assets/sfx/) or a direct file path to a clip on disk. */
|
||||
// fallow-ignore-next-line complexity
|
||||
function resolveClipPath(projectDir: string, idOrPath: string): string | undefined {
|
||||
if (existsSync(idOrPath) && statSync(idOrPath).isFile()) return resolve(idOrPath);
|
||||
const dir = sfxDir(projectDir);
|
||||
for (const ext of AUDIO_EXTS) {
|
||||
const p = join(dir, `${idOrPath}${ext}`);
|
||||
if (existsSync(p)) return p;
|
||||
}
|
||||
return undefined;
|
||||
}
|
||||
|
||||
export default defineCommand({
|
||||
meta: {
|
||||
name: "inspect",
|
||||
description:
|
||||
"Measure a downloaded sound effect: loudness, true peak, peak time, onset/tail silence",
|
||||
},
|
||||
args: {
|
||||
id: {
|
||||
type: "positional",
|
||||
description: "SFX id (downloaded via `sfx add`) or a path to an audio file",
|
||||
required: true,
|
||||
},
|
||||
dir: { type: "string", description: "Project directory (default: current directory)" },
|
||||
json: { type: "boolean", description: "Output the analysis as JSON", default: false },
|
||||
},
|
||||
// CLI entrypoint: resolve + json/human + ffmpeg error paths.
|
||||
// fallow-ignore-next-line complexity
|
||||
async run({ args }) {
|
||||
const projectDir = resolve(args.dir ?? ".");
|
||||
const clip = resolveClipPath(projectDir, args.id);
|
||||
if (!clip) {
|
||||
errorBox(
|
||||
"Clip not found",
|
||||
`No downloaded clip "${args.id}" in ${sfxDir(projectDir)} and no file at that path.`,
|
||||
"Run `hyperframes sfx add <id>` first, or pass a path to an audio file.",
|
||||
);
|
||||
process.exit(1);
|
||||
}
|
||||
|
||||
try {
|
||||
const analysis = await analyzeAudio(clip);
|
||||
if (args.json) {
|
||||
console.log(JSON.stringify(analysis));
|
||||
return;
|
||||
}
|
||||
console.log(` ${c.bold(args.id)}`);
|
||||
for (const line of formatAnalysisLines(analysis)) console.log(` ${c.dim(line)}`);
|
||||
} catch (err) {
|
||||
if (err instanceof FFmpegNotFoundError) {
|
||||
errorBox("ffmpeg not found", "Analysis needs ffmpeg.", `Install it: ${err.hint}`);
|
||||
process.exit(1);
|
||||
}
|
||||
errorBox("Analysis failed", err instanceof Error ? err.message : String(err));
|
||||
process.exit(1);
|
||||
}
|
||||
},
|
||||
});
|
||||
@@ -0,0 +1,45 @@
|
||||
import { defineCommand } from "citty";
|
||||
import type { Example } from "../_examples.js";
|
||||
import { c } from "../../ui/colors.js";
|
||||
import { CATALOG_MANIFEST } from "./catalog-manifest.js";
|
||||
|
||||
export const examples: Example[] = [
|
||||
["See what's in the catalog before searching", "hyperframes sfx list"],
|
||||
["Machine-readable map (for agents)", "hyperframes sfx list --json"],
|
||||
];
|
||||
|
||||
export default defineCommand({
|
||||
meta: {
|
||||
name: "list",
|
||||
description:
|
||||
"List the sound-effect families in the catalog — see what's available before searching",
|
||||
},
|
||||
args: {
|
||||
json: {
|
||||
type: "boolean",
|
||||
description: "Output the catalog map as JSON (for agents)",
|
||||
default: false,
|
||||
},
|
||||
},
|
||||
run({ args }) {
|
||||
if (args.json) {
|
||||
console.log(JSON.stringify(CATALOG_MANIFEST));
|
||||
return;
|
||||
}
|
||||
const { totalClips, families } = CATALOG_MANIFEST;
|
||||
console.log(` ${c.bold(`${totalClips} sound effects across ${families.length} families`)}`);
|
||||
console.log(
|
||||
c.dim(
|
||||
" Pick the family that fits the moment, then search within it — don't default to the same whoosh.\n",
|
||||
),
|
||||
);
|
||||
const pad = Math.max(...families.map((f) => f.category.length));
|
||||
for (const f of families) {
|
||||
console.log(` ${c.accent(f.category.padEnd(pad))} ${c.dim(`(${f.count})`)} ${f.use}`);
|
||||
console.log(` ${" ".repeat(pad)} ${c.dim(`e.g. ${f.examples.join(", ")}`)}`);
|
||||
}
|
||||
console.log(
|
||||
`\n ${c.dim("Then:")} ${c.accent('hyperframes sfx search "<function + character>"')}`,
|
||||
);
|
||||
},
|
||||
});
|
||||
@@ -0,0 +1,111 @@
|
||||
import { defineCommand } from "citty";
|
||||
import { resolve } from "node:path";
|
||||
import * as clack from "@clack/prompts";
|
||||
import type { Example } from "../_examples.js";
|
||||
import { c } from "../../ui/colors.js";
|
||||
import { createCloudClient } from "../../cloud/index.js";
|
||||
import { reportApiError } from "../../cloud/errors.js";
|
||||
import { writeSearchCache, type CachedSfx } from "./state.js";
|
||||
import { requireSfxKey } from "./auth.js";
|
||||
|
||||
export const examples: Example[] = [
|
||||
["Search the catalog by meaning", 'hyperframes sfx search "whoosh for a scene change"'],
|
||||
["Punchy UI sound, fewer results", 'hyperframes sfx search "punchy ui tap" --limit 5'],
|
||||
["JSON output (for agents)", 'hyperframes sfx search "cinematic impact boom" --json'],
|
||||
];
|
||||
|
||||
export default defineCommand({
|
||||
meta: {
|
||||
name: "search",
|
||||
description: "Semantic search over the HeyGen sound-effects catalog",
|
||||
},
|
||||
args: {
|
||||
query: {
|
||||
type: "positional",
|
||||
description: "Natural-language description of the sound you want (function + character)",
|
||||
required: true,
|
||||
},
|
||||
limit: { type: "string", description: "Max results (default 10)", alias: "n" },
|
||||
minScore: {
|
||||
type: "string",
|
||||
description: "Minimum similarity score 0–1 (default 0.2, exploratory)",
|
||||
},
|
||||
dir: { type: "string", description: "Project directory (default: current directory)" },
|
||||
json: { type: "boolean", description: "Output results as JSON (for agents)", default: false },
|
||||
},
|
||||
// CLI entrypoint: arg parsing + spinner + json/human + error paths.
|
||||
// fallow-ignore-next-line complexity
|
||||
async run({ args }) {
|
||||
const projectDir = resolve(args.dir ?? ".");
|
||||
const limit = args.limit ? Number.parseInt(args.limit, 10) : 10;
|
||||
const minScore = args.minScore ? Number.parseFloat(args.minScore) : 0.2;
|
||||
|
||||
if (!(await requireSfxKey(args.json))) return;
|
||||
|
||||
const spin = args.json ? null : clack.spinner();
|
||||
spin?.start(`Searching sound effects for "${args.query}"...`);
|
||||
try {
|
||||
const client = await createCloudClient();
|
||||
const res = await client.searchSounds({
|
||||
query: args.query,
|
||||
type: "sound_effects",
|
||||
limit,
|
||||
min_score: minScore,
|
||||
});
|
||||
const now = Date.now();
|
||||
const items: CachedSfx[] = (res.data ?? []).map((it) => ({
|
||||
id: it.id,
|
||||
name: it.name,
|
||||
description: it.description,
|
||||
duration: it.duration ?? null,
|
||||
score: it.score,
|
||||
audio_url: it.audio_url,
|
||||
query: args.query,
|
||||
fetchedAt: now,
|
||||
}));
|
||||
|
||||
// Cache so `sfx add <id>` can resolve the (expiring) download URL.
|
||||
try {
|
||||
writeSearchCache(projectDir, items);
|
||||
} catch {
|
||||
/* cache is best-effort; add can still refresh by re-searching */
|
||||
}
|
||||
|
||||
if (args.json) {
|
||||
console.log(
|
||||
JSON.stringify({
|
||||
items: items.map(({ query: _q, fetchedAt: _f, ...rest }) => rest),
|
||||
has_more: res.has_more ?? false,
|
||||
next_token: res.next_token ?? null,
|
||||
}),
|
||||
);
|
||||
return;
|
||||
}
|
||||
|
||||
spin?.stop(
|
||||
c.success(
|
||||
`Found ${items.length} sound${items.length === 1 ? "" : "s"} for "${args.query}"`,
|
||||
),
|
||||
);
|
||||
if (items.length === 0) {
|
||||
console.log(c.dim(" No matches — try a broader description, or lower --minScore."));
|
||||
return;
|
||||
}
|
||||
for (const it of items) {
|
||||
const dur = it.duration != null ? `${it.duration.toFixed(2)}s` : "?";
|
||||
console.log(
|
||||
` ${c.accent(it.id)} ${c.bold(it.name)} ${c.dim(`${dur} · score ${it.score.toFixed(3)}`)}`,
|
||||
);
|
||||
const desc =
|
||||
it.description.length > 96 ? `${it.description.slice(0, 96)}…` : it.description;
|
||||
console.log(` ${c.dim(desc)}`);
|
||||
}
|
||||
console.log(
|
||||
`\n ${c.dim("Add one to your project:")} ${c.accent("hyperframes sfx add <id>")}`,
|
||||
);
|
||||
} catch (err) {
|
||||
spin?.stop(c.error("Search failed"));
|
||||
reportApiError("SFX search failed", err);
|
||||
}
|
||||
},
|
||||
});
|
||||
@@ -0,0 +1,65 @@
|
||||
/**
|
||||
* Local state for the `sfx` commands.
|
||||
*
|
||||
* The public catalog API is search-only (no fetch-by-id), and presigned
|
||||
* `audio_url`s expire (~15 min). So `sfx search` caches each result's
|
||||
* download URL + the query that produced it, and `sfx add <id>` reads that
|
||||
* cache — refreshing a stale URL by re-running the cached query. The cache
|
||||
* is co-located with the downloaded clips under `<assets>/sfx/`.
|
||||
*/
|
||||
|
||||
import { mkdirSync, readFileSync, writeFileSync } from "node:fs";
|
||||
import { join, resolve } from "node:path";
|
||||
import { loadProjectConfig } from "../../utils/projectConfig.js";
|
||||
|
||||
export interface CachedSfx {
|
||||
id: string;
|
||||
name: string;
|
||||
description: string;
|
||||
duration: number | null;
|
||||
score: number;
|
||||
audio_url: string;
|
||||
/** The search query that surfaced this clip — used to refresh an expired URL. */
|
||||
query: string;
|
||||
/** Epoch ms when audio_url was fetched (presigned URLs expire). */
|
||||
fetchedAt: number;
|
||||
}
|
||||
|
||||
const CACHE_FILE = ".catalog-cache.json";
|
||||
|
||||
/** Directory SFX clips download into: `<project>/<assets>/sfx`. */
|
||||
export function sfxDir(projectDir: string): string {
|
||||
const config = loadProjectConfig(projectDir);
|
||||
return join(resolve(projectDir), config.paths.assets, "sfx");
|
||||
}
|
||||
|
||||
function cachePath(projectDir: string): string {
|
||||
return join(sfxDir(projectDir), CACHE_FILE);
|
||||
}
|
||||
|
||||
/** Merge `items` into the per-project search cache (keyed by id). */
|
||||
export function writeSearchCache(projectDir: string, items: CachedSfx[]): void {
|
||||
const dir = sfxDir(projectDir);
|
||||
mkdirSync(dir, { recursive: true });
|
||||
let all: Record<string, CachedSfx> = {};
|
||||
try {
|
||||
all = JSON.parse(readFileSync(cachePath(projectDir), "utf-8")) as Record<string, CachedSfx>;
|
||||
} catch {
|
||||
/* no cache yet */
|
||||
}
|
||||
for (const it of items) all[it.id] = it;
|
||||
writeFileSync(cachePath(projectDir), JSON.stringify(all, null, 2) + "\n", "utf-8");
|
||||
}
|
||||
|
||||
/** Look up a cached clip by id, or undefined if not in the cache. */
|
||||
export function readCachedSfx(projectDir: string, id: string): CachedSfx | undefined {
|
||||
try {
|
||||
const all = JSON.parse(readFileSync(cachePath(projectDir), "utf-8")) as Record<
|
||||
string,
|
||||
CachedSfx
|
||||
>;
|
||||
return all[id];
|
||||
} catch {
|
||||
return undefined;
|
||||
}
|
||||
}
|
||||
@@ -68,6 +68,7 @@ const GROUPS: Group[] = [
|
||||
"Transcribe audio/video to word-level timestamps, or import an existing transcript",
|
||||
],
|
||||
["tts", "Generate speech audio from text using a local AI model (Kokoro-82M)"],
|
||||
["sfx", "Search and download sound effects from the HeyGen catalog"],
|
||||
["remove-background", "Remove background from a video or image to produce transparent media"],
|
||||
],
|
||||
},
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
---
|
||||
name: hyperframes
|
||||
description: Create video compositions, animations, title cards, overlays, captions, voiceovers, audio-reactive visuals, and scene transitions in HyperFrames HTML. Use when asked to build any HTML-based video content, add captions or subtitles synced to audio, generate text-to-speech narration, create audio-reactive animation (beat sync, glow, pulse driven by music), add animated text highlighting (marker sweeps, hand-drawn circles, burst lines, scribble, sketchout), or add transitions between scenes (crossfades, wipes, reveals, shader transitions). Covers composition authoring, timing, media, and the full video production workflow. For dev-loop CLI commands (init, lint, inspect, preview, render) see the hyperframes-cli skill; for asset preprocessing commands (tts, transcribe, remove-background) see the hyperframes-media skill.
|
||||
description: Create video compositions, animations, title cards, overlays, captions, voiceovers, audio-reactive visuals, sound effects, and scene transitions in HyperFrames HTML. Use when asked to build any HTML-based video content, add captions or subtitles synced to audio, generate text-to-speech narration, create audio-reactive animation (beat sync, glow, pulse driven by music), add animated text highlighting (marker sweeps, hand-drawn circles, burst lines, scribble, sketchout), add sound effects (whooshes, impacts, risers, UI taps) searched from the HeyGen catalog, or add transitions between scenes (crossfades, wipes, reveals, shader transitions). Covers composition authoring, timing, media, and the full video production workflow. For dev-loop CLI commands (init, lint, inspect, preview, render) see the hyperframes-cli skill; for asset preprocessing commands (tts, transcribe, remove-background) see the hyperframes-media skill.
|
||||
---
|
||||
|
||||
# HyperFrames
|
||||
@@ -468,6 +468,7 @@ Skip on small edits (fixing a color, adjusting one duration). Run on new composi
|
||||
|
||||
- **[references/captions.md](references/captions.md)** — Captions, subtitles, lyrics, karaoke synced to audio. Tone-adaptive style detection, per-word styling, text overflow prevention, caption exit guarantees, word grouping. Read when adding any text synced to audio timing.
|
||||
- **[references/audio-reactive.md](references/audio-reactive.md)** — Audio-reactive animation: map frequency bands and amplitude to GSAP properties. Read when visuals should respond to music, voice, or sound.
|
||||
- **[references/sound-effects.md](references/sound-effects.md)** — Sound effects: search the HeyGen catalog, download a clip, embed as an audio element. When to punctuate, choosing by function + character, the volume hierarchy, and trigger timing. Read when adding SFX (whooshes, impacts, risers, UI taps) to moments.
|
||||
- **[references/css-patterns.md](references/css-patterns.md)** — CSS+GSAP marker highlighting: highlight, circle, burst, scribble, sketchout. Deterministic, fully seekable. Read when adding visual emphasis to text.
|
||||
- **[references/video-composition.md](references/video-composition.md)** — Video-medium rules: density, color presence, scale, frame composition, the design spec as brand not layout. **Always read** — these override web instincts.
|
||||
- **[references/beat-direction.md](references/beat-direction.md)** — Beat planning: concept, mood, choreography verbs, rhythm templates, transition decisions, depth layers. **Always read for multi-scene compositions.**
|
||||
|
||||
@@ -142,7 +142,7 @@ What's in foreground, midground, and background. Every beat should have at least
|
||||
|
||||
### SFX cues
|
||||
|
||||
What sounds at what moment:
|
||||
What sounds at what moment (see [sound-effects.md](sound-effects.md) to search the catalog, choose by function + character, and time them):
|
||||
|
||||
- "On the capture pulse — a soft, warm analog shutter click."
|
||||
- "Left side carries a faint low drone. On fold: drone cuts. Silence. Then a single clean chime."
|
||||
|
||||
@@ -0,0 +1,183 @@
|
||||
# Sound Effects
|
||||
|
||||
Sound effects are searched from HeyGen's catalog by meaning, downloaded locally, and embedded as ordinary audio clips. What a sound _does_ depends on its shape in time — some punctuate a single frame, some build across a stretch, some run quietly under a whole scene — and the real craft is combining them: sequencing, layering, and chaining clips into something that feels designed rather than dropped in.
|
||||
|
||||
## The five things a sound can do
|
||||
|
||||
A clip's **duration tells you its job.** Don't treat them all as one-frame stingers.
|
||||
|
||||
| Mode | What it needs | Examples | Typical len | How it's placed |
|
||||
| --------- | ----------------------------------- | --------------------------------------------------------------- | ----------- | --------------------------------------- |
|
||||
| **Hit** | a beat (one frame) | whoosh, impact, pop, ui click, bell, glitch, notification, coin | 0.1–2s | `data-start` = the exact event frame |
|
||||
| **Build** | a span _into_ a beat | riser (ascends), downlifter (descends) | 3–12s | so it resolves on the beat (see Timing) |
|
||||
| **Bed** | a scene + a mood — **not** an event | ambience, drone, wind, rain, sci-fi hum, crowd roar | 8–20s | runs _under_ the whole scene, very low |
|
||||
| **Sting** | a reveal / lock moment | logo reveal, suspense sting, brand intro | 1.5–8s | `data-start` = the reveal frame |
|
||||
| **Meme** | a comedic beat | vine boom, record scratch, air horn, sad trombone, "sus", boing | 0.4–3s | on the comedic moment — sparingly |
|
||||
|
||||
The key thing: **not every sound is tied to an event.** A _bed_ needs a scene with a mood, not a beat — it's a texture you lay underneath. Hits, builds, stings, and memes are punctuation and _do_ land on beats.
|
||||
|
||||
## Two layers: a bed underneath, punctuation on top
|
||||
|
||||
Think of a clip's sound as (at most) two layers:
|
||||
|
||||
1. **A bed (optional, continuous).** If a scene has an _environment or mood_ worth establishing — an outdoor shot, a tense reveal, a futuristic UI, a stadium moment — lay **one** bed under it at very low volume. It isn't tied to any frame; it sets the floor and gets out of the way. Most plain talking-head / UGC clips don't need one; atmospheric ones come alive with one. Never stack two beds — they muddy each other.
|
||||
|
||||
2. **Punctuation (on beats).** On top of the bed — or with no bed at all — punctuate the real beats with hits, builds, stings, or the occasional meme.
|
||||
|
||||
## Punctuation follows beats, not motion
|
||||
|
||||
This rule governs the _punctuation_ layer (hits / builds / stings / memes), not beds.
|
||||
|
||||
A **beat** is a moment the viewer is meant to _register_ — the edit snaps, lands, or turns: a hard cut, a key element popping in, a number hitting, a build peaking, a CTA, a comedic turn. These earn a punctuation sound.
|
||||
|
||||
**Motion is not a beat.** A card drifting across the frame, a slow zoom, a parallax pan, a looping idle animation, decorative movement — these carry themselves visually. A whoosh on every one of them turns polish into noise. When unsure, ask: _would an editor cut, flash, or snap something here?_ If not, no hit.
|
||||
|
||||
The rule cuts both ways:
|
||||
|
||||
- **No beat left silent.** A reveal montage is a _series_ of beats — skipping SFX there entirely is a miss, not restraint.
|
||||
- **No hit on mere motion.** Don't punctuate drifts, ambient loops, or decorative animation. If a scene needs _something_ under that motion, that's a job for a quiet bed, not a hit.
|
||||
|
||||
## Same beat, same sound (the motif rule)
|
||||
|
||||
Repeated beats of the **same kind share one sound.** Six cards entering one after another are six instances of _one_ beat → _one_ whoosh, reused: embed the same file in multiple `<audio>` tags with different `data-start` values. Giving each repeated beat a _different_ sound is the opposite mistake from skipping them — it reads as busy and amateur and pulls attention to the audio. Reserve a **new** sound only for a **new kind** of beat: the final logo lock, the CTA, the one big impact. Aim for breadth of **function**, not a pile of variants.
|
||||
|
||||
## Combining sounds
|
||||
|
||||
The catalog is a kit, not a jukebox — the interesting results come from putting clips _together_. You have four knobs (`data-start`, `data-duration`, `data-volume`, `data-media-start`) and the freedom to use one file many times. Experiment whenever you see an opportunity; the patterns below are starting points, not limits. (Pro sound design is exactly this — stacking and sequencing layers, not dropping single one-shots.)
|
||||
|
||||
- **Sequence — turn one hit into a rhythm.** A "ta-ta-ta-ta" burst is _one_ short impact reused: five `<audio>` tags pointing at the same file, `data-start`s on each rhythmic frame, and one of them louder (`data-volume`) to accent the downbeat. Use `data-media-start` to trim the attack so each repeat is tight.
|
||||
- **Layer — stack clips at the same frame for weight.** Fire two or three clips at the _same_ `data-start` to thicken one moment: an impact + a sub-drop for a heavier hit, a whoosh + a click for a snappier transition. Keep the total under control with `data-volume` so the stack doesn't clip.
|
||||
- **Chain — connect a build to its payoff.** A riser into an impact/bass-drop: the drop fires at the riser's **actual peak**, which is often _before_ the file ends (an 8s riser may peak at 6.0s). Set the drop's `data-start` to the real peak time, not `riser_start + file_duration`. They can even overlap — the tail of the riser under the head of the drop reads as one continuous gesture. (Use `sfx inspect` / the `add` output to find the peak — see below.)
|
||||
- **Walk — fake motion from foley.** Two footstep variants alternated at a steady cadence (step every ~0.4–0.5s), each step's `data-volume` nudged slightly so it doesn't sound machine-stamped, makes a walk cycle. (The engine has no stereo pan, so cadence + level variation — not L/R placement — is what sells it.)
|
||||
|
||||
## Knowing what a clip does (you can't hear, but you can measure)
|
||||
|
||||
You can't listen to a clip, so don't _guess_ where a riser peaks or how loud one sound is next to another. **`sfx add` prints an analysis after download** (and `sfx inspect <id>` re-runs it on demand). Two things cover almost everything, with the clip's catalog `description` filling in _character_:
|
||||
|
||||
**1. Numbers — loudness and timing.** Integrated loudness (LUFS), true peak, **peak time** (loudest moment), and **onset / tail** (first/last audible moment):
|
||||
|
||||
- **peak time** → where to fire the next clip in a chain (the drop after a riser).
|
||||
- **onset** → set `data-media-start` to skip dead air at the head of a slow-attack clip; **tail** → the clip's real end.
|
||||
- **loudness (LUFS)** → set `data-volume` by _measured_ level, not vibes: a clip several LU louder than another should sit a notch lower to match.
|
||||
|
||||
**2. The shape — a loudness sparkline** (`shape ▁▂▅█▇▃▁`). This is the _energy contour over time_ — a build ramps up, a drop falls off a cliff, a break dips to silence mid-clip. It tells you how the **level** moves (which is what `data-volume` and chaining care about).
|
||||
|
||||
For _character_ — bright tick vs. dull thud, the feel of the sound — read the clip's `description` / `name` from the search result; that's what it's for, and it's already text.
|
||||
|
||||
### Reading the numbers correctly
|
||||
|
||||
All times in the analysis are **clip-relative** (0 = the clip's own start), _not_ positions on the composition timeline. Work in this order:
|
||||
|
||||
1. **Trim dead air first.** A catalog clip can carry silence padding — a "5s riser" may be 2s silence + 5s rise + 1s silence — and if you place the raw file it fires late and wastes its tail. Set `data-media-start = onset` and `data-duration = tail − onset` so only the real content plays. (`sfx add`/`inspect` print this as a ready `trim` line.) After trimming, the audible sound begins **exactly at `data-start`**.
|
||||
2. **Anchor to the beat.** A **hit** (click, impact, whoosh) aligns by its attack — now at the start — so `data-start = beat`. A **build** (riser) aligns by its peak, which sits `peak − onset` into the trimmed clip, so `data-start = beat − (peak − onset)`. Example: a riser with onset 2.0s, peak 6.0s landing on a beat at 10.0s → `data-media-start="2.0"`, `data-start="6.0"` (= 10.0 − 4.0); fire the drop at 10.0s. They overlap — that's the chain.
|
||||
3. **Level** — the catalog is loudness-normalized, so the volume hierarchy is your baseline; you usually don't touch it. Use measured LUFS only to _nudge_: if two layered clips clash, drop `data-volume` on the one with the higher (louder, less-negative) LUFS.
|
||||
4. **Shape** — the sparkline spans the whole clip; block `i` sits at `t ≈ (i + 0.5) / length × duration`. Read it for _where_ the energy builds, drops, or breaks — then place your trigger there.
|
||||
|
||||
(There's no need to _cut_ the audio into a new file — `data-media-start` + `data-duration` trim at render time, losslessly and reversibly. Only mid-clip gaps, which a single window can't remove, would call for a different clip.)
|
||||
|
||||
## How many — count beats, not animations
|
||||
|
||||
A typical 15–30s social clip has roughly **3–7 real beats**, and motifs mean even fewer _distinct_ sounds. Beds are separate — about **one per scene**, not in that budget. If your hit count is climbing toward your _animation_ count, you're scoring motion: stop. Contrast is the real tool — a quiet stretch is what makes the next hit land. "Silence is a tool" means _let beats breathe_, not _prefer none_.
|
||||
|
||||
## Get a sound: search, add, embed
|
||||
|
||||
The catalog is semantic — search by what the sound _does_ and how it _feels_, not by a filename. But **don't reflexively search the same "whoosh" every time** — the catalog has ~180 clips across ~22 families (impacts, foley, whooshes, **memes**, stingers/logos, bells, UI, **ambiences**, **downlifters**, risers, glitches, …). Most agents never reach past the obvious. So **start by browsing the families**, then search within the right one.
|
||||
|
||||
```bash
|
||||
# 0. See what's actually in the catalog — families, counts, examples (start here)
|
||||
hyperframes sfx list
|
||||
hyperframes sfx list --json # for agents
|
||||
|
||||
# 1. Search by description (function + character), within the right family
|
||||
hyperframes sfx search "whoosh for a scene change"
|
||||
hyperframes sfx search "low tense ambience drone bed" # a bed, not a hit
|
||||
hyperframes sfx search "sad trombone fail meme" --json # for agents
|
||||
|
||||
# 2. Download the chosen clip into the project (assets/sfx/) — prints an analysis
|
||||
hyperframes sfx add <id>
|
||||
|
||||
# 3. (optional) Re-print the analysis for a clip you already have
|
||||
hyperframes sfx inspect <id>
|
||||
```
|
||||
|
||||
Search hits return `name`, `description`, `duration`, and `score`. Use the **duration to tell the mode apart** — a 0.5s result is a hit, a 10s result is a bed. Always `add` (download) and embed the **local file**; never embed the search result's `audio_url` (it's a presigned URL that expires before render):
|
||||
|
||||
```html
|
||||
<audio src="assets/sfx/<id>.mp3" data-start="3.2" data-duration="1.5" data-volume="0.3"></audio>
|
||||
```
|
||||
|
||||
### Access (free, but needs a key)
|
||||
|
||||
Sound effects come from HeyGen's **free** catalog, which needs a HeyGen API key (same credential chain as `tts` — `HEYGEN_API_KEY` or `hyperframes auth login`). `sfx list` and `sfx inspect` work offline; `sfx search` and `sfx add` need the key.
|
||||
|
||||
**If the key isn't set, don't silently skip SFX — ask the user**, the way VO asks for a voice key:
|
||||
|
||||
> "I can add sound effects from HeyGen's free catalog — it needs a HeyGen API key. Paste one here, or add `HEYGEN_API_KEY=your-key` to a `.env` file in the project root (get a free key at https://app.heygen.com/developers/api). Or say _skip_ and I'll build the video without SFX."
|
||||
|
||||
`sfx search`/`add` print this same guidance if the key is missing, so a missing key is a clear prompt, never a crash.
|
||||
|
||||
## Choosing the right one
|
||||
|
||||
Search with **function + character**, then read the returned `name`, `description`, and `score`:
|
||||
|
||||
- "whoosh for a scene change" surfaces transition whooshes; "deep cinematic impact" vs "soft ui tap" — character matters as much as type.
|
||||
- Match **duration to the mode**: a short swipe for a quick cut (hit), a longer riser for a build, a long drone for a scene bed. Use the returned `duration` as the clip's `data-duration`.
|
||||
|
||||
## The audio contract (what the engine honors)
|
||||
|
||||
SFX are plain audio clips — no special engine handling. The mixer reads exactly:
|
||||
|
||||
- **`data-start`** — when the clip fires (seconds): the trigger frame (or the scene start, for a bed).
|
||||
- **`data-duration`** — how long it plays; use the catalog duration. (`data-end` also works; duration is end minus start.)
|
||||
- **`data-volume`** — 0 to 1. This is the **only** knob that sets level under voiceover. There is **no pan** — volume and timing are the only audio controls.
|
||||
- **`data-media-start`** — trim into the source file (skip the first N seconds), for clips with a slow attack.
|
||||
|
||||
`data-track-index` does **not** affect audio mixing — every clip mixes together regardless of its index. It only organizes the visual timeline. So `data-volume`, not the track index, is what keeps SFX under the VO.
|
||||
|
||||
## Volume hierarchy
|
||||
|
||||
Layer levels so nothing fights the voice:
|
||||
|
||||
| Layer | data-volume |
|
||||
| -------------------- | ----------- |
|
||||
| Narration / VO | 1.0 |
|
||||
| Background music | 0.4 – 0.6 |
|
||||
| Sound effects (hits) | 0.2 – 0.35 |
|
||||
| Ambient bed | 0.08 – 0.2 |
|
||||
|
||||
A bed sits at the very bottom — it should be _felt_, not heard. A whoosh at 1.0 over narration is jarring; at 0.3 it punctuates without masking the words. When two SFX should match in level, compare their measured LUFS (from `sfx add`/`inspect`) rather than assuming equal `data-volume` sounds equal.
|
||||
|
||||
## Timing wisdom
|
||||
|
||||
- **Hits peak at the start** — set `data-start` to the exact event frame.
|
||||
- **Risers peak at the end of their _rise_, not their file** — fire the payoff at the measured peak time (from `inspect`), and set a riser's own `data-start` to (climax minus rise length) so the swell lands on the hit. **Downlifters** start _at_ the beat and fall away after it.
|
||||
- **Beds span the scene** — `data-start` at the scene's first frame, `data-duration` to cover it; extend or repeat the clip to fill a long scene.
|
||||
- **Sync to beats** — align punctuation to the same frames your motion lands on (see [beat-direction.md](beat-direction.md)). A whoosh on a cut, with the visual cut on the same frame, reads as one event.
|
||||
- **Anticipation** — a few frames early often feels tighter than dead-on.
|
||||
|
||||
## Worked examples
|
||||
|
||||
**A reveal composition** — six cards enter from different angles, then the logo locks in:
|
||||
|
||||
- The six entrances are **one beat class** → one whoosh, reused (`sfx add` once, six `<audio>` tags at the same file, each `data-start` on that card's entrance frame, small `data-volume` variation). _Not_ six different whooshes.
|
||||
- The logo lock is a **different beat** → one sting or impact on the lock frame.
|
||||
- The drift and parallax between entrances → nothing. That's motion.
|
||||
- Result: **two distinct sounds, seven hits** — deliberate, not a soundboard.
|
||||
|
||||
**A build into a drop** — a stat sweeps up, then slams onto screen:
|
||||
|
||||
- Add a riser and an impact. Run `sfx inspect` on the riser → say it peaks at 6.0s of its 8s file.
|
||||
- Riser `data-start` so 6.0s-in lands on the slam frame; impact `data-start` on that same frame (overlap the riser's tail under it).
|
||||
- Result: one continuous build-and-hit gesture from two clips.
|
||||
|
||||
**An atmospheric scene** — a tense product teaser over a dark background:
|
||||
|
||||
- Lay **one bed** (a low tension drone, ~10s) under the whole scene at `data-volume="0.12"`. No beat needed — it sets mood.
|
||||
- Punctuate only the real beats on top: a riser into the product reveal, one impact on the reveal frame.
|
||||
- Result: a continuous floor + two punctuation sounds.
|
||||
|
||||
## Constraints
|
||||
|
||||
- Embed the **local** downloaded file, never the presigned `audio_url` (it expires).
|
||||
- One catalog clip is one file; re-use it across moments by adding multiple audio tags pointing at it with different `data-start` values — this is how the motif rule, a rhythmic sequence, and a repeated bed are all implemented.
|
||||
- Deterministic like everything else — no runtime fetching at render; the file is on disk via `sfx add`.
|
||||
@@ -1,35 +0,0 @@
|
||||
# SFX Credits
|
||||
|
||||
All sound effects in this directory are sourced from [Pixabay](https://pixabay.com/sound-effects/) and used under the [Pixabay Content License](https://pixabay.com/service/license-summary/).
|
||||
|
||||
The Pixabay license allows free use for commercial and non-commercial purposes without attribution, but attribution is appreciated and given here for transparency.
|
||||
|
||||
## Files
|
||||
|
||||
The following `.mp3` files are bundled with this skill:
|
||||
|
||||
- `chime.mp3`
|
||||
- `click.mp3` / `click-soft.mp3`
|
||||
- `error.mp3`
|
||||
- `glitch-1.mp3` / `glitch-2.mp3` / `glitch-3.mp3`
|
||||
- `impact-bass-1.mp3` / `impact-bass-2.mp3`
|
||||
- `key-press.mp3`
|
||||
- `notification.mp3`
|
||||
- `ping.mp3`
|
||||
- `pop.mp3`
|
||||
- `riser.mp3`
|
||||
- `sparkle.mp3`
|
||||
- `typing.mp3`
|
||||
- `whoosh.mp3` / `whoosh-short.mp3` / `whoosh-cinematic.mp3`
|
||||
|
||||
See `manifest.json` for per-file metadata (duration, energy character, recommended use).
|
||||
|
||||
## License
|
||||
|
||||
All files are distributed under the [Pixabay Content License](https://pixabay.com/service/license-summary/), which permits:
|
||||
|
||||
- Commercial and non-commercial use
|
||||
- Modification and remixing
|
||||
- Redistribution as part of derivative works (such as videos rendered with HyperFrames)
|
||||
|
||||
without any attribution requirement.
|
||||
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
@@ -1,97 +0,0 @@
|
||||
{
|
||||
"chime": {
|
||||
"file": "chime.mp3",
|
||||
"duration": 2.5,
|
||||
"description": "Soft melodic chime — gentle positive beat: success/confirmation or a lighthearted transition. Sync to the visual moment."
|
||||
},
|
||||
"click-soft": {
|
||||
"file": "click-soft.mp3",
|
||||
"duration": 0.37,
|
||||
"description": "Quiet short click — low-key UI tap / soft selection. Short accent, sync exactly to the on-screen action."
|
||||
},
|
||||
"click": {
|
||||
"file": "click.mp3",
|
||||
"duration": 0.37,
|
||||
"description": "Crisp UI click — button press, toggle, selection. Short accent, sync exactly to the on-screen action."
|
||||
},
|
||||
"error": {
|
||||
"file": "error.mp3",
|
||||
"duration": 1.62,
|
||||
"description": "Negative / error tone — failure state, a 'wrong' beat, or a glitchy interruption."
|
||||
},
|
||||
"glitch-1": {
|
||||
"file": "glitch-1.mp3",
|
||||
"duration": 2.64,
|
||||
"description": "Punchy digital glitch — hard-cut accent or sudden reveal. Trigger on the hit; let the decay bleed into the next shot (J-cut)."
|
||||
},
|
||||
"glitch-2": {
|
||||
"file": "glitch-2.mp3",
|
||||
"duration": 3.5,
|
||||
"description": "Harsh, longer glitch — chaotic / jarring transition or a distorted reveal."
|
||||
},
|
||||
"glitch-3": {
|
||||
"file": "glitch-3.mp3",
|
||||
"duration": 3.1,
|
||||
"description": "Low-key glitch texture — subtle digital shift, minimal transition that sits under other audio."
|
||||
},
|
||||
"impact-bass-1": {
|
||||
"file": "impact-bass-1.mp3",
|
||||
"duration": 2.12,
|
||||
"description": "Bass impact hit — logo/hero snap, headline slam. Trigger on the visual landing; decay carries into the next shot (J-cut)."
|
||||
},
|
||||
"impact-bass-2": {
|
||||
"file": "impact-bass-2.mp3",
|
||||
"duration": 2.59,
|
||||
"description": "Bass impact with a short swell — brief anticipation then a deep hit. Place so the peak lands on the reveal."
|
||||
},
|
||||
"key-press": {
|
||||
"file": "key-press.mp3",
|
||||
"duration": 0.4,
|
||||
"description": "Single key press — one keystroke / terminal-input beat. Short accent, sync to the typed character."
|
||||
},
|
||||
"notification": {
|
||||
"file": "notification.mp3",
|
||||
"duration": 2.46,
|
||||
"description": "Notification chime — alert, message-in, toast/badge appears. Sync to the element entering."
|
||||
},
|
||||
"ping": {
|
||||
"file": "ping.mp3",
|
||||
"duration": 1.32,
|
||||
"description": "Sharp electronic ping — punchy accent on a key reveal or data point. Sync to the beat."
|
||||
},
|
||||
"pop": {
|
||||
"file": "pop.mp3",
|
||||
"duration": 0.72,
|
||||
"description": "Quick pop — element appear/spawn, chip/tag/badge in. Small precise accent, sync to the pop-in."
|
||||
},
|
||||
"riser": {
|
||||
"file": "riser.mp3",
|
||||
"duration": 10.03,
|
||||
"description": "Long cinematic riser (~10s build, peak at the end). Trigger at (climax_time − 10.03s) so it crests exactly on the reveal."
|
||||
},
|
||||
"sparkle": {
|
||||
"file": "sparkle.mp3",
|
||||
"duration": 1.8,
|
||||
"description": "Bright sparkle / shimmer — magical reveal or 'shine' highlight on a hero element. Sync to the highlight."
|
||||
},
|
||||
"typing": {
|
||||
"file": "typing.mp3",
|
||||
"duration": 1.5,
|
||||
"description": "Typing burst (~1.5s of keys) — keyboard / code typing reveal, text-being-typed beat. Start as the text begins typing."
|
||||
},
|
||||
"whoosh-cinematic": {
|
||||
"file": "whoosh-cinematic.mp3",
|
||||
"duration": 5.54,
|
||||
"description": "Cinematic whoosh build (~5.5s) — sweeping scene transition. Align so the swell peaks on the cut."
|
||||
},
|
||||
"whoosh-short": {
|
||||
"file": "whoosh-short.mp3",
|
||||
"duration": 0.57,
|
||||
"description": "Short whoosh — quick swipe/slide accent, fast element move, snappy transition. Sync to the motion."
|
||||
},
|
||||
"whoosh": {
|
||||
"file": "whoosh.mp3",
|
||||
"duration": 0.57,
|
||||
"description": "Punchy whoosh/impact — fast reveal or hard transition accent. Sync to the motion."
|
||||
}
|
||||
}
|
||||
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
@@ -152,24 +152,20 @@ When planning beats, decide which ones deserve an HTML-in-Canvas treatment vs. a
|
||||
|
||||
### SFX assignment — happens here, not in Step 5
|
||||
|
||||
**Before writing beats,** read `skills/website-to-hyperframes/assets/sfx/manifest.json` (or your local copy at `sfx/manifest.json` if already copied to the project). Each entry has a filename, duration in seconds, and description. Assign **specific SFX files** to exact moments in the storyboard. Step 5 implements what you specify here — it makes no SFX decisions.
|
||||
SFX come from HeyGen's global catalog via the `hyperframes sfx` CLI — **not** a bundled file set. The full model (when to use them, the five families, the volume hierarchy, the trim/anchor recipe) lives in [`../../hyperframes/references/sound-effects.md`](../../hyperframes/references/sound-effects.md); read it. This step only decides _which moments get sound and what kind_ — Step 5 searches the catalog and wires the clips.
|
||||
|
||||
Per beat, specify SFX like:
|
||||
**Browse first, then assign by function — not by filename.** Run `hyperframes sfx list` to see what families exist (impacts, whooshes, risers, memes, stingers, ambiences, …). For each beat that earns a sound, specify it by _what it does + how it feels_ (the catalog is searched by meaning), the moment, and a volume:
|
||||
|
||||
- `sfx/impact-bass-1.mp3` at `0.2s`, volume `0.35` — on the hero image snapping into frame
|
||||
- `sfx/chime.mp3` at `3.8s`, volume `0.5` — on the logo appearing
|
||||
- a punchy transition whoosh at `0.2s`, volume `0.3` — on the hero image snapping in
|
||||
- a soft success chime at `3.8s`, volume `0.5` — on the logo appearing
|
||||
|
||||
**Less is more.** Most beats need zero SFX. One SFX per beat is typical; multiple only if the beat has genuinely distinct punctuation moments. Never place SFX on shader transitions directly — shader transitions are already an audio-visual event.
|
||||
**Less is more.** Most beats need zero SFX; one per beat is typical (see the global doc's "count beats, not animations"). Never place SFX on shader transitions — they're already an audio-visual event.
|
||||
|
||||
**How to place each sound type** (industry-standard rules):
|
||||
**Placement, trimming, and chains are all in the global doc — don't restate per-file timing here.** Step 5 follows it: `sfx add` measures each clip (peak, onset/tail, loudness), then trims dead air and anchors hits/risers precisely. (This replaces the old "peak at start/end, never trim" rules — catalog clips aren't a fixed hand-tuned set, so trimming to the _measured_ onset/tail is how you avoid late or cut-off sounds.)
|
||||
|
||||
- **Impact/hit sounds** (`impact-bass-1`, `ping`, `pop`, `glitch-*`): peak is at the start of the clip. Trigger exactly at the visual moment. Let the decay tail bleed into the next scene — this is normal, called a J-Cut, and sounds professional. `data-duration` = full manifest duration, never trimmed.
|
||||
- **Riser/build-up sounds** (`riser`, `whoosh-cinematic`): peak is at the END of the clip. To make the peak land on a climax moment (a transition, a reveal), trigger at `climax_time - sfx_duration`. For `riser.mp3` (10.03s) peaking at a t=20s transition: trigger at t=9.97s.
|
||||
- **Short accent sounds** (`click`, `click-soft`, `chime`, `sparkle`, `ping`): trigger at the exact visual punctuation moment. Duration is short, no tail concern.
|
||||
**Volume under narration:** HyperFrames has no auto-ducking — keep SFX at 0.2–0.3 under VO. Note the intended volume per entry so Step 5 wires it.
|
||||
|
||||
**Volume when SFX overlaps narration:** HyperFrames has no automatic audio ducking. If an SFX plays under spoken narration, set its volume to 0.2–0.3 max, not 0.5+. Specify this in the storyboard entry so Step 5 wires it correctly.
|
||||
|
||||
**data-duration rule** (for Step 5 to implement): always equals the manifest's duration field exactly. Never set it shorter to "fit" the remaining beat time — truncating an impact mid-decay is the exact problem causing the cut-off sounds in v2 videos.
|
||||
**Access:** the catalog is free but needs a HeyGen API key. If it isn't set when Step 5 runs `sfx`, ask the user for one (free at https://app.heygen.com/developers/api) or build without SFX — never silently drop them.
|
||||
|
||||
### Architecture Constraint: Each Beat is an Independent Composition
|
||||
|
||||
|
||||
@@ -15,14 +15,18 @@ Load the `hyperframes` skill — it has the rules for data attributes, timeline
|
||||
|
||||
---
|
||||
|
||||
## 1. Copy SFX to project
|
||||
## 1. Get SFX from the catalog
|
||||
|
||||
SFX come from HeyGen's free catalog via the `hyperframes sfx` CLI — see [`../../hyperframes/references/sound-effects.md`](../../hyperframes/references/sound-effects.md). For each SFX the storyboard (Step 3) assigned, search by its description and download it into the project:
|
||||
|
||||
```bash
|
||||
cp -r skills/website-to-hyperframes/assets/sfx/ <project-dir>/sfx/
|
||||
# If skill is installed elsewhere:
|
||||
find . -path "*/website-to-hyperframes/assets/sfx" -exec cp -r {} <project-dir>/sfx/ \;
|
||||
hyperframes sfx list # the families, to orient (optional)
|
||||
hyperframes sfx search "punchy transition whoosh" --json
|
||||
hyperframes sfx add <id> # → assets/sfx/<id>.mp3, prints loudness/peak/onset/tail
|
||||
```
|
||||
|
||||
`sfx add` writes the clip to `assets/sfx/` and prints its analysis (peak time, onset/tail, loudness) — use it to trim and anchor per the global doc. If `HEYGEN_API_KEY` isn't set, `sfx` prints how to get a free key; ask the user for one or build without SFX — don't silently drop them.
|
||||
|
||||
## Known landmines — read before writing each beat composition
|
||||
|
||||
The rules below came out of cell-A through cell-H website-capture builds across heygen.com, raycast.com, huly.io, and airbnb.com in 2026-04 → 2026-05. Beat compositions that lint clean still ship broken without these — text that mask-reveals to nothing, sub-agents that silently overwrite each other's HTML. The linter cannot catch most of these; the author must.
|
||||
@@ -52,6 +56,7 @@ Every beat composition lives inside `<template id="<beat-id>-template">…</temp
|
||||
</template>
|
||||
```
|
||||
|
||||
|
||||
```html
|
||||
<!-- ❌ BAD: data-duration on the host div in index.html (legacy cell-A pattern) -->
|
||||
<!-- The assembler emits this host div; workers must NOT write it themselves. -->
|
||||
|
||||
Reference in New Issue
Block a user