diff --git a/skills-manifest.json b/skills-manifest.json index f0ca9ef95..d99337698 100644 --- a/skills-manifest.json +++ b/skills-manifest.json @@ -46,8 +46,8 @@ "files": 10 }, "media-use": { - "hash": "969e6ed350111d63", - "files": 87 + "hash": "db5787c5ff2bd852", + "files": 97 }, "motion-graphics": { "hash": "73771b2d1300236d", diff --git a/skills/media-use/scripts/audio-duck.mjs b/skills/media-use/scripts/audio-duck.mjs new file mode 100644 index 000000000..4fd1f342d --- /dev/null +++ b/skills/media-use/scripts/audio-duck.mjs @@ -0,0 +1,121 @@ +#!/usr/bin/env node + +import { readFileSync } from "node:fs"; +import { resolve } from "node:path"; +import { parseArgs } from "node:util"; +import { duckKeyframes, speechSpans } from "./lib/duck.mjs"; +import { track } from "./lib/telemetry.mjs"; + +const { values: args } = parseArgs({ + options: { + meta: { type: "string" }, + target: { type: "string" }, + duck: { type: "string", default: "0.25" }, + attack: { type: "string", default: "0.15" }, + release: { type: "string", default: "0.4" }, + "merge-gap": { type: "string", default: "0.6" }, + sequential: { type: "boolean", default: false }, + gap: { type: "string", default: "0" }, + offsets: { type: "string" }, + composition: { type: "string" }, + json: { type: "boolean", default: false }, + help: { type: "boolean", short: "h", default: false }, + }, + strict: true, +}); + +if (args.help) { + console.log(`media-use audio-duck — generate GSAP volume ducking keyframes + +Usage: + node audio-duck.mjs --meta audio_meta.json --target "#bgm" + +Options: + --meta audio_meta.json or JSON word transcript + --target GSAP selector for the background audio element + --duck Duck multiplier (default: 0.25) + --attack Duck-in duration seconds (default: 0.15) + --release Restore duration seconds (default: 0.4) + --merge-gap Bridge speech gaps smaller than this many seconds (default: 0.6) + --sequential Place multi-line meta back to back at composition time + --gap Extra seconds between sequential lines (default: 0) + --offsets Explicit placement, "l1=0,l2=3.4" (voice id = start seconds) + --composition Read target data-volume from this HTML file + --json Output { spans, keyframes } + --help, -h Show this help`); + process.exit(0); +} + +try { + run(); + await track("media_use_duck", { sequential: !!args.sequential }); +} catch (err) { + if (args.json) console.log(JSON.stringify({ ok: false, error: err.message })); + else console.error(`error: ${err.message}`); + process.exit(1); +} + +function run() { + if (!args.meta || !args.target) throw new Error("--meta and --target are required"); + const meta = JSON.parse(readFileSync(resolve(args.meta), "utf8")); + const target = args.target; + const baseVolume = readBaseVolume(args.composition, target); + const offsets = args.offsets + ? Object.fromEntries( + args.offsets.split(",").map((pair) => { + const [id, t] = pair.split("="); + return [id.trim(), Number(t)]; + }), + ) + : undefined; + const spans = speechSpans(meta, { + mergeGap: Number(args["merge-gap"]), + sequential: args.sequential, + gap: Number(args.gap), + offsets, + }); + const keyframes = duckKeyframes(spans, { + duck: Number(args.duck), + attack: Number(args.attack), + release: Number(args.release), + baseVolume, + }); + + if (args.json) { + console.log(JSON.stringify({ spans, keyframes })); + return; + } + + console.log( + `// auto-duck: ${target} under narration (generated; base volume ${fmt(baseVolume)})`, + ); + for (const keyframe of keyframes) { + console.log( + `tl.to(${JSON.stringify(target)}, { volume: ${fmt(keyframe.volume)}, duration: ${fmt( + keyframe.duration, + )} }, ${fmt(keyframe.time)});`, + ); + } +} + +function readBaseVolume(composition, target) { + if (!composition || !target.startsWith("#")) return 1; + const id = target.slice(1); + const html = readFileSync(resolve(composition), "utf8"); + // ponytail: regex is enough here because this only reads one attribute from + // one user-authored composition element, not arbitrary HTML. + const tag = html.match(new RegExp(`<[^>]*\\bid=["']${escapeRegExp(id)}["'][^>]*>`, "i"))?.[0]; + const raw = tag?.match(/\bdata-volume=["']([^"']+)["']/i)?.[1]; + const volume = Number(raw); + return Number.isFinite(volume) ? volume : 1; +} + +function escapeRegExp(value) { + return value.replace(/[.*+?^${}()|[\]\\]/g, "\\$&"); +} + +function fmt(n) { + return Number(n) + .toFixed(3) + .replace(/\.?0+$/, ""); +} diff --git a/skills/media-use/scripts/lib/coverage.test.mjs b/skills/media-use/scripts/lib/coverage.test.mjs new file mode 100644 index 000000000..ab293d8ea --- /dev/null +++ b/skills/media-use/scripts/lib/coverage.test.mjs @@ -0,0 +1,112 @@ +import { strict as assert } from "node:assert"; +import { test } from "node:test"; +import { existsSync } from "node:fs"; +import { join, dirname } from "node:path"; +import { fileURLToPath } from "node:url"; +import { listTypes, getProviders } from "./registry.mjs"; +import { CAPABILITIES, listModels } from "./local-models.mjs"; + +// Capstone: media-use must actually OWN each hyperframes media weakness. This +// test enforces the weakness→owner matrix in SKILL.md so a claim can't rot — if +// a capability's entrypoint disappears, this fails. + +const SKILL = join(dirname(fileURLToPath(import.meta.url)), "..", ".."); + +test("weakness: audio-only → media-use resolves image + icon", () => { + for (const t of ["image", "icon"]) { + assert.ok(getProviders(t).length > 0, `no provider for ${t}`); + } +}); + +test("weakness: no voice/audio gen → media-use exposes voice + the audio engine", () => { + assert.ok(listTypes().includes("voice"), "voice type missing"); + assert.ok(getProviders("voice").length > 0, "no enabled voice provider (Bin approved)"); + assert.ok(existsSync(join(SKILL, "audio", "scripts", "audio.mjs")), "audio engine missing"); +}); + +test("weakness: scattered audio engine → consolidated under media-use (hyperframes-media gone)", () => { + assert.ok(existsSync(join(SKILL, "audio", "scripts", "lib", "tts.mjs")), "tts engine missing"); + assert.ok( + existsSync(join(SKILL, "audio", "assets", "sfx", "manifest.json")), + "bundled SFX missing", + ); +}); + +test("weakness: no media-ops → ops guidance reference exists", () => { + assert.ok(existsSync(join(SKILL, "references", "operations.md")), "operations.md missing"); +}); + +test("weakness: no transcript-driven cutting → cut compiler entrypoints exist", async () => { + assert.ok(existsSync(join(SKILL, "scripts", "transcript-cut.mjs")), "transcript-cut missing"); + assert.ok(existsSync(join(SKILL, "scripts", "lib", "cutlist.mjs")), "cutlist lib missing"); + const cutlist = await import("./cutlist.mjs"); + assert.equal(typeof cutlist.compileCutList, "function"); +}); + +test("weakness: whisper.cpp is weak → better local ASR (Parakeet) entrypoint exists", async () => { + assert.ok(existsSync(join(SKILL, "scripts", "transcribe.mjs")), "transcribe.mjs missing"); + const pw = await import("./parakeet-words.mjs"); + assert.equal(typeof pw.mergeTokensToWords, "function", "token->word merge missing"); + const lm = await import("./local-models.mjs"); + const asr = lm.listModels("asr"); + const parakeet = asr.find((m) => m.id === "parakeet-mlx"); + assert.ok(parakeet && parakeet.rank === 0, "Parakeet must be the rank-0 preferred ASR"); +}); + +test("weakness: no auto-duck/loudness → duck compiler and recipes exist", async () => { + assert.ok(existsSync(join(SKILL, "scripts", "audio-duck.mjs")), "audio-duck missing"); + assert.ok(existsSync(join(SKILL, "scripts", "lib", "duck.mjs")), "duck lib missing"); + assert.ok(existsSync(join(SKILL, "references", "operations.md")), "operations.md missing"); + const duck = await import("./duck.mjs"); + assert.equal(typeof duck.speechSpans, "function"); + assert.equal(typeof duck.duckKeyframes, "function"); +}); + +test("weakness: no cross-project memory → global cache + ingest entrypoints exist", async () => { + const cache = await import("./cache.mjs"); + assert.equal(typeof cache.cachePut, "function"); + assert.equal(typeof cache.promote, "function"); + assert.equal(typeof cache.globalMediaDir, "function"); + const freeze = await import("./freeze.mjs"); + assert.equal(typeof freeze.isDirectMediaUrl, "function", "ingest URL guard missing"); +}); + +// Wenbo (06-29): heygen free-usage is the default; local models are the opt-out +// fallback ("if user no, then local"). We still assert the fallback table is +// populated so the opt-out path stays real. +test("weakness: weak local defaults → local models exist as the opt-out fallback (tts/asr/upscale)", () => { + for (const cap of ["tts", "asr", "upscale"]) { + assert.ok(CAPABILITIES.includes(cap), `capability ${cap} missing`); + assert.ok(listModels(cap).length > 0, `no local models for ${cap}`); + } +}); + +test("weakness: no image generation → local mflux (RAM-graded) + codex upsell", async () => { + const ps = getProviders("image"); + assert.ok( + ps.some((p) => p.name === "mflux.local" && typeof p.generate === "function"), + "local image gen missing", + ); + assert.ok( + ps.some((p) => p.name === "codex.image_gen" && typeof p.generate === "function"), + "codex image upsell missing", + ); + const lm = await import("./local-models.mjs"); + assert.ok(lm.CAPABILITIES.includes("imagegen"), "imagegen capability missing"); + assert.ok(lm.listModels("imagegen").length >= 3, "imagegen RAM ladder too small"); + assert.equal(typeof lm.describeModelLadder, "function", "agent-facing ladder missing"); +}); + +test("weakness: no video generation → local videogen ladder + heygen avatar upsell", async () => { + const lm = await import("./local-models.mjs"); + assert.ok(lm.CAPABILITIES.includes("videogen"), "videogen capability missing"); + assert.ok(lm.listModels("videogen").length >= 2, "videogen ladder too small"); + const ops = existsSync(join(SKILL, "references", "operations.md")); + assert.ok(ops, "operations.md (avatar-upsell recipe) missing"); +}); + +test("every resolve type has at least one enabled provider", () => { + for (const t of listTypes()) { + assert.ok(getProviders(t).length > 0, `type ${t} has no enabled provider`); + } +}); diff --git a/skills/media-use/scripts/lib/cutlist.mjs b/skills/media-use/scripts/lib/cutlist.mjs new file mode 100644 index 000000000..07d0d3415 --- /dev/null +++ b/skills/media-use/scripts/lib/cutlist.mjs @@ -0,0 +1,184 @@ +import { normalizeWords } from "./words.mjs"; + +const MIN_SEGMENT_SECONDS = 0.2; +const SILENCE_PAD_SECONDS = 0.15; + +export function compileCutList(transcript, opts = {}) { + const words = normalizeWords(transcript); + if (opts.keep != null && hasRemovalSource(opts)) { + throw new Error("--keep is mutually exclusive with removal options"); + } + + if (opts.keep != null) { + const duration = durationFrom(words, opts); + const ranges = parseTimeRanges(opts.keep); + return finalizeKept(duration != null ? clampRanges(ranges, duration) : ranges); + } + + const duration = durationFrom(words, opts); + if (!duration) return []; + + const removals = [ + ...parseTimeRanges(opts.remove), + ...wordIndexRanges(words, opts.removeWords), + ...fillerRanges(words, opts.removeFillers), + ...silenceRanges(words, opts.cutSilence), + ]; + const mergedRemovals = mergeRanges(clampRanges(removals, duration)); + return finalizeKept(invertRanges(mergedRemovals, duration)); +} + +function hasRemovalSource(opts) { + return ( + opts.remove != null || + opts.removeWords != null || + opts.removeFillers != null || + opts.cutSilence != null + ); +} + +function durationFrom(words, opts) { + const explicit = Number(opts.duration ?? opts.totalDuration); + if (Number.isFinite(explicit) && explicit > 0) return explicit; + const last = words.at(-1); + return last && Number.isFinite(last.end) && last.end > 0 ? last.end : null; +} + +function parseTimeRanges(value) { + if (value == null || value === false || value === "") return []; + if (typeof value === "string") { + return value + .split(",") + .map((part) => part.trim()) + .filter(Boolean) + .map(parseRangeString); + } + if (!Array.isArray(value)) throw new Error("range list must be a string or array"); + return value.map((range) => { + if (Array.isArray(range)) return cleanRange(Number(range[0]), Number(range[1])); + return cleanRange(Number(range?.start), Number(range?.end)); + }); +} + +function parseRangeString(value) { + const match = value.match(/^([0-9]*\.?[0-9]+)\s*-\s*([0-9]*\.?[0-9]+)$/); + if (!match) throw new Error(`invalid range: ${value}`); + return cleanRange(Number(match[1]), Number(match[2])); +} + +function cleanRange(start, end) { + if (!Number.isFinite(start) || !Number.isFinite(end)) { + throw new Error("range start/end must be finite numbers"); + } + if (end < start) throw new Error(`range end ${end} is before start ${start}`); + return { start, end }; +} + +function wordIndexRanges(words, value) { + if (value == null || value === false || value === "") return []; + const ranges = typeof value === "string" ? value.split(",") : value; + if (!Array.isArray(ranges)) throw new Error("--remove-words must be a string or array"); + return ranges + .map((range) => (typeof range === "string" ? range.trim() : range)) + .filter(Boolean) + .map((range) => { + const [first, last = first] = + typeof range === "string" ? range.split("-").map((n) => n.trim()) : range; + const startIndex = Number(first); + const endIndex = Number(last); + if (!Number.isInteger(startIndex) || !Number.isInteger(endIndex)) { + throw new Error(`invalid word range: ${range}`); + } + if (startIndex < 0 || endIndex < startIndex || endIndex >= words.length) { + throw new Error(`word range out of bounds: ${range}`); + } + return { start: words[startIndex].start, end: words[endIndex].end }; + }); +} + +function fillerRanges(words, value) { + if (value == null || value === false || value === "") return []; + const fillers = Array.isArray(value) + ? value + : String(value) + .split(",") + .map((s) => s.trim()); + const set = new Set(fillers.filter(Boolean).map(bareToken)); + if (set.size === 0) return []; + // Whisper emits words with attached punctuation and arbitrary case + // ("UM," / "Um."), so compare bare tokens. + return words + .filter((word) => set.has(bareToken(word.text))) + .map((word) => ({ start: word.start, end: word.end })); +} + +function bareToken(text) { + return String(text) + .toLowerCase() + .replace(/^[^\p{L}\p{N}]+|[^\p{L}\p{N}]+$/gu, ""); +} + +function silenceRanges(words, value) { + if (value == null || value === false || value === "") return []; + const threshold = Number(value); + if (!Number.isFinite(threshold) || threshold <= 0) { + throw new Error("--cut-silence must be a positive number"); + } + const ranges = []; + for (let i = 0; i < words.length - 1; i++) { + const current = words[i]; + const next = words[i + 1]; + const gap = next.start - current.end; + if (gap <= threshold) continue; + const start = current.end + SILENCE_PAD_SECONDS; + const end = next.start - SILENCE_PAD_SECONDS; + if (end > start) ranges.push({ start, end }); + } + return ranges; +} + +function clampRanges(ranges, duration) { + return ranges + .map((range) => ({ + start: Math.max(0, Math.min(duration, range.start)), + end: Math.max(0, Math.min(duration, range.end)), + })) + .filter((range) => range.end > range.start); +} + +function mergeRanges(ranges) { + const sorted = ranges + .map((range) => ({ start: round3(range.start), end: round3(range.end) })) + .sort((a, b) => a.start - b.start || a.end - b.end); + const merged = []; + for (const range of sorted) { + const prev = merged.at(-1); + if (prev && range.start <= prev.end) { + prev.end = Math.max(prev.end, range.end); + } else { + merged.push({ ...range }); + } + } + return merged; +} + +function invertRanges(removals, duration) { + const kept = []; + let cursor = 0; + for (const range of removals) { + if (range.start > cursor) kept.push({ start: cursor, end: range.start }); + cursor = Math.max(cursor, range.end); + } + if (cursor < duration) kept.push({ start: cursor, end: duration }); + return kept; +} + +function finalizeKept(ranges) { + return mergeRanges(ranges) + .map((range) => ({ start: round3(range.start), end: round3(range.end) })) + .filter((range) => round3(range.end - range.start) >= MIN_SEGMENT_SECONDS); +} + +function round3(n) { + return Math.round(Number(n) * 1000) / 1000; +} diff --git a/skills/media-use/scripts/lib/cutlist.test.mjs b/skills/media-use/scripts/lib/cutlist.test.mjs new file mode 100644 index 000000000..72e6f2b0d --- /dev/null +++ b/skills/media-use/scripts/lib/cutlist.test.mjs @@ -0,0 +1,148 @@ +import { strict as assert } from "node:assert"; +import { execFileSync } from "node:child_process"; +import { mkdtempSync, writeFileSync, rmSync } from "node:fs"; +import { join, dirname } from "node:path"; +import { tmpdir } from "node:os"; +import { fileURLToPath } from "node:url"; +import { test } from "node:test"; +import { compileCutList } from "./cutlist.mjs"; + +const HERE = dirname(fileURLToPath(import.meta.url)); +const SCRIPT = join(HERE, "..", "transcript-cut.mjs"); + +test("explicit --remove ranges invert to kept segments", () => { + const transcript = [ + word("w0", "alpha", 0, 1), + word("w1", "beta", 1.2, 2), + word("w2", "gamma", 2.2, 5), + ]; + + assert.deepEqual(compileCutList(transcript, { remove: "1-2.5" }), [ + { start: 0, end: 1 }, + { start: 2.5, end: 5 }, + ]); +}); + +test("--remove-words resolves inclusive word-index ranges to time ranges", () => { + const transcript = [ + word("w0", "zero", 0, 0.5), + word("w1", "one", 0.6, 1), + word("w2", "two", 1.1, 1.5), + word("w3", "three", 2, 3), + ]; + + assert.deepEqual(compileCutList(transcript, { removeWords: "1-2" }), [ + { start: 0, end: 0.6 }, + { start: 1.5, end: 3 }, + ]); +}); + +test("--remove-fillers drops case-insensitive matching words", () => { + const transcript = [ + word("w0", "Hello", 0, 0.5), + word("w1", "Um", 0.5, 0.7), + word("w2", "world", 0.8, 1.2), + word("w3", "LIKE", 1.3, 1.5), + word("w4", "done", 1.6, 2), + ]; + + assert.deepEqual(compileCutList(transcript, { removeFillers: "um,like" }), [ + { start: 0, end: 0.5 }, + { start: 0.7, end: 1.3 }, + { start: 1.5, end: 2 }, + ]); +}); + +test("--cut-silence removes only the center of long inter-word gaps", () => { + const transcript = [word("w0", "a", 0, 0.5), word("w1", "b", 2, 2.5), word("w2", "c", 2.7, 3)]; + + assert.deepEqual(compileCutList(transcript, { cutSilence: 0.8 }), [ + { start: 0, end: 0.65 }, + { start: 1.85, end: 3 }, + ]); +}); + +test("overlapping removal sources merge before inversion", () => { + const transcript = [ + word("w0", "start", 0, 0.5), + word("w1", "um", 0.9, 1.1), + word("w2", "middle", 2.5, 2.8), + word("w3", "more", 3.1, 3.4), + word("w4", "end", 5.5, 6), + ]; + + assert.deepEqual( + compileCutList(transcript, { + remove: "1-2.7", + removeWords: "2-3", + removeFillers: "um", + }), + [ + { start: 0, end: 0.9 }, + { start: 3.4, end: 6 }, + ], + ); +}); + +test("kept slivers shorter than 0.2s are dropped", () => { + const transcript = [word("w0", "start", 0, 0.5), word("w1", "end", 2.5, 3)]; + + assert.deepEqual(compileCutList(transcript, { remove: "0.1-2.95" }), []); +}); + +test("--keep is inverse mode and coalesces direct kept ranges", () => { + const transcript = [word("w0", "start", 0, 0.5), word("w1", "end", 4.5, 5)]; + + assert.deepEqual(compileCutList(transcript, { keep: "3-4,1-2,1.5-2.5,4.1-4.2" }), [ + { start: 1, end: 2.5 }, + { start: 3, end: 4 }, + ]); +}); + +test("--plan on a fixture transcript prints the exact segment JSON", () => { + const dir = mkdtempSync(join(tmpdir(), "media-use-cutlist-")); + try { + const transcriptPath = join(dir, "fixture.json"); + writeFileSync( + transcriptPath, + JSON.stringify([ + word("w0", "hello", 0, 0.4), + word("w1", "um", 0.5, 0.65), + word("w2", "there", 0.7, 1), + word("w3", "pause", 2.2, 2.5), + word("w4", "end", 2.7, 3.2), + ]), + ); + + const out = execFileSync( + process.execPath, + [ + SCRIPT, + "--input", + "ignored.mp4", + "--transcript", + transcriptPath, + "--remove", + "0.9-1.2", + "--remove-fillers", + "um", + "--cut-silence", + "0.8", + "--plan", + ], + { encoding: "utf8" }, + ); + + assert.deepEqual(JSON.parse(out), [ + { start: 0, end: 0.5 }, + { start: 0.65, end: 0.9 }, + { start: 2.05, end: 3.2 }, + ]); + } finally { + rmSync(dir, { recursive: true, force: true }); + } +}); + +function word(id, text, start, end) { + return { id, text, start, end }; +} diff --git a/skills/media-use/scripts/lib/duck.mjs b/skills/media-use/scripts/lib/duck.mjs new file mode 100644 index 000000000..5330e8e35 --- /dev/null +++ b/skills/media-use/scripts/lib/duck.mjs @@ -0,0 +1,89 @@ +import { wordListsFromMediaMeta } from "./words.mjs"; + +/** + * Speech spans from word timestamps. + * + * audio_meta.json word times are relative to EACH LINE'S OWN FILE, not to the + * composition. Without placement info, multiple lines would overlap at t=0 and + * merge into one bogus span. Placement options: + * offsets: { [voiceId]: startSeconds } explicit composition placement + * sequential: stack lines back to back (plus `gap` seconds between lines) + * A single word list (bare transcript) needs neither. + */ +export function speechSpans(meta, { mergeGap = 0.6, offsets, sequential = false, gap = 0 } = {}) { + const merge = Number(mergeGap); + const lists = wordListsFromMediaMeta(meta); + const voices = Array.isArray(meta?.voices) ? meta.voices : []; + if (lists.length > 1 && !offsets && !sequential) { + throw new Error( + "audio_meta has multiple voice lines with file-relative times; pass --sequential or --offsets so spans land at composition time", + ); + } + const intervals = []; + let cursor = 0; + for (let i = 0; i < lists.length; i++) { + const voice = voices[i]; + let offset = 0; + if (offsets) { + const id = voice?.id ?? String(i); + if (!(id in offsets)) throw new Error(`--offsets is missing voice "${id}"`); + offset = Number(offsets[id]) || 0; + } else if (sequential) { + offset = cursor; + const lineDuration = Number(voice?.duration_s) || Math.max(...lists[i].map((w) => w.end), 0); + cursor += lineDuration + (Number(gap) || 0); + } + for (const word of lists[i]) { + if (word.end > word.start) + intervals.push({ start: word.start + offset, end: word.end + offset }); + } + } + return mergeIntervals(intervals, Number.isFinite(merge) && merge >= 0 ? merge : 0.6); +} + +export function duckKeyframes( + spans, + { duck = 0.25, attack = 0.15, release = 0.4, baseVolume = 1 } = {}, +) { + const base = finiteOr(baseVolume, 1); + const ducked = round3(base * finiteOr(duck, 0.25)); + const keyframes = []; + for (const span of spans) { + keyframes.push({ + time: round3(Math.max(0, finiteOr(span.start, 0))), + volume: ducked, + duration: round3(finiteOr(attack, 0.15)), + }); + keyframes.push({ + time: round3(Math.max(0, finiteOr(span.end, 0))), + volume: round3(base), + duration: round3(finiteOr(release, 0.4)), + }); + } + return keyframes.sort((a, b) => a.time - b.time); +} + +function mergeIntervals(intervals, mergeGap) { + const sorted = intervals + .map((range) => ({ start: round3(range.start), end: round3(range.end) })) + .sort((a, b) => a.start - b.start || a.end - b.end); + const merged = []; + for (const range of sorted) { + const prev = merged.at(-1); + if (prev && (range.start <= prev.end || range.start - prev.end < mergeGap)) { + prev.end = Math.max(prev.end, range.end); + } else { + merged.push({ ...range }); + } + } + return merged; +} + +function finiteOr(value, fallback) { + const n = Number(value); + return Number.isFinite(n) ? n : fallback; +} + +function round3(n) { + return Math.round(Number(n) * 1000) / 1000; +} diff --git a/skills/media-use/scripts/lib/duck.test.mjs b/skills/media-use/scripts/lib/duck.test.mjs new file mode 100644 index 000000000..7609d6ce5 --- /dev/null +++ b/skills/media-use/scripts/lib/duck.test.mjs @@ -0,0 +1,118 @@ +import { strict as assert } from "node:assert"; +import { execFileSync } from "node:child_process"; +import { mkdtempSync, writeFileSync, rmSync } from "node:fs"; +import { join, dirname } from "node:path"; +import { tmpdir } from "node:os"; +import { fileURLToPath } from "node:url"; +import { test } from "node:test"; +import { duckKeyframes, speechSpans } from "./duck.mjs"; + +const HERE = dirname(fileURLToPath(import.meta.url)); +const SCRIPT = join(HERE, "..", "audio-duck.mjs"); + +test("speechSpans bridges gaps smaller than mergeGap", () => { + const meta = { + words: [word("w0", "one", 0, 0.5), word("w1", "two", 0.8, 1), word("w2", "three", 2, 2.2)], + }; + + assert.deepEqual(speechSpans(meta, { mergeGap: 0.4 }), [ + { start: 0, end: 1 }, + { start: 2, end: 2.2 }, + ]); +}); + +test("speechSpans refuses multi-line meta without placement (file-relative times)", () => { + const meta = { + voices: [ + { id: "a", words: [word("w0", "one", 0, 1)] }, + { id: "b", words: [word("w1", "two", 0, 1)] }, + ], + }; + assert.throws(() => speechSpans(meta, { mergeGap: 0.2 }), /--sequential or --offsets/); +}); + +test("speechSpans sequential stacks lines by duration plus gap", () => { + const meta = { + voices: [ + { id: "a", duration_s: 2, words: [word("w0", "one", 0.1, 1.9)] }, + { id: "b", duration_s: 1, words: [word("w1", "two", 0.1, 0.9)] }, + ], + }; + assert.deepEqual(speechSpans(meta, { mergeGap: 0.2, sequential: true, gap: 0.5 }), [ + { start: 0.1, end: 1.9 }, + { start: 2.6, end: 3.4 }, + ]); +}); + +test("speechSpans explicit offsets place each line at composition time", () => { + const meta = { + voices: [ + { id: "a", words: [word("w0", "one", 0, 1)] }, + { id: "b", words: [word("w1", "two", 0, 1)] }, + ], + }; + assert.deepEqual(speechSpans(meta, { mergeGap: 0.2, offsets: { a: 0, b: 4 } }), [ + { start: 0, end: 1 }, + { start: 4, end: 5 }, + ]); + assert.throws(() => speechSpans(meta, { offsets: { a: 0 } }), /missing voice "b"/); +}); + +test("speechSpans returns empty spans for empty input", () => { + assert.deepEqual(speechSpans({ voices: [] }, { mergeGap: 0.6 }), []); +}); + +test("duckKeyframes shapes attack and release from base volume", () => { + assert.deepEqual( + duckKeyframes([{ start: 3, end: 5 }], { + duck: 0.25, + attack: 0.15, + release: 0.4, + baseVolume: 0.6, + }), + [ + { time: 3, volume: 0.15, duration: 0.15 }, + { time: 5, volume: 0.6, duration: 0.4 }, + ], + ); +}); + +test("--json spans match --merge-gap semantics exactly", () => { + const dir = mkdtempSync(join(tmpdir(), "media-use-duck-")); + try { + const metaPath = join(dir, "audio_meta.json"); + writeFileSync( + metaPath, + JSON.stringify({ + voices: [ + { + id: "narration", + words: [ + word("w0", "one", 0, 0.4), + word("w1", "two", 0.9, 1.2), + word("w2", "three", 1.8, 2.1), + ], + }, + ], + }), + ); + + const out = execFileSync( + process.execPath, + [SCRIPT, "--meta", metaPath, "--target", "#bgm", "--merge-gap", "0.6", "--json"], + { encoding: "utf8" }, + ); + + const parsed = JSON.parse(out); + assert.deepEqual(parsed.spans, [ + { start: 0, end: 1.2 }, + { start: 1.8, end: 2.1 }, + ]); + } finally { + rmSync(dir, { recursive: true, force: true }); + } +}); + +function word(id, text, start, end) { + return { id, text, start, end }; +} diff --git a/skills/media-use/scripts/lib/parakeet-words.mjs b/skills/media-use/scripts/lib/parakeet-words.mjs new file mode 100644 index 000000000..04df79c7d --- /dev/null +++ b/skills/media-use/scripts/lib/parakeet-words.mjs @@ -0,0 +1,27 @@ +// Merge Parakeet-MLX token timestamps into word timestamps. +// +// parakeet-mlx JSON emits SUB-WORD tokens (" H", "ello", ...) with per-token +// start/end. Captions + transcript-cut need WORD timestamps, so join tokens +// into words on the space boundary: a token whose text starts with a space +// (or the very first token) begins a new word; the rest append. Output matches +// the { words: [{ text, start, end }] } shape the rest of media-use consumes +// (see words.mjs / cutlist.mjs). + +export function mergeTokensToWords(parakeet) { + const sentences = Array.isArray(parakeet?.sentences) ? parakeet.sentences : []; + const words = []; + for (const s of sentences) { + for (const t of s.tokens ?? []) { + const raw = typeof t.text === "string" ? t.text : ""; + const startsWord = raw.startsWith(" ") || words.length === 0; + if (startsWord) { + words.push({ text: raw.trim(), start: t.start, end: t.end }); + } else { + const w = words[words.length - 1]; + w.text += raw; + w.end = t.end; + } + } + } + return { text: (parakeet?.text ?? "").trim(), words: words.filter((w) => w.text.length > 0) }; +} diff --git a/skills/media-use/scripts/lib/parakeet-words.test.mjs b/skills/media-use/scripts/lib/parakeet-words.test.mjs new file mode 100644 index 000000000..f74e78481 --- /dev/null +++ b/skills/media-use/scripts/lib/parakeet-words.test.mjs @@ -0,0 +1,44 @@ +import { strict as assert } from "node:assert"; +import { test } from "node:test"; +import { mergeTokensToWords } from "./parakeet-words.mjs"; + +test("mergeTokensToWords joins sub-word tokens on the space boundary", () => { + const parakeet = { + text: "Hello everyone. Um,", + sentences: [ + { + tokens: [ + { text: " H", start: 0.0, end: 0.24 }, + { text: "ello", start: 0.24, end: 0.48 }, + { text: " everyone.", start: 0.48, end: 1.28 }, + { text: " Um,", start: 1.28, end: 1.92 }, + ], + }, + ], + }; + const { words } = mergeTokensToWords(parakeet); + assert.deepEqual(words, [ + { text: "Hello", start: 0.0, end: 0.48 }, + { text: "everyone.", start: 0.48, end: 1.28 }, + { text: "Um,", start: 1.28, end: 1.92 }, + ]); +}); + +test("mergeTokensToWords spans multiple sentences and drops empties", () => { + const parakeet = { + text: "Hi there", + sentences: [ + { tokens: [{ text: "Hi", start: 0, end: 0.2 }] }, + { tokens: [{ text: " there", start: 0.5, end: 0.9 }] }, + ], + }; + const { words } = mergeTokensToWords(parakeet); + assert.equal(words.length, 2); + assert.equal(words[1].text, "there"); + assert.equal(words[1].start, 0.5); +}); + +test("mergeTokensToWords tolerates missing sentences/tokens", () => { + assert.deepEqual(mergeTokensToWords({}).words, []); + assert.deepEqual(mergeTokensToWords({ sentences: [{}] }).words, []); +}); diff --git a/skills/media-use/scripts/transcribe.mjs b/skills/media-use/scripts/transcribe.mjs new file mode 100644 index 000000000..22baa15a9 --- /dev/null +++ b/skills/media-use/scripts/transcribe.mjs @@ -0,0 +1,172 @@ +#!/usr/bin/env node + +import { execFileSync } from "node:child_process"; +import { + existsSync, + readFileSync, + writeFileSync, + copyFileSync, + renameSync, + mkdtempSync, + rmSync, +} from "node:fs"; +import { homedir, tmpdir } from "node:os"; +import { basename, extname, join, resolve } from "node:path"; +import { parseArgs } from "node:util"; +import { mergeTokensToWords } from "./lib/parakeet-words.mjs"; +import { track } from "./lib/telemetry.mjs"; + +// The DEFAULT local transcription path. Prefers NVIDIA Parakeet-TDT via +// parakeet-mlx, which beats whisper.cpp on the Open ASR Leaderboard (~6.05% vs +// 7.44% avg WER, and 4.73% vs 5.96% on noisy test-other) and is 5-10x faster +// with native punctuation. Emits { text, words:[{text,start,end}] } (word +// timestamps merged from Parakeet's sub-word tokens) for transcript-cut / +// captions / the audio engine. +// +// Parakeet v3 covers English + 25 European languages. For other languages, or +// when parakeet-mlx is not installed, it falls back to the packaged whisper.cpp +// (`hyperframes transcribe`, 99 languages). `--engine` forces one. + +const { values: args } = parseArgs({ + options: { + input: { type: "string", short: "i" }, + out: { type: "string", short: "o" }, + engine: { type: "string", default: "auto" }, // auto | parakeet | whisper + model: { type: "string", default: "mlx-community/parakeet-tdt-0.6b-v3" }, + json: { type: "boolean", default: false }, + help: { type: "boolean", short: "h", default: false }, + }, + strict: true, +}); + +if (args.help) { + console.log(`media-use transcribe: better-than-whisper local ASR (Parakeet), whisper.cpp fallback + +Usage: + node transcribe.mjs --input audio.wav [--out audio.transcribe.json] [--engine auto|parakeet|whisper] + +Parakeet (default) beats whisper.cpp on accuracy + speed for English/European +languages; whisper.cpp (99 languages) is the fallback. Install Parakeet once: + uv venv ~/.venvs/parakeet && VIRTUAL_ENV=~/.venvs/parakeet uv pip install parakeet-mlx`); + process.exit(0); +} + +if (!args.input) { + console.error("error: --input is required"); + process.exit(2); +} +const inputPath = resolve(args.input); +if (!existsSync(inputPath)) { + console.error(`error: input not found: ${inputPath}`); + process.exit(2); +} +const outPath = resolve( + args.out || `${inputPath.slice(0, -extname(inputPath).length)}.transcribe.json`, +); + +// Locate the parakeet-mlx runner the same way the CLI does: env override, then +// the documented ~/.venvs/parakeet install, then PATH. Checking the venv (not +// just PATH) is what keeps a user who followed the install docs verbatim from +// silently falling through to whisper. Returns the runner path, or null. +function resolveParakeet() { + for (const p of [ + process.env.HYPERFRAMES_PARAKEET, + join(homedir(), ".venvs", "parakeet", "bin", "parakeet-mlx"), + ]) { + if (p && existsSync(p)) return p; + } + try { + execFileSync("parakeet-mlx", ["--help"], { + stdio: ["ignore", "ignore", "ignore"], + timeout: 20000, + }); + return "parakeet-mlx"; + } catch { + return null; + } +} + +// Write via a sibling temp + atomic rename so a SIGKILL mid-write can't leave a +// truncated transcript at outPath (downstream reads it as valid JSON). +function atomicWrite(target, data) { + const tmp = `${target}.tmp-${process.pid}`; + writeFileSync(tmp, data); + renameSync(tmp, target); +} + +function report(engine, wordCount) { + if (args.json) console.log(JSON.stringify({ ok: true, out: outPath, engine, words: wordCount })); + else + console.log( + `transcribed ${basename(inputPath)} -> ${outPath}${wordCount != null ? ` (${wordCount} words,` : " ("}${engine})`, + ); +} + +function runParakeet(runner) { + const workDir = mkdtempSync(join(tmpdir(), "media-use-asr-")); + try { + execFileSync( + runner, + [inputPath, "--model", args.model, "--output-format", "json", "--output-dir", workDir], + { stdio: ["ignore", "pipe", "pipe"], timeout: 1_800_000 }, + ); + const jsonPath = join(workDir, `${basename(inputPath, extname(inputPath))}.json`); + if (!existsSync(jsonPath)) throw new Error("parakeet produced no JSON"); + const merged = mergeTokensToWords(JSON.parse(readFileSync(jsonPath, "utf8"))); + atomicWrite(outPath, JSON.stringify(merged, null, 2)); + report("parakeet", merged.words.length); + } finally { + rmSync(workDir, { recursive: true, force: true }); + } +} + +// whisper.cpp via the packaged CLI: writes transcript.json into --dir; relocate to --out. +function runWhisper() { + const workDir = mkdtempSync(join(tmpdir(), "media-use-whisper-")); + try { + execFileSync("npx", ["hyperframes", "transcribe", inputPath, "--dir", workDir], { + stdio: ["ignore", "pipe", "pipe"], + timeout: 1_800_000, + }); + const produced = join(workDir, "transcript.json"); + if (!existsSync(produced)) throw new Error("whisper produced no transcript.json"); + const tmp = `${outPath}.tmp-${process.pid}`; + copyFileSync(produced, tmp); + renameSync(tmp, outPath); // atomic publish + let words; + try { + const t = JSON.parse(readFileSync(outPath, "utf8")); + words = Array.isArray(t?.words) ? t.words.length : undefined; + } catch { + /* leave undefined */ + } + report("whisper", words); + } finally { + rmSync(workDir, { recursive: true, force: true }); + } +} + +try { + const parakeetBin = resolveParakeet(); + const engine = + args.engine === "parakeet" || args.engine === "whisper" + ? args.engine + : parakeetBin + ? "parakeet" + : "whisper"; + if (engine === "parakeet") { + if (!parakeetBin) { + throw new Error( + "parakeet-mlx not found (checked $HYPERFRAMES_PARAKEET, ~/.venvs/parakeet, and PATH). Install: uv venv ~/.venvs/parakeet && VIRTUAL_ENV=~/.venvs/parakeet uv pip install parakeet-mlx (or use --engine whisper)", + ); + } + runParakeet(parakeetBin); + } else { + runWhisper(); + } + await track("media_use_transcribe", { engine }); +} catch (err) { + if (args.json) console.log(JSON.stringify({ ok: false, error: err.message })); + else console.error(`error: transcription failed: ${err.message}`); + process.exit(1); +} diff --git a/skills/media-use/scripts/transcript-cut.mjs b/skills/media-use/scripts/transcript-cut.mjs new file mode 100644 index 000000000..49210abc9 --- /dev/null +++ b/skills/media-use/scripts/transcript-cut.mjs @@ -0,0 +1,238 @@ +#!/usr/bin/env node + +import { execFileSync } from "node:child_process"; +import { mkdtempSync, mkdirSync, readFileSync, renameSync, rmSync, writeFileSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { dirname, extname, join, resolve } from "node:path"; +import { parseArgs } from "node:util"; +import { compileCutList } from "./lib/cutlist.mjs"; +import { track } from "./lib/telemetry.mjs"; + +const { values: args } = parseArgs({ + options: { + input: { type: "string" }, + transcript: { type: "string" }, + remove: { type: "string" }, + "remove-words": { type: "string" }, + "remove-fillers": { type: "string" }, + "cut-silence": { type: "string" }, + keep: { type: "string" }, + copy: { type: "boolean", default: false }, + plan: { type: "boolean", default: false }, + out: { type: "string" }, + json: { type: "boolean", default: false }, + help: { type: "boolean", short: "h", default: false }, + }, + strict: true, +}); + +if (args.help) { + console.log(`media-use transcript-cut — compile transcript edits into video cuts + +Usage: + node transcript-cut.mjs --input in.mp4 --transcript transcript.json --remove "12-15" --out out.mp4 + +Options: + --input Source video/audio file + --transcript JSON word transcript, array or { words: [...] } + --remove Time ranges to remove, seconds: a-b,c-d + --remove-words Word-index ranges to remove: 12-18,40-41 + --remove-fillers Comma list of filler words to remove + --cut-silence Remove inter-word gaps longer than this many seconds + --keep Inverse mode: direct kept ranges, mutually exclusive with removal + --copy Use stream copy for faster, keyframe-snapped cuts + --plan Print kept segment JSON and exit without ffmpeg + --out Output file + --json Output JSON status + --help, -h Show this help`); + process.exit(0); +} + +try { + run(); + await track("media_use_transcript_cut", { + mode: args.plan ? "plan" : "encode", + remove_fillers: !!args["remove-fillers"], + cut_silence: !!args["cut-silence"], + ranges: !!args.remove, + keep: !!args.keep, + }); +} catch (err) { + if (args.json) console.log(JSON.stringify({ ok: false, error: err.message })); + else console.error(`error: ${err.message}`); + process.exit(1); +} + +function run() { + if (!args.transcript) throw new Error("--transcript is required"); + const transcript = JSON.parse(readFileSync(resolve(args.transcript), "utf8")); + const segments = compileCutList(transcript, { + remove: args.remove, + removeWords: args["remove-words"], + removeFillers: args["remove-fillers"], + cutSilence: args["cut-silence"], + keep: args.keep, + }); + + if (args.plan) { + console.log(JSON.stringify(segments)); + return; + } + + if (!args.input || !args.out) + throw new Error("--input and --out are required unless --plan is set"); + if (segments.length === 0) throw new Error("cut list has no kept segments"); + + const inputPath = resolve(args.input); + const outPath = resolve(args.out); + mkdirSync(dirname(outPath), { recursive: true }); + const tmpDir = mkdtempSync(join(tmpdir(), "media-use-cut-")); + const keptSeconds = sumDurations(segments); + const totalSeconds = probeDuration(inputPath); + + try { + const parts = segments.map((segment, index) => { + const out = join( + tmpDir, + `segment-${String(index).padStart(4, "0")}${extname(outPath) || ".mp4"}`, + ); + cutSegment(inputPath, segment, out, Boolean(args.copy)); + return out; + }); + const listPath = join(tmpDir, "list.txt"); + writeFileSync( + listPath, + parts.map((part) => `file '${escapeConcatPath(part)}'`).join("\n") + "\n", + ); + // Encode to a sibling temp (same extension so ffmpeg picks the right muxer), + // then atomic-rename so a SIGKILL mid-encode can't leave a truncated outPath. + const tmpOut = `${outPath}.part${extname(outPath) || ".mp4"}`; + execFileSync( + "ffmpeg", + ["-y", "-f", "concat", "-safe", "0", "-i", listPath, "-c", "copy", tmpOut], + { + stdio: "ignore", + }, + ); + renameSync(tmpOut, outPath); + } finally { + rmSync(tmpDir, { recursive: true, force: true }); + } + + // Stream copy can only cut on keyframes; on sparse-keyframe footage the snap + // can silently swallow the whole cut. Compare, then surface the drift in BOTH + // the stderr warning (human) and the --json result (pipelines). + let copyDrift = null; + if (args.copy) { + const outSeconds = probeDuration(outPath); + if (Math.abs(outSeconds - keptSeconds) > 1) { + copyDrift = { produced_s: round3(outSeconds), expected_s: round3(keptSeconds) }; + if (!args.json) { + console.error( + `warning: --copy keyframe snapping produced ${round3(outSeconds)}s instead of ${round3(keptSeconds)}s kept; drop --copy for frame-accurate cuts`, + ); + } + } + } + + if (args.json) { + console.log( + JSON.stringify({ + ok: true, + input: inputPath, + out: outPath, + segments, + kept_s: round3(keptSeconds), + total_s: round3(totalSeconds), + ...(copyDrift && { copy_drift: copyDrift }), + }), + ); + return; + } + + console.log( + `cut ${inputPath} -> ${outPath} (${segments.length} segments, ${fmt(keptSeconds)}s kept of ${fmt( + totalSeconds, + )}s)`, + ); + console.log(`next: resolve --from ${outPath} --type `); +} + +function cutSegment(inputPath, segment, outPath, copy) { + const argv = [ + "-y", + "-nostdin", + "-ss", + fmt(segment.start), + "-i", + inputPath, + "-to", + fmt(segment.end - segment.start), + ]; + if (copy) { + argv.push("-c", "copy", "-avoid_negative_ts", "make_zero"); + } else { + argv.push(...encodeArgsFor(extname(outPath).toLowerCase())); + } + argv.push(outPath); + execFileSync("ffmpeg", argv, { stdio: "ignore" }); +} + +// Codec set per output container. Audio-only outputs must not get the +// video-centric aac/x264 set (aac inside .wav breaks timing entirely). +function encodeArgsFor(ext) { + if (ext === ".wav") return ["-c:a", "pcm_s16le"]; + if (ext === ".mp3") return ["-c:a", "libmp3lame", "-q:a", "2"]; + if (ext === ".m4a" || ext === ".aac") return ["-c:a", "aac"]; + if (ext === ".flac") return ["-c:a", "flac"]; + return [ + "-c:v", + "libx264", + "-preset", + "veryfast", + "-crf", + "18", + "-c:a", + "aac", + "-movflags", + "+faststart", + ]; +} + +function probeDuration(filePath) { + const raw = execFileSync( + "ffprobe", + [ + "-v", + "error", + "-show_entries", + "format=duration", + "-of", + "default=noprint_wrappers=1:nokey=1", + filePath, + ], + { encoding: "utf8" }, + ); + const duration = Number(raw.trim()); + if (!Number.isFinite(duration) || duration <= 0) + throw new Error(`could not probe duration: ${filePath}`); + return duration; +} + +function escapeConcatPath(filePath) { + return filePath.replace(/'/g, "'\\''"); +} + +function sumDurations(segments) { + return segments.reduce((sum, segment) => sum + (segment.end - segment.start), 0); +} + +function fmt(n) { + return round3(n) + .toFixed(3) + .replace(/\.?0+$/, ""); +} + +function round3(n) { + return Math.round(Number(n) * 1000) / 1000; +}