From 14e598e5a17cbf541c2201f5c84dbbde3c25ce1c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Miguel=20=C3=81ngel?= Date: Sat, 7 Nov 2026 20:10:38 +0000 Subject: [PATCH] feat(media-use): your own voice recording is a voice line, and speed applies to it (#7377) * feat(media-use): your own voice recording is a voice line, and speed applies to it * fix(media-use): regenerate the skills manifest and keep audio.mjs's header at its limit --- skills-manifest.json | 4 +- skills/media-use/audio/scripts/audio.mjs | 78 ++++++---- .../audio/scripts/lib/media-record.mjs | 6 +- .../audio/scripts/lib/media-record.test.mjs | 17 ++- .../media-use/audio/scripts/lib/recording.mjs | 53 +++++++ .../audio/scripts/recording.test.mjs | 138 ++++++++++++++++++ skills/media-use/references/audio.md | 1 + 7 files changed, 266 insertions(+), 31 deletions(-) create mode 100644 skills/media-use/audio/scripts/lib/recording.mjs create mode 100644 skills/media-use/audio/scripts/recording.test.mjs diff --git a/skills-manifest.json b/skills-manifest.json index 03e91da346..bdb2800855 100644 --- a/skills-manifest.json +++ b/skills-manifest.json @@ -18,8 +18,8 @@ "files": 13 }, "media-use": { - "hash": "44ef2d115fa8bebd", - "files": 118 + "hash": "d4221a4cd4118c29", + "files": 120 }, "remotion-to-hyperframes": { "hash": "73b82d19db2ed8c9", diff --git a/skills/media-use/audio/scripts/audio.mjs b/skills/media-use/audio/scripts/audio.mjs index 06cd7665e5..e86c17a5dd 100644 --- a/skills/media-use/audio/scripts/audio.mjs +++ b/skills/media-use/audio/scripts/audio.mjs @@ -52,6 +52,7 @@ import { } from "./lib/tts.mjs"; import { generateBgmDetached, inferBgmPrompt, retrieveBgm } from "./lib/bgm.mjs"; import { BUNDLED_SFX_DIR, resolveSfx } from "./lib/sfx.mjs"; +import { parseWords, scaleWords, stretchToWav } from "./lib/recording.mjs"; import { mapWithConcurrency } from "./lib/concurrency.mjs"; import { openAudioMeta } from "./lib/audio-meta.mjs"; import { recordInManifest, voicePaths, writtenAssets } from "./lib/media-record.mjs"; @@ -127,42 +128,63 @@ let voices = prev.voices ?? []; let ttsProvider = prev.tts_provider ?? null; let voiceId = prev.voice_id ?? null; if (only.has("tts") && lines.length) { - try { - ttsProvider = pickProvider( - providerOverride || (request.provider === "auto" ? null : request.provider), - ); - } catch (e) { - die(e.message); + if (lines.some((l) => !l.audio)) { + try { + ttsProvider = pickProvider( + providerOverride || (request.provider === "auto" ? null : request.provider), + ); + } catch (e) { + die(e.message); + } + voiceId = await resolveVoiceId({ + provider: ttsProvider, + userVoice: voiceOverride || request.voice, + lang, + }); + } else { + ttsProvider = "recording"; // every line is the person's own recording: no TTS provider to pick + voiceId = null; } - voiceId = await resolveVoiceId({ - provider: ttsProvider, - userVoice: voiceOverride || request.voice, - lang, - }); console.error(`· tts: ${ttsProvider} · voice ${voiceId} · ${lines.length} line(s)`); const paths = voicePaths(hyperframesDir, lines, anomalies); + // A line with `audio` is the person's own recording (path relative to the project), with optional + // `words`; `speed` applies to it like to a generated voice. Returns synthesizeOne's shape. + const useRecording = (line, abs) => { + const made = stretchToWav(resolve(hyperframesDir, String(line.audio)), abs, speed); + if (!made.ok) return made; + try { + const given = parseWords(line.words); + return { ok: true, words: given && scaleWords(given, speed), source: "recording" }; + } catch (e) { + return { ok: false, error: e.message }; + } + }; const synthLine = async (line) => { const id = String(line.id); const text = String(line.text ?? "").trim(); - if (!text) { + if (!text && !line.audio) { anomalies.push(`line ${id}: empty text — skipped`); return null; } const rel = paths.get(id); const abs = join(hyperframesDir, rel); - const { ok, words, error } = await synthesizeOne({ - provider: ttsProvider, - text, - voiceId, - lang, - speed, - model: flag("tts-model", request.tts_model), - style: flag("style", line.style ?? request.style), - wavAbs: abs, - hyperframesDir, - }); + const { ok, words, error, source } = line.audio + ? useRecording(line, abs) + : await synthesizeOne({ + provider: ttsProvider, + text, + voiceId, + lang, + speed, + model: flag("tts-model", request.tts_model), + style: flag("style", line.style ?? request.style), + wavAbs: abs, + hyperframesDir, + }); if (!ok) { - anomalies.push(`line ${id}: TTS failed — omitted${error ? ` (${error})` : ""}`); + anomalies.push( + `line ${id}: ${line.audio ? "recording" : "TTS"} failed — omitted${error ? ` (${error})` : ""}`, + ); return null; } let wordArr = words; // heygen: native; else transcribe @@ -172,7 +194,13 @@ if (only.has("tts") && lines.length) { anomalies.push(`line ${id}: bad voice duration — omitted`); return null; } - return { id, path: rel, duration_s: r3(dur), words: withWordIds(wordArr) }; + return { + id, + path: rel, + duration_s: r3(dur), + words: withWordIds(wordArr), + ...(source && { source }), + }; }; const results = await mapWithConcurrency(lines, ttsConcurrency, synthLine); voices = results.filter(Boolean); diff --git a/skills/media-use/audio/scripts/lib/media-record.mjs b/skills/media-use/audio/scripts/lib/media-record.mjs index a11433cc20..888bf5d6f8 100644 --- a/skills/media-use/audio/scripts/lib/media-record.mjs +++ b/skills/media-use/audio/scripts/lib/media-record.mjs @@ -35,7 +35,7 @@ export function agentWritePath( export function voicePaths(hyperframesDir, lines, anomalies) { const taken = new Set(); const paths = new Map(); - for (const line of lines.filter((l) => String(l.text ?? "").trim())) { + for (const line of lines.filter((l) => String(l.text ?? "").trim() || l.audio)) { const rel = agentWritePath(hyperframesDir, `assets/voice/${line.id}.wav`, { anomalies, taken }); taken.add(rel); paths.set(String(line.id), rel); @@ -60,9 +60,9 @@ export function writtenAssets({ only, lines, voices, ttsProvider, bgm, bgmFields path: voice.path, type: "voice", source: "generated", - intent: textById.get(voice.id), + intent: textById.get(voice.id) || "your own recording", duration: voice.duration_s, - provider: ttsProvider, + provider: voice.source ?? ttsProvider, }); } } diff --git a/skills/media-use/audio/scripts/lib/media-record.test.mjs b/skills/media-use/audio/scripts/lib/media-record.test.mjs index d6e7458241..e3ea9e8353 100644 --- a/skills/media-use/audio/scripts/lib/media-record.test.mjs +++ b/skills/media-use/audio/scripts/lib/media-record.test.mjs @@ -3,7 +3,7 @@ import { test } from "node:test"; import { mkdirSync, mkdtempSync, readdirSync, readFileSync, rmSync, writeFileSync } from "node:fs"; import { join } from "node:path"; import { tmpdir } from "node:os"; -import { findByPrompt, readManifest } from "../../../scripts/lib/manifest.mjs"; +import { AGENT_SOURCES, findByPrompt, readManifest } from "../../../scripts/lib/manifest.mjs"; import { agentWritePath, recordInManifest, voicePaths, writtenAssets } from "./media-record.mjs"; const noBgm = { bgm: null, bgmFields: { bgm_pending: false } }; @@ -222,3 +222,18 @@ test("a record never takes the id of a download still in flight", (t) => { ["bgm_002"], ); }); + +// An engine-written copy stays an agent source, or every rerun would write a numbered copy beside it +// (AGENT_SOURCES, manifest.mjs); what says it is the person's own voice is the provider and intent. +test("a recording line is recorded as your own recording, still an engine-written file", () => { + const [asset] = writtenAssets({ + only: new Set(["tts"]), + lines: [{ id: "01", audio: "take.m4a" }], + voices: [{ id: "01", path: "assets/voice/01.wav", duration_s: 2, source: "recording" }], + ttsProvider: "recording", + }); + + assert.equal(asset.provider, "recording"); + assert.equal(asset.intent, "your own recording"); + assert.ok(AGENT_SOURCES.includes(asset.source)); +}); diff --git a/skills/media-use/audio/scripts/lib/recording.mjs b/skills/media-use/audio/scripts/lib/recording.mjs new file mode 100644 index 0000000000..a92e554ba3 --- /dev/null +++ b/skills/media-use/audio/scripts/lib/recording.mjs @@ -0,0 +1,53 @@ +// recording.mjs — the person's own voice recording as a voice line: converted to the engine's wav, +// time-stretched by `speed` with the pitch kept, with word timings scaled to match. +import { spawnSync } from "node:child_process"; +import { existsSync, mkdirSync } from "node:fs"; +import { dirname } from "node:path"; + +// ffmpeg's atempo takes 0.5..2 per stage; chain stages for the rest of the 0.1..3 speed range. +export function atempoChain(speed) { + const stages = []; + let left = speed; + while (left > 2) { + stages.push(2); + left /= 2; + } + while (left < 0.5) { + stages.push(0.5); + left /= 0.5; + } + if (Math.abs(left - 1) > 1e-9 || stages.length === 0) stages.push(left); + return stages.map((s) => `atempo=${Number(s.toFixed(6))}`).join(","); +} + +/** src -> 44.1k mono wav at wavAbs, played `speed` times faster. Returns { ok, error? }. */ +export function stretchToWav(src, wavAbs, speed = 1) { + if (!existsSync(src)) return { ok: false, error: `recording not found: ${src}` }; + mkdirSync(dirname(wavAbs), { recursive: true }); + const filter = speed === 1 ? [] : ["-af", atempoChain(speed)]; + const r = spawnSync( + "ffmpeg", + ["-y", "-loglevel", "error", "-i", src, ...filter, "-ar", "44100", "-ac", "1", wavAbs], + { encoding: "utf8" }, + ); + if (r.error || r.status !== 0 || !existsSync(wavAbs)) { + return { ok: false, error: `ffmpeg could not convert ${src}: ${r.error?.message ?? r.stderr}` }; + } + return { ok: true }; +} + +/** Caller-supplied word timings ({text|word, start, end}) as [{text,start,end}], or null when absent. */ +export function parseWords(words) { + if (!Array.isArray(words) || words.length === 0) return null; + return words.map((w, i) => { + const text = w?.text ?? w?.word; + if (typeof text !== "string" || !Number.isFinite(w.start) || !Number.isFinite(w.end)) { + throw new Error(`words[${i}] needs text (or word), start and end in seconds`); + } + return { text, start: w.start, end: w.end }; + }); +} + +/** Words of a recording played `speed` times faster land at start/speed. */ +export const scaleWords = (words, speed) => + words.map((w) => ({ ...w, start: w.start / speed, end: w.end / speed })); diff --git a/skills/media-use/audio/scripts/recording.test.mjs b/skills/media-use/audio/scripts/recording.test.mjs new file mode 100644 index 0000000000..0bce5d37ef --- /dev/null +++ b/skills/media-use/audio/scripts/recording.test.mjs @@ -0,0 +1,138 @@ +import { test } from "node:test"; +import assert from "node:assert/strict"; +import { spawnSync } from "node:child_process"; +import { mkdtempSync, readFileSync, writeFileSync, rmSync } from "node:fs"; +import { join, dirname } from "node:path"; +import { tmpdir } from "node:os"; +import { fileURLToPath } from "node:url"; +import { atempoChain, parseWords, scaleWords } from "./lib/recording.mjs"; + +const AUDIO = join(dirname(fileURLToPath(import.meta.url)), "audio.mjs"); + +test("atempoChain splits speeds outside ffmpeg's 0.5..2 stage range", () => { + assert.equal(atempoChain(1.5), "atempo=1.5"); + assert.equal(atempoChain(3), "atempo=2,atempo=1.5"); + assert.equal(atempoChain(0.25), "atempo=0.5,atempo=0.5"); +}); + +test("parseWords takes text or word and rejects a missing time", () => { + assert.deepEqual(parseWords([{ word: "hi", start: 0, end: 1 }]), [ + { text: "hi", start: 0, end: 1 }, + ]); + assert.equal(parseWords(undefined), null); + assert.throws(() => parseWords([{ text: "hi", start: 0 }]), /words\[0\]/); +}); + +test("scaleWords moves words by 1/speed", () => { + assert.deepEqual(scaleWords([{ text: "a", start: 1, end: 2 }], 2), [ + { text: "a", start: 0.5, end: 1 }, + ]); +}); + +test("a recording line with malformed words is omitted with the reason, not a crash", () => { + const dir = mkdtempSync(join(tmpdir(), "hf-rec-")); + try { + const make = spawnSync( + "ffmpeg", + ["-y", "-loglevel", "error", "-f", "lavfi", "-i", "sine=duration=1", join(dir, "take.wav")], + { encoding: "utf8" }, + ); + assert.equal(make.status, 0, make.stderr); + writeFileSync( + join(dir, "audio_request.json"), + JSON.stringify({ + lines: [{ id: "01", audio: "take.wav", words: [{ text: "x", start: 0 }] }], + }), + ); + const run = spawnSync( + process.execPath, + [ + AUDIO, + "--request", + join(dir, "audio_request.json"), + "--hyperframes", + dir, + "--out", + join(dir, "m.json"), + "--only", + "tts", + ], + { encoding: "utf8", env: { ...process.env, HOME: dir } }, + ); + assert.equal(run.status, 0, run.stderr); + assert.match(run.stdout, /line 01: recording failed — omitted \(words\[0\] needs text/); + assert.deepEqual(JSON.parse(readFileSync(join(dir, "m.json"), "utf8")).voices, []); + } finally { + rmSync(dir, { recursive: true, force: true }); + } +}); + +// A 1 s gap then a 1 s tone is a "word" at 1..2 s. At speed 2 the tone must start at 0.5 s in the +// stretched file, which is where the scaled word says it is. +test("a recording at speed 2: the file is stretched and the words still land on the audio", () => { + const dir = mkdtempSync(join(tmpdir(), "hf-rec-")); + try { + const make = spawnSync( + "ffmpeg", + [ + "-y", + "-loglevel", + "error", + "-f", + "lavfi", + "-i", + "anullsrc=r=44100:cl=mono", + "-f", + "lavfi", + "-i", + "sine=frequency=440:duration=1:sample_rate=44100", + "-filter_complex", + "[0]atrim=duration=1[s];[s][1]concat=n=2:v=0:a=1", + join(dir, "take.wav"), + ], + { encoding: "utf8" }, + ); + assert.equal(make.status, 0, make.stderr); + writeFileSync( + join(dir, "audio_request.json"), + JSON.stringify({ + speed: 2, + lines: [{ id: "01", audio: "take.wav", words: [{ text: "tone", start: 1, end: 2 }] }], + }), + ); + const run = spawnSync( + process.execPath, + [ + AUDIO, + "--request", + join(dir, "audio_request.json"), + "--hyperframes", + dir, + "--out", + join(dir, "audio_meta.json"), + "--only", + "tts", + ], + { encoding: "utf8", env: { ...process.env, HOME: dir } }, + ); + assert.equal(run.status, 0, run.stderr + run.stdout); + const meta = JSON.parse(readFileSync(join(dir, "audio_meta.json"), "utf8")); + const voice = meta.voices[0]; + assert.equal(meta.tts_provider, "recording"); + assert.equal(voice.source, "recording"); + assert.ok(Math.abs(voice.duration_s - 1) < 0.05, `duration ${voice.duration_s}`); + assert.deepEqual([voice.words[0].start, voice.words[0].end], [0.5, 1]); + const probe = spawnSync( + "ffmpeg", + ["-i", join(dir, voice.path), "-af", "silencedetect=noise=-40dB:d=0.2", "-f", "null", "-"], + { encoding: "utf8" }, + ); + const silenceEnd = Number(/silence_end: ([\d.]+)/.exec(probe.stderr)?.[1]); + assert.ok( + Math.abs(silenceEnd - voice.words[0].start) < 0.08, + `tone starts at ${silenceEnd}s, word says ${voice.words[0].start}s`, + ); + } finally { + rmSync(dir, { recursive: true, force: true }); + } +}); diff --git a/skills/media-use/references/audio.md b/skills/media-use/references/audio.md index 350185512b..5566dbc351 100644 --- a/skills/media-use/references/audio.md +++ b/skills/media-use/references/audio.md @@ -10,6 +10,7 @@ node /audio/scripts/audio.mjs --request ./audio_request.json --out ./ ``` - **Request** `{ provider?, voice?, lang?, speed?, tts_model?, style?, lines: [{ id, text, style?, sfx?: [names] }], bgm: { mode?, query?, prompt? } }`: `id` joins each line back to your model; `bgm.mode` = `retrieve | generate | none` (omit for auto). `--only tts,bgm,sfx` runs a subset and merges into an existing `--out`. +- **Your own recording**: a line with `audio` (a path in the project) instead of `text` is used as the voice: `{ id, audio: "take.m4a", words?: [{ text, start, end }] }`. It is converted to `assets/voice/.wav`, `speed` stretches it with the pitch kept and scales `words` the same (without `words` the recording is transcribed). Its `audio_meta.json` voice has `source: "recording"`; with only recording lines `tts_provider` is `"recording"` and no provider or HeyGen sign-in is needed. - **Gemini narration** is opt-in with `provider: "gemini"` and an API key or service-account credentials. `tts_model` defaults to `gemini-3.8-flash-tts`; 3.8 Flash-Lite, 3.1 Flash Preview, and 2.5 Pro/Flash Preview TTS are also supported. `style` directs delivery; a line's style overrides the request's style. CLI overrides: `--tts-model`, `--style`. See `audio/references/tts.md` for a complete request. The existing automatic provider order is unchanged. - **Output** `audio_meta.json` (id-keyed): `voices[].{path,duration_s,words[]}` (word timestamps for captions), `sfx[]`, `bgm`, `total_duration_s`. - **HeyGen free-usage path**: HeyGen CLI auth unlocks TTS plus music retrieval; SFX come from Openverse with no account (`audio/references/sfx.md`). Local/provider-specific generators are explicit alternatives where installed; run `npx hyperframes media-use resolve --doctor` before assuming retrieval or TTS will work.