feat(media-use): your own voice recording is a voice line, and speed applies to it (#7377)

* feat(media-use): your own voice recording is a voice line, and speed applies to it

* fix(media-use): regenerate the skills manifest and keep audio.mjs's header at its limit
This commit is contained in:
Miguel Ángel
2026-11-07 20:10:38 +00:00
committed by GitHub
parent b418bb24fc
commit 14e598e5a1
7 changed files with 266 additions and 31 deletions
+2 -2
View File
@@ -18,8 +18,8 @@
"files": 13
},
"media-use": {
"hash": "44ef2d115fa8bebd",
"files": 118
"hash": "d4221a4cd4118c29",
"files": 120
},
"remotion-to-hyperframes": {
"hash": "73b82d19db2ed8c9",
+53 -25
View File
@@ -52,6 +52,7 @@ import {
} from "./lib/tts.mjs";
import { generateBgmDetached, inferBgmPrompt, retrieveBgm } from "./lib/bgm.mjs";
import { BUNDLED_SFX_DIR, resolveSfx } from "./lib/sfx.mjs";
import { parseWords, scaleWords, stretchToWav } from "./lib/recording.mjs";
import { mapWithConcurrency } from "./lib/concurrency.mjs";
import { openAudioMeta } from "./lib/audio-meta.mjs";
import { recordInManifest, voicePaths, writtenAssets } from "./lib/media-record.mjs";
@@ -127,42 +128,63 @@ let voices = prev.voices ?? [];
let ttsProvider = prev.tts_provider ?? null;
let voiceId = prev.voice_id ?? null;
if (only.has("tts") && lines.length) {
try {
ttsProvider = pickProvider(
providerOverride || (request.provider === "auto" ? null : request.provider),
);
} catch (e) {
die(e.message);
if (lines.some((l) => !l.audio)) {
try {
ttsProvider = pickProvider(
providerOverride || (request.provider === "auto" ? null : request.provider),
);
} catch (e) {
die(e.message);
}
voiceId = await resolveVoiceId({
provider: ttsProvider,
userVoice: voiceOverride || request.voice,
lang,
});
} else {
ttsProvider = "recording"; // every line is the person's own recording: no TTS provider to pick
voiceId = null;
}
voiceId = await resolveVoiceId({
provider: ttsProvider,
userVoice: voiceOverride || request.voice,
lang,
});
console.error(`· tts: ${ttsProvider} · voice ${voiceId} · ${lines.length} line(s)`);
const paths = voicePaths(hyperframesDir, lines, anomalies);
// A line with `audio` is the person's own recording (path relative to the project), with optional
// `words`; `speed` applies to it like to a generated voice. Returns synthesizeOne's shape.
const useRecording = (line, abs) => {
const made = stretchToWav(resolve(hyperframesDir, String(line.audio)), abs, speed);
if (!made.ok) return made;
try {
const given = parseWords(line.words);
return { ok: true, words: given && scaleWords(given, speed), source: "recording" };
} catch (e) {
return { ok: false, error: e.message };
}
};
const synthLine = async (line) => {
const id = String(line.id);
const text = String(line.text ?? "").trim();
if (!text) {
if (!text && !line.audio) {
anomalies.push(`line ${id}: empty text — skipped`);
return null;
}
const rel = paths.get(id);
const abs = join(hyperframesDir, rel);
const { ok, words, error } = await synthesizeOne({
provider: ttsProvider,
text,
voiceId,
lang,
speed,
model: flag("tts-model", request.tts_model),
style: flag("style", line.style ?? request.style),
wavAbs: abs,
hyperframesDir,
});
const { ok, words, error, source } = line.audio
? useRecording(line, abs)
: await synthesizeOne({
provider: ttsProvider,
text,
voiceId,
lang,
speed,
model: flag("tts-model", request.tts_model),
style: flag("style", line.style ?? request.style),
wavAbs: abs,
hyperframesDir,
});
if (!ok) {
anomalies.push(`line ${id}: TTS failed — omitted${error ? ` (${error})` : ""}`);
anomalies.push(
`line ${id}: ${line.audio ? "recording" : "TTS"} failed — omitted${error ? ` (${error})` : ""}`,
);
return null;
}
let wordArr = words; // heygen: native; else transcribe
@@ -172,7 +194,13 @@ if (only.has("tts") && lines.length) {
anomalies.push(`line ${id}: bad voice duration — omitted`);
return null;
}
return { id, path: rel, duration_s: r3(dur), words: withWordIds(wordArr) };
return {
id,
path: rel,
duration_s: r3(dur),
words: withWordIds(wordArr),
...(source && { source }),
};
};
const results = await mapWithConcurrency(lines, ttsConcurrency, synthLine);
voices = results.filter(Boolean);
@@ -35,7 +35,7 @@ export function agentWritePath(
export function voicePaths(hyperframesDir, lines, anomalies) {
const taken = new Set();
const paths = new Map();
for (const line of lines.filter((l) => String(l.text ?? "").trim())) {
for (const line of lines.filter((l) => String(l.text ?? "").trim() || l.audio)) {
const rel = agentWritePath(hyperframesDir, `assets/voice/${line.id}.wav`, { anomalies, taken });
taken.add(rel);
paths.set(String(line.id), rel);
@@ -60,9 +60,9 @@ export function writtenAssets({ only, lines, voices, ttsProvider, bgm, bgmFields
path: voice.path,
type: "voice",
source: "generated",
intent: textById.get(voice.id),
intent: textById.get(voice.id) || "your own recording",
duration: voice.duration_s,
provider: ttsProvider,
provider: voice.source ?? ttsProvider,
});
}
}
@@ -3,7 +3,7 @@ import { test } from "node:test";
import { mkdirSync, mkdtempSync, readdirSync, readFileSync, rmSync, writeFileSync } from "node:fs";
import { join } from "node:path";
import { tmpdir } from "node:os";
import { findByPrompt, readManifest } from "../../../scripts/lib/manifest.mjs";
import { AGENT_SOURCES, findByPrompt, readManifest } from "../../../scripts/lib/manifest.mjs";
import { agentWritePath, recordInManifest, voicePaths, writtenAssets } from "./media-record.mjs";
const noBgm = { bgm: null, bgmFields: { bgm_pending: false } };
@@ -222,3 +222,18 @@ test("a record never takes the id of a download still in flight", (t) => {
["bgm_002"],
);
});
// An engine-written copy stays an agent source, or every rerun would write a numbered copy beside it
// (AGENT_SOURCES, manifest.mjs); what says it is the person's own voice is the provider and intent.
test("a recording line is recorded as your own recording, still an engine-written file", () => {
const [asset] = writtenAssets({
only: new Set(["tts"]),
lines: [{ id: "01", audio: "take.m4a" }],
voices: [{ id: "01", path: "assets/voice/01.wav", duration_s: 2, source: "recording" }],
ttsProvider: "recording",
});
assert.equal(asset.provider, "recording");
assert.equal(asset.intent, "your own recording");
assert.ok(AGENT_SOURCES.includes(asset.source));
});
@@ -0,0 +1,53 @@
// recording.mjs — the person's own voice recording as a voice line: converted to the engine's wav,
// time-stretched by `speed` with the pitch kept, with word timings scaled to match.
import { spawnSync } from "node:child_process";
import { existsSync, mkdirSync } from "node:fs";
import { dirname } from "node:path";
// ffmpeg's atempo takes 0.5..2 per stage; chain stages for the rest of the 0.1..3 speed range.
export function atempoChain(speed) {
const stages = [];
let left = speed;
while (left > 2) {
stages.push(2);
left /= 2;
}
while (left < 0.5) {
stages.push(0.5);
left /= 0.5;
}
if (Math.abs(left - 1) > 1e-9 || stages.length === 0) stages.push(left);
return stages.map((s) => `atempo=${Number(s.toFixed(6))}`).join(",");
}
/** src -> 44.1k mono wav at wavAbs, played `speed` times faster. Returns { ok, error? }. */
export function stretchToWav(src, wavAbs, speed = 1) {
if (!existsSync(src)) return { ok: false, error: `recording not found: ${src}` };
mkdirSync(dirname(wavAbs), { recursive: true });
const filter = speed === 1 ? [] : ["-af", atempoChain(speed)];
const r = spawnSync(
"ffmpeg",
["-y", "-loglevel", "error", "-i", src, ...filter, "-ar", "44100", "-ac", "1", wavAbs],
{ encoding: "utf8" },
);
if (r.error || r.status !== 0 || !existsSync(wavAbs)) {
return { ok: false, error: `ffmpeg could not convert ${src}: ${r.error?.message ?? r.stderr}` };
}
return { ok: true };
}
/** Caller-supplied word timings ({text|word, start, end}) as [{text,start,end}], or null when absent. */
export function parseWords(words) {
if (!Array.isArray(words) || words.length === 0) return null;
return words.map((w, i) => {
const text = w?.text ?? w?.word;
if (typeof text !== "string" || !Number.isFinite(w.start) || !Number.isFinite(w.end)) {
throw new Error(`words[${i}] needs text (or word), start and end in seconds`);
}
return { text, start: w.start, end: w.end };
});
}
/** Words of a recording played `speed` times faster land at start/speed. */
export const scaleWords = (words, speed) =>
words.map((w) => ({ ...w, start: w.start / speed, end: w.end / speed }));
@@ -0,0 +1,138 @@
import { test } from "node:test";
import assert from "node:assert/strict";
import { spawnSync } from "node:child_process";
import { mkdtempSync, readFileSync, writeFileSync, rmSync } from "node:fs";
import { join, dirname } from "node:path";
import { tmpdir } from "node:os";
import { fileURLToPath } from "node:url";
import { atempoChain, parseWords, scaleWords } from "./lib/recording.mjs";
const AUDIO = join(dirname(fileURLToPath(import.meta.url)), "audio.mjs");
test("atempoChain splits speeds outside ffmpeg's 0.5..2 stage range", () => {
assert.equal(atempoChain(1.5), "atempo=1.5");
assert.equal(atempoChain(3), "atempo=2,atempo=1.5");
assert.equal(atempoChain(0.25), "atempo=0.5,atempo=0.5");
});
test("parseWords takes text or word and rejects a missing time", () => {
assert.deepEqual(parseWords([{ word: "hi", start: 0, end: 1 }]), [
{ text: "hi", start: 0, end: 1 },
]);
assert.equal(parseWords(undefined), null);
assert.throws(() => parseWords([{ text: "hi", start: 0 }]), /words\[0\]/);
});
test("scaleWords moves words by 1/speed", () => {
assert.deepEqual(scaleWords([{ text: "a", start: 1, end: 2 }], 2), [
{ text: "a", start: 0.5, end: 1 },
]);
});
test("a recording line with malformed words is omitted with the reason, not a crash", () => {
const dir = mkdtempSync(join(tmpdir(), "hf-rec-"));
try {
const make = spawnSync(
"ffmpeg",
["-y", "-loglevel", "error", "-f", "lavfi", "-i", "sine=duration=1", join(dir, "take.wav")],
{ encoding: "utf8" },
);
assert.equal(make.status, 0, make.stderr);
writeFileSync(
join(dir, "audio_request.json"),
JSON.stringify({
lines: [{ id: "01", audio: "take.wav", words: [{ text: "x", start: 0 }] }],
}),
);
const run = spawnSync(
process.execPath,
[
AUDIO,
"--request",
join(dir, "audio_request.json"),
"--hyperframes",
dir,
"--out",
join(dir, "m.json"),
"--only",
"tts",
],
{ encoding: "utf8", env: { ...process.env, HOME: dir } },
);
assert.equal(run.status, 0, run.stderr);
assert.match(run.stdout, /line 01: recording failed — omitted \(words\[0\] needs text/);
assert.deepEqual(JSON.parse(readFileSync(join(dir, "m.json"), "utf8")).voices, []);
} finally {
rmSync(dir, { recursive: true, force: true });
}
});
// A 1 s gap then a 1 s tone is a "word" at 1..2 s. At speed 2 the tone must start at 0.5 s in the
// stretched file, which is where the scaled word says it is.
test("a recording at speed 2: the file is stretched and the words still land on the audio", () => {
const dir = mkdtempSync(join(tmpdir(), "hf-rec-"));
try {
const make = spawnSync(
"ffmpeg",
[
"-y",
"-loglevel",
"error",
"-f",
"lavfi",
"-i",
"anullsrc=r=44100:cl=mono",
"-f",
"lavfi",
"-i",
"sine=frequency=440:duration=1:sample_rate=44100",
"-filter_complex",
"[0]atrim=duration=1[s];[s][1]concat=n=2:v=0:a=1",
join(dir, "take.wav"),
],
{ encoding: "utf8" },
);
assert.equal(make.status, 0, make.stderr);
writeFileSync(
join(dir, "audio_request.json"),
JSON.stringify({
speed: 2,
lines: [{ id: "01", audio: "take.wav", words: [{ text: "tone", start: 1, end: 2 }] }],
}),
);
const run = spawnSync(
process.execPath,
[
AUDIO,
"--request",
join(dir, "audio_request.json"),
"--hyperframes",
dir,
"--out",
join(dir, "audio_meta.json"),
"--only",
"tts",
],
{ encoding: "utf8", env: { ...process.env, HOME: dir } },
);
assert.equal(run.status, 0, run.stderr + run.stdout);
const meta = JSON.parse(readFileSync(join(dir, "audio_meta.json"), "utf8"));
const voice = meta.voices[0];
assert.equal(meta.tts_provider, "recording");
assert.equal(voice.source, "recording");
assert.ok(Math.abs(voice.duration_s - 1) < 0.05, `duration ${voice.duration_s}`);
assert.deepEqual([voice.words[0].start, voice.words[0].end], [0.5, 1]);
const probe = spawnSync(
"ffmpeg",
["-i", join(dir, voice.path), "-af", "silencedetect=noise=-40dB:d=0.2", "-f", "null", "-"],
{ encoding: "utf8" },
);
const silenceEnd = Number(/silence_end: ([\d.]+)/.exec(probe.stderr)?.[1]);
assert.ok(
Math.abs(silenceEnd - voice.words[0].start) < 0.08,
`tone starts at ${silenceEnd}s, word says ${voice.words[0].start}s`,
);
} finally {
rmSync(dir, { recursive: true, force: true });
}
});
+1
View File
@@ -10,6 +10,7 @@ node <SKILL_DIR>/audio/scripts/audio.mjs --request ./audio_request.json --out ./
```
- **Request** `{ provider?, voice?, lang?, speed?, tts_model?, style?, lines: [{ id, text, style?, sfx?: [names] }], bgm: { mode?, query?, prompt? } }`: `id` joins each line back to your model; `bgm.mode` = `retrieve | generate | none` (omit for auto). `--only tts,bgm,sfx` runs a subset and merges into an existing `--out`.
- **Your own recording**: a line with `audio` (a path in the project) instead of `text` is used as the voice: `{ id, audio: "take.m4a", words?: [{ text, start, end }] }`. It is converted to `assets/voice/<id>.wav`, `speed` stretches it with the pitch kept and scales `words` the same (without `words` the recording is transcribed). Its `audio_meta.json` voice has `source: "recording"`; with only recording lines `tts_provider` is `"recording"` and no provider or HeyGen sign-in is needed.
- **Gemini narration** is opt-in with `provider: "gemini"` and an API key or service-account credentials. `tts_model` defaults to `gemini-3.8-flash-tts`; 3.8 Flash-Lite, 3.1 Flash Preview, and 2.5 Pro/Flash Preview TTS are also supported. `style` directs delivery; a line's style overrides the request's style. CLI overrides: `--tts-model`, `--style`. See `audio/references/tts.md` for a complete request. The existing automatic provider order is unchanged.
- **Output** `audio_meta.json` (id-keyed): `voices[].{path,duration_s,words[]}` (word timestamps for captions), `sfx[]`, `bgm`, `total_duration_s`.
- **HeyGen free-usage path**: HeyGen CLI auth unlocks TTS plus music retrieval; SFX come from Openverse with no account (`audio/references/sfx.md`). Local/provider-specific generators are explicit alternatives where installed; run `npx hyperframes media-use resolve --doctor` before assuming retrieval or TTS will work.