mirror of
https://github.com/heygen-com/hyperframes.git
synced 2026-11-18 07:50:13 +00:00
feat(media-use): your own voice recording is a voice line, and speed applies to it (#7377)
* feat(media-use): your own voice recording is a voice line, and speed applies to it * fix(media-use): regenerate the skills manifest and keep audio.mjs's header at its limit
This commit is contained in:
@@ -18,8 +18,8 @@
|
||||
"files": 13
|
||||
},
|
||||
"media-use": {
|
||||
"hash": "44ef2d115fa8bebd",
|
||||
"files": 118
|
||||
"hash": "d4221a4cd4118c29",
|
||||
"files": 120
|
||||
},
|
||||
"remotion-to-hyperframes": {
|
||||
"hash": "73b82d19db2ed8c9",
|
||||
|
||||
@@ -52,6 +52,7 @@ import {
|
||||
} from "./lib/tts.mjs";
|
||||
import { generateBgmDetached, inferBgmPrompt, retrieveBgm } from "./lib/bgm.mjs";
|
||||
import { BUNDLED_SFX_DIR, resolveSfx } from "./lib/sfx.mjs";
|
||||
import { parseWords, scaleWords, stretchToWav } from "./lib/recording.mjs";
|
||||
import { mapWithConcurrency } from "./lib/concurrency.mjs";
|
||||
import { openAudioMeta } from "./lib/audio-meta.mjs";
|
||||
import { recordInManifest, voicePaths, writtenAssets } from "./lib/media-record.mjs";
|
||||
@@ -127,42 +128,63 @@ let voices = prev.voices ?? [];
|
||||
let ttsProvider = prev.tts_provider ?? null;
|
||||
let voiceId = prev.voice_id ?? null;
|
||||
if (only.has("tts") && lines.length) {
|
||||
try {
|
||||
ttsProvider = pickProvider(
|
||||
providerOverride || (request.provider === "auto" ? null : request.provider),
|
||||
);
|
||||
} catch (e) {
|
||||
die(e.message);
|
||||
if (lines.some((l) => !l.audio)) {
|
||||
try {
|
||||
ttsProvider = pickProvider(
|
||||
providerOverride || (request.provider === "auto" ? null : request.provider),
|
||||
);
|
||||
} catch (e) {
|
||||
die(e.message);
|
||||
}
|
||||
voiceId = await resolveVoiceId({
|
||||
provider: ttsProvider,
|
||||
userVoice: voiceOverride || request.voice,
|
||||
lang,
|
||||
});
|
||||
} else {
|
||||
ttsProvider = "recording"; // every line is the person's own recording: no TTS provider to pick
|
||||
voiceId = null;
|
||||
}
|
||||
voiceId = await resolveVoiceId({
|
||||
provider: ttsProvider,
|
||||
userVoice: voiceOverride || request.voice,
|
||||
lang,
|
||||
});
|
||||
console.error(`· tts: ${ttsProvider} · voice ${voiceId} · ${lines.length} line(s)`);
|
||||
const paths = voicePaths(hyperframesDir, lines, anomalies);
|
||||
// A line with `audio` is the person's own recording (path relative to the project), with optional
|
||||
// `words`; `speed` applies to it like to a generated voice. Returns synthesizeOne's shape.
|
||||
const useRecording = (line, abs) => {
|
||||
const made = stretchToWav(resolve(hyperframesDir, String(line.audio)), abs, speed);
|
||||
if (!made.ok) return made;
|
||||
try {
|
||||
const given = parseWords(line.words);
|
||||
return { ok: true, words: given && scaleWords(given, speed), source: "recording" };
|
||||
} catch (e) {
|
||||
return { ok: false, error: e.message };
|
||||
}
|
||||
};
|
||||
const synthLine = async (line) => {
|
||||
const id = String(line.id);
|
||||
const text = String(line.text ?? "").trim();
|
||||
if (!text) {
|
||||
if (!text && !line.audio) {
|
||||
anomalies.push(`line ${id}: empty text — skipped`);
|
||||
return null;
|
||||
}
|
||||
const rel = paths.get(id);
|
||||
const abs = join(hyperframesDir, rel);
|
||||
const { ok, words, error } = await synthesizeOne({
|
||||
provider: ttsProvider,
|
||||
text,
|
||||
voiceId,
|
||||
lang,
|
||||
speed,
|
||||
model: flag("tts-model", request.tts_model),
|
||||
style: flag("style", line.style ?? request.style),
|
||||
wavAbs: abs,
|
||||
hyperframesDir,
|
||||
});
|
||||
const { ok, words, error, source } = line.audio
|
||||
? useRecording(line, abs)
|
||||
: await synthesizeOne({
|
||||
provider: ttsProvider,
|
||||
text,
|
||||
voiceId,
|
||||
lang,
|
||||
speed,
|
||||
model: flag("tts-model", request.tts_model),
|
||||
style: flag("style", line.style ?? request.style),
|
||||
wavAbs: abs,
|
||||
hyperframesDir,
|
||||
});
|
||||
if (!ok) {
|
||||
anomalies.push(`line ${id}: TTS failed — omitted${error ? ` (${error})` : ""}`);
|
||||
anomalies.push(
|
||||
`line ${id}: ${line.audio ? "recording" : "TTS"} failed — omitted${error ? ` (${error})` : ""}`,
|
||||
);
|
||||
return null;
|
||||
}
|
||||
let wordArr = words; // heygen: native; else transcribe
|
||||
@@ -172,7 +194,13 @@ if (only.has("tts") && lines.length) {
|
||||
anomalies.push(`line ${id}: bad voice duration — omitted`);
|
||||
return null;
|
||||
}
|
||||
return { id, path: rel, duration_s: r3(dur), words: withWordIds(wordArr) };
|
||||
return {
|
||||
id,
|
||||
path: rel,
|
||||
duration_s: r3(dur),
|
||||
words: withWordIds(wordArr),
|
||||
...(source && { source }),
|
||||
};
|
||||
};
|
||||
const results = await mapWithConcurrency(lines, ttsConcurrency, synthLine);
|
||||
voices = results.filter(Boolean);
|
||||
|
||||
@@ -35,7 +35,7 @@ export function agentWritePath(
|
||||
export function voicePaths(hyperframesDir, lines, anomalies) {
|
||||
const taken = new Set();
|
||||
const paths = new Map();
|
||||
for (const line of lines.filter((l) => String(l.text ?? "").trim())) {
|
||||
for (const line of lines.filter((l) => String(l.text ?? "").trim() || l.audio)) {
|
||||
const rel = agentWritePath(hyperframesDir, `assets/voice/${line.id}.wav`, { anomalies, taken });
|
||||
taken.add(rel);
|
||||
paths.set(String(line.id), rel);
|
||||
@@ -60,9 +60,9 @@ export function writtenAssets({ only, lines, voices, ttsProvider, bgm, bgmFields
|
||||
path: voice.path,
|
||||
type: "voice",
|
||||
source: "generated",
|
||||
intent: textById.get(voice.id),
|
||||
intent: textById.get(voice.id) || "your own recording",
|
||||
duration: voice.duration_s,
|
||||
provider: ttsProvider,
|
||||
provider: voice.source ?? ttsProvider,
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
@@ -3,7 +3,7 @@ import { test } from "node:test";
|
||||
import { mkdirSync, mkdtempSync, readdirSync, readFileSync, rmSync, writeFileSync } from "node:fs";
|
||||
import { join } from "node:path";
|
||||
import { tmpdir } from "node:os";
|
||||
import { findByPrompt, readManifest } from "../../../scripts/lib/manifest.mjs";
|
||||
import { AGENT_SOURCES, findByPrompt, readManifest } from "../../../scripts/lib/manifest.mjs";
|
||||
import { agentWritePath, recordInManifest, voicePaths, writtenAssets } from "./media-record.mjs";
|
||||
|
||||
const noBgm = { bgm: null, bgmFields: { bgm_pending: false } };
|
||||
@@ -222,3 +222,18 @@ test("a record never takes the id of a download still in flight", (t) => {
|
||||
["bgm_002"],
|
||||
);
|
||||
});
|
||||
|
||||
// An engine-written copy stays an agent source, or every rerun would write a numbered copy beside it
|
||||
// (AGENT_SOURCES, manifest.mjs); what says it is the person's own voice is the provider and intent.
|
||||
test("a recording line is recorded as your own recording, still an engine-written file", () => {
|
||||
const [asset] = writtenAssets({
|
||||
only: new Set(["tts"]),
|
||||
lines: [{ id: "01", audio: "take.m4a" }],
|
||||
voices: [{ id: "01", path: "assets/voice/01.wav", duration_s: 2, source: "recording" }],
|
||||
ttsProvider: "recording",
|
||||
});
|
||||
|
||||
assert.equal(asset.provider, "recording");
|
||||
assert.equal(asset.intent, "your own recording");
|
||||
assert.ok(AGENT_SOURCES.includes(asset.source));
|
||||
});
|
||||
|
||||
@@ -0,0 +1,53 @@
|
||||
// recording.mjs — the person's own voice recording as a voice line: converted to the engine's wav,
|
||||
// time-stretched by `speed` with the pitch kept, with word timings scaled to match.
|
||||
import { spawnSync } from "node:child_process";
|
||||
import { existsSync, mkdirSync } from "node:fs";
|
||||
import { dirname } from "node:path";
|
||||
|
||||
// ffmpeg's atempo takes 0.5..2 per stage; chain stages for the rest of the 0.1..3 speed range.
|
||||
export function atempoChain(speed) {
|
||||
const stages = [];
|
||||
let left = speed;
|
||||
while (left > 2) {
|
||||
stages.push(2);
|
||||
left /= 2;
|
||||
}
|
||||
while (left < 0.5) {
|
||||
stages.push(0.5);
|
||||
left /= 0.5;
|
||||
}
|
||||
if (Math.abs(left - 1) > 1e-9 || stages.length === 0) stages.push(left);
|
||||
return stages.map((s) => `atempo=${Number(s.toFixed(6))}`).join(",");
|
||||
}
|
||||
|
||||
/** src -> 44.1k mono wav at wavAbs, played `speed` times faster. Returns { ok, error? }. */
|
||||
export function stretchToWav(src, wavAbs, speed = 1) {
|
||||
if (!existsSync(src)) return { ok: false, error: `recording not found: ${src}` };
|
||||
mkdirSync(dirname(wavAbs), { recursive: true });
|
||||
const filter = speed === 1 ? [] : ["-af", atempoChain(speed)];
|
||||
const r = spawnSync(
|
||||
"ffmpeg",
|
||||
["-y", "-loglevel", "error", "-i", src, ...filter, "-ar", "44100", "-ac", "1", wavAbs],
|
||||
{ encoding: "utf8" },
|
||||
);
|
||||
if (r.error || r.status !== 0 || !existsSync(wavAbs)) {
|
||||
return { ok: false, error: `ffmpeg could not convert ${src}: ${r.error?.message ?? r.stderr}` };
|
||||
}
|
||||
return { ok: true };
|
||||
}
|
||||
|
||||
/** Caller-supplied word timings ({text|word, start, end}) as [{text,start,end}], or null when absent. */
|
||||
export function parseWords(words) {
|
||||
if (!Array.isArray(words) || words.length === 0) return null;
|
||||
return words.map((w, i) => {
|
||||
const text = w?.text ?? w?.word;
|
||||
if (typeof text !== "string" || !Number.isFinite(w.start) || !Number.isFinite(w.end)) {
|
||||
throw new Error(`words[${i}] needs text (or word), start and end in seconds`);
|
||||
}
|
||||
return { text, start: w.start, end: w.end };
|
||||
});
|
||||
}
|
||||
|
||||
/** Words of a recording played `speed` times faster land at start/speed. */
|
||||
export const scaleWords = (words, speed) =>
|
||||
words.map((w) => ({ ...w, start: w.start / speed, end: w.end / speed }));
|
||||
@@ -0,0 +1,138 @@
|
||||
import { test } from "node:test";
|
||||
import assert from "node:assert/strict";
|
||||
import { spawnSync } from "node:child_process";
|
||||
import { mkdtempSync, readFileSync, writeFileSync, rmSync } from "node:fs";
|
||||
import { join, dirname } from "node:path";
|
||||
import { tmpdir } from "node:os";
|
||||
import { fileURLToPath } from "node:url";
|
||||
import { atempoChain, parseWords, scaleWords } from "./lib/recording.mjs";
|
||||
|
||||
const AUDIO = join(dirname(fileURLToPath(import.meta.url)), "audio.mjs");
|
||||
|
||||
test("atempoChain splits speeds outside ffmpeg's 0.5..2 stage range", () => {
|
||||
assert.equal(atempoChain(1.5), "atempo=1.5");
|
||||
assert.equal(atempoChain(3), "atempo=2,atempo=1.5");
|
||||
assert.equal(atempoChain(0.25), "atempo=0.5,atempo=0.5");
|
||||
});
|
||||
|
||||
test("parseWords takes text or word and rejects a missing time", () => {
|
||||
assert.deepEqual(parseWords([{ word: "hi", start: 0, end: 1 }]), [
|
||||
{ text: "hi", start: 0, end: 1 },
|
||||
]);
|
||||
assert.equal(parseWords(undefined), null);
|
||||
assert.throws(() => parseWords([{ text: "hi", start: 0 }]), /words\[0\]/);
|
||||
});
|
||||
|
||||
test("scaleWords moves words by 1/speed", () => {
|
||||
assert.deepEqual(scaleWords([{ text: "a", start: 1, end: 2 }], 2), [
|
||||
{ text: "a", start: 0.5, end: 1 },
|
||||
]);
|
||||
});
|
||||
|
||||
test("a recording line with malformed words is omitted with the reason, not a crash", () => {
|
||||
const dir = mkdtempSync(join(tmpdir(), "hf-rec-"));
|
||||
try {
|
||||
const make = spawnSync(
|
||||
"ffmpeg",
|
||||
["-y", "-loglevel", "error", "-f", "lavfi", "-i", "sine=duration=1", join(dir, "take.wav")],
|
||||
{ encoding: "utf8" },
|
||||
);
|
||||
assert.equal(make.status, 0, make.stderr);
|
||||
writeFileSync(
|
||||
join(dir, "audio_request.json"),
|
||||
JSON.stringify({
|
||||
lines: [{ id: "01", audio: "take.wav", words: [{ text: "x", start: 0 }] }],
|
||||
}),
|
||||
);
|
||||
const run = spawnSync(
|
||||
process.execPath,
|
||||
[
|
||||
AUDIO,
|
||||
"--request",
|
||||
join(dir, "audio_request.json"),
|
||||
"--hyperframes",
|
||||
dir,
|
||||
"--out",
|
||||
join(dir, "m.json"),
|
||||
"--only",
|
||||
"tts",
|
||||
],
|
||||
{ encoding: "utf8", env: { ...process.env, HOME: dir } },
|
||||
);
|
||||
assert.equal(run.status, 0, run.stderr);
|
||||
assert.match(run.stdout, /line 01: recording failed — omitted \(words\[0\] needs text/);
|
||||
assert.deepEqual(JSON.parse(readFileSync(join(dir, "m.json"), "utf8")).voices, []);
|
||||
} finally {
|
||||
rmSync(dir, { recursive: true, force: true });
|
||||
}
|
||||
});
|
||||
|
||||
// A 1 s gap then a 1 s tone is a "word" at 1..2 s. At speed 2 the tone must start at 0.5 s in the
|
||||
// stretched file, which is where the scaled word says it is.
|
||||
test("a recording at speed 2: the file is stretched and the words still land on the audio", () => {
|
||||
const dir = mkdtempSync(join(tmpdir(), "hf-rec-"));
|
||||
try {
|
||||
const make = spawnSync(
|
||||
"ffmpeg",
|
||||
[
|
||||
"-y",
|
||||
"-loglevel",
|
||||
"error",
|
||||
"-f",
|
||||
"lavfi",
|
||||
"-i",
|
||||
"anullsrc=r=44100:cl=mono",
|
||||
"-f",
|
||||
"lavfi",
|
||||
"-i",
|
||||
"sine=frequency=440:duration=1:sample_rate=44100",
|
||||
"-filter_complex",
|
||||
"[0]atrim=duration=1[s];[s][1]concat=n=2:v=0:a=1",
|
||||
join(dir, "take.wav"),
|
||||
],
|
||||
{ encoding: "utf8" },
|
||||
);
|
||||
assert.equal(make.status, 0, make.stderr);
|
||||
writeFileSync(
|
||||
join(dir, "audio_request.json"),
|
||||
JSON.stringify({
|
||||
speed: 2,
|
||||
lines: [{ id: "01", audio: "take.wav", words: [{ text: "tone", start: 1, end: 2 }] }],
|
||||
}),
|
||||
);
|
||||
const run = spawnSync(
|
||||
process.execPath,
|
||||
[
|
||||
AUDIO,
|
||||
"--request",
|
||||
join(dir, "audio_request.json"),
|
||||
"--hyperframes",
|
||||
dir,
|
||||
"--out",
|
||||
join(dir, "audio_meta.json"),
|
||||
"--only",
|
||||
"tts",
|
||||
],
|
||||
{ encoding: "utf8", env: { ...process.env, HOME: dir } },
|
||||
);
|
||||
assert.equal(run.status, 0, run.stderr + run.stdout);
|
||||
const meta = JSON.parse(readFileSync(join(dir, "audio_meta.json"), "utf8"));
|
||||
const voice = meta.voices[0];
|
||||
assert.equal(meta.tts_provider, "recording");
|
||||
assert.equal(voice.source, "recording");
|
||||
assert.ok(Math.abs(voice.duration_s - 1) < 0.05, `duration ${voice.duration_s}`);
|
||||
assert.deepEqual([voice.words[0].start, voice.words[0].end], [0.5, 1]);
|
||||
const probe = spawnSync(
|
||||
"ffmpeg",
|
||||
["-i", join(dir, voice.path), "-af", "silencedetect=noise=-40dB:d=0.2", "-f", "null", "-"],
|
||||
{ encoding: "utf8" },
|
||||
);
|
||||
const silenceEnd = Number(/silence_end: ([\d.]+)/.exec(probe.stderr)?.[1]);
|
||||
assert.ok(
|
||||
Math.abs(silenceEnd - voice.words[0].start) < 0.08,
|
||||
`tone starts at ${silenceEnd}s, word says ${voice.words[0].start}s`,
|
||||
);
|
||||
} finally {
|
||||
rmSync(dir, { recursive: true, force: true });
|
||||
}
|
||||
});
|
||||
@@ -10,6 +10,7 @@ node <SKILL_DIR>/audio/scripts/audio.mjs --request ./audio_request.json --out ./
|
||||
```
|
||||
|
||||
- **Request** `{ provider?, voice?, lang?, speed?, tts_model?, style?, lines: [{ id, text, style?, sfx?: [names] }], bgm: { mode?, query?, prompt? } }`: `id` joins each line back to your model; `bgm.mode` = `retrieve | generate | none` (omit for auto). `--only tts,bgm,sfx` runs a subset and merges into an existing `--out`.
|
||||
- **Your own recording**: a line with `audio` (a path in the project) instead of `text` is used as the voice: `{ id, audio: "take.m4a", words?: [{ text, start, end }] }`. It is converted to `assets/voice/<id>.wav`, `speed` stretches it with the pitch kept and scales `words` the same (without `words` the recording is transcribed). Its `audio_meta.json` voice has `source: "recording"`; with only recording lines `tts_provider` is `"recording"` and no provider or HeyGen sign-in is needed.
|
||||
- **Gemini narration** is opt-in with `provider: "gemini"` and an API key or service-account credentials. `tts_model` defaults to `gemini-3.8-flash-tts`; 3.8 Flash-Lite, 3.1 Flash Preview, and 2.5 Pro/Flash Preview TTS are also supported. `style` directs delivery; a line's style overrides the request's style. CLI overrides: `--tts-model`, `--style`. See `audio/references/tts.md` for a complete request. The existing automatic provider order is unchanged.
|
||||
- **Output** `audio_meta.json` (id-keyed): `voices[].{path,duration_s,words[]}` (word timestamps for captions), `sfx[]`, `bgm`, `total_duration_s`.
|
||||
- **HeyGen free-usage path**: HeyGen CLI auth unlocks TTS plus music retrieval; SFX come from Openverse with no account (`audio/references/sfx.md`). Local/provider-specific generators are explicit alternatives where installed; run `npx hyperframes media-use resolve --doctor` before assuming retrieval or TTS will work.
|
||||
|
||||
Reference in New Issue
Block a user