mirror of
https://github.com/heygen-com/hyperframes.git
synced 2026-10-30 09:16:55 +00:00
fix(media-use): never replace a voice, music or sound file the person put in the project (#4665)
* fix(media-use): record agent-made voice, music and sound effects in the project's media manifest * fix(media-use): record in place only when --source says the agent made the file * test(media-use): a rewritten agent file keeps one manifest record * fix(media-use): leave a person's own sound effect unlabelled and mark locally made music generated * refactor(media-use): record agent audio in-process through the shared manifest library * style(media-use): format the in-process recording and refresh the skills manifest * chore(skills): refresh the skills manifest for the media-use changes * fix(media-use): report a broken ffprobe setting plainly and document the project sfx source * style(skills): format the pipeline manifest assertion * chore(skills): refresh the skills manifest * fix(media-use): never replace a voice, music or sound file the person put in the project * style(media-use): format the keep-person-files change * chore(skills): refresh the skills manifest * fix(media-use): a regenerated file gets a fresh record, and lookups skip the replaced one * test(media-use): a replaced record no longer answers a prompt lookup in the CLI * chore(skills): refresh the skills manifest * chore(skills): refresh the skills manifest from a clean tree * fix(media-use): never give a recorded file the id of a download in flight * chore(skills): refresh the media-use manifest hash * fix(media-use): say when audio lands on a new name, and never let two lines share one * style(media-use): format the keep-files change * chore(skills): refresh the media-use manifest hash * fix(media-use): keep two sound effects off one file when a name is taken * refactor(media-use): drop a set the one-at-a-time downloads never needed * chore(skills): refresh the media-use manifest hash * fix(media-use): keep two sound effects off one recorded file on a rerun * style(media-use): format the rerun test * chore(skills): refresh the media-use manifest hash * fix(media-use): list only a file's newest record in candidates and the media index * revert(media-use): leave promote as it was, nothing calls it * test(media-use): give the index test's two assets their own files * chore(skills): refresh the media-use manifest hash * chore(skills): regenerate the manifest after merging the base branch * fix(core): a figma import lists each media file once, like media-use
This commit is contained in:
@@ -79,6 +79,9 @@ export function appendRecord(projectDir, record) {
|
||||
appendFileSync(p, line);
|
||||
}
|
||||
|
||||
/** Sources that mean the agent made or fetched the file; any other file is the person's own. */
|
||||
export const AGENT_SOURCES = ["generated", "search", "bundled"];
|
||||
|
||||
/** The record a path has now: the manifest only appends, so the last one for a path wins. */
|
||||
export function latestRecordFor(projectDir, path) {
|
||||
return readManifest(projectDir).findLast((record) => record.path === path);
|
||||
|
||||
@@ -5,6 +5,7 @@ import { existsSync, statSync, writeFileSync, renameSync, rmSync, realpathSync }
|
||||
import { resolve, join, extname, basename, relative, isAbsolute, sep } from "node:path";
|
||||
import { parseArgs } from "node:util";
|
||||
import {
|
||||
AGENT_SOURCES,
|
||||
appendRecord,
|
||||
latestRecordFor,
|
||||
recordInPlace,
|
||||
@@ -60,7 +61,6 @@ import {
|
||||
} from "./lib/local-media-search.mjs";
|
||||
|
||||
const INGEST_TYPES = listTypes();
|
||||
const RECORDED_SOURCES = ["generated", "search", "bundled"];
|
||||
const DEFAULT_EXT = {
|
||||
bgm: ".wav",
|
||||
sfx: ".mp3",
|
||||
@@ -130,7 +130,7 @@ Options:
|
||||
--reuse <sha> Import a specific global-cache asset (by content sha/prefix,
|
||||
from --candidates) into this project
|
||||
--from <file> Freeze a local file or direct public URL (ingest)
|
||||
--source <how> With --from: how the file was made (${RECORDED_SOURCES.join(" | ")}).
|
||||
--source <how> With --from: how the file was made (${AGENT_SOURCES.join(" | ")}).
|
||||
A file already inside the project is then recorded where it is
|
||||
--params <json> Build an explicit parametric LUT (lut/grade only)
|
||||
--for <media> Analyze a local image/video and add measured grade adjust
|
||||
@@ -918,8 +918,8 @@ async function ingest(src) {
|
||||
console.error(`error: refusing to ingest a 0-byte file: ${src}`);
|
||||
process.exit(2);
|
||||
}
|
||||
if (args.source && !RECORDED_SOURCES.includes(args.source)) {
|
||||
console.error(`error: --source takes one of: ${RECORDED_SOURCES.join(", ")}`);
|
||||
if (args.source && !AGENT_SOURCES.includes(args.source)) {
|
||||
console.error(`error: --source takes one of: ${AGENT_SOURCES.join(", ")}`);
|
||||
process.exit(2);
|
||||
}
|
||||
if (args.source && (type === "lut" || type === "grade")) {
|
||||
|
||||
@@ -54,7 +54,7 @@
|
||||
"files": 1
|
||||
},
|
||||
"media-use": {
|
||||
"hash": "1871ad36628ac54b",
|
||||
"hash": "0744050e3114e60a",
|
||||
"files": 104
|
||||
},
|
||||
"motion-graphics": {
|
||||
|
||||
@@ -15,6 +15,8 @@ One music bed per composition, produced by the shared audio engine (`scripts/aud
|
||||
- **`query`** — the mood, used for retrieval and as a fallback prompt seed (e.g. a storyboard's `music:` field, falling back to `message` → `arc` → `"calm cinematic underscore"`).
|
||||
- **`prompt`** — an explicit full prompt for generation; omit and the engine infers one (see Mood inference). Optional `blob` / `archetype` / `arc` feed that inference.
|
||||
|
||||
Both routes keep a file of yours already at the output name: the engine writes the next free name (`track-2.mp3`), reports it as an anomaly, and `bgm.path` carries the real path.
|
||||
|
||||
## HeyGen retrieval (default)
|
||||
|
||||
`searchSounds(query, "music", { limit: 5 })` → `GET /audio/sounds?query=<mood>&type=music&limit=5`. Take the top result (ranked by `score`), download its presigned `audio_url` → `assets/bgm/track.mp3`. Synchronous. No match → skip (BGM is optional; never fail the render over it). Cue written to `audio_meta.json`:
|
||||
|
||||
@@ -16,15 +16,14 @@ Each line names the effects it wants: `lines[].sfx: ["whoosh", "ui click"]`. The
|
||||
"id": "3", // joins the cue to the caller's model (frame / scene / segment)
|
||||
"name": "whoosh",
|
||||
"file": "assets/sfx/whoosh.mp3", // downloaded or copied, relative to project root
|
||||
"source": "heygen" | "local" | "project", // which route resolved it; "project": a file already
|
||||
// in assets/sfx under that name that is not the library's copy
|
||||
"source": "heygen" | "local", // which route resolved it
|
||||
"offset_s": 0, // delay from the line's start
|
||||
"duration_s": 0.57,
|
||||
"volume": 0.35 // SFX sit UNDER voice + BGM
|
||||
}
|
||||
```
|
||||
|
||||
A cue that matches nothing is **skipped** (recorded as an anomaly); SFX never blocks a render.
|
||||
A cue that matches nothing is **skipped** (recorded as an anomaly); SFX never blocks a render. Neither route replaces a file of yours already at the output name: the cue gets the next free name (`whoosh-2.mp3`), reported as an anomaly, and `file` carries the real path.
|
||||
|
||||
## HeyGen retrieval (credentialed)
|
||||
|
||||
|
||||
@@ -135,7 +135,7 @@ node <SKILL_DIR>/audio/scripts/audio.mjs \
|
||||
--request ./audio_request.json --hyperframes . --out ./audio_meta.json --only tts
|
||||
```
|
||||
|
||||
The engine saves `assets/voice/intro.wav`, measures its duration, and transcribes
|
||||
The engine saves `assets/voice/intro.wav` (or `intro-2.wav` when a file of yours already has that name; `voices[].path` says which), measures its duration, and transcribes
|
||||
it into `voices[].words` in `audio_meta.json`. Check that every requested line
|
||||
has audio and nonempty word timings before building a captioned video. Review
|
||||
the timings against the actual audio; transcription is estimated alignment,
|
||||
|
||||
@@ -55,7 +55,7 @@ import { generateBgmDetached, inferBgmPrompt, retrieveBgm } from "./lib/bgm.mjs"
|
||||
import { resolveSfx } from "./lib/sfx.mjs";
|
||||
import { mapWithConcurrency } from "./lib/concurrency.mjs";
|
||||
import { openAudioMeta } from "./lib/audio-meta.mjs";
|
||||
import { recordInManifest, writtenAssets } from "./lib/media-record.mjs";
|
||||
import { recordInManifest, voicePaths, writtenAssets } from "./lib/media-record.mjs";
|
||||
|
||||
const HERE = dirname(fileURLToPath(import.meta.url));
|
||||
const argv = process.argv.slice(2);
|
||||
@@ -138,6 +138,7 @@ if (only.has("tts") && lines.length) {
|
||||
lang,
|
||||
});
|
||||
console.error(`· tts: ${ttsProvider} · voice ${voiceId} · ${lines.length} line(s)`);
|
||||
const paths = voicePaths(hyperframesDir, lines, anomalies);
|
||||
const synthLine = async (line) => {
|
||||
const id = String(line.id);
|
||||
const text = String(line.text ?? "").trim();
|
||||
@@ -145,7 +146,7 @@ if (only.has("tts") && lines.length) {
|
||||
anomalies.push(`line ${id}: empty text — skipped`);
|
||||
return null;
|
||||
}
|
||||
const rel = `assets/voice/${id}.wav`;
|
||||
const rel = paths.get(id);
|
||||
const abs = join(hyperframesDir, rel);
|
||||
const { ok, words, error } = await synthesizeOne({
|
||||
provider: ttsProvider,
|
||||
@@ -217,6 +218,7 @@ if (only.has("bgm")) {
|
||||
headers: heygenAuthHeaders(),
|
||||
hyperframesDir,
|
||||
hasVoice,
|
||||
anomalies,
|
||||
});
|
||||
if (bgm) {
|
||||
bgmFields.bgm_provider = "heygen";
|
||||
@@ -243,6 +245,7 @@ if (only.has("bgm")) {
|
||||
lyriaRecipe: existsSync(lyriaRecipe) ? lyriaRecipe : null,
|
||||
seedSeconds,
|
||||
hasVoice,
|
||||
anomalies,
|
||||
});
|
||||
if (gen.disabled) {
|
||||
anomalies.push(`bgm: ${gen.reason}`);
|
||||
|
||||
@@ -1,9 +1,10 @@
|
||||
import { strict as assert } from "node:assert";
|
||||
import { test } from "node:test";
|
||||
import { mkdtempSync, rmSync, existsSync, writeFileSync } from "node:fs";
|
||||
import { mkdtempSync, mkdirSync, readFileSync, rmSync, existsSync, writeFileSync } from "node:fs";
|
||||
import { join, dirname } from "node:path";
|
||||
import { tmpdir } from "node:os";
|
||||
import { fileURLToPath } from "node:url";
|
||||
import { recordInManifest } from "./lib/media-record.mjs";
|
||||
import { resolveSfx } from "./lib/sfx.mjs";
|
||||
|
||||
// Proves the relocated engine (skills/media-use/audio/) still resolves its
|
||||
@@ -48,9 +49,12 @@ test("an unknown cue is reported, not fatal", async () => {
|
||||
}
|
||||
});
|
||||
|
||||
test("a person's own file under a bundled name is not labelled as the library's", async () => {
|
||||
test("a person's own file under a bundled name survives, and the cue gets the next free name", async () => {
|
||||
const dir = mkdtempSync(join(tmpdir(), "mu-audio-"));
|
||||
try {
|
||||
const own = join(dir, "assets", "sfx", "whoosh.mp3");
|
||||
mkdirSync(dirname(own), { recursive: true });
|
||||
writeFileSync(own, "the person's own whoosh");
|
||||
const resolve = () =>
|
||||
resolveSfx({
|
||||
cues: [{ id: "1", name: "whoosh" }],
|
||||
@@ -58,16 +62,65 @@ test("a person's own file under a bundled name is not labelled as the library's"
|
||||
hyperframesDir: dir,
|
||||
sfxLibDir,
|
||||
});
|
||||
const copied = await resolve();
|
||||
const reused = await resolve();
|
||||
writeFileSync(join(dir, copied.sfx[0].file), "the person's own whoosh");
|
||||
const own = await resolve();
|
||||
|
||||
const first = await resolve();
|
||||
const second = await resolve();
|
||||
|
||||
assert.equal(readFileSync(own, "utf8"), "the person's own whoosh");
|
||||
assert.deepEqual(
|
||||
[copied, reused, own].map(({ sfx }) => sfx[0].source),
|
||||
["local", "local", "project"],
|
||||
[first, second].map(({ sfx }) => [sfx[0].file, sfx[0].source]),
|
||||
[
|
||||
["assets/sfx/whoosh-2.mp3", "local"],
|
||||
["assets/sfx/whoosh-2.mp3", "local"],
|
||||
],
|
||||
);
|
||||
assert.deepEqual(
|
||||
readFileSync(join(dir, "assets/sfx/whoosh-2.mp3")),
|
||||
readFileSync(join(sfxLibDir, "whoosh.mp3")),
|
||||
);
|
||||
} finally {
|
||||
rmSync(dir, { recursive: true, force: true });
|
||||
}
|
||||
});
|
||||
|
||||
test("two effects never share a file when one's name is taken by the person, run after run", async () => {
|
||||
const dir = mkdtempSync(join(tmpdir(), "mu-audio-"));
|
||||
try {
|
||||
mkdirSync(join(dir, "assets", "sfx"), { recursive: true });
|
||||
writeFileSync(join(dir, "assets", "sfx", "glitch.mp3"), "the person's own glitch");
|
||||
const realFetch = globalThis.fetch;
|
||||
globalThis.fetch = async (url) => {
|
||||
const query = new URL(url).searchParams.get("query");
|
||||
if (query)
|
||||
return Response.json({ data: [{ audio_url: `https://sound.test/${query}`, score: 0.6 }] });
|
||||
return new Response(`bytes of ${new URL(url).pathname}`);
|
||||
};
|
||||
const run = async (names) => {
|
||||
const { sfx } = await resolveSfx({
|
||||
cues: names.map((name, index) => ({ id: String(index), name })),
|
||||
heygenOK: true,
|
||||
headers: {},
|
||||
hyperframesDir: dir,
|
||||
sfxLibDir,
|
||||
});
|
||||
const files = sfx.map(({ file }) => file);
|
||||
recordInManifest(
|
||||
dir,
|
||||
[...new Set(files)].map((path) => ({ path, type: "sfx", source: "search" })),
|
||||
);
|
||||
return files;
|
||||
};
|
||||
try {
|
||||
assert.deepEqual(await run(["glitch"]), ["assets/sfx/glitch-2.mp3"]);
|
||||
assert.deepEqual(await run(["glitch", "glitch 2", "glitch"]), [
|
||||
"assets/sfx/glitch-2.mp3",
|
||||
"assets/sfx/glitch-2-2.mp3",
|
||||
"assets/sfx/glitch-2.mp3",
|
||||
]);
|
||||
} finally {
|
||||
globalThis.fetch = realFetch;
|
||||
}
|
||||
} finally {
|
||||
rmSync(dir, { recursive: true, force: true });
|
||||
}
|
||||
});
|
||||
|
||||
@@ -9,9 +9,15 @@ import { spawnSync } from "node:child_process";
|
||||
// external audio tools are fixtures. This is not a live synthesis/render test.
|
||||
for (const expired of [false, true]) {
|
||||
for (const only of ["tts", "tts,bgm,sfx"]) {
|
||||
// With expired credentials the person already has an assets/voice/intro.wav, which the engine must keep.
|
||||
const voicePath = expired ? "assets/voice/intro-2.wav" : "assets/voice/intro.wav";
|
||||
test(`Gemini engine returns caption metadata with ${expired ? "expired" : "absent"} HeyGen credentials (${only})`, (t) => {
|
||||
const dir = mkdtempSync(join(tmpdir(), "hf-gemini-pipeline-"));
|
||||
t.after(() => rmSync(dir, { recursive: true, force: true }));
|
||||
if (expired) {
|
||||
mkdirSync(join(dir, "assets/voice"), { recursive: true });
|
||||
writeFileSync(join(dir, "assets/voice/intro.wav"), "the person's own intro");
|
||||
}
|
||||
const config = join(dir, "heygen");
|
||||
mkdirSync(config);
|
||||
if (expired)
|
||||
@@ -48,7 +54,7 @@ const assert = require('node:assert/strict');
|
||||
const fs = require('node:fs');
|
||||
const path = require('node:path');
|
||||
const args = process.argv.slice(2);
|
||||
assert.deepEqual(args.slice(0, 3), ['hyperframes', 'transcribe', 'assets/voice/intro.wav']);
|
||||
assert.deepEqual(args.slice(0, 3), ['hyperframes', 'transcribe', '${voicePath}']);
|
||||
assert.ok(fs.existsSync(args[2]));
|
||||
assert.equal(args[args.indexOf('--model')+1], 'small.en');
|
||||
fs.writeFileSync(path.join(args[args.indexOf('--dir')+1], 'transcript.json'), JSON.stringify([
|
||||
@@ -99,7 +105,7 @@ fs.writeFileSync(path.join(args[args.indexOf('--dir')+1], 'transcript.json'), JS
|
||||
assert.deepEqual(meta.voices, [
|
||||
{
|
||||
id: "intro",
|
||||
path: "assets/voice/intro.wav",
|
||||
path: voicePath,
|
||||
duration_s: 1.25,
|
||||
words: [
|
||||
{ id: "w0", text: "Hello", start: 0.1, end: 0.4 },
|
||||
@@ -113,8 +119,13 @@ fs.writeFileSync(path.join(args[args.indexOf('--dir')+1], 'transcript.json'), JS
|
||||
manifest
|
||||
.map((line) => JSON.parse(line))
|
||||
.map(({ path, type, source }) => [path, type, source]),
|
||||
[["assets/voice/intro.wav", "voice", "generated"]],
|
||||
[[voicePath, "voice", "generated"]],
|
||||
);
|
||||
if (expired)
|
||||
assert.equal(
|
||||
readFileSync(join(dir, "assets/voice/intro.wav"), "utf8"),
|
||||
"the person's own intro",
|
||||
);
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
@@ -12,9 +12,10 @@
|
||||
// Missing/failed BGM never blocks a render.
|
||||
|
||||
import { spawn, spawnSync } from "node:child_process";
|
||||
import { existsSync, mkdirSync, openSync, closeSync } from "node:fs";
|
||||
import { existsSync, mkdirSync, openSync, closeSync, rmSync } from "node:fs";
|
||||
import { join } from "node:path";
|
||||
import { downloadTo, searchSounds } from "./heygen.mjs";
|
||||
import { agentWritePath } from "./media-record.mjs";
|
||||
import { pythonInvocation } from "./python.mjs";
|
||||
|
||||
const r3 = (x) => Number(x.toFixed(3));
|
||||
@@ -45,12 +46,12 @@ function pipInstall(deps) {
|
||||
}
|
||||
|
||||
// ── retrieval (HeyGen music library) ──────────────────────────────────────────
|
||||
export async function retrieveBgm({ query, headers, hyperframesDir, hasVoice }) {
|
||||
export async function retrieveBgm({ query, headers, hyperframesDir, hasVoice, anomalies }) {
|
||||
const q = query || "calm cinematic underscore";
|
||||
const results = await searchSounds(q, "music", headers, { limit: 5 });
|
||||
if (!results.length) return null;
|
||||
const top = results[0];
|
||||
const rel = "assets/bgm/track.mp3";
|
||||
const rel = agentWritePath(hyperframesDir, "assets/bgm/track.mp3", { anomalies });
|
||||
await downloadTo(top.audio_url, join(hyperframesDir, rel));
|
||||
return {
|
||||
path: rel,
|
||||
@@ -113,8 +114,9 @@ export function generateBgmDetached({
|
||||
lyriaRecipe,
|
||||
seedSeconds = 28,
|
||||
hasVoice,
|
||||
anomalies,
|
||||
}) {
|
||||
const rel = "assets/bgm/track.wav";
|
||||
const rel = agentWritePath(hyperframesDir, "assets/bgm/track.wav", { anomalies });
|
||||
const abs = join(hyperframesDir, rel);
|
||||
mkdirSync(join(hyperframesDir, "assets", "bgm"), { recursive: true });
|
||||
const log = join(hyperframesDir, "assets", "bgm", `bgm-${Date.now()}.log`);
|
||||
@@ -141,6 +143,7 @@ export function generateBgmDetached({
|
||||
"--prompt",
|
||||
prompt,
|
||||
]);
|
||||
rmSync(abs, { force: true }); // wait-bgm takes any file here as the finished track
|
||||
const proc = spawn(cmd, args, { detached: true, stdio: ["ignore", fd, fd] });
|
||||
proc.unref();
|
||||
closeSync(fd);
|
||||
@@ -159,6 +162,7 @@ export function generateBgmDetached({
|
||||
const loops = targetS > seedS ? Math.ceil(targetS / seedS) : 1;
|
||||
const script = musicgenScript({ prompt, abs, targetS, seedS });
|
||||
const { cmd, args } = pythonInvocation(["-c", script]);
|
||||
rmSync(abs, { force: true });
|
||||
const proc = spawn(cmd, args, { detached: true, stdio: ["ignore", fd, fd] });
|
||||
proc.unref();
|
||||
closeSync(fd);
|
||||
|
||||
@@ -1,6 +1,15 @@
|
||||
import { test } from "node:test";
|
||||
import assert from "node:assert/strict";
|
||||
import { BGM_BED_VOLUME, BGM_SILENT_VOLUME, bgmDefaultVolume } from "./bgm.mjs";
|
||||
import { existsSync, mkdirSync, mkdtempSync, rmSync, writeFileSync } from "node:fs";
|
||||
import { join } from "node:path";
|
||||
import { tmpdir } from "node:os";
|
||||
import { appendRecord } from "../../../scripts/lib/manifest.mjs";
|
||||
import {
|
||||
BGM_BED_VOLUME,
|
||||
BGM_SILENT_VOLUME,
|
||||
bgmDefaultVolume,
|
||||
generateBgmDetached,
|
||||
} from "./bgm.mjs";
|
||||
|
||||
// Regression: narrated pipelines used to ship BGM at 0.8 (≈ -2 dB), ~16 dB
|
||||
// hotter than a music bed under a voice should be. The default under narration
|
||||
@@ -28,3 +37,38 @@ test("the narrated default is well below the voice (≈ 0 dBFS)", () => {
|
||||
`bed should sit ≥16 dB under the voice, got ${separation.toFixed(1)} dB`,
|
||||
);
|
||||
});
|
||||
|
||||
test(
|
||||
"generating music again clears the engine's old track, so waiting cannot mistake it for the new one",
|
||||
{ skip: process.platform === "win32" && "the fake python is a shell script" },
|
||||
(t) => {
|
||||
const dir = mkdtempSync(join(tmpdir(), "hf-bgm-"));
|
||||
const path = process.env.PATH;
|
||||
t.after(() => {
|
||||
process.env.PATH = path;
|
||||
rmSync(dir, { recursive: true, force: true });
|
||||
});
|
||||
mkdirSync(join(dir, "bin"));
|
||||
for (const name of ["python3", "python"])
|
||||
writeFileSync(join(dir, "bin", name), "#!/bin/sh\nexit 0\n", { mode: 0o755 });
|
||||
process.env.PATH = `${join(dir, "bin")}:${path}`;
|
||||
mkdirSync(join(dir, "assets/bgm"), { recursive: true });
|
||||
writeFileSync(join(dir, "assets/bgm/track.wav"), "last run's track");
|
||||
appendRecord(dir, {
|
||||
id: "bgm_001",
|
||||
type: "bgm",
|
||||
path: "assets/bgm/track.wav",
|
||||
source: "generated",
|
||||
});
|
||||
|
||||
const gen = generateBgmDetached({
|
||||
prompt: "calm",
|
||||
durationS: 5,
|
||||
hyperframesDir: dir,
|
||||
anomalies: [],
|
||||
});
|
||||
|
||||
assert.equal(gen.path, "assets/bgm/track.wav");
|
||||
assert.equal(existsSync(join(dir, "assets/bgm/track.wav")), false);
|
||||
},
|
||||
);
|
||||
|
||||
@@ -1,6 +1,48 @@
|
||||
import { recordInPlace } from "../../../scripts/lib/manifest.mjs";
|
||||
import { existsSync } from "node:fs";
|
||||
import { extname, join } from "node:path";
|
||||
import { AGENT_SOURCES, latestRecordFor, recordInPlace } from "../../../scripts/lib/manifest.mjs";
|
||||
import { regenerateIndex } from "../../../scripts/lib/index-gen.mjs";
|
||||
|
||||
/**
|
||||
* Where the engine may write `rel`: there, unless another file of this run has `taken` it or the file there is
|
||||
* the person's (not `reusable` and not recorded as agent-made). Then the first free `name-2.ext`, with an anomaly.
|
||||
*/
|
||||
export function agentWritePath(
|
||||
hyperframesDir,
|
||||
rel,
|
||||
{ anomalies, taken = new Set(), reusable = () => false },
|
||||
) {
|
||||
const free = (path) =>
|
||||
!taken.has(path) &&
|
||||
(!existsSync(join(hyperframesDir, path)) ||
|
||||
reusable(path) ||
|
||||
AGENT_SOURCES.includes(latestRecordFor(hyperframesDir, path)?.source));
|
||||
if (free(rel)) return rel;
|
||||
const ext = extname(rel);
|
||||
const stem = rel.slice(0, rel.length - ext.length);
|
||||
let n = 2;
|
||||
while (!free(`${stem}-${n}${ext}`)) n++;
|
||||
const path = `${stem}-${n}${ext}`;
|
||||
const why = taken.has(rel)
|
||||
? "another file of this run goes there"
|
||||
: "the file there is yours (the media manifest does not record it as made by the engine)";
|
||||
const note = `${rel}: kept, because ${why}; writing ${path} instead (audio_meta.json has the path used)`;
|
||||
if (!anomalies.includes(note)) anomalies.push(note);
|
||||
return path;
|
||||
}
|
||||
|
||||
/** Each spoken line's file, picked before lines synthesize concurrently so two never land on one free name. */
|
||||
export function voicePaths(hyperframesDir, lines, anomalies) {
|
||||
const taken = new Set();
|
||||
const paths = new Map();
|
||||
for (const line of lines.filter((l) => String(l.text ?? "").trim())) {
|
||||
const rel = agentWritePath(hyperframesDir, `assets/voice/${line.id}.wav`, { anomalies, taken });
|
||||
taken.add(rel);
|
||||
paths.set(String(line.id), rel);
|
||||
}
|
||||
return paths;
|
||||
}
|
||||
|
||||
const SFX_SOURCES = { heygen: "search", local: "bundled" };
|
||||
|
||||
// The files one engine run wrote, each with how it was made, for the project's media manifest.
|
||||
@@ -32,7 +74,6 @@ export function writtenAssets({ only, lines, voices, ttsProvider, bgm, bgmFields
|
||||
if (only.has("sfx")) {
|
||||
for (const cue of new Map(sfx.map((entry) => [entry.file, entry])).values()) {
|
||||
const source = SFX_SOURCES[cue.source];
|
||||
if (!source) continue;
|
||||
const provider = source === "search" ? "heygen" : "bundled.sfx";
|
||||
assets.push({
|
||||
path: cue.file,
|
||||
|
||||
@@ -4,7 +4,7 @@ import { mkdirSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from "nod
|
||||
import { join } from "node:path";
|
||||
import { tmpdir } from "node:os";
|
||||
import { findByPrompt, readManifest } from "../../../scripts/lib/manifest.mjs";
|
||||
import { recordInManifest, writtenAssets } from "./media-record.mjs";
|
||||
import { agentWritePath, recordInManifest, voicePaths, writtenAssets } from "./media-record.mjs";
|
||||
|
||||
const noBgm = { bgm: null, bgmFields: { bgm_pending: false } };
|
||||
|
||||
@@ -98,7 +98,6 @@ test("music and sound effects are marked by where they came from", () => {
|
||||
{ file: "assets/sfx/whoosh.mp3", name: "whoosh", source: "local" },
|
||||
{ file: "assets/sfx/whoosh.mp3", name: "whoosh", source: "local" },
|
||||
{ file: "assets/sfx/glass.mp3", name: "glass", source: "heygen" },
|
||||
{ file: "assets/sfx/pop.mp3", name: "pop", source: "project" },
|
||||
],
|
||||
});
|
||||
|
||||
@@ -139,6 +138,59 @@ test("a file that cannot be recorded becomes an anomaly, not a failure", (t) =>
|
||||
assert.match(anomalies[0], /^assets\/voice\/01\.wav: not recorded in the media manifest/);
|
||||
});
|
||||
|
||||
test("the engine writes over only its own files, else the next free name", (t) => {
|
||||
const dir = project(t);
|
||||
mkdirSync(join(dir, "assets/sfx"), { recursive: true });
|
||||
const names = ["mine", "made", "adopted", "twice", "twice-2", "kept", "kept-2"];
|
||||
for (const name of names) writeFileSync(join(dir, `assets/sfx/${name}.mp3`), name);
|
||||
for (const path of ["assets/sfx/made.mp3", "assets/sfx/kept-2.mp3"])
|
||||
recordInManifest(dir, [{ path, type: "sfx", source: "search" }]);
|
||||
recordInManifest(dir, [{ path: "assets/sfx/adopted.mp3", type: "sfx", source: "existing" }]);
|
||||
const anomalies = [];
|
||||
const at = (name, reusable) =>
|
||||
agentWritePath(dir, `assets/sfx/${name}.mp3`, { anomalies, reusable });
|
||||
|
||||
assert.equal(at("new"), "assets/sfx/new.mp3");
|
||||
assert.equal(at("made"), "assets/sfx/made.mp3");
|
||||
assert.equal(
|
||||
at("mine", (path) => path === "assets/sfx/mine.mp3"),
|
||||
"assets/sfx/mine.mp3",
|
||||
);
|
||||
assert.deepEqual(anomalies, []);
|
||||
assert.equal(at("mine"), "assets/sfx/mine-2.mp3");
|
||||
assert.equal(at("adopted"), "assets/sfx/adopted-2.mp3");
|
||||
assert.equal(at("twice"), "assets/sfx/twice-3.mp3");
|
||||
assert.equal(at("kept"), "assets/sfx/kept-2.mp3");
|
||||
assert.equal(anomalies.length, 4);
|
||||
assert.match(
|
||||
anomalies[0],
|
||||
/^assets\/sfx\/mine\.mp3: kept, .* writing assets\/sfx\/mine-2\.mp3 instead/,
|
||||
);
|
||||
});
|
||||
|
||||
test("two spoken lines never share a file when one's name is taken by the person", (t) => {
|
||||
const dir = project(t);
|
||||
mkdirSync(join(dir, "assets/voice"), { recursive: true });
|
||||
writeFileSync(join(dir, "assets/voice/hook.wav"), "the person's own hook");
|
||||
const anomalies = [];
|
||||
|
||||
const paths = voicePaths(
|
||||
dir,
|
||||
[
|
||||
{ id: "hook", text: "First" },
|
||||
{ id: "hook-2", text: "Second" },
|
||||
{ id: "blank", text: " " },
|
||||
],
|
||||
anomalies,
|
||||
);
|
||||
|
||||
assert.deepEqual(Object.fromEntries(paths), {
|
||||
hook: "assets/voice/hook-2.wav",
|
||||
"hook-2": "assets/voice/hook-2-2.wav",
|
||||
});
|
||||
assert.equal(anomalies.length, 2);
|
||||
});
|
||||
|
||||
test("a record never takes the id of a download still in flight", (t) => {
|
||||
const dir = project(t);
|
||||
mkdirSync(join(dir, ".media/audio/bgm"), { recursive: true });
|
||||
|
||||
@@ -15,6 +15,8 @@
|
||||
import { copyFileSync, existsSync, mkdirSync, readFileSync } from "node:fs";
|
||||
import { join } from "node:path";
|
||||
import { downloadTo, searchSounds } from "./heygen.mjs";
|
||||
import { latestRecordFor } from "../../../scripts/lib/manifest.mjs";
|
||||
import { agentWritePath } from "./media-record.mjs";
|
||||
|
||||
const SFX_VOLUME = 0.35;
|
||||
const slug = (s) =>
|
||||
@@ -31,6 +33,8 @@ export async function resolveSfx({ cues, heygenOK, headers, hyperframesDir, sfxL
|
||||
const sfx = [];
|
||||
const anomalies = [];
|
||||
const destDir = join(hyperframesDir, "assets", "sfx");
|
||||
// Each effect's file this run: one effect named twice gets one file, and two effects never share one.
|
||||
const fileFor = new Map();
|
||||
|
||||
// Dedupe identical (id,name) cues — the same effect named twice in one line
|
||||
// downloads/copies once.
|
||||
@@ -57,8 +61,14 @@ export async function resolveSfx({ cues, heygenOK, headers, hyperframesDir, sfxL
|
||||
continue;
|
||||
}
|
||||
const top = results[0];
|
||||
const file = `assets/sfx/${slug(name)}.mp3`;
|
||||
const file =
|
||||
fileFor.get(slug(name)) ??
|
||||
agentWritePath(hyperframesDir, `assets/sfx/${slug(name)}.mp3`, {
|
||||
anomalies,
|
||||
taken: new Set(fileFor.values()),
|
||||
});
|
||||
await downloadTo(top.audio_url, join(hyperframesDir, file));
|
||||
fileFor.set(slug(name), file);
|
||||
sfx.push({
|
||||
id,
|
||||
name,
|
||||
@@ -111,32 +121,41 @@ export async function resolveSfx({ cues, heygenOK, headers, hyperframesDir, sfxL
|
||||
continue;
|
||||
}
|
||||
const src = join(sfxLibDir, hit.file);
|
||||
const destRel = `assets/sfx/${hit.file}`;
|
||||
const library = existsSync(src) ? readFileSync(src) : null;
|
||||
const isLibraryCopy = (rel) =>
|
||||
library &&
|
||||
existsSync(join(hyperframesDir, rel)) &&
|
||||
readFileSync(join(hyperframesDir, rel)).equals(library);
|
||||
const destRel =
|
||||
fileFor.get(hit.file) ??
|
||||
agentWritePath(hyperframesDir, `assets/sfx/${hit.file}`, {
|
||||
anomalies,
|
||||
taken: new Set(fileFor.values()),
|
||||
// An unrecorded copy of the library file is one an engine run made before the manifest had it.
|
||||
reusable: (rel) => !latestRecordFor(hyperframesDir, rel) && isLibraryCopy(rel),
|
||||
});
|
||||
const dest = join(hyperframesDir, destRel);
|
||||
// The bundled library may be incomplete: some installs of the skill ship
|
||||
// manifest.json without the actual mp3s. Pushing an sfx entry that points at
|
||||
// a file we never copied produces a dangling reference that silently drops
|
||||
// downstream ("not on disk"). Surface it as a loud anomaly and skip the cue
|
||||
// instead, so the audio_meta never references a missing file.
|
||||
const reused = existsSync(dest);
|
||||
if (!reused) {
|
||||
if (!existsSync(src)) {
|
||||
anomalies.push(
|
||||
`sfx "${name}" (id ${id}): bundled file ${hit.file} missing from the offline ` +
|
||||
`library (${sfxLibDir}) — skipped. Reinstall the media-use skill to ` +
|
||||
`restore assets/sfx/*.mp3, or configure a HeyGen credential for retrieval.`,
|
||||
);
|
||||
continue;
|
||||
}
|
||||
copyFileSync(src, dest);
|
||||
}
|
||||
// A file already there that is not the library's copy is the person's own.
|
||||
const own = reused && !(existsSync(src) && readFileSync(dest).equals(readFileSync(src)));
|
||||
if (library) {
|
||||
if (!isLibraryCopy(destRel)) copyFileSync(src, dest);
|
||||
} else if (!existsSync(dest)) {
|
||||
anomalies.push(
|
||||
`sfx "${name}" (id ${id}): bundled file ${hit.file} missing from the offline ` +
|
||||
`library (${sfxLibDir}) — skipped. Reinstall the media-use skill to ` +
|
||||
`restore assets/sfx/*.mp3, or configure a HeyGen credential for retrieval.`,
|
||||
);
|
||||
continue;
|
||||
} // else the engine's earlier copy at dest stands in for the file this install lacks
|
||||
fileFor.set(hit.file, destRel);
|
||||
sfx.push({
|
||||
id,
|
||||
name,
|
||||
file: destRel,
|
||||
source: own ? "project" : "local",
|
||||
source: "local",
|
||||
offset_s: 0,
|
||||
duration_s: r3(hit.duration),
|
||||
volume: SFX_VOLUME,
|
||||
|
||||
@@ -1,8 +1,9 @@
|
||||
import { test } from "node:test";
|
||||
import assert from "node:assert/strict";
|
||||
import { mkdtempSync, mkdirSync, writeFileSync, existsSync, rmSync } from "node:fs";
|
||||
import { mkdtempSync, mkdirSync, writeFileSync, existsSync, readFileSync, rmSync } from "node:fs";
|
||||
import { join } from "node:path";
|
||||
import { tmpdir } from "node:os";
|
||||
import { appendRecord } from "../../../scripts/lib/manifest.mjs";
|
||||
import { resolveSfx } from "./sfx.mjs";
|
||||
|
||||
// Offline (no HeyGen) SFX resolution: the bundled library may ship manifest.json
|
||||
@@ -67,3 +68,54 @@ test("offline: a matched-but-missing bundled file yields an anomaly and NO dangl
|
||||
assert.ok(!existsSync(join(projDir, "assets/sfx/whoosh.mp3")), "nothing copied");
|
||||
});
|
||||
});
|
||||
|
||||
const whooshLibrary = (libDir, withFile) => {
|
||||
writeFileSync(
|
||||
join(libDir, "manifest.json"),
|
||||
JSON.stringify({ whoosh: { file: "whoosh.mp3", duration: 0.8 } }),
|
||||
);
|
||||
if (withFile) writeFileSync(join(libDir, "whoosh.mp3"), "ID3-library-bytes");
|
||||
};
|
||||
const record = (projDir, source) =>
|
||||
appendRecord(projDir, { id: "sfx_001", type: "sfx", path: "assets/sfx/whoosh.mp3", source });
|
||||
const offline = (libDir, projDir) =>
|
||||
resolveSfx({
|
||||
cues: [{ id: "s1", name: "whoosh" }],
|
||||
heygenOK: false,
|
||||
hyperframesDir: projDir,
|
||||
sfxLibDir: libDir,
|
||||
});
|
||||
|
||||
test("offline: the engine's earlier copy stands in for a library file this install lacks", async () => {
|
||||
await withDirs(async ({ libDir, projDir }) => {
|
||||
whooshLibrary(libDir, false);
|
||||
mkdirSync(join(projDir, "assets/sfx"), { recursive: true });
|
||||
writeFileSync(join(projDir, "assets/sfx/whoosh.mp3"), "engine copy");
|
||||
record(projDir, "bundled");
|
||||
|
||||
const { sfx, anomalies } = await offline(libDir, projDir);
|
||||
|
||||
assert.deepEqual(
|
||||
sfx.map(({ file }) => file),
|
||||
["assets/sfx/whoosh.mp3"],
|
||||
);
|
||||
assert.deepEqual(anomalies, []);
|
||||
});
|
||||
});
|
||||
|
||||
test("offline: a person's file with the library's bytes stays theirs once recorded as their own", async () => {
|
||||
await withDirs(async ({ libDir, projDir }) => {
|
||||
whooshLibrary(libDir, true);
|
||||
mkdirSync(join(projDir, "assets/sfx"), { recursive: true });
|
||||
writeFileSync(join(projDir, "assets/sfx/whoosh.mp3"), "ID3-library-bytes");
|
||||
record(projDir, "existing");
|
||||
|
||||
const { sfx } = await offline(libDir, projDir);
|
||||
|
||||
assert.deepEqual(
|
||||
sfx.map(({ file }) => file),
|
||||
["assets/sfx/whoosh-2.mp3"],
|
||||
);
|
||||
assert.equal(readFileSync(join(projDir, "assets/sfx/whoosh.mp3"), "utf8"), "ID3-library-bytes");
|
||||
});
|
||||
});
|
||||
|
||||
@@ -79,6 +79,9 @@ export function appendRecord(projectDir, record) {
|
||||
appendFileSync(p, line);
|
||||
}
|
||||
|
||||
/** Sources that mean the agent made or fetched the file; any other file is the person's own. */
|
||||
export const AGENT_SOURCES = ["generated", "search", "bundled"];
|
||||
|
||||
/** The record a path has now: the manifest only appends, so the last one for a path wins. */
|
||||
export function latestRecordFor(projectDir, path) {
|
||||
return readManifest(projectDir).findLast((record) => record.path === path);
|
||||
|
||||
Reference in New Issue
Block a user