mirror of
https://github.com/heygen-com/hyperframes.git
synced 2026-10-30 09:16:55 +00:00
feat(cli): route media search through media-use (#4230)
* feat(cli): move media-use engine into the CLI * fix(cli): replace media-use compatibility asset * fix(cli): replace moved media-use tree * fix(cli): copy moved media-use files cleanly * fix(cli): copy media-use engine files individually * fix(cli): keep audio helper compatibility import * fix(cli): clear copied skill destinations * test(cli): preserve moved media-use test paths * fix(cli): preserve media-use script compatibility imports * fix(cli): preserve media-use telemetry shim * fix(cli): resolve packaged media-use LUT assets * fix(cli): resolve packaged bundled SFX assets * chore(skills): regenerate media-use manifest * fix(cli): preserve media-use parity coverage * fix(cli): cover relocated media-use tests * chore(skills): regenerate media-use manifest * fix(cli): format bundled media provider * fix(ci): run relocated media-use tests with node * fix(ci): preserve fallow script exemptions * fix(ci): scope media-use fallow exemption * refactor(cli): trim media-use essay comments * feat(cli): index media-use assets for catalog search * feat(catalog): commit media-use search vectors * chore(catalog): track media vector metadata * fix(catalog): validate media vector row order * fix(catalog): validate media artifact freshness * fix(catalog): cover media metadata and credits * fix(cli): reject malformed media rows * fix(catalog): compare complete media rows * feat(cli): route media search through media-use * refactor(cli): trim relocated media-use comments * fix(cli): make media search runtime-safe * fix(cli): preserve sfx fallback diagnostics * fix(cli): probe sfx fallback provider explicitly * chore(skills): regenerate manifest * refactor(cli): keep typed media ranker source * fix(cli): align media search checks with local-first flow * fix(cli): scope vector audit exceptions * fix(cli): ship resolver registry beside skill * fix(cli): keep media-use skill scripts standalone * test(cli): cover local vector validation * test(cli): correct media vector fixtures * fix(cli): prefer exact media keys * fix(cli): stem exact media keys * test(cli): skip media lint for snippets without media * fix(cli): align skills test discovery Keep the CI skills job's bash discovery as the single source of truth so its empty-list guard covers the exact files Node executes. Make hasMediaVectors private because it has no external callers and needs no Fallow suppression. * fix(cli): close media-use review gaps Prioritize literal ids before stemmed ids and cover the collision alongside the plural regression. Add command seam tests for engine discovery, actionable failure, and argv passthrough. Add a deterministic parity check for standalone media-use copies, with documented allowlists for the standalone media-fetch shim and self-contained npx resolver. Keep the bash-discovered skills test list as the CI source of truth. * fix(cli): lower parity check complexity Split missing-copy and unallowlisted-drift checks so the parity guard keeps its behavior without a threshold CRAP score. * fix(cli): close media-use review seams Run the parity test in the skills job so skills-only edits exercise the same check. Export the verb mapping and inject command dispatch for direct coverage. Keep missing-copy diagnostics separate from drift reads.
This commit is contained in:
Miguel Ángel
@@ -72,6 +72,9 @@
|
||||
"packages/engine/tests/**",
|
||||
"skills/**/test-corpus/**",
|
||||
"skills/**/scripts/**",
|
||||
// The relocated media-use .mjs files are agent-invoked scripts outside the
|
||||
// import graph; keep the exemption scoped to that shipped script type.
|
||||
"packages/cli/src/media-use/**/*.mjs",
|
||||
// Agent-invoked motion-graphics tools co-located with their docs (run via
|
||||
// `node <path>` per grounding/PROTOCOL.md / categories/maps/module.md
|
||||
// prose), not import-graph reachable.
|
||||
|
||||
@@ -57,6 +57,7 @@ jobs:
|
||||
catalog_index:
|
||||
- "registry/registry.json"
|
||||
- "registry/catalog-artifact/**"
|
||||
- "skills/media-use/audio/assets/sfx/**"
|
||||
- "scripts/catalog/check-artifact-coverage.ts"
|
||||
docs_catalog:
|
||||
- "docs/**"
|
||||
@@ -375,8 +376,8 @@ jobs:
|
||||
# `node:` built-in imports. They aren't part of any workspace package, and
|
||||
# the main `Test` job's `code` path filter excludes `skills/**`, so without
|
||||
# this dedicated job they'd never run in CI. Examples:
|
||||
# * skills/media-use/scripts/resolve.test.mjs
|
||||
# * skills/media-use/scripts/lib/manifest.test.mjs
|
||||
# * packages/cli/src/media-use/resolve.test.mjs
|
||||
# * packages/cli/src/media-use/lib/manifest.test.mjs
|
||||
# Several of these are regression guards (e.g. shell-injection cases), so
|
||||
# the whole point is that they fire on PRs that touch skills/.
|
||||
test-skills:
|
||||
@@ -396,11 +397,12 @@ jobs:
|
||||
# would defeat the whole point of this job).
|
||||
run: |
|
||||
set -euo pipefail
|
||||
mapfile -t SKILLS_TESTS < <(find skills -type f -name "*.test.mjs" | sort)
|
||||
mapfile -t SKILLS_TESTS < <(find skills packages/cli/src/media-use -type f -name "*.test.mjs" | sort)
|
||||
if [ "${#SKILLS_TESTS[@]}" -eq 0 ]; then
|
||||
echo "::error::No skills/**/*.test.mjs files found. Did the layout change?"
|
||||
exit 1
|
||||
fi
|
||||
SKILLS_TESTS+=(scripts/check-media-use-copy-parity.test.mjs)
|
||||
printf 'Running %d skills test file(s):\n' "${#SKILLS_TESTS[@]}"
|
||||
printf ' * %s\n' "${SKILLS_TESTS[@]}"
|
||||
node --test "${SKILLS_TESTS[@]}"
|
||||
|
||||
+2
-2
@@ -51,9 +51,9 @@
|
||||
"player:perf": "bun run --filter @hyperframes/player perf",
|
||||
"format:check": "oxfmt --check .",
|
||||
"knip": "knip",
|
||||
"test:scripts": "node --import tsx --test scripts/animejs-v4-guidance.test.mjs scripts/check-tracked-artifacts.test.mjs scripts/check-no-main-deletions.test.mjs scripts/check-pr-captures.test.mjs scripts/check-docs-snippet-motion.test.mjs scripts/registry-target-paths.test.mjs scripts/check-workspace-contracts.test.mjs scripts/check-package-cycles.test.mjs scripts/check-cli-process-ownership.test.mjs scripts/check-large-files.test.mjs scripts/package-subpaths.test.mjs scripts/validate-release-channel.test.mjs scripts/publish-workflow.test.mjs scripts/install-workspace-dependencies.test.mjs scripts/draft-changelog.test.ts scripts/set-version.test.ts scripts/release-prepare.test.ts scripts/cli-options.test.ts scripts/changelog-weekly.test.ts scripts/claude-plugin-compression.test.ts scripts/catalog-payload-assets.test.ts scripts/host-registry-assets.test.ts scripts/catalog-preview-temp.test.ts scripts/catalog-hosted-files.test.ts scripts/player-cdn-pin.test.ts scripts/studio-runtime-smoke.test.mjs scripts/verify-packed-manifests.test.mjs scripts/lint-skills.test.mjs scripts/creator-editing-recipes.test.mjs packages/gcp-cloud-run/check-dockerfile-workspaces.test.mjs packages/core/scripts/writeGeneratedFile.test.ts scripts/catalog-drift.test.ts scripts/catalog-fetch-mirror.test.ts scripts/catalog-script-inlining.test.ts scripts/catalog-detail.test.ts scripts/generate-catalog-pages.test.ts scripts/verify-catalog-payloads.test.ts scripts/registry-skill-files.test.ts && vitest run scripts/catalog/",
|
||||
"test:scripts": "node --import tsx --test scripts/animejs-v4-guidance.test.mjs scripts/check-tracked-artifacts.test.mjs scripts/check-no-main-deletions.test.mjs scripts/check-pr-captures.test.mjs scripts/check-docs-snippet-motion.test.mjs scripts/registry-target-paths.test.mjs scripts/check-workspace-contracts.test.mjs scripts/check-media-use-copy-parity.test.mjs scripts/check-package-cycles.test.mjs scripts/check-cli-process-ownership.test.mjs scripts/check-large-files.test.mjs scripts/package-subpaths.test.mjs scripts/validate-release-channel.test.mjs scripts/publish-workflow.test.mjs scripts/install-workspace-dependencies.test.mjs scripts/draft-changelog.test.ts scripts/set-version.test.ts scripts/release-prepare.test.ts scripts/cli-options.test.ts scripts/changelog-weekly.test.ts scripts/claude-plugin-compression.test.ts scripts/catalog-payload-assets.test.ts scripts/host-registry-assets.test.ts scripts/catalog-preview-temp.test.ts scripts/catalog-hosted-files.test.ts scripts/player-cdn-pin.test.ts scripts/studio-runtime-smoke.test.mjs scripts/verify-packed-manifests.test.mjs scripts/lint-skills.test.mjs scripts/creator-editing-recipes.test.mjs packages/gcp-cloud-run/check-dockerfile-workspaces.test.mjs packages/core/scripts/writeGeneratedFile.test.ts scripts/catalog-drift.test.ts scripts/catalog-fetch-mirror.test.ts scripts/catalog-script-inlining.test.ts scripts/catalog-detail.test.ts scripts/generate-catalog-pages.test.ts scripts/verify-catalog-payloads.test.ts scripts/registry-skill-files.test.ts && vitest run scripts/catalog/",
|
||||
"typecheck:scripts": "tsc --noEmit -p scripts/tsconfig.json",
|
||||
"test:skills": "node --test 'skills/**/*.test.mjs'",
|
||||
"test:skills": "node --test 'skills/**/*.test.mjs' 'packages/cli/src/media-use/**/*.test.mjs'",
|
||||
"generate:previews": "tsx scripts/generate-template-previews.ts",
|
||||
"generate:catalog-previews": "tsx scripts/generate-catalog-previews.ts",
|
||||
"package:codex-plugin": "node scripts/package-codex-plugin.mjs",
|
||||
|
||||
@@ -1,7 +1,7 @@
|
||||
// Cross-platform replacement for the previous `mkdir -p … && cp -r …` shell
|
||||
// chain, which failed on Windows because `cp` doesn't accept `-r` there.
|
||||
|
||||
import { cpSync, existsSync, mkdirSync, readdirSync } from "node:fs";
|
||||
import { cpSync, existsSync, mkdirSync, readdirSync, rmSync } from "node:fs";
|
||||
import { dirname, join, resolve } from "node:path";
|
||||
import { fileURLToPath } from "node:url";
|
||||
import { setTimeout as sleep } from "node:timers/promises";
|
||||
@@ -80,15 +80,33 @@ async function main() {
|
||||
// Skills bundled into the published CLI. Branches don't all carry the same
|
||||
// skills/ tree (it gets restructured), so each entry is existsSync-guarded:
|
||||
// a missing skill dir warns + skips instead of crashing the build.
|
||||
for (const skill of ["hyperframes", "hyperframes-cli", "gsap"]) {
|
||||
for (const skill of ["hyperframes", "hyperframes-cli", "gsap", "media-use"]) {
|
||||
const src = join(REPO_ROOT, "skills", skill);
|
||||
if (!existsSync(src)) {
|
||||
console.warn(`[build-copy] skill not found, skipping: skills/${skill}`);
|
||||
continue;
|
||||
}
|
||||
copyDir(src, join(DIST, "skills", skill));
|
||||
const destination = join(DIST, "skills", skill);
|
||||
rmSync(destination, { recursive: true, force: true });
|
||||
copyDir(src, destination);
|
||||
}
|
||||
|
||||
// The media-use engine lives with the CLI source, but keeps its published
|
||||
// skill-relative layout so the moved .mjs tree can run without a rewrite.
|
||||
const mediaEngine = join(CLI_ROOT, "src", "media-use");
|
||||
const publishedMediaLib = join(DIST, "skills", "media-use", "scripts", "lib");
|
||||
rmSync(publishedMediaLib, { recursive: true, force: true });
|
||||
mkdirSync(publishedMediaLib, { recursive: true });
|
||||
copyDirContents(join(mediaEngine, "lib"), publishedMediaLib);
|
||||
cpSync(
|
||||
join(mediaEngine, "resolve.mjs"),
|
||||
join(DIST, "skills", "media-use", "scripts", "resolve.mjs"),
|
||||
);
|
||||
mkdirSync(join(DIST, "skills", "registry"), { recursive: true });
|
||||
copyDirContents(join(DIST, "registry"), join(DIST, "skills", "registry"));
|
||||
mkdirSync(join(DIST, "skills", "media-use", "registry"), { recursive: true });
|
||||
copyDirContents(join(DIST, "registry"), join(DIST, "skills", "media-use", "registry"));
|
||||
|
||||
const dockerfile = join(CLI_ROOT, "src", "docker", "Dockerfile.render");
|
||||
if (existsSync(dockerfile)) {
|
||||
cpSync(dockerfile, join(DIST, "docker", "Dockerfile.render"));
|
||||
|
||||
Symlink
+1
@@ -0,0 +1 @@
|
||||
../../../../skills/media-use/audio/scripts
|
||||
@@ -143,6 +143,7 @@ const commandLoaders = {
|
||||
init: () => import("./commands/init.js").then((m) => m.default),
|
||||
add: () => import("./commands/add.js").then((m) => m.default),
|
||||
catalog: () => import("./commands/catalog.js").then((m) => m.default),
|
||||
"media-use": () => import("./commands/media-use.js").then((m) => m.default),
|
||||
play: () => import("./commands/play.js").then((m) => m.default),
|
||||
present: () => import("./commands/present.js").then((m) => m.default),
|
||||
preview: () =>
|
||||
|
||||
@@ -143,6 +143,7 @@ vi.mock("../registry/localEmbedder.js", () => ({
|
||||
}));
|
||||
|
||||
vi.mock("../registry/localSemantic.js", () => ({
|
||||
mediaSemanticRanking: async () => null,
|
||||
localSemanticRanking: async () => {
|
||||
if (state.rankingError) throw state.rankingError;
|
||||
return state.ranking;
|
||||
|
||||
@@ -0,0 +1,86 @@
|
||||
import { describe, expect, it } from "vitest";
|
||||
import { mkdtempSync, rmSync } from "node:fs";
|
||||
import { tmpdir } from "node:os";
|
||||
import { join } from "node:path";
|
||||
import {
|
||||
createMediaUseCommand,
|
||||
MEDIA_USE_VERBS,
|
||||
mediaUsePassthroughArgs,
|
||||
mediaUseVerbFlags,
|
||||
resolveMediaUseEnginePath,
|
||||
} from "./media-use.js";
|
||||
|
||||
function tempCommandDir(): string {
|
||||
return mkdtempSync(join(tmpdir(), "hyperframes-media-use-command-"));
|
||||
}
|
||||
|
||||
describe("media-use command wiring", () => {
|
||||
it("uses the bundled engine when the first candidate exists", () => {
|
||||
const here = tempCommandDir();
|
||||
const engine = join(here, "..", "media-use", "resolve.mjs");
|
||||
try {
|
||||
expect(resolveMediaUseEnginePath(here, (candidate) => candidate === engine)).toBe(engine);
|
||||
} finally {
|
||||
rmSync(here, { recursive: true, force: true });
|
||||
}
|
||||
});
|
||||
|
||||
it("falls back to the source-tree skill engine", () => {
|
||||
const here = tempCommandDir();
|
||||
const engine = join(here, "skills", "media-use", "scripts", "resolve.mjs");
|
||||
try {
|
||||
expect(resolveMediaUseEnginePath(here, (candidate) => candidate === engine)).toBe(engine);
|
||||
} finally {
|
||||
rmSync(here, { recursive: true, force: true });
|
||||
}
|
||||
});
|
||||
|
||||
it("explains how to recover when the engine is absent", () => {
|
||||
const here = tempCommandDir();
|
||||
try {
|
||||
expect(() => resolveMediaUseEnginePath(here)).toThrow(
|
||||
"media-use engine is missing from this CLI build; reinstall the CLI or run from a source checkout",
|
||||
);
|
||||
} finally {
|
||||
rmSync(here, { recursive: true, force: true });
|
||||
}
|
||||
});
|
||||
|
||||
it("passes flags after the media-use verb through unchanged", () => {
|
||||
expect(
|
||||
mediaUsePassthroughArgs([
|
||||
"/usr/bin/node",
|
||||
"cli.js",
|
||||
"media-use",
|
||||
"resolve",
|
||||
"--type",
|
||||
"sfx",
|
||||
"--intent",
|
||||
"cat",
|
||||
]),
|
||||
).toEqual(["--type", "sfx", "--intent", "cat"]);
|
||||
});
|
||||
|
||||
it("maps every verb to the engine flag, leaving resolve unflagged", () => {
|
||||
for (const verb of MEDIA_USE_VERBS) {
|
||||
expect(mediaUseVerbFlags(verb)).toEqual(verb === "resolve" ? [] : [`--${verb}`]);
|
||||
}
|
||||
});
|
||||
|
||||
it("wires every subcommand to invoke its matching verb", () => {
|
||||
const invoked: string[] = [];
|
||||
const command = createMediaUseCommand((verb) => {
|
||||
invoked.push(verb);
|
||||
throw new Error("stop after dispatch");
|
||||
});
|
||||
|
||||
for (const verb of MEDIA_USE_VERBS) {
|
||||
const factory = (command.subCommands as Record<string, unknown> | undefined)?.[verb];
|
||||
expect(factory).toBeTypeOf("function");
|
||||
const child = (factory as () => { run?: () => never })();
|
||||
expect(() => child.run?.()).toThrow("stop after dispatch");
|
||||
}
|
||||
|
||||
expect(invoked).toEqual(MEDIA_USE_VERBS);
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,107 @@
|
||||
import { defineCommand } from "citty";
|
||||
import { dirname, join } from "node:path";
|
||||
import { fileURLToPath } from "node:url";
|
||||
import { spawnSync } from "node:child_process";
|
||||
import { existsSync } from "node:fs";
|
||||
import { finishCommand } from "../utils/commandResult.js";
|
||||
|
||||
const MEDIA_USE_ARGS = {
|
||||
type: { type: "string" },
|
||||
intent: { type: "string" },
|
||||
entity: { type: "string" },
|
||||
project: { type: "string", alias: "p" },
|
||||
adopt: { type: "boolean" },
|
||||
candidates: { type: "boolean" },
|
||||
doctor: { type: "boolean" },
|
||||
stats: { type: "boolean" },
|
||||
days: { type: "string" },
|
||||
"dry-run": { type: "boolean" },
|
||||
reuse: { type: "string" },
|
||||
from: { type: "string" },
|
||||
params: { type: "string" },
|
||||
for: { type: "string" },
|
||||
analyze: { type: "boolean" },
|
||||
"local-only": { type: "boolean" },
|
||||
provider: { type: "string" },
|
||||
"avatar-id": { type: "string" },
|
||||
"voice-id": { type: "string" },
|
||||
json: { type: "boolean" },
|
||||
help: { type: "boolean", alias: "h" },
|
||||
} as const;
|
||||
|
||||
export const MEDIA_USE_VERBS = [
|
||||
"resolve",
|
||||
"doctor",
|
||||
"stats",
|
||||
"adopt",
|
||||
"candidates",
|
||||
"reuse",
|
||||
"from",
|
||||
"params",
|
||||
"analyze",
|
||||
] as const;
|
||||
export type MediaUseVerb = (typeof MEDIA_USE_VERBS)[number];
|
||||
|
||||
export function resolveMediaUseEnginePath(
|
||||
here: string,
|
||||
fileExists: (path: string) => boolean = existsSync,
|
||||
): string {
|
||||
const candidates = [
|
||||
join(here, "..", "media-use", "resolve.mjs"),
|
||||
join(here, "skills", "media-use", "scripts", "resolve.mjs"),
|
||||
];
|
||||
const engine = candidates.find((candidate) => fileExists(candidate));
|
||||
if (!engine) {
|
||||
throw new Error(
|
||||
"media-use engine is missing from this CLI build; reinstall the CLI or run from a source checkout",
|
||||
);
|
||||
}
|
||||
return engine;
|
||||
}
|
||||
|
||||
export function mediaUsePassthroughArgs(argv: readonly string[]): string[] {
|
||||
const commandIndex = argv.indexOf("media-use");
|
||||
return argv.slice(commandIndex + 2);
|
||||
}
|
||||
|
||||
export function mediaUseVerbFlags(verb: MediaUseVerb): string[] {
|
||||
return verb === "resolve" ? [] : [`--${verb}`];
|
||||
}
|
||||
|
||||
type InvokeMediaUse = (verb: MediaUseVerb) => never;
|
||||
|
||||
function invokeEngine(verb: MediaUseVerb): never {
|
||||
const here = dirname(fileURLToPath(import.meta.url));
|
||||
const passed = mediaUsePassthroughArgs(process.argv);
|
||||
const flag = mediaUseVerbFlags(verb);
|
||||
const result = spawnSync(
|
||||
process.execPath,
|
||||
[resolveMediaUseEnginePath(here), ...flag, ...passed],
|
||||
{
|
||||
stdio: "inherit",
|
||||
},
|
||||
);
|
||||
if (result.error) throw result.error;
|
||||
finishCommand(result.status ?? 1);
|
||||
}
|
||||
|
||||
function subcommand(name: MediaUseVerb, invoke: InvokeMediaUse) {
|
||||
return defineCommand({
|
||||
meta: { name, description: `media-use ${name}` },
|
||||
args: MEDIA_USE_ARGS,
|
||||
run: () => invoke(name),
|
||||
});
|
||||
}
|
||||
|
||||
export function createMediaUseCommand(invoke: InvokeMediaUse = invokeEngine) {
|
||||
const subCommands = Object.fromEntries(
|
||||
MEDIA_USE_VERBS.map((name) => [name, () => subcommand(name, invoke)]),
|
||||
);
|
||||
return defineCommand({
|
||||
meta: { name: "media-use", description: "Resolve and operate on project media" },
|
||||
subCommands,
|
||||
run: () => console.log("Run `hyperframes media-use <resolve|doctor|stats|...> --help`"),
|
||||
});
|
||||
}
|
||||
|
||||
export default createMediaUseCommand();
|
||||
+31
-20
@@ -1,8 +1,25 @@
|
||||
import { existsSync, readFileSync } from "node:fs";
|
||||
import { extname, join } from "node:path";
|
||||
import { rankMediaRows } from "./media-search.mjs";
|
||||
|
||||
const LIB_DIR =
|
||||
process.env.HYPERFRAMES_MEDIA_USE_SFX_DIR ||
|
||||
[
|
||||
join(import.meta.dirname, "..", "..", "audio", "assets", "sfx"),
|
||||
join(
|
||||
import.meta.dirname,
|
||||
"..",
|
||||
"..",
|
||||
"..",
|
||||
"..",
|
||||
"..",
|
||||
"skills",
|
||||
"media-use",
|
||||
"audio",
|
||||
"assets",
|
||||
"sfx",
|
||||
),
|
||||
].find((candidate) => existsSync(candidate)) ||
|
||||
join(import.meta.dirname, "..", "..", "audio", "assets", "sfx");
|
||||
|
||||
export const BUNDLED_SFX_RECOVERY_COMMAND = "npx hyperframes skills update media-use";
|
||||
@@ -59,25 +76,10 @@ export function inspectBundledSfxAssets(libraryDir = LIB_DIR) {
|
||||
};
|
||||
}
|
||||
|
||||
const normalize = (value) =>
|
||||
String(value)
|
||||
.toLowerCase()
|
||||
.replace(/[^a-z0-9]+/g, " ")
|
||||
.trim();
|
||||
|
||||
export function extensionForBundledSfxFile(filename) {
|
||||
return extname(filename) || ".mp3";
|
||||
}
|
||||
|
||||
function score(intent, key, entry) {
|
||||
const query = normalize(intent);
|
||||
const name = normalize(key);
|
||||
if (query === name) return 100;
|
||||
if (query.includes(name) || name.includes(query)) return 50;
|
||||
const haystack = new Set(normalize(`${key} ${entry.description || ""}`).split(/\s+/));
|
||||
return query.split(/\s+/).filter((token) => token && haystack.has(token)).length;
|
||||
}
|
||||
|
||||
export const bundledSfxProvider = {
|
||||
async search(intent, ctx = {}) {
|
||||
const libraryDir = ctx.libraryDir || LIB_DIR;
|
||||
@@ -85,11 +87,20 @@ export const bundledSfxProvider = {
|
||||
if (!health.ok) throw new BundledSfxAssetsError(health);
|
||||
const manifest = JSON.parse(readFileSync(join(libraryDir, "manifest.json"), "utf8"));
|
||||
|
||||
const ranked = Object.entries(manifest)
|
||||
.map(([key, entry]) => ({ key, entry, score: score(intent, key, entry) }))
|
||||
.filter(({ entry, score }) => entry?.file && score > 0)
|
||||
.sort((a, b) => b.score - a.score || a.key.localeCompare(b.key));
|
||||
const best = ranked[0];
|
||||
const rows = Object.entries(manifest).map(([key, entry]) => ({
|
||||
id: key,
|
||||
kind: "sfx",
|
||||
title: key,
|
||||
description: entry?.description || key,
|
||||
tags: ["sfx"],
|
||||
file: entry?.file || "",
|
||||
duration: entry?.duration,
|
||||
}));
|
||||
const bestRow = rankMediaRows(intent, rows)[0];
|
||||
const best =
|
||||
bestRow && manifest[bestRow.id]?.file
|
||||
? { key: bestRow.id, entry: manifest[bestRow.id] }
|
||||
: null;
|
||||
if (!best) return null;
|
||||
|
||||
const localPath = join(libraryDir, best.entry.file);
|
||||
+54
@@ -10,6 +10,7 @@ import {
|
||||
extensionForBundledSfxFile,
|
||||
inspectBundledSfxAssets,
|
||||
} from "./bundled-sfx-provider.mjs";
|
||||
import { rankMediaRows } from "./media-search.mjs";
|
||||
|
||||
test("derives bundled SFX extension from the manifest filename", () => {
|
||||
assert.equal(extensionForBundledSfxFile("impact.wav"), ".wav");
|
||||
@@ -83,3 +84,56 @@ test("accepts a complete bundled SFX library", () => {
|
||||
rmSync(libraryDir, { recursive: true, force: true });
|
||||
}
|
||||
});
|
||||
|
||||
test("prefers an exact key over a longer key with the same words", async () => {
|
||||
const libraryDir = mkdtempSync(join(tmpdir(), "media-use-sfx-exact-key-"));
|
||||
try {
|
||||
writeFileSync(
|
||||
join(libraryDir, "manifest.json"),
|
||||
JSON.stringify({
|
||||
"whoosh-cinematic": { file: "whoosh-cinematic.mp3", description: "long whoosh" },
|
||||
whoosh: { file: "whoosh.mp3", description: "short whoosh" },
|
||||
}),
|
||||
);
|
||||
writeFileSync(join(libraryDir, "whoosh-cinematic.mp3"), "cinematic audio");
|
||||
writeFileSync(join(libraryDir, "whoosh.mp3"), "exact audio");
|
||||
|
||||
const result = await bundledSfxProvider.search("whoosh", { libraryDir });
|
||||
assert.equal(result?.localPath, join(libraryDir, "whoosh.mp3"));
|
||||
assert.equal(result?.metadata.provenance.library_key, "whoosh");
|
||||
|
||||
const stemmed = await bundledSfxProvider.search("whooshes", { libraryDir });
|
||||
assert.equal(stemmed?.localPath, join(libraryDir, "whoosh.mp3"));
|
||||
assert.equal(stemmed?.metadata.provenance.library_key, "whoosh");
|
||||
} finally {
|
||||
rmSync(libraryDir, { recursive: true, force: true });
|
||||
}
|
||||
});
|
||||
|
||||
test("literal ids outrank distinct ids with the same stem", () => {
|
||||
const rows = [
|
||||
{
|
||||
id: "cats",
|
||||
title: "Cats Fighting",
|
||||
description: "fighting cats",
|
||||
tags: ["yowl", "fight"],
|
||||
kind: "sfx",
|
||||
},
|
||||
{
|
||||
id: "cat",
|
||||
title: "Cat",
|
||||
description: "single cat",
|
||||
tags: ["meow"],
|
||||
kind: "sfx",
|
||||
},
|
||||
];
|
||||
|
||||
assert.deepEqual(
|
||||
rankMediaRows("cat", rows).map((row) => row.id),
|
||||
["cat", "cats"],
|
||||
);
|
||||
assert.deepEqual(
|
||||
rankMediaRows("cats", rows).map((row) => row.id),
|
||||
["cats", "cat"],
|
||||
);
|
||||
});
|
||||
+18
-3
@@ -10,7 +10,16 @@ import { CAPABILITIES, listModels } from "./local-models.mjs";
|
||||
// test enforces the weakness→owner matrix in references/meta.md so a claim can't rot — if
|
||||
// a capability's entrypoint disappears, this fails.
|
||||
|
||||
const SKILL = join(dirname(fileURLToPath(import.meta.url)), "..", "..");
|
||||
const SKILL = join(
|
||||
dirname(fileURLToPath(import.meta.url)),
|
||||
"..",
|
||||
"..",
|
||||
"..",
|
||||
"..",
|
||||
"..",
|
||||
"skills",
|
||||
"media-use",
|
||||
);
|
||||
|
||||
test("weakness: audio-only → media-use resolves image + icon", () => {
|
||||
for (const t of ["image", "icon"]) {
|
||||
@@ -43,7 +52,10 @@ test("weakness: no media-ops → ops guidance reference exists", () => {
|
||||
|
||||
test("weakness: no transcript-driven cutting → cut compiler entrypoints exist", async () => {
|
||||
assert.ok(existsSync(join(SKILL, "scripts", "transcript-cut.mjs")), "transcript-cut missing");
|
||||
assert.ok(existsSync(join(SKILL, "scripts", "lib", "cutlist.mjs")), "cutlist lib missing");
|
||||
assert.ok(
|
||||
existsSync(join(dirname(fileURLToPath(import.meta.url)), "cutlist.mjs")),
|
||||
"cutlist lib missing",
|
||||
);
|
||||
const cutlist = await import("./cutlist.mjs");
|
||||
assert.equal(typeof cutlist.compileCutList, "function");
|
||||
});
|
||||
@@ -60,7 +72,10 @@ test("weakness: whisper.cpp is weak → better local ASR (Parakeet) entrypoint e
|
||||
|
||||
test("weakness: no auto-duck/loudness → duck compiler and recipes exist", async () => {
|
||||
assert.ok(existsSync(join(SKILL, "scripts", "audio-duck.mjs")), "audio-duck missing");
|
||||
assert.ok(existsSync(join(SKILL, "scripts", "lib", "duck.mjs")), "duck lib missing");
|
||||
assert.ok(
|
||||
existsSync(join(dirname(fileURLToPath(import.meta.url)), "duck.mjs")),
|
||||
"duck lib missing",
|
||||
);
|
||||
assert.ok(existsSync(join(SKILL, "references", "operations.md")), "operations.md missing");
|
||||
const duck = await import("./duck.mjs");
|
||||
assert.equal(typeof duck.speechSpans, "function");
|
||||
@@ -0,0 +1,184 @@
|
||||
import { normalizeWords } from "./words.mjs";
|
||||
|
||||
const MIN_SEGMENT_SECONDS = 0.2;
|
||||
const SILENCE_PAD_SECONDS = 0.15;
|
||||
|
||||
export function compileCutList(transcript, opts = {}) {
|
||||
const words = normalizeWords(transcript);
|
||||
if (opts.keep != null && hasRemovalSource(opts)) {
|
||||
throw new Error("--keep is mutually exclusive with removal options");
|
||||
}
|
||||
|
||||
if (opts.keep != null) {
|
||||
const duration = durationFrom(words, opts);
|
||||
const ranges = parseTimeRanges(opts.keep);
|
||||
return finalizeKept(duration != null ? clampRanges(ranges, duration) : ranges);
|
||||
}
|
||||
|
||||
const duration = durationFrom(words, opts);
|
||||
if (!duration) return [];
|
||||
|
||||
const removals = [
|
||||
...parseTimeRanges(opts.remove),
|
||||
...wordIndexRanges(words, opts.removeWords),
|
||||
...fillerRanges(words, opts.removeFillers),
|
||||
...silenceRanges(words, opts.cutSilence),
|
||||
];
|
||||
const mergedRemovals = mergeRanges(clampRanges(removals, duration));
|
||||
return finalizeKept(invertRanges(mergedRemovals, duration));
|
||||
}
|
||||
|
||||
function hasRemovalSource(opts) {
|
||||
return (
|
||||
opts.remove != null ||
|
||||
opts.removeWords != null ||
|
||||
opts.removeFillers != null ||
|
||||
opts.cutSilence != null
|
||||
);
|
||||
}
|
||||
|
||||
function durationFrom(words, opts) {
|
||||
const explicit = Number(opts.duration ?? opts.totalDuration);
|
||||
if (Number.isFinite(explicit) && explicit > 0) return explicit;
|
||||
const last = words.at(-1);
|
||||
return last && Number.isFinite(last.end) && last.end > 0 ? last.end : null;
|
||||
}
|
||||
|
||||
function parseTimeRanges(value) {
|
||||
if (value == null || value === false || value === "") return [];
|
||||
if (typeof value === "string") {
|
||||
return value
|
||||
.split(",")
|
||||
.map((part) => part.trim())
|
||||
.filter(Boolean)
|
||||
.map(parseRangeString);
|
||||
}
|
||||
if (!Array.isArray(value)) throw new Error("range list must be a string or array");
|
||||
return value.map((range) => {
|
||||
if (Array.isArray(range)) return cleanRange(Number(range[0]), Number(range[1]));
|
||||
return cleanRange(Number(range?.start), Number(range?.end));
|
||||
});
|
||||
}
|
||||
|
||||
function parseRangeString(value) {
|
||||
const match = value.match(/^([0-9]*\.?[0-9]+)\s*-\s*([0-9]*\.?[0-9]+)$/);
|
||||
if (!match) throw new Error(`invalid range: ${value}`);
|
||||
return cleanRange(Number(match[1]), Number(match[2]));
|
||||
}
|
||||
|
||||
function cleanRange(start, end) {
|
||||
if (!Number.isFinite(start) || !Number.isFinite(end)) {
|
||||
throw new Error("range start/end must be finite numbers");
|
||||
}
|
||||
if (end < start) throw new Error(`range end ${end} is before start ${start}`);
|
||||
return { start, end };
|
||||
}
|
||||
|
||||
function wordIndexRanges(words, value) {
|
||||
if (value == null || value === false || value === "") return [];
|
||||
const ranges = typeof value === "string" ? value.split(",") : value;
|
||||
if (!Array.isArray(ranges)) throw new Error("--remove-words must be a string or array");
|
||||
return ranges
|
||||
.map((range) => (typeof range === "string" ? range.trim() : range))
|
||||
.filter(Boolean)
|
||||
.map((range) => {
|
||||
const [first, last = first] =
|
||||
typeof range === "string" ? range.split("-").map((n) => n.trim()) : range;
|
||||
const startIndex = Number(first);
|
||||
const endIndex = Number(last);
|
||||
if (!Number.isInteger(startIndex) || !Number.isInteger(endIndex)) {
|
||||
throw new Error(`invalid word range: ${range}`);
|
||||
}
|
||||
if (startIndex < 0 || endIndex < startIndex || endIndex >= words.length) {
|
||||
throw new Error(`word range out of bounds: ${range}`);
|
||||
}
|
||||
return { start: words[startIndex].start, end: words[endIndex].end };
|
||||
});
|
||||
}
|
||||
|
||||
function fillerRanges(words, value) {
|
||||
if (value == null || value === false || value === "") return [];
|
||||
const fillers = Array.isArray(value)
|
||||
? value
|
||||
: String(value)
|
||||
.split(",")
|
||||
.map((s) => s.trim());
|
||||
const set = new Set(fillers.filter(Boolean).map(bareToken));
|
||||
if (set.size === 0) return [];
|
||||
// Whisper emits words with attached punctuation and arbitrary case
|
||||
// ("UM," / "Um."), so compare bare tokens.
|
||||
return words
|
||||
.filter((word) => set.has(bareToken(word.text)))
|
||||
.map((word) => ({ start: word.start, end: word.end }));
|
||||
}
|
||||
|
||||
function bareToken(text) {
|
||||
return String(text)
|
||||
.toLowerCase()
|
||||
.replace(/^[^\p{L}\p{N}]+|[^\p{L}\p{N}]+$/gu, "");
|
||||
}
|
||||
|
||||
function silenceRanges(words, value) {
|
||||
if (value == null || value === false || value === "") return [];
|
||||
const threshold = Number(value);
|
||||
if (!Number.isFinite(threshold) || threshold <= 0) {
|
||||
throw new Error("--cut-silence must be a positive number");
|
||||
}
|
||||
const ranges = [];
|
||||
for (let i = 0; i < words.length - 1; i++) {
|
||||
const current = words[i];
|
||||
const next = words[i + 1];
|
||||
const gap = next.start - current.end;
|
||||
if (gap <= threshold) continue;
|
||||
const start = current.end + SILENCE_PAD_SECONDS;
|
||||
const end = next.start - SILENCE_PAD_SECONDS;
|
||||
if (end > start) ranges.push({ start, end });
|
||||
}
|
||||
return ranges;
|
||||
}
|
||||
|
||||
function clampRanges(ranges, duration) {
|
||||
return ranges
|
||||
.map((range) => ({
|
||||
start: Math.max(0, Math.min(duration, range.start)),
|
||||
end: Math.max(0, Math.min(duration, range.end)),
|
||||
}))
|
||||
.filter((range) => range.end > range.start);
|
||||
}
|
||||
|
||||
function mergeRanges(ranges) {
|
||||
const sorted = ranges
|
||||
.map((range) => ({ start: round3(range.start), end: round3(range.end) }))
|
||||
.sort((a, b) => a.start - b.start || a.end - b.end);
|
||||
const merged = [];
|
||||
for (const range of sorted) {
|
||||
const prev = merged.at(-1);
|
||||
if (prev && range.start <= prev.end) {
|
||||
prev.end = Math.max(prev.end, range.end);
|
||||
} else {
|
||||
merged.push({ ...range });
|
||||
}
|
||||
}
|
||||
return merged;
|
||||
}
|
||||
|
||||
function invertRanges(removals, duration) {
|
||||
const kept = [];
|
||||
let cursor = 0;
|
||||
for (const range of removals) {
|
||||
if (range.start > cursor) kept.push({ start: cursor, end: range.start });
|
||||
cursor = Math.max(cursor, range.end);
|
||||
}
|
||||
if (cursor < duration) kept.push({ start: cursor, end: duration });
|
||||
return kept;
|
||||
}
|
||||
|
||||
function finalizeKept(ranges) {
|
||||
return mergeRanges(ranges)
|
||||
.map((range) => ({ start: round3(range.start), end: round3(range.end) }))
|
||||
.filter((range) => round3(range.end - range.start) >= MIN_SEGMENT_SECONDS);
|
||||
}
|
||||
|
||||
function round3(n) {
|
||||
return Math.round(Number(n) * 1000) / 1000;
|
||||
}
|
||||
+15
-1
@@ -8,7 +8,21 @@ import { test } from "node:test";
|
||||
import { compileCutList } from "./cutlist.mjs";
|
||||
|
||||
const HERE = dirname(fileURLToPath(import.meta.url));
|
||||
const SCRIPT = join(HERE, "..", "transcript-cut.mjs");
|
||||
// These legacy wrappers remain under skills/media-use/scripts because the
|
||||
// skill's public audio entrypoints still invoke them; the moved engine tests
|
||||
// their implementation here while preserving that compatibility surface.
|
||||
const SCRIPT = join(
|
||||
HERE,
|
||||
"..",
|
||||
"..",
|
||||
"..",
|
||||
"..",
|
||||
"..",
|
||||
"skills",
|
||||
"media-use",
|
||||
"scripts",
|
||||
"transcript-cut.mjs",
|
||||
);
|
||||
|
||||
test("explicit --remove ranges invert to kept segments", () => {
|
||||
const transcript = [
|
||||
@@ -0,0 +1,95 @@
|
||||
import { wordListsFromMediaMeta } from "./words.mjs";
|
||||
|
||||
// audio_meta.json word times are relative to each line's own file.
|
||||
export function speechSpans(meta, { mergeGap = 0.6, offsets, sequential = false, gap = 0 } = {}) {
|
||||
const merge = Number(mergeGap);
|
||||
const lists = wordListsFromMediaMeta(meta);
|
||||
const voices = Array.isArray(meta?.voices) ? meta.voices : [];
|
||||
if (lists.length > 1 && !offsets && !sequential) {
|
||||
throw new Error(
|
||||
"audio_meta has multiple voice lines with file-relative times; pass --sequential or --offsets so spans land at composition time",
|
||||
);
|
||||
}
|
||||
const intervals = [];
|
||||
let cursor = 0;
|
||||
for (let i = 0; i < lists.length; i++) {
|
||||
const voice = voices[i];
|
||||
let offset = 0;
|
||||
if (offsets) {
|
||||
const id = voice?.id ?? String(i);
|
||||
if (!(id in offsets)) throw new Error(`--offsets is missing voice "${id}"`);
|
||||
offset = Number(offsets[id]) || 0;
|
||||
} else if (sequential) {
|
||||
offset = cursor;
|
||||
const lineDuration = Number(voice?.duration_s) || Math.max(...lists[i].map((w) => w.end), 0);
|
||||
cursor += lineDuration + (Number(gap) || 0);
|
||||
}
|
||||
for (const word of lists[i]) {
|
||||
if (word.end > word.start)
|
||||
intervals.push({ start: word.start + offset, end: word.end + offset });
|
||||
}
|
||||
}
|
||||
return mergeIntervals(intervals, Number.isFinite(merge) && merge >= 0 ? merge : 0.6);
|
||||
}
|
||||
|
||||
export function duckKeyframes(
|
||||
spans,
|
||||
{ duck = 0.25, attack = 0.15, release = 0.4, baseVolume = 1 } = {},
|
||||
) {
|
||||
const base = finiteOr(baseVolume, 1);
|
||||
const ducked = round3(base * finiteOr(duck, 0.25));
|
||||
const keyframes = [];
|
||||
for (const span of spans) {
|
||||
keyframes.push({
|
||||
time: round3(Math.max(0, finiteOr(span.start, 0))),
|
||||
volume: ducked,
|
||||
duration: round3(finiteOr(attack, 0.15)),
|
||||
});
|
||||
keyframes.push({
|
||||
time: round3(Math.max(0, finiteOr(span.end, 0))),
|
||||
volume: round3(base),
|
||||
duration: round3(finiteOr(release, 0.4)),
|
||||
});
|
||||
}
|
||||
return keyframes.sort((a, b) => a.time - b.time);
|
||||
}
|
||||
|
||||
/** Volume lane for `data-automation`: composition-time keyframes as clip-local ramps. */
|
||||
export function duckLane(keyframes, { clipStart = 0, baseVolume = 1 } = {}) {
|
||||
const start = finiteOr(clipStart, 0);
|
||||
const points = [{ t: 0, v: round3(finiteOr(baseVolume, 1)) }];
|
||||
const push = (t, v) => {
|
||||
if (t > points.at(-1).t) points.push({ t: round3(t), v });
|
||||
};
|
||||
for (const kf of keyframes) {
|
||||
const t = Math.max(0, kf.time - start);
|
||||
push(t, points.at(-1).v);
|
||||
push(t + kf.duration, kf.volume);
|
||||
}
|
||||
return { version: 1, lanes: [{ target: "volume", points }] };
|
||||
}
|
||||
|
||||
function mergeIntervals(intervals, mergeGap) {
|
||||
const sorted = intervals
|
||||
.map((range) => ({ start: round3(range.start), end: round3(range.end) }))
|
||||
.sort((a, b) => a.start - b.start || a.end - b.end);
|
||||
const merged = [];
|
||||
for (const range of sorted) {
|
||||
const prev = merged.at(-1);
|
||||
if (prev && (range.start <= prev.end || range.start - prev.end < mergeGap)) {
|
||||
prev.end = Math.max(prev.end, range.end);
|
||||
} else {
|
||||
merged.push({ ...range });
|
||||
}
|
||||
}
|
||||
return merged;
|
||||
}
|
||||
|
||||
function finiteOr(value, fallback) {
|
||||
const n = Number(value);
|
||||
return Number.isFinite(n) ? n : fallback;
|
||||
}
|
||||
|
||||
function round3(n) {
|
||||
return Math.round(Number(n) * 1000) / 1000;
|
||||
}
|
||||
+14
-1
@@ -8,7 +8,20 @@ import { test } from "node:test";
|
||||
import { duckKeyframes, duckLane, speechSpans } from "./duck.mjs";
|
||||
|
||||
const HERE = dirname(fileURLToPath(import.meta.url));
|
||||
const SCRIPT = join(HERE, "..", "audio-duck.mjs");
|
||||
// Keep the skill-owned wrapper path under test until the compatibility layer
|
||||
// is removed in a later release.
|
||||
const SCRIPT = join(
|
||||
HERE,
|
||||
"..",
|
||||
"..",
|
||||
"..",
|
||||
"..",
|
||||
"..",
|
||||
"skills",
|
||||
"media-use",
|
||||
"scripts",
|
||||
"audio-duck.mjs",
|
||||
);
|
||||
|
||||
test("speechSpans bridges gaps smaller than mergeGap", () => {
|
||||
const meta = {
|
||||
@@ -0,0 +1,230 @@
|
||||
export const ERROR_DIFFUSION_ALGORITHMS = {
|
||||
"floyd-steinberg": {
|
||||
kernel: [
|
||||
[1, 0, 7],
|
||||
[-1, 1, 3],
|
||||
[0, 1, 5],
|
||||
[1, 1, 1],
|
||||
],
|
||||
divisor: 16,
|
||||
},
|
||||
atkinson: {
|
||||
kernel: [
|
||||
[1, 0, 1],
|
||||
[2, 0, 1],
|
||||
[-1, 1, 1],
|
||||
[0, 1, 1],
|
||||
[1, 1, 1],
|
||||
[0, 2, 1],
|
||||
],
|
||||
divisor: 8,
|
||||
},
|
||||
"jarvis-judice-ninke": {
|
||||
kernel: [
|
||||
[1, 0, 7],
|
||||
[2, 0, 5],
|
||||
[-2, 1, 3],
|
||||
[-1, 1, 5],
|
||||
[0, 1, 7],
|
||||
[1, 1, 5],
|
||||
[2, 1, 3],
|
||||
[-2, 2, 1],
|
||||
[-1, 2, 3],
|
||||
[0, 2, 5],
|
||||
[1, 2, 3],
|
||||
[2, 2, 1],
|
||||
],
|
||||
divisor: 48,
|
||||
},
|
||||
stucki: {
|
||||
kernel: [
|
||||
[1, 0, 8],
|
||||
[2, 0, 4],
|
||||
[-2, 1, 2],
|
||||
[-1, 1, 4],
|
||||
[0, 1, 8],
|
||||
[1, 1, 4],
|
||||
[2, 1, 2],
|
||||
[-2, 2, 1],
|
||||
[-1, 2, 2],
|
||||
[0, 2, 4],
|
||||
[1, 2, 2],
|
||||
[2, 2, 1],
|
||||
],
|
||||
divisor: 42,
|
||||
},
|
||||
burkes: {
|
||||
kernel: [
|
||||
[1, 0, 8],
|
||||
[2, 0, 4],
|
||||
[-2, 1, 2],
|
||||
[-1, 1, 4],
|
||||
[0, 1, 8],
|
||||
[1, 1, 4],
|
||||
[2, 1, 2],
|
||||
],
|
||||
divisor: 32,
|
||||
},
|
||||
sierra: {
|
||||
kernel: [
|
||||
[1, 0, 5],
|
||||
[2, 0, 3],
|
||||
[-2, 1, 2],
|
||||
[-1, 1, 4],
|
||||
[0, 1, 5],
|
||||
[1, 1, 4],
|
||||
[2, 1, 2],
|
||||
[-1, 2, 2],
|
||||
[0, 2, 3],
|
||||
[1, 2, 2],
|
||||
],
|
||||
divisor: 32,
|
||||
},
|
||||
"sierra-lite": {
|
||||
kernel: [
|
||||
[1, 0, 2],
|
||||
[-1, 1, 1],
|
||||
[0, 1, 1],
|
||||
],
|
||||
divisor: 4,
|
||||
},
|
||||
"two-row-sierra": {
|
||||
kernel: [
|
||||
[1, 0, 4],
|
||||
[2, 0, 3],
|
||||
[-2, 1, 1],
|
||||
[-1, 1, 2],
|
||||
[0, 1, 3],
|
||||
[1, 1, 2],
|
||||
[2, 1, 1],
|
||||
],
|
||||
divisor: 16,
|
||||
},
|
||||
};
|
||||
|
||||
const DEFAULTS = {
|
||||
algorithm: "floyd-steinberg",
|
||||
brightness: 1,
|
||||
contrast: 1.2,
|
||||
detail: 1,
|
||||
palette: ["#000000", "#ffffff"],
|
||||
pointSize: 3,
|
||||
};
|
||||
|
||||
export function errorDiffusionBufferLength(width, height, pointSize) {
|
||||
return Math.ceil(width / pointSize) * Math.ceil(height / pointSize) * 3;
|
||||
}
|
||||
|
||||
export function applyErrorDiffusionRgba(data, width, height, options = {}, errorBuffer) {
|
||||
if (!Number.isInteger(width) || width < 1 || !Number.isInteger(height) || height < 1) {
|
||||
throw new Error("width and height must be positive integers");
|
||||
}
|
||||
if (!data || data.length !== width * height * 4) {
|
||||
throw new Error(`RGBA data must contain ${width * height * 4} bytes`);
|
||||
}
|
||||
|
||||
const algorithm = options.algorithm ?? DEFAULTS.algorithm;
|
||||
const diffusion = ERROR_DIFFUSION_ALGORITHMS[algorithm];
|
||||
if (!diffusion) throw new Error(`unknown error-diffusion algorithm: ${algorithm}`);
|
||||
|
||||
const pointSize = integerInRange(options.pointSize ?? DEFAULTS.pointSize, 1, 20, "pointSize");
|
||||
const brightness = numberInRange(options.brightness ?? DEFAULTS.brightness, 0.5, 2, "brightness");
|
||||
const contrast = numberInRange(options.contrast ?? DEFAULTS.contrast, 0.5, 2, "contrast");
|
||||
const detail = numberInRange(options.detail ?? DEFAULTS.detail, 0.1, 1, "detail");
|
||||
const palette = parsePalette(options.palette ?? DEFAULTS.palette);
|
||||
const blockColumns = Math.ceil(width / pointSize);
|
||||
const blockRows = Math.ceil(height / pointSize);
|
||||
const errorLength = errorDiffusionBufferLength(width, height, pointSize);
|
||||
const errors = errorBuffer ?? new Float32Array(errorLength);
|
||||
if (!(errors instanceof Float32Array) || errors.length !== errorLength) {
|
||||
throw new Error(`errorBuffer must be a Float32Array of length ${errorLength}`);
|
||||
}
|
||||
errors.fill(0);
|
||||
|
||||
const centerOffset = Math.floor(pointSize / 2);
|
||||
for (let blockRow = 0; blockRow < blockRows; blockRow++) {
|
||||
const blockY = blockRow * pointSize;
|
||||
for (let blockColumn = 0; blockColumn < blockColumns; blockColumn++) {
|
||||
const blockX = blockColumn * pointSize;
|
||||
const centerX = Math.min(blockX + centerOffset, width - 1);
|
||||
const centerY = Math.min(blockY + centerOffset, height - 1);
|
||||
const rgbaIndex = (centerY * width + centerX) * 4;
|
||||
const errorIndex = (blockRow * blockColumns + blockColumn) * 3;
|
||||
const red = correctedChannel(data[rgbaIndex], errors[errorIndex], brightness, contrast);
|
||||
const green = correctedChannel(
|
||||
data[rgbaIndex + 1],
|
||||
errors[errorIndex + 1],
|
||||
brightness,
|
||||
contrast,
|
||||
);
|
||||
const blue = correctedChannel(
|
||||
data[rgbaIndex + 2],
|
||||
errors[errorIndex + 2],
|
||||
brightness,
|
||||
contrast,
|
||||
);
|
||||
const luminance = 0.299 * red + 0.587 * green + 0.114 * blue;
|
||||
const output = palette[Math.min(palette.length - 1, Math.floor(luminance * palette.length))];
|
||||
|
||||
for (let y = blockY; y < Math.min(blockY + pointSize, height); y++) {
|
||||
for (let x = blockX; x < Math.min(blockX + pointSize, width); x++) {
|
||||
const outputIndex = (y * width + x) * 4;
|
||||
data[outputIndex] = Math.round(output[0] * 255);
|
||||
data[outputIndex + 1] = Math.round(output[1] * 255);
|
||||
data[outputIndex + 2] = Math.round(output[2] * 255);
|
||||
}
|
||||
}
|
||||
|
||||
for (const [dx, dy, weight] of diffusion.kernel) {
|
||||
const targetColumn = blockColumn + dx;
|
||||
const targetRow = blockRow + dy;
|
||||
if (
|
||||
targetColumn < 0 ||
|
||||
targetColumn >= blockColumns ||
|
||||
targetRow < 0 ||
|
||||
targetRow >= blockRows
|
||||
) {
|
||||
continue;
|
||||
}
|
||||
const target = (targetRow * blockColumns + targetColumn) * 3;
|
||||
const scale = (weight / diffusion.divisor) * detail;
|
||||
errors[target] += (red - output[0]) * scale;
|
||||
errors[target + 1] += (green - output[1]) * scale;
|
||||
errors[target + 2] += (blue - output[2]) * scale;
|
||||
}
|
||||
}
|
||||
}
|
||||
return data;
|
||||
}
|
||||
|
||||
function correctedChannel(byte, error, brightness, contrast) {
|
||||
return Math.min(1, Math.max(0, ((byte / 255 - 0.5) * contrast + 0.5) * brightness + error));
|
||||
}
|
||||
|
||||
function parsePalette(colors) {
|
||||
if (!Array.isArray(colors) || colors.length < 2 || colors.length > 6) {
|
||||
throw new Error("palette must contain 2 to 6 colors");
|
||||
}
|
||||
return colors.map((color) => {
|
||||
const match = /^#([0-9a-f]{6})$/i.exec(color);
|
||||
if (!match) throw new Error(`palette color must use #rrggbb: ${color}`);
|
||||
const value = Number.parseInt(match[1], 16);
|
||||
return [(value >> 16) / 255, ((value >> 8) & 255) / 255, (value & 255) / 255];
|
||||
});
|
||||
}
|
||||
|
||||
function numberInRange(value, min, max, name) {
|
||||
const number = Number(value);
|
||||
if (!Number.isFinite(number) || number < min || number > max) {
|
||||
throw new Error(`${name} must be between ${min} and ${max}`);
|
||||
}
|
||||
return number;
|
||||
}
|
||||
|
||||
function integerInRange(value, min, max, name) {
|
||||
const number = Number(value);
|
||||
if (!Number.isInteger(number) || number < min || number > max) {
|
||||
throw new Error(`${name} must be an integer between ${min} and ${max}`);
|
||||
}
|
||||
return number;
|
||||
}
|
||||
@@ -0,0 +1,63 @@
|
||||
import { writeFileSync, mkdirSync } from "node:fs";
|
||||
import { dirname } from "node:path";
|
||||
import { readManifest, indexPath } from "./manifest.mjs";
|
||||
|
||||
function pad(str, len) {
|
||||
return String(str ?? "").padEnd(len);
|
||||
}
|
||||
|
||||
function formatDur(record) {
|
||||
if (record.duration == null) return "—";
|
||||
return `${record.duration}s`;
|
||||
}
|
||||
|
||||
function formatDims(record) {
|
||||
if (record.width && record.height) return `${record.width}×${record.height}`;
|
||||
if (record.type === "icon" && record.transparent) return "svg";
|
||||
return "—";
|
||||
}
|
||||
|
||||
export function generateIndexContent(records) {
|
||||
const count = records.length;
|
||||
const header = `# .media · ${count} asset${count === 1 ? "" : "s"}\n`;
|
||||
if (count === 0) return header;
|
||||
|
||||
const cols = { id: 4, type: 5, dur: 4, dims: 5, path: 5, desc: 11 };
|
||||
for (const r of records) {
|
||||
cols.id = Math.max(cols.id, (r.id ?? "").length);
|
||||
cols.type = Math.max(cols.type, (r.type ?? "").length);
|
||||
cols.dur = Math.max(cols.dur, formatDur(r).length);
|
||||
cols.dims = Math.max(cols.dims, formatDims(r).length);
|
||||
cols.path = Math.max(cols.path, (r.path ?? "").length);
|
||||
}
|
||||
|
||||
const heading =
|
||||
pad("id", cols.id + 2) +
|
||||
pad("type", cols.type + 2) +
|
||||
pad("dur", cols.dur + 2) +
|
||||
pad("dims", cols.dims + 2) +
|
||||
pad("path", cols.path + 2) +
|
||||
"description";
|
||||
|
||||
const lines = [header, heading];
|
||||
for (const r of records) {
|
||||
lines.push(
|
||||
pad(r.id, cols.id + 2) +
|
||||
pad(r.type, cols.type + 2) +
|
||||
pad(formatDur(r), cols.dur + 2) +
|
||||
pad(formatDims(r), cols.dims + 2) +
|
||||
pad(r.path, cols.path + 2) +
|
||||
(r.description ?? ""),
|
||||
);
|
||||
}
|
||||
return lines.join("\n") + "\n";
|
||||
}
|
||||
|
||||
export function regenerateIndex(projectDir) {
|
||||
const records = readManifest(projectDir);
|
||||
const content = generateIndexContent(records);
|
||||
const p = indexPath(projectDir);
|
||||
mkdirSync(dirname(p), { recursive: true });
|
||||
writeFileSync(p, content);
|
||||
return content;
|
||||
}
|
||||
@@ -0,0 +1,32 @@
|
||||
import { existsSync, mkdirSync, readFileSync, writeFileSync } from "node:fs";
|
||||
import { homedir } from "node:os";
|
||||
import { join } from "node:path";
|
||||
|
||||
const DEFAULT_DIRECTORY = join(homedir(), ".hyperframes", "catalog");
|
||||
|
||||
function directory() {
|
||||
return process.env.HYPERFRAMES_CATALOG_ARTIFACT_DIR || DEFAULT_DIRECTORY;
|
||||
}
|
||||
|
||||
export async function fetchMediaVectors(registryBaseUrl, options = {}) {
|
||||
const target = options.directory || directory();
|
||||
const base = registryBaseUrl.replace(/\/+$/, "");
|
||||
mkdirSync(target, { recursive: true, mode: 0o700 });
|
||||
const fetched = [];
|
||||
for (const file of ["media-vectors.json", "media-vectors.bin"]) {
|
||||
const response = await fetch(`${base}/catalog-artifact/${file}`);
|
||||
if (!response.ok) return false;
|
||||
fetched.push([file, Buffer.from(await response.arrayBuffer())]);
|
||||
}
|
||||
for (const [file, bytes] of fetched) writeFileSync(join(target, file), bytes, { mode: 0o600 });
|
||||
return true;
|
||||
}
|
||||
|
||||
export function mediaVectorRows(target = directory()) {
|
||||
const metadataPath = join(target, "media-vectors.json");
|
||||
if (!existsSync(metadataPath)) return [];
|
||||
const metadata = JSON.parse(readFileSync(metadataPath, "utf8"));
|
||||
return Array.isArray(metadata.rows) ? metadata.rows : [];
|
||||
}
|
||||
|
||||
export { rankMediaRowsWithVectors } from "./media-search.mjs";
|
||||
+5
-1
@@ -6,7 +6,11 @@ import { tokenOverlap } from "./match.mjs";
|
||||
import { buildCube } from "./cube-build.mjs";
|
||||
import { validateCube, validateCubeFile } from "./cube-validate.mjs";
|
||||
|
||||
const SKILL_DIR = join(import.meta.dirname, "..", "..");
|
||||
const SKILL_DIR = [
|
||||
join(import.meta.dirname, ".."),
|
||||
join(import.meta.dirname, "..", "..", "..", "..", "skills", "media-use"),
|
||||
join(import.meta.dirname, "..", "..", "..", "..", "..", "skills", "media-use"),
|
||||
].find((candidate) => existsSync(join(candidate, "luts", "index.json")));
|
||||
const LUT_DIR = join(SKILL_DIR, "luts");
|
||||
const LUT_INDEX = join(LUT_DIR, "index.json");
|
||||
export const LIBRARY_LUT_OFFLINE_CODE = "MEDIA_USE_LIBRARY_LUT_OFFLINE";
|
||||
+1
-1
@@ -22,7 +22,7 @@ import {
|
||||
import { buildCube } from "./cube-build.mjs";
|
||||
import { validateCube, validateCubeFile } from "./cube-validate.mjs";
|
||||
|
||||
const REPO_ROOT = join(import.meta.dirname, "..", "..", "..", "..");
|
||||
const REPO_ROOT = join(import.meta.dirname, "..", "..", "..", "..", "..");
|
||||
|
||||
function corePresetIdsFromSource() {
|
||||
const src = readFileSync(join(REPO_ROOT, "packages/core/src/colorGrading.ts"), "utf8");
|
||||
@@ -0,0 +1,229 @@
|
||||
import {
|
||||
readFileSync,
|
||||
appendFileSync,
|
||||
mkdirSync,
|
||||
existsSync,
|
||||
readdirSync,
|
||||
openSync,
|
||||
closeSync,
|
||||
writeFileSync,
|
||||
rmSync,
|
||||
statSync,
|
||||
} from "node:fs";
|
||||
import { join } from "node:path";
|
||||
|
||||
const MANIFEST_FILE = "manifest.jsonl";
|
||||
const INDEX_FILE = "index.md";
|
||||
|
||||
const TYPE_DIRS = {
|
||||
bgm: "audio/bgm",
|
||||
sfx: "audio/sfx",
|
||||
voice: "audio/voice",
|
||||
image: "images",
|
||||
icon: "images",
|
||||
logo: "images",
|
||||
brand: "images",
|
||||
video: "video",
|
||||
grade: "luts",
|
||||
lut: "luts",
|
||||
recipe: "recipes",
|
||||
};
|
||||
|
||||
export function mediaDir(projectDir) {
|
||||
return join(projectDir, ".media");
|
||||
}
|
||||
|
||||
export function manifestPath(projectDir) {
|
||||
return join(mediaDir(projectDir), MANIFEST_FILE);
|
||||
}
|
||||
|
||||
export function indexPath(projectDir) {
|
||||
return join(mediaDir(projectDir), INDEX_FILE);
|
||||
}
|
||||
|
||||
export function typeSubdir(type) {
|
||||
const sub = TYPE_DIRS[type];
|
||||
if (!sub) throw new Error(`unknown media type: ${type}`);
|
||||
return sub;
|
||||
}
|
||||
|
||||
export function typeDirPath(projectDir, type) {
|
||||
return join(mediaDir(projectDir), typeSubdir(type));
|
||||
}
|
||||
|
||||
export function readManifest(projectDir) {
|
||||
const p = manifestPath(projectDir);
|
||||
if (!existsSync(p)) return [];
|
||||
const raw = readFileSync(p, "utf8");
|
||||
const records = [];
|
||||
for (const line of raw.split(/\r?\n/)) {
|
||||
const trimmed = line.trim();
|
||||
if (!trimmed) continue;
|
||||
try {
|
||||
records.push(JSON.parse(trimmed));
|
||||
} catch {
|
||||
// ponytail: skip malformed lines, don't crash
|
||||
}
|
||||
}
|
||||
return records;
|
||||
}
|
||||
|
||||
export function appendRecord(projectDir, record) {
|
||||
const dir = mediaDir(projectDir);
|
||||
mkdirSync(dir, { recursive: true });
|
||||
const typeDir = typeDirPath(projectDir, record.type);
|
||||
mkdirSync(typeDir, { recursive: true });
|
||||
|
||||
const p = manifestPath(projectDir);
|
||||
const line = JSON.stringify(record) + "\n";
|
||||
appendFileSync(p, line);
|
||||
}
|
||||
|
||||
// Match prompts forgivingly. Agents rarely re-emit a byte-identical intent, so
|
||||
// keying cache lookups on exact equality meant "Calm piano" and "calm piano"
|
||||
// re-searched and re-downloaded. Normalize (trim, lowercase, collapse internal
|
||||
// whitespace) on both sides; the raw prompt is still stored for audit.
|
||||
export function normalizePrompt(prompt) {
|
||||
return String(prompt ?? "")
|
||||
.trim()
|
||||
.toLowerCase()
|
||||
.replace(/\s+/g, " ");
|
||||
}
|
||||
|
||||
export function findByPrompt(projectDir, prompt, type) {
|
||||
const key = normalizePrompt(prompt);
|
||||
if (!key) return null;
|
||||
const records = readManifest(projectDir);
|
||||
return (
|
||||
records.find(
|
||||
(r) => normalizePrompt(r.provenance?.prompt) === key && (type == null || r.type === type),
|
||||
) || null
|
||||
);
|
||||
}
|
||||
|
||||
export function findByEntity(projectDir, entity) {
|
||||
const lower = entity.toLowerCase();
|
||||
const records = readManifest(projectDir);
|
||||
return records.find((r) => r.entity && r.entity.toLowerCase() === lower) || null;
|
||||
}
|
||||
|
||||
export function nextId(projectDir, type) {
|
||||
const records = readManifest(projectDir);
|
||||
const prefix = type;
|
||||
let max = 0;
|
||||
for (const r of records) {
|
||||
if (r.type !== type) continue;
|
||||
const m = r.id?.match(new RegExp(`^${prefix}_(\\d+)$`));
|
||||
if (m) max = Math.max(max, parseInt(m[1], 10));
|
||||
}
|
||||
return `${prefix}_${String(max + 1).padStart(3, "0")}`;
|
||||
}
|
||||
|
||||
// Sync sleep (no busy-spin) for the allocation lock retry.
|
||||
function sleepMs(ms) {
|
||||
Atomics.wait(new Int32Array(new SharedArrayBuffer(4)), 0, 0, ms);
|
||||
}
|
||||
|
||||
// Coarse per-project lock so concurrent resolves don't race on id allocation.
|
||||
// ponytail: one lock file with a 15s stale-steal (a crashed holder can't wedge
|
||||
// the project); fine for agent-scale concurrency — revisit if throughput needs
|
||||
// finer locking. Date.now() is available here (a normal Node CLI, not a
|
||||
// workflow DSL), so mtime-based staleness is safe.
|
||||
const LOCK_STALE_MS = 15000;
|
||||
const LOCK_TIMEOUT_MS = 20000;
|
||||
|
||||
function withLock(dir, fn) {
|
||||
const lock = join(dir, ".lock");
|
||||
const start = Date.now();
|
||||
for (;;) {
|
||||
try {
|
||||
closeSync(openSync(lock, "wx")); // O_EXCL: atomic acquire
|
||||
break;
|
||||
} catch (err) {
|
||||
if (err.code !== "EEXIST") throw err;
|
||||
try {
|
||||
if (Date.now() - statSync(lock).mtimeMs > LOCK_STALE_MS) {
|
||||
rmSync(lock, { force: true }); // steal a stale lock from a dead holder
|
||||
continue;
|
||||
}
|
||||
} catch {
|
||||
continue; // lock vanished between check and stat — retry the acquire
|
||||
}
|
||||
if (Date.now() - start > LOCK_TIMEOUT_MS) {
|
||||
throw new Error("media-use: timed out acquiring .media/.lock");
|
||||
}
|
||||
sleepMs(25);
|
||||
}
|
||||
}
|
||||
try {
|
||||
return fn();
|
||||
} finally {
|
||||
rmSync(lock, { force: true });
|
||||
}
|
||||
}
|
||||
|
||||
// Atomically allocate the next free id for `type` AND reserve its file, so a
|
||||
// slow download/copy between allocation and appendRecord can't let a concurrent
|
||||
// caller grab the same id (the MU-23 clobber). Under the lock we take the max id
|
||||
// across BOTH the manifest and any already-reserved files in the type dir, then
|
||||
// O_EXCL-create an empty placeholder at the target path; freeze/copy overwrites
|
||||
// it. Returns { id, localPath }.
|
||||
export function allocateId(projectDir, type, ext) {
|
||||
mkdirSync(mediaDir(projectDir), { recursive: true });
|
||||
const typeDir = typeDirPath(projectDir, type);
|
||||
mkdirSync(typeDir, { recursive: true });
|
||||
return withLock(mediaDir(projectDir), () => {
|
||||
const re = new RegExp(`^${type}_(\\d+)`);
|
||||
let max = 0;
|
||||
for (const r of readManifest(projectDir)) {
|
||||
if (r.type !== type) continue;
|
||||
const m = r.id?.match(re);
|
||||
if (m) max = Math.max(max, parseInt(m[1], 10));
|
||||
}
|
||||
for (const f of readdirSync(typeDir)) {
|
||||
const m = f.match(re);
|
||||
if (m) max = Math.max(max, parseInt(m[1], 10)); // skip ids reserved but not yet appended
|
||||
}
|
||||
const id = `${type}_${String(max + 1).padStart(3, "0")}`;
|
||||
const localPath = `.media/${typeSubdir(type)}/${id}${ext}`;
|
||||
writeFileSync(join(projectDir, localPath), "", { flag: "wx" }); // durable reservation
|
||||
return { id, localPath };
|
||||
});
|
||||
}
|
||||
|
||||
function reservedFile(projectDir, type, ext) {
|
||||
const allocation = allocateId(projectDir, type, ext);
|
||||
return { ...allocation, fullPath: join(projectDir, allocation.localPath) };
|
||||
}
|
||||
|
||||
function rollbackReservation(reservation) {
|
||||
rmSync(reservation.fullPath, { force: true });
|
||||
}
|
||||
|
||||
// A reservation is committed only when populate returns a non-null value.
|
||||
// Throwing/rejecting or returning null means no usable asset was produced, so
|
||||
// the placeholder must be released. Keeping this transaction beside allocateId
|
||||
// prevents individual provider/cache/LUT paths from forgetting the rollback.
|
||||
export function withReservedFileSync(projectDir, type, ext, populate) {
|
||||
const reservation = reservedFile(projectDir, type, ext);
|
||||
try {
|
||||
const result = populate(reservation);
|
||||
if (result == null) rollbackReservation(reservation);
|
||||
return result;
|
||||
} catch (error) {
|
||||
rollbackReservation(reservation);
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
|
||||
export async function withReservedFile(projectDir, type, ext, populate) {
|
||||
const reservation = reservedFile(projectDir, type, ext);
|
||||
try {
|
||||
const result = await populate(reservation);
|
||||
if (result == null) rollbackReservation(reservation);
|
||||
return result;
|
||||
} catch (error) {
|
||||
rollbackReservation(reservation);
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,71 @@
|
||||
// Media downloads use public HTTP(S) URLs. Validate every redirect target;
|
||||
// a provider result must meet the same host policy as a direct ingest URL.
|
||||
// Public HTTPS-to-HTTP redirects are allowed, matching direct HTTP support.
|
||||
// This is a literal-host policy, not DNS pinning: DNS resolution remains trusted.
|
||||
|
||||
import { BlockList, isIP } from "node:net";
|
||||
|
||||
const blocked = new BlockList();
|
||||
for (const [network, prefix] of [
|
||||
["0.0.0.0", 8],
|
||||
["10.0.0.0", 8],
|
||||
["100.64.0.0", 10],
|
||||
["127.0.0.0", 8],
|
||||
["169.254.0.0", 16],
|
||||
["172.16.0.0", 12],
|
||||
["192.0.0.0", 24],
|
||||
["192.0.2.0", 24],
|
||||
["192.88.99.0", 24],
|
||||
["192.168.0.0", 16],
|
||||
["198.18.0.0", 15],
|
||||
["198.51.100.0", 24],
|
||||
["203.0.113.0", 24],
|
||||
["224.0.0.0", 4],
|
||||
["240.0.0.0", 4],
|
||||
])
|
||||
blocked.addSubnet(network, prefix, "ipv4");
|
||||
for (const [network, prefix] of [
|
||||
["::", 128],
|
||||
["::1", 128],
|
||||
["fc00::", 7],
|
||||
["fe80::", 10],
|
||||
["fec0::", 10],
|
||||
["ff00::", 8],
|
||||
["2001:db8::", 32],
|
||||
])
|
||||
blocked.addSubnet(network, prefix, "ipv6");
|
||||
|
||||
export function isPublicMediaUrl(value) {
|
||||
try {
|
||||
const url = new URL(value);
|
||||
if (url.protocol !== "http:" && url.protocol !== "https:") return false;
|
||||
const host = url.hostname.replace(/\.$/, "");
|
||||
if (
|
||||
host === "localhost" ||
|
||||
host.endsWith(".localhost") ||
|
||||
host.endsWith(".local") ||
|
||||
host.endsWith(".internal")
|
||||
)
|
||||
return false;
|
||||
const address = host.replace(/^\[|\]$/g, "");
|
||||
const family = isIP(address);
|
||||
return family === 0 || !blocked.check(address, family === 4 ? "ipv4" : "ipv6");
|
||||
} catch {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
export async function fetchMedia(url, { method = "GET", signal, fetchImpl = fetch } = {}) {
|
||||
let current = String(url);
|
||||
for (let hop = 0; hop <= 5; hop++) {
|
||||
if (!isPublicMediaUrl(current))
|
||||
throw new Error("Media download blocked: URL is not public HTTP(S)");
|
||||
const response = await fetchImpl(current, { method, signal, redirect: "manual" });
|
||||
if (!(response.status >= 300 && response.status < 400)) return response;
|
||||
const location = response.headers.get("location");
|
||||
if (!location) return response;
|
||||
await response.body?.cancel();
|
||||
current = new URL(location, current).href;
|
||||
}
|
||||
throw new Error("Media download exceeded redirect limit");
|
||||
}
|
||||
@@ -0,0 +1,9 @@
|
||||
export function rankMediaRows<
|
||||
T extends {
|
||||
id: string;
|
||||
kind: string;
|
||||
title: string;
|
||||
description: string;
|
||||
tags: string[];
|
||||
},
|
||||
>(query: string, rows: T[]): T[];
|
||||
@@ -0,0 +1,107 @@
|
||||
const STOP = new Set(
|
||||
(
|
||||
"the a an and or of to in on at is are be it its for with that this as by from into " +
|
||||
"one two must not no all over under across while when where which who whom whose they " +
|
||||
"them their we our you your he she his her but if then than so such can may might will " +
|
||||
"would should each other another same both few more most some any every"
|
||||
).split(" "),
|
||||
);
|
||||
|
||||
const PLURAL_RULES = [
|
||||
[/([^aeiou])ies$/, "$1y"],
|
||||
[/(ch|sh|ss|x)es$/, "$1"],
|
||||
[/(ss|us|is)$/, "$&"],
|
||||
[/s$/, ""],
|
||||
];
|
||||
const STRONG_FIELD_WEIGHT = 3;
|
||||
const INFERRED_TOKEN_WEIGHT = 0.35;
|
||||
|
||||
function singularize(word) {
|
||||
if (word.length <= 3) return word;
|
||||
for (const [pattern, replacement] of PLURAL_RULES) {
|
||||
if (pattern.test(word)) return word.replace(pattern, replacement);
|
||||
}
|
||||
return word;
|
||||
}
|
||||
|
||||
function tokenize(text) {
|
||||
const words =
|
||||
String(text)
|
||||
.toLowerCase()
|
||||
.match(/[a-z]+/g) || [];
|
||||
return words.filter((word) => word.length > 2 && !STOP.has(word)).map(singularize);
|
||||
}
|
||||
|
||||
export function rankMediaRows(query, rows) {
|
||||
const asked = tokenize(query);
|
||||
const normalizedLiteralQuery = String(query).trim().toLowerCase();
|
||||
const normalizedQuery = asked.join(" ");
|
||||
const want = new Map(asked.map((token) => [token, 1]));
|
||||
if (want.size === 0) return [];
|
||||
|
||||
const parsed = rows.map((row) => {
|
||||
const strongTokens = new Set(tokenize(`${row.id} ${row.title}`));
|
||||
const allTokens = new Set([
|
||||
...strongTokens,
|
||||
...tokenize(`${row.description} ${row.tags.join(" ")} ${row.kind}`),
|
||||
]);
|
||||
return {
|
||||
row,
|
||||
strongTokens,
|
||||
allTokens,
|
||||
literalId: row.id.trim().toLowerCase() === normalizedLiteralQuery,
|
||||
stemmedId: tokenize(row.id).join(" ") === normalizedQuery,
|
||||
};
|
||||
});
|
||||
const vocabulary = new Set(parsed.flatMap(({ allTokens }) => [...allTokens]));
|
||||
for (const token of asked) {
|
||||
if (token.length < 6) continue;
|
||||
for (let cut = 3; cut <= token.length - 3; cut++) {
|
||||
const head = token.slice(0, cut);
|
||||
const tail = token.slice(cut);
|
||||
if (vocabulary.has(head) && vocabulary.has(tail)) {
|
||||
want.set(head, INFERRED_TOKEN_WEIGHT);
|
||||
want.set(tail, INFERRED_TOKEN_WEIGHT);
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
for (let index = 0; index < asked.length - 1; index++) {
|
||||
const joined = `${asked[index]}${asked[index + 1]}`;
|
||||
if (vocabulary.has(joined)) want.set(joined, INFERRED_TOKEN_WEIGHT);
|
||||
}
|
||||
|
||||
const idf = new Map();
|
||||
for (const token of want.keys()) {
|
||||
const documentFrequency = parsed.filter(({ allTokens }) => allTokens.has(token)).length;
|
||||
idf.set(token, Math.log((parsed.length + 1) / (documentFrequency + 1)) + 1);
|
||||
}
|
||||
return parsed
|
||||
.map(({ row, strongTokens, allTokens, literalId, stemmedId }) => {
|
||||
let score = 0;
|
||||
for (const [token, asking] of want) {
|
||||
const weight = (idf.get(token) || 1) * asking;
|
||||
if (strongTokens.has(token)) score += STRONG_FIELD_WEIGHT * weight;
|
||||
else if (allTokens.has(token)) score += weight;
|
||||
}
|
||||
return { row, score, literalId, stemmedId };
|
||||
})
|
||||
.filter(({ score }) => score > 0)
|
||||
.sort(
|
||||
(a, b) =>
|
||||
Number(b.literalId) - Number(a.literalId) ||
|
||||
Number(b.stemmedId) - Number(a.stemmedId) ||
|
||||
b.score - a.score,
|
||||
)
|
||||
.map(({ row }) => row);
|
||||
}
|
||||
|
||||
export async function rankMediaRowsWithVectors(query, rows, semanticRanking) {
|
||||
const words = rankMediaRows(query, rows);
|
||||
if (words.length > 0) return { rows: words, tier: "words" };
|
||||
if (!semanticRanking) return { rows: [], tier: "words" };
|
||||
const semantic = await semanticRanking(query, rows);
|
||||
return semantic
|
||||
? { rows: semantic.map(({ row }) => row), tier: "on-device" }
|
||||
: { rows: [], tier: "words" };
|
||||
}
|
||||
@@ -0,0 +1,31 @@
|
||||
import { existsSync } from "node:fs";
|
||||
import { resolveSpawnCommand } from "../../audio/scripts/lib/tts.mjs";
|
||||
|
||||
// Sync-spawn analog of the audio engine's spawnP, for execFileSync call sites
|
||||
// that must hard-fail (rather than fall through to another provider) when npx
|
||||
// cannot be resolved. On Windows a bare "npx" is npx.cmd, which
|
||||
// execFileSync/spawnSync cannot exec (spawnSync npx ENOENT) —
|
||||
// resolveSpawnCommand reroutes it through node + npx-cli.js, no shell:true.
|
||||
//
|
||||
// `platform`/`env`/`pathExists` params (defaulting to the real values) exist
|
||||
// so tests can exercise the win32 branch without mocking node:child_process
|
||||
// (its ESM exports are non-configurable) — same idiom as spawnP and
|
||||
// localTtsGenerate.
|
||||
export function resolveNpxInvocation(
|
||||
argv,
|
||||
opts,
|
||||
platform = process.platform,
|
||||
env = process.env,
|
||||
pathExists = existsSync,
|
||||
) {
|
||||
const resolved = resolveSpawnCommand("npx", argv, opts, platform, env, pathExists);
|
||||
if (!resolved) {
|
||||
// npx-on-win32 with no resolvable npx-cli.js — same terminal condition
|
||||
// spawnP warns about, surfaced as a throw for callers with no fallback.
|
||||
throw new Error(
|
||||
"cannot run npx on Windows: npm's npx-cli.js was not found " +
|
||||
"(install npm with Node, or run via npx/npm run so npm_execpath is set)",
|
||||
);
|
||||
}
|
||||
return resolved;
|
||||
}
|
||||
@@ -0,0 +1,27 @@
|
||||
// Merge Parakeet-MLX token timestamps into word timestamps.
|
||||
//
|
||||
// parakeet-mlx JSON emits SUB-WORD tokens (" H", "ello", ...) with per-token
|
||||
// start/end. Captions + transcript-cut need WORD timestamps, so join tokens
|
||||
// into words on the space boundary: a token whose text starts with a space
|
||||
// (or the very first token) begins a new word; the rest append. Output matches
|
||||
// the { words: [{ text, start, end }] } shape the rest of media-use consumes
|
||||
// (see words.mjs / cutlist.mjs).
|
||||
|
||||
export function mergeTokensToWords(parakeet) {
|
||||
const sentences = Array.isArray(parakeet?.sentences) ? parakeet.sentences : [];
|
||||
const words = [];
|
||||
for (const s of sentences) {
|
||||
for (const t of s.tokens ?? []) {
|
||||
const raw = typeof t.text === "string" ? t.text : "";
|
||||
const startsWord = raw.startsWith(" ") || words.length === 0;
|
||||
if (startsWord) {
|
||||
words.push({ text: raw.trim(), start: t.start, end: t.end });
|
||||
} else {
|
||||
const w = words[words.length - 1];
|
||||
w.text += raw;
|
||||
w.end = t.end;
|
||||
}
|
||||
}
|
||||
}
|
||||
return { text: (parakeet?.text ?? "").trim(), words: words.filter((w) => w.text.length > 0) };
|
||||
}
|
||||
@@ -0,0 +1,180 @@
|
||||
import { existsSync, mkdirSync, readFileSync, renameSync, writeFileSync } from "node:fs";
|
||||
import { homedir } from "node:os";
|
||||
import { basename, dirname, join, resolve } from "node:path";
|
||||
|
||||
/**
|
||||
* Remembered defaults — the lightweight tier of HyperFrames user memory.
|
||||
*
|
||||
* Two files, same shape as the rest of media-use's storage split:
|
||||
* - project `.media/preferences.json` — committed with the repo, so the whole
|
||||
* team inherits it; written every time a brief answer is confirmed.
|
||||
* - user `~/.media/preferences.json` — personal, cross-repo. A key is promoted
|
||||
* here only once the same value has been confirmed in two different projects
|
||||
* (`PROMOTE_AT`), so a one-off choice never pollutes the global defaults.
|
||||
* Pre-promotion evidence accumulates in the user file's `sightings` ledger —
|
||||
* project files can't see each other, so the cross-project count has to live
|
||||
* user-side.
|
||||
*
|
||||
* Consumption contract (brief-contract § 2, Remembered defaults): a remembered
|
||||
* value becomes the recommended option with a receipt naming its source — it
|
||||
* never skips a question, and explicit request content always wins.
|
||||
*/
|
||||
|
||||
const PREFS_FILE = "preferences.json";
|
||||
|
||||
/** Keys the brief contract records; `style_preset` is stored per workflow. */
|
||||
export const PREFERENCE_KEYS = [
|
||||
"destination",
|
||||
"aspect",
|
||||
"language",
|
||||
"flow",
|
||||
"storyboard",
|
||||
"voice",
|
||||
"style_preset",
|
||||
];
|
||||
|
||||
/** A value must be confirmed in this many distinct projects to go user-tier. */
|
||||
export const PROMOTE_AT = 2;
|
||||
|
||||
export function projectPrefsPath(projectDir) {
|
||||
return join(resolve(projectDir), ".media", PREFS_FILE);
|
||||
}
|
||||
|
||||
export function userPrefsPath() {
|
||||
return join(homedir(), ".media", PREFS_FILE);
|
||||
}
|
||||
|
||||
function emptyFile() {
|
||||
return { version: 1, preferences: {}, sightings: {} };
|
||||
}
|
||||
|
||||
function isRecord(value) {
|
||||
return typeof value === "object" && value !== null && !Array.isArray(value);
|
||||
}
|
||||
|
||||
/** Tolerant read — a missing or malformed file counts as empty. */
|
||||
function readPrefsFile(path) {
|
||||
try {
|
||||
if (!existsSync(path)) return emptyFile();
|
||||
const parsed = JSON.parse(readFileSync(path, "utf8"));
|
||||
if (!isRecord(parsed)) return emptyFile();
|
||||
return {
|
||||
version: 1,
|
||||
preferences: isRecord(parsed.preferences) ? parsed.preferences : {},
|
||||
sightings: isRecord(parsed.sightings) ? parsed.sightings : {},
|
||||
};
|
||||
} catch {
|
||||
return emptyFile();
|
||||
}
|
||||
}
|
||||
|
||||
/** Atomic write (tmp + rename) so a crash never leaves a torn file. */
|
||||
function writePrefsFile(path, file) {
|
||||
mkdirSync(dirname(path), { recursive: true });
|
||||
const tmp = `${path}.tmp`;
|
||||
writeFileSync(tmp, `${JSON.stringify(file, null, 2)}\n`);
|
||||
renameSync(tmp, path);
|
||||
}
|
||||
|
||||
/** `style_preset` entries are stored per workflow as `style_preset.<workflow>`. */
|
||||
export function preferenceKeyFor(key, workflow) {
|
||||
return key === "style_preset" && workflow ? `style_preset.${workflow}` : key;
|
||||
}
|
||||
|
||||
function validEntry(entry) {
|
||||
return isRecord(entry) && typeof entry.value === "string" && entry.value.length > 0;
|
||||
}
|
||||
|
||||
/**
|
||||
* The merged view the brief reads: user-tier promoted entries first, project
|
||||
* entries on top (project wins). Each entry carries `source` plus the receipt
|
||||
* material (`confirmed_in`, `updated_at`).
|
||||
*/
|
||||
export function mergedPreferences(projectDir) {
|
||||
const user = readPrefsFile(userPrefsPath());
|
||||
const project = readPrefsFile(projectPrefsPath(projectDir));
|
||||
const merged = {};
|
||||
for (const [key, entry] of Object.entries(user.preferences)) {
|
||||
if (validEntry(entry)) merged[key] = { ...entry, source: "user" };
|
||||
}
|
||||
for (const [key, entry] of Object.entries(project.preferences)) {
|
||||
if (validEntry(entry)) merged[key] = { ...entry, source: "project" };
|
||||
}
|
||||
return merged;
|
||||
}
|
||||
|
||||
function dedupe(list) {
|
||||
return [...new Set(list)];
|
||||
}
|
||||
|
||||
/**
|
||||
* Project tier: same value accumulates confirmations; a changed value starts
|
||||
* provenance over (the old confirmations vouched for the old value).
|
||||
*/
|
||||
function recordProjectTier(projectDir, fullKey, value, projectName, now) {
|
||||
const path = projectPrefsPath(projectDir);
|
||||
const file = readPrefsFile(path);
|
||||
const previous = file.preferences[fullKey];
|
||||
const keepProvenance = validEntry(previous) && previous.value === value;
|
||||
const confirmedIn = keepProvenance
|
||||
? dedupe([...(Array.isArray(previous.confirmed_in) ? previous.confirmed_in : []), projectName])
|
||||
: [projectName];
|
||||
file.preferences[fullKey] = { value, confirmed_in: confirmedIn, updated_at: now };
|
||||
writePrefsFile(path, file);
|
||||
return confirmedIn;
|
||||
}
|
||||
|
||||
/**
|
||||
* User tier: accumulate this sighting in the ledger, and promote the key once
|
||||
* the same value has been confirmed in PROMOTE_AT distinct projects.
|
||||
*/
|
||||
function recordUserSighting(fullKey, value, projectName, now) {
|
||||
const path = userPrefsPath();
|
||||
const file = readPrefsFile(path);
|
||||
const keySightings = isRecord(file.sightings[fullKey]) ? file.sightings[fullKey] : {};
|
||||
const seenIn = dedupe([
|
||||
...(Array.isArray(keySightings[value]) ? keySightings[value] : []),
|
||||
projectName,
|
||||
]);
|
||||
keySightings[value] = seenIn;
|
||||
file.sightings[fullKey] = keySightings;
|
||||
const promoted = seenIn.length >= PROMOTE_AT;
|
||||
if (promoted) {
|
||||
file.preferences[fullKey] = { value, confirmed_in: seenIn, updated_at: now };
|
||||
}
|
||||
writePrefsFile(path, file);
|
||||
return promoted;
|
||||
}
|
||||
|
||||
/**
|
||||
* Record one confirmed brief answer. Always writes the project tier; feeds the
|
||||
* user tier's sightings ledger and promotes once the same value has been
|
||||
* confirmed in PROMOTE_AT distinct projects. Idempotent per project.
|
||||
*/
|
||||
export function recordPreference({ projectDir, key, value, workflow }) {
|
||||
if (!PREFERENCE_KEYS.includes(key)) {
|
||||
throw new Error(`unknown preference key: "${key}" (known: ${PREFERENCE_KEYS.join(", ")})`);
|
||||
}
|
||||
if (typeof value !== "string" || !value.trim()) {
|
||||
throw new Error("a preference needs a non-empty string value");
|
||||
}
|
||||
if (key === "style_preset" && (!workflow || !String(workflow).trim())) {
|
||||
throw new Error("style_preset is stored per workflow — pass --workflow <w>");
|
||||
}
|
||||
const fullKey = preferenceKeyFor(key, workflow);
|
||||
const projectName = basename(resolve(projectDir));
|
||||
const trimmed = value.trim();
|
||||
const now = new Date().toISOString();
|
||||
|
||||
const confirmedIn = recordProjectTier(projectDir, fullKey, trimmed, projectName, now);
|
||||
|
||||
// Best-effort — a read-only home directory must never fail a brief.
|
||||
let promoted = false;
|
||||
try {
|
||||
promoted = recordUserSighting(fullKey, trimmed, projectName, now);
|
||||
} catch {
|
||||
// The project record already landed; promotion just waits for next time.
|
||||
}
|
||||
|
||||
return { key: fullKey, value: trimmed, confirmed_in: confirmedIn, promoted };
|
||||
}
|
||||
@@ -0,0 +1,367 @@
|
||||
import {
|
||||
cpSync,
|
||||
existsSync,
|
||||
mkdirSync,
|
||||
readdirSync,
|
||||
readFileSync,
|
||||
renameSync,
|
||||
rmSync,
|
||||
writeFileSync,
|
||||
} from "node:fs";
|
||||
import { homedir } from "node:os";
|
||||
import { basename, join, resolve } from "node:path";
|
||||
import { appendRecord, mediaDir, nextId } from "./manifest.mjs";
|
||||
import { regenerateIndex } from "./index-gen.mjs";
|
||||
import { mergedPreferences } from "./prefs-store.mjs";
|
||||
|
||||
/**
|
||||
* Recipes — the heavyweight tier of HyperFrames user memory.
|
||||
*
|
||||
* A recipe is the full confirmed bundle for one video type: the frozen design
|
||||
* spec (`frame.md`), the storyboard skeleton (structure with the content
|
||||
* blanked), and the confirmed brief values — frozen after the run's final
|
||||
* approval, reused to start the next video of the same type from everything
|
||||
* already approved.
|
||||
*
|
||||
* Storage is **named folders**, not content-addressed cache entries: a recipe
|
||||
* is an evolving bundle with a `version`, so re-freezing the same name bumps
|
||||
* the version and archives the old folder as `<name>@v<N>`. Two tiers, same
|
||||
* split as everything else in media-use: project `.media/recipes/<name>/`
|
||||
* (committed) and user `~/.media/recipes/<name>/` (a freeze is already a
|
||||
* confirmed bundle, so it promotes immediately — no two-project rule here).
|
||||
*/
|
||||
|
||||
/** Frontmatter keys that describe THIS video, not the reusable type. */
|
||||
const FRONTMATTER_CONTENT_KEYS = new Set(["message", "audience", "mode"]);
|
||||
|
||||
/** BRIEF.md frontmatter keys that describe this run, not the reusable type —
|
||||
* a recipe never locks the run's shape, so the intent layer always re-asks. */
|
||||
const BRIEF_CONTENT_KEYS = new Set(["flow", "storyboard", "message", "audience"]);
|
||||
|
||||
/** Per-frame metadata that is content, not structure. */
|
||||
const FRAME_CONTENT_KEYS = new Set([
|
||||
"voiceover",
|
||||
"vo",
|
||||
"voice_over",
|
||||
"narration",
|
||||
"scene",
|
||||
"description",
|
||||
"summary",
|
||||
"caption",
|
||||
"asset_candidates",
|
||||
]);
|
||||
|
||||
const FRAME_HEADING_RE = /^(#{2,3})\s+(?:frame|beat|scene)\s+\d+/i;
|
||||
|
||||
export function projectRecipesDir(projectDir) {
|
||||
return join(mediaDir(projectDir), "recipes");
|
||||
}
|
||||
|
||||
export function userRecipesDir() {
|
||||
return join(homedir(), ".media", "recipes");
|
||||
}
|
||||
|
||||
export function slugifyRecipeName(name) {
|
||||
const slug = String(name ?? "")
|
||||
.trim()
|
||||
.toLowerCase()
|
||||
.replace(/[\s_]+/g, "-")
|
||||
.replace(/[^a-z0-9-]/g, "")
|
||||
.replace(/-+/g, "-")
|
||||
.replace(/^-|-$/g, "");
|
||||
if (!slug) throw new Error(`recipe name "${name}" has no usable characters`);
|
||||
return slug;
|
||||
}
|
||||
|
||||
function frameTitle(headingLine) {
|
||||
const dash = headingLine.split(/\s+—\s+/)[1];
|
||||
if (dash && dash.trim()) return dash.trim();
|
||||
return headingLine.replace(/^#+\s*/, "").trim();
|
||||
}
|
||||
|
||||
/** Frontmatter: drop the content keys, keep structure/style keys verbatim. */
|
||||
function skeletonFrontmatter(lines, out, contentKeys = FRONTMATTER_CONTENT_KEYS) {
|
||||
if (lines[0]?.trim() !== "---") return 0;
|
||||
out.push(lines[0]);
|
||||
let i = 1;
|
||||
while (i < lines.length && lines[i].trim() !== "---") {
|
||||
const key = lines[i].match(/^(\w+)\s*:/)?.[1]?.toLowerCase();
|
||||
if (!key || !contentKeys.has(key)) out.push(lines[i]);
|
||||
i++;
|
||||
}
|
||||
if (i < lines.length) {
|
||||
out.push(lines[i]); // closing ---
|
||||
i++;
|
||||
}
|
||||
return i;
|
||||
}
|
||||
|
||||
/** One line inside a frame section — returns the replacement lines (may be none). */
|
||||
function skeletonFrameLine(line, state, out) {
|
||||
const bulletKey = line.match(/^-\s+(\w+)\s*:/)?.[1]?.toLowerCase();
|
||||
if (bulletKey) {
|
||||
if (bulletKey === "status") out.push("- status: outline");
|
||||
else if (!FRAME_CONTENT_KEYS.has(bulletKey)) out.push(line);
|
||||
return;
|
||||
}
|
||||
if (!line.trim()) {
|
||||
out.push(line);
|
||||
return;
|
||||
}
|
||||
// Frame prose: one placeholder per frame in place of the narrative.
|
||||
if (!state.proseReplaced) {
|
||||
out.push(
|
||||
`<fill in: this video's content for the "${state.title}" beat — keep the layout role, replace the words.>`,
|
||||
);
|
||||
state.proseReplaced = true;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Skeletonize a STORYBOARD.md: keep the reusable structure (frame count,
|
||||
* durations, transitions, src paths, the Video direction block, style-ish
|
||||
* frontmatter), reset every status to `outline`, and blank the content
|
||||
* (message/audience, narration guides, per-frame prose) down to a fill-in
|
||||
* placeholder that names the frame's role.
|
||||
*/
|
||||
/**
|
||||
* Skeletonize a BRIEF.md: keep the frontmatter's reusable keys (workflow,
|
||||
* destination, aspect, language, length, angle…), drop the run-shape and
|
||||
* content keys (flow, storyboard, message, audience), and blank each body
|
||||
* section down to a fill-in placeholder under its kept heading.
|
||||
*/
|
||||
export function skeletonizeBrief(source) {
|
||||
const lines = String(source ?? "").split(/\r?\n/);
|
||||
const out = [];
|
||||
let i = skeletonFrontmatter(lines, out, BRIEF_CONTENT_KEYS);
|
||||
for (; i < lines.length; i++) {
|
||||
const heading = lines[i].match(/^##\s+(.+)$/);
|
||||
if (heading) {
|
||||
out.push(lines[i], "");
|
||||
out.push(
|
||||
`<fill in: this video's ${heading[1].trim().toLowerCase()} — the recipe keeps the shape, this run supplies the specifics.>`,
|
||||
);
|
||||
out.push("");
|
||||
}
|
||||
}
|
||||
return out.join("\n").replace(/\n{3,}/g, "\n\n");
|
||||
}
|
||||
|
||||
export function skeletonizeStoryboard(source) {
|
||||
const lines = String(source ?? "").split(/\r?\n/);
|
||||
const out = [];
|
||||
const state = { inFrame: false, proseReplaced: false, title: "" };
|
||||
for (let i = skeletonFrontmatter(lines, out); i < lines.length; i++) {
|
||||
const line = lines[i];
|
||||
if (/^#{2,3}\s/.test(line)) {
|
||||
state.inFrame = FRAME_HEADING_RE.test(line);
|
||||
state.proseReplaced = false;
|
||||
state.title = state.inFrame ? frameTitle(line) : "";
|
||||
out.push(line);
|
||||
} else if (!state.inFrame) {
|
||||
out.push(line);
|
||||
} else {
|
||||
skeletonFrameLine(line, state, out);
|
||||
}
|
||||
}
|
||||
return out.join("\n").replace(/\n{3,}/g, "\n\n");
|
||||
}
|
||||
|
||||
/** The run's workflow as BRIEF.md records it — the source of truth a freeze
|
||||
* must not contradict. Undefined when no BRIEF.md (or no `workflow:`) exists. */
|
||||
function briefWorkflow(root) {
|
||||
const brief = join(root, "BRIEF.md");
|
||||
if (!existsSync(brief)) return undefined;
|
||||
const lines = readFileSync(brief, "utf8").split(/\r?\n/);
|
||||
if (lines[0]?.trim() !== "---") return undefined;
|
||||
for (let i = 1; i < lines.length && lines[i].trim() !== "---"; i++) {
|
||||
const match = lines[i].match(/^workflow\s*:\s*(.+?)\s*$/);
|
||||
if (match) return match[1].replace(/^["']|["']$/g, "") || undefined;
|
||||
}
|
||||
return undefined;
|
||||
}
|
||||
|
||||
function readRecipeJson(dir) {
|
||||
try {
|
||||
const parsed = JSON.parse(readFileSync(join(dir, "recipe.json"), "utf8"));
|
||||
if (typeof parsed !== "object" || parsed === null || typeof parsed.name !== "string") {
|
||||
return null;
|
||||
}
|
||||
return parsed;
|
||||
} catch {
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
function prefValue(prefs, key) {
|
||||
return prefs[key]?.value;
|
||||
}
|
||||
|
||||
/**
|
||||
* Freeze the current project's approved run as a named recipe. Writes the
|
||||
* project-tier folder + a manifest record, then copies to the user tier (a
|
||||
* freeze is already confirmed — it promotes immediately).
|
||||
*/
|
||||
export function freezeRecipe({ projectDir, name, workflow, blocks }) {
|
||||
const slug = slugifyRecipeName(name);
|
||||
const root = resolve(projectDir);
|
||||
const fromBrief = briefWorkflow(root);
|
||||
const fromFlag = workflow && String(workflow).trim() ? String(workflow).trim() : undefined;
|
||||
// BRIEF.md decides; the flag only covers projects briefed before it existed.
|
||||
const resolvedWorkflow = fromBrief ?? fromFlag;
|
||||
if (!resolvedWorkflow) {
|
||||
throw new Error("no workflow found — BRIEF.md names none and no --workflow was given");
|
||||
}
|
||||
const frameSpec = join(root, "frame.md");
|
||||
const storyboard = join(root, "STORYBOARD.md");
|
||||
if (!existsSync(frameSpec)) throw new Error("no frame.md to freeze — run the design step first");
|
||||
if (!existsSync(storyboard)) throw new Error("no STORYBOARD.md to freeze");
|
||||
|
||||
const dir = join(projectRecipesDir(root), slug);
|
||||
let version = 1;
|
||||
const previous = existsSync(dir) ? readRecipeJson(dir) : null;
|
||||
if (previous) {
|
||||
version = (Number.isInteger(previous.version) ? previous.version : 1) + 1;
|
||||
const archive = `${dir}@v${previous.version ?? 1}`;
|
||||
rmSync(archive, { recursive: true, force: true });
|
||||
renameSync(dir, archive);
|
||||
}
|
||||
mkdirSync(dir, { recursive: true });
|
||||
|
||||
const prefs = mergedPreferences(root);
|
||||
const recipe = {
|
||||
version,
|
||||
name: slug,
|
||||
workflow: resolvedWorkflow,
|
||||
approved_at: new Date().toISOString(),
|
||||
source_project: basename(root),
|
||||
destination: prefValue(prefs, "destination"),
|
||||
aspect: prefValue(prefs, "aspect"),
|
||||
language: prefValue(prefs, "language"),
|
||||
voice: prefValue(prefs, "voice"),
|
||||
// The bare-key fallback tolerates records made before the store required
|
||||
// style_preset to be workflow-scoped.
|
||||
style_preset:
|
||||
prefValue(prefs, `style_preset.${resolvedWorkflow}`) ?? prefValue(prefs, "style_preset"),
|
||||
blocks: Array.isArray(blocks) && blocks.length > 0 ? blocks : undefined,
|
||||
};
|
||||
|
||||
writeFileSync(join(dir, "recipe.json"), `${JSON.stringify(recipe, null, 2)}\n`);
|
||||
cpSync(frameSpec, join(dir, "frame.md"));
|
||||
writeFileSync(
|
||||
join(dir, "storyboard-skeleton.md"),
|
||||
`${skeletonizeStoryboard(readFileSync(storyboard, "utf8")).trimEnd()}\n`,
|
||||
);
|
||||
|
||||
// Best-effort fourth artifact — projects briefed before BRIEF.md existed
|
||||
// (or by workflows that don't write one) freeze fine without it.
|
||||
const brief = join(root, "BRIEF.md");
|
||||
const briefSkeleton = existsSync(brief);
|
||||
if (briefSkeleton) {
|
||||
writeFileSync(
|
||||
join(dir, "brief-skeleton.md"),
|
||||
`${skeletonizeBrief(readFileSync(brief, "utf8")).trimEnd()}\n`,
|
||||
);
|
||||
}
|
||||
|
||||
const id = nextId(root, "recipe");
|
||||
appendRecord(root, {
|
||||
id,
|
||||
type: "recipe",
|
||||
path: `.media/recipes/${slug}/recipe.json`,
|
||||
entity: slug,
|
||||
description: `recipe: ${slug} (${recipe.workflow}, v${version})`,
|
||||
provenance: { provider: "recipe.freeze", version, source_project: recipe.source_project },
|
||||
});
|
||||
regenerateIndex(root);
|
||||
|
||||
// User tier — best-effort, like every other promotion.
|
||||
try {
|
||||
const userDir = join(userRecipesDir(), slug);
|
||||
mkdirSync(userDir, { recursive: true });
|
||||
cpSync(dir, userDir, { recursive: true, force: true });
|
||||
} catch {
|
||||
// The project-tier freeze already landed.
|
||||
}
|
||||
|
||||
return {
|
||||
id,
|
||||
slug,
|
||||
version,
|
||||
dir,
|
||||
briefSkeleton,
|
||||
workflow: resolvedWorkflow,
|
||||
workflowOverridden: Boolean(fromBrief && fromFlag && fromBrief !== fromFlag),
|
||||
};
|
||||
}
|
||||
|
||||
function scanRecipesDir(dir, source) {
|
||||
if (!existsSync(dir)) return [];
|
||||
const found = [];
|
||||
for (const entry of readdirSync(dir, { withFileTypes: true })) {
|
||||
if (!entry.isDirectory() || entry.name.includes("@v")) continue;
|
||||
const recipe = readRecipeJson(join(dir, entry.name));
|
||||
if (recipe) found.push({ ...recipe, source, dir: join(dir, entry.name) });
|
||||
}
|
||||
return found;
|
||||
}
|
||||
|
||||
/** Two-tier merged listing (project wins), newest approval first. */
|
||||
export function listRecipes({ projectDir, workflow }) {
|
||||
const merged = new Map();
|
||||
for (const recipe of scanRecipesDir(userRecipesDir(), "user")) merged.set(recipe.name, recipe);
|
||||
for (const recipe of scanRecipesDir(projectRecipesDir(resolve(projectDir)), "project")) {
|
||||
merged.set(recipe.name, recipe);
|
||||
}
|
||||
let list = [...merged.values()];
|
||||
if (workflow) list = list.filter((r) => r.workflow === workflow);
|
||||
return list.sort((a, b) =>
|
||||
String(b.approved_at ?? "").localeCompare(String(a.approved_at ?? "")),
|
||||
);
|
||||
}
|
||||
|
||||
/**
|
||||
* Adopt a recipe into the current project: import the folder from the user
|
||||
* tier when the project doesn't have it, copy its frame.md over the project's,
|
||||
* and hand back the values + the skeleton path for the storyboard draft.
|
||||
*/
|
||||
export function useRecipe({ projectDir, name }) {
|
||||
const slug = slugifyRecipeName(name);
|
||||
const root = resolve(projectDir);
|
||||
let dir = join(projectRecipesDir(root), slug);
|
||||
|
||||
if (!readRecipeJson(dir)) {
|
||||
const userDir = join(userRecipesDir(), slug);
|
||||
if (!readRecipeJson(userDir)) {
|
||||
const known = listRecipes({ projectDir: root }).map((r) => r.name);
|
||||
throw new Error(
|
||||
`no recipe named "${slug}"${known.length ? ` (known: ${known.join(", ")})` : ""}`,
|
||||
);
|
||||
}
|
||||
mkdirSync(dir, { recursive: true });
|
||||
cpSync(userDir, dir, { recursive: true, force: true });
|
||||
const imported = readRecipeJson(dir);
|
||||
appendRecord(root, {
|
||||
id: nextId(root, "recipe"),
|
||||
type: "recipe",
|
||||
path: `.media/recipes/${slug}/recipe.json`,
|
||||
entity: slug,
|
||||
description: `recipe: ${slug} (${imported.workflow}, v${imported.version})`,
|
||||
provenance: { provider: "recipe.local", imported_from: "user-tier" },
|
||||
});
|
||||
regenerateIndex(root);
|
||||
}
|
||||
|
||||
const recipe = readRecipeJson(dir);
|
||||
cpSync(join(dir, "frame.md"), join(root, "frame.md"));
|
||||
return {
|
||||
recipe,
|
||||
dir,
|
||||
frameSpecPath: "frame.md",
|
||||
skeletonPath: `.media/recipes/${slug}/storyboard-skeleton.md`,
|
||||
// Recipes frozen before BRIEF.md existed have no brief skeleton — degrade.
|
||||
briefSkeletonPath: existsSync(join(dir, "brief-skeleton.md"))
|
||||
? `.media/recipes/${slug}/brief-skeleton.md`
|
||||
: undefined,
|
||||
};
|
||||
}
|
||||
+1
-1
@@ -51,8 +51,8 @@ const P = (name, caps) => ({ name, network: true, paid: true, ...caps }); // rem
|
||||
const REGISTRY = {
|
||||
bgm: [N("heygen.audio.sounds", { search: bgmProvider.search })],
|
||||
sfx: [
|
||||
N("heygen.audio.sounds", { search: sfxProvider.search }),
|
||||
A("bundled.sfx", { search: bundledSfxProvider.search }),
|
||||
N("heygen.audio.sounds", { search: sfxProvider.search }),
|
||||
],
|
||||
image: [
|
||||
N("heygen.asset.search", { search: imageProvider.search }),
|
||||
+9
-8
@@ -32,12 +32,13 @@ test("listTypes exposes the v2 media types", () => {
|
||||
}
|
||||
});
|
||||
|
||||
test("heygen provider is first for every type it serves", () => {
|
||||
for (const t of ["bgm", "sfx", "image", "icon"]) {
|
||||
test("heygen provider is first for every type it serves except bundled sfx", () => {
|
||||
for (const t of ["bgm", "image", "icon"]) {
|
||||
const first = getProviders(t)[0];
|
||||
assert.ok(first, `no enabled provider for ${t}`);
|
||||
assert.match(first.name, /^heygen/, `${t} first provider is ${first.name}`);
|
||||
}
|
||||
assert.equal(getProviders("sfx")[0].name, "bundled.sfx");
|
||||
});
|
||||
|
||||
test("sanctioned providers only: heygen, local mflux/kokoro/ltx, codex, design spec, logo tiers", () => {
|
||||
@@ -84,13 +85,13 @@ test("video cascade: HeyGen first, LTX local fallback, generate-only", async ()
|
||||
assert.equal(await runCapability("video", "search", "x", {}), null);
|
||||
});
|
||||
|
||||
test("sfx cascade: HeyGen catalog first, bundled library remains the local fallback", () => {
|
||||
test("sfx cascade: bundled library first, HeyGen catalog remains the network fallback", () => {
|
||||
const ps = getProviders("sfx");
|
||||
assert.equal(ps[0].name, "heygen.audio.sounds");
|
||||
assert.ok(ps[0].network, "HeyGen SFX catalog is network-only");
|
||||
assert.equal(ps[1].name, "bundled.sfx");
|
||||
assert.equal(typeof ps[1].search, "function");
|
||||
assert.ok(!ps[1].network, "bundled SFX remain available offline");
|
||||
assert.equal(ps[0].name, "bundled.sfx");
|
||||
assert.equal(typeof ps[0].search, "function");
|
||||
assert.ok(!ps[0].network, "bundled SFX remain available offline");
|
||||
assert.equal(ps[1].name, "heygen.audio.sounds");
|
||||
assert.ok(ps[1].network, "HeyGen SFX catalog is network-only");
|
||||
});
|
||||
|
||||
test("ctx.provider forces one generator (e.g. 'make an image WITH codex')", async () => {
|
||||
@@ -0,0 +1,193 @@
|
||||
// Usage tracking shares the CLI and Studio identity. Properties stay coarse and
|
||||
// never carry intent text, file names, or paths.
|
||||
|
||||
import { randomUUID } from "node:crypto";
|
||||
import { existsSync, mkdirSync, readFileSync, writeFileSync } from "node:fs";
|
||||
import { homedir } from "node:os";
|
||||
import { join } from "node:path";
|
||||
|
||||
const POSTHOG_API_KEY = "phc_zjjbX0PnWxERXrMHhkEJWj9A9BhGVLRReICgsfTMmpx";
|
||||
const POSTHOG_HOST = "https://us.i.posthog.com";
|
||||
const TIMEOUT_MS = 1500;
|
||||
let identifiedAccount = false;
|
||||
let warnedNonDefaultHost = false;
|
||||
|
||||
function isTestOrCiContext() {
|
||||
return (
|
||||
process.env.CI === "true" ||
|
||||
process.env.CI === "1" ||
|
||||
process.env.NODE_ENV === "test" ||
|
||||
process.env.NODE_ENV === "development"
|
||||
);
|
||||
}
|
||||
|
||||
function posthogHost() {
|
||||
const override = process.env.MEDIA_USE_TELEMETRY_HOST;
|
||||
if (override && !warnedNonDefaultHost && !isTestOrCiContext()) {
|
||||
warnedNonDefaultHost = true;
|
||||
console.error(
|
||||
`media-use: telemetry is redirected to a non-default host via MEDIA_USE_TELEMETRY_HOST (${override}) — unset it unless this is intentional.`,
|
||||
);
|
||||
}
|
||||
return override || POSTHOG_HOST;
|
||||
}
|
||||
|
||||
/** True when telemetry must NOT be sent (opt-out envs, CI, dev). */
|
||||
export function optedOut() {
|
||||
return (
|
||||
process.env.HYPERFRAMES_NO_TELEMETRY === "1" ||
|
||||
process.env.DO_NOT_TRACK === "1" ||
|
||||
process.env.CI === "true" ||
|
||||
process.env.CI === "1" ||
|
||||
process.env.NODE_ENV === "development"
|
||||
);
|
||||
}
|
||||
|
||||
// Read and write the shared config so media-use keeps one identity per install.
|
||||
function sharedConfigPath() {
|
||||
return join(homedir(), ".hyperframes", "config.json");
|
||||
}
|
||||
|
||||
function readSharedConfig() {
|
||||
try {
|
||||
const file = sharedConfigPath();
|
||||
if (existsSync(file)) {
|
||||
const parsed = JSON.parse(readFileSync(file, "utf8"));
|
||||
if (parsed && typeof parsed === "object" && !Array.isArray(parsed)) return parsed;
|
||||
}
|
||||
} catch {
|
||||
// unreadable config → treat as empty; never throw
|
||||
}
|
||||
return {};
|
||||
}
|
||||
|
||||
function writeSharedConfig(config) {
|
||||
const dir = join(homedir(), ".hyperframes");
|
||||
if (!existsSync(dir)) mkdirSync(dir, { recursive: true });
|
||||
writeFileSync(join(dir, "config.json"), JSON.stringify(config, null, 2) + "\n");
|
||||
}
|
||||
|
||||
// Adopt a pre-existing media-use-only id (~/.media/anon-id from before this
|
||||
// change) so upgraders keep their PostHog persona instead of resetting to a new
|
||||
// one — otherwise cross-surface continuity would start over on upgrade.
|
||||
function legacyMediaAnonId() {
|
||||
try {
|
||||
const file = join(homedir(), ".media", "anon-id");
|
||||
if (existsSync(file)) {
|
||||
const id = readFileSync(file, "utf8").trim();
|
||||
if (id) return id;
|
||||
}
|
||||
} catch {
|
||||
// ignore
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
// Stable per-machine id from the shared config; seeds it (adopting a legacy
|
||||
// media-use id when present) if absent.
|
||||
function anonymousId() {
|
||||
try {
|
||||
const config = readSharedConfig();
|
||||
if (typeof config.anonymousId === "string" && config.anonymousId.trim()) {
|
||||
return config.anonymousId.trim();
|
||||
}
|
||||
const id = legacyMediaAnonId() || randomUUID();
|
||||
writeSharedConfig({ ...config, anonymousId: id });
|
||||
return id;
|
||||
} catch {
|
||||
return "anon"; // best-effort; a shared bucket is fine if the fs is read-only
|
||||
}
|
||||
}
|
||||
|
||||
function heygenAccountDistinctId() {
|
||||
const file = join(process.env.HEYGEN_CONFIG_DIR || join(homedir(), ".heygen"), "credentials");
|
||||
try {
|
||||
if (!existsSync(file)) return null;
|
||||
const raw = readFileSync(file, "utf8").trim();
|
||||
if (!raw.startsWith("{")) return null;
|
||||
const parsed = JSON.parse(raw);
|
||||
if (!parsed || typeof parsed !== "object" || Array.isArray(parsed)) return null;
|
||||
const user = parsed.user;
|
||||
if (!user || typeof user !== "object" || Array.isArray(user)) return null;
|
||||
const id = typeof user.email === "string" && user.email.trim() ? user.email : user.username;
|
||||
// Lowercased so this joins with the CLI's own identify call regardless of
|
||||
// the account's stored email casing — two different-case distinct ids
|
||||
// would otherwise split one person across two PostHog profiles.
|
||||
return typeof id === "string" && id.trim() ? id.trim().toLowerCase() : null;
|
||||
} catch {
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
function showTelemetryNotice() {
|
||||
if (optedOut()) return;
|
||||
try {
|
||||
const config = readSharedConfig();
|
||||
// Shared with the CLI (config.telemetryNoticeShown): shown once per person
|
||||
// across surfaces, not once per tool.
|
||||
if (config.telemetryNoticeShown === true) return;
|
||||
console.error(
|
||||
[
|
||||
"media-use sends usage telemetry: media type, resolution source, and provider; never intent text, file names, or paths.",
|
||||
"If you sign in to HeyGen, usage links to your account email or username. Opt out with HYPERFRAMES_NO_TELEMETRY=1 or DO_NOT_TRACK=1.",
|
||||
].join("\n"),
|
||||
);
|
||||
writeSharedConfig({ ...config, telemetryNoticeShown: true });
|
||||
} catch {
|
||||
// notice is best-effort; never surface into the command
|
||||
}
|
||||
}
|
||||
|
||||
async function postBatch(batch) {
|
||||
try {
|
||||
await fetch(`${posthogHost()}/batch/`, {
|
||||
method: "POST",
|
||||
headers: { "Content-Type": "application/json", Connection: "close" },
|
||||
body: JSON.stringify({ api_key: POSTHOG_API_KEY, batch }),
|
||||
signal: AbortSignal.timeout(TIMEOUT_MS),
|
||||
});
|
||||
} catch {
|
||||
// telemetry is best-effort; never surface into the command
|
||||
}
|
||||
}
|
||||
|
||||
async function postEvent(event, properties, distinctId) {
|
||||
await postBatch([
|
||||
{
|
||||
event,
|
||||
properties: { ...properties, surface: "media-use", $ip: null },
|
||||
distinct_id: distinctId,
|
||||
timestamp: new Date().toISOString(),
|
||||
},
|
||||
]);
|
||||
}
|
||||
|
||||
async function identifyAccount(anonId) {
|
||||
if (optedOut() || identifiedAccount) return;
|
||||
const distinctId = heygenAccountDistinctId();
|
||||
if (!distinctId) return;
|
||||
identifiedAccount = true;
|
||||
await postEvent("$identify", { $anon_distinct_id: anonId }, distinctId);
|
||||
}
|
||||
|
||||
/**
|
||||
* Fire-and-forget a single event to PostHog. Best-effort: awaited with a short
|
||||
* timeout so a short-lived script flushes before exit, but any failure (offline,
|
||||
* opted out) is swallowed. `properties` must be non-PII (no intent/paths).
|
||||
*/
|
||||
export async function track(event, properties = {}) {
|
||||
if (optedOut()) return;
|
||||
showTelemetryNotice();
|
||||
const anonId = anonymousId();
|
||||
await identifyAccount(anonId);
|
||||
await postEvent(event, properties, anonId);
|
||||
}
|
||||
|
||||
export function __anonymousIdForTest() {
|
||||
return anonymousId();
|
||||
}
|
||||
|
||||
export function __resetTelemetryForTest() {
|
||||
identifiedAccount = false;
|
||||
warnedNonDefaultHost = false;
|
||||
}
|
||||
@@ -0,0 +1,18 @@
|
||||
export function normalizeWords(input) {
|
||||
const raw = Array.isArray(input) ? input : Array.isArray(input?.words) ? input.words : [];
|
||||
return raw
|
||||
.map((w, index) => {
|
||||
const text = String(w?.text ?? w?.word ?? "").trim();
|
||||
const start = Number(w?.start);
|
||||
const end = Number(w?.end);
|
||||
if (!text || !Number.isFinite(start) || !Number.isFinite(end)) return null;
|
||||
return { id: w?.id ?? `w${index}`, text, start, end };
|
||||
})
|
||||
.filter(Boolean);
|
||||
}
|
||||
|
||||
export function wordListsFromMediaMeta(input) {
|
||||
if (Array.isArray(input) || Array.isArray(input?.words)) return [normalizeWords(input)];
|
||||
if (!Array.isArray(input?.voices)) return [];
|
||||
return input.voices.map((voice) => normalizeWords(voice)).filter((words) => words.length > 0);
|
||||
}
|
||||
File diff suppressed because it is too large
Load Diff
+1
-1
@@ -21,7 +21,7 @@ import { freezeLocalFile } from "./lib/freeze.mjs";
|
||||
import { cachePut, cacheGet, importFromCache } from "./lib/cache.mjs";
|
||||
import { validateCubeFile } from "./lib/cube-validate.mjs";
|
||||
|
||||
const REPO_ROOT = join(import.meta.dirname, "..", "..", "..");
|
||||
const REPO_ROOT = join(import.meta.dirname, "..", "..", "..", "..");
|
||||
const RESOLVE_CLI = join(import.meta.dirname, "resolve.mjs");
|
||||
// The "Test: skills" CI job has no ffmpeg on PATH (by design). The smart-grade
|
||||
// test shells to ffmpeg, so it's skipped there and runs where ffmpeg exists.
|
||||
@@ -3,7 +3,13 @@ import { existsSync, mkdirSync, mkdtempSync, readFileSync, rmSync, writeFileSync
|
||||
import { tmpdir } from "node:os";
|
||||
import { join } from "node:path";
|
||||
|
||||
import { cachedLocalVectorRevision, fetchLocalVectors } from "./localSemantic.js";
|
||||
import {
|
||||
cachedLocalVectorRevision,
|
||||
fetchLocalVectors,
|
||||
isMediaVectorRow,
|
||||
mediaSemanticRanking,
|
||||
vectorPairAgrees,
|
||||
} from "./localSemantic.js";
|
||||
import { LOCAL_MODEL_DIMENSIONS } from "./localModel.js";
|
||||
|
||||
describe("fetchLocalVectors", () => {
|
||||
@@ -29,6 +35,28 @@ describe("fetchLocalVectors", () => {
|
||||
})),
|
||||
);
|
||||
|
||||
const mediaRow = {
|
||||
id: "click",
|
||||
kind: "sfx",
|
||||
title: "click",
|
||||
description: "short click",
|
||||
tags: ["ui"],
|
||||
file: "click.mp3",
|
||||
duration: 0.2,
|
||||
};
|
||||
|
||||
const serveMediaPair = (metadata: unknown, floats = LOCAL_MODEL_DIMENSIONS) =>
|
||||
vi.stubGlobal(
|
||||
"fetch",
|
||||
vi.fn(async (url: string) => ({
|
||||
ok: true,
|
||||
arrayBuffer: async () =>
|
||||
url.endsWith(".json")
|
||||
? new TextEncoder().encode(JSON.stringify(metadata)).buffer
|
||||
: new Float32Array(floats).buffer,
|
||||
})),
|
||||
);
|
||||
|
||||
it("writes both files into the cache directory", async () => {
|
||||
servePair(["whip-pan"], LOCAL_MODEL_DIMENSIONS, LOCAL_MODEL_DIMENSIONS);
|
||||
expect(await fetchLocalVectors("http://registry.test/", { directory: dir })).toBe(true);
|
||||
@@ -107,4 +135,140 @@ describe("fetchLocalVectors", () => {
|
||||
writeFileSync(join(dir, "local-vectors.bin"), new Float32Array(LOCAL_MODEL_DIMENSIONS));
|
||||
expect(cachedLocalVectorRevision(dir)).toBe("r1");
|
||||
});
|
||||
|
||||
it("reports no semantic result without a media-vector cache", async () => {
|
||||
expect(await mediaSemanticRanking("click", dir)).toBeNull();
|
||||
});
|
||||
|
||||
it("accepts a complete media-vector pair with validated rows", async () => {
|
||||
serveMediaPair({
|
||||
names: ["click"],
|
||||
dimensions: LOCAL_MODEL_DIMENSIONS,
|
||||
rows: [mediaRow],
|
||||
});
|
||||
|
||||
expect(
|
||||
await fetchLocalVectors("http://registry.test", {
|
||||
directory: dir,
|
||||
artifactBasename: "media-vectors",
|
||||
}),
|
||||
).toBe(true);
|
||||
expect(existsSync(join(dir, "media-vectors.json"))).toBe(true);
|
||||
expect(existsSync(join(dir, "media-vectors.bin"))).toBe(true);
|
||||
});
|
||||
|
||||
it.each([
|
||||
["missing rows", {}],
|
||||
["wrong row count", { rows: [] }],
|
||||
["wrong row shape", { rows: [{ ...mediaRow, tags: ["ui", 3] }] }],
|
||||
["wrong row id", { rows: [{ ...mediaRow, id: "other" }] }],
|
||||
["invalid duration", { rows: [{ ...mediaRow, duration: -1 }] }],
|
||||
["invalid dimensions", { rows: [{ ...mediaRow, dimensions: { width: 0, height: 10 } }] }],
|
||||
])("rejects media-vector metadata with %s", async (_reason, extra) => {
|
||||
serveMediaPair({
|
||||
names: ["click"],
|
||||
dimensions: LOCAL_MODEL_DIMENSIONS,
|
||||
...extra,
|
||||
});
|
||||
|
||||
expect(
|
||||
await fetchLocalVectors("http://registry.test", {
|
||||
directory: dir,
|
||||
artifactBasename: "media-vectors",
|
||||
}),
|
||||
).toBe(false);
|
||||
expect(existsSync(join(dir, "media-vectors.json"))).toBe(false);
|
||||
});
|
||||
|
||||
it("rejects an unreadable media-vector metadata response", async () => {
|
||||
vi.stubGlobal(
|
||||
"fetch",
|
||||
vi.fn(async (url: string) => ({
|
||||
ok: true,
|
||||
arrayBuffer: async () =>
|
||||
url.endsWith(".json")
|
||||
? new TextEncoder().encode("not json").buffer
|
||||
: new Float32Array(LOCAL_MODEL_DIMENSIONS).buffer,
|
||||
})),
|
||||
);
|
||||
|
||||
expect(
|
||||
await fetchLocalVectors("http://registry.test", {
|
||||
directory: dir,
|
||||
artifactBasename: "media-vectors",
|
||||
}),
|
||||
).toBe(false);
|
||||
});
|
||||
});
|
||||
|
||||
describe("media-vector validation", () => {
|
||||
const row = {
|
||||
id: "click",
|
||||
kind: "sfx",
|
||||
title: "click",
|
||||
description: "short click",
|
||||
tags: ["ui"],
|
||||
file: "click.mp3",
|
||||
duration: 0.2,
|
||||
};
|
||||
|
||||
it.each([
|
||||
["null", null],
|
||||
["missing id", { ...row, id: undefined }],
|
||||
["non-string tags", { ...row, tags: ["ui", 3] }],
|
||||
["negative duration", { ...row, duration: -1 }],
|
||||
["zero width", { ...row, dimensions: { width: 0, height: 10 } }],
|
||||
])("rejects %s rows", (_name, candidate) => {
|
||||
expect(isMediaVectorRow(candidate)).toBe(false);
|
||||
});
|
||||
|
||||
it("accepts a complete media row", () => {
|
||||
expect(isMediaVectorRow(row)).toBe(true);
|
||||
expect(isMediaVectorRow({ ...row, dimensions: { width: 1920, height: 1080 } })).toBe(true);
|
||||
});
|
||||
|
||||
const invalidPairs: Array<[string, Array<[string, Buffer]>]> = [
|
||||
["missing metadata", [["media-vectors.bin", Buffer.alloc(4)]]],
|
||||
["missing matrix", [["media-vectors.json", Buffer.from("{}")]]],
|
||||
[
|
||||
"invalid json",
|
||||
[
|
||||
["media-vectors.json", Buffer.from("not json")],
|
||||
["media-vectors.bin", Buffer.alloc(4)],
|
||||
],
|
||||
],
|
||||
[
|
||||
"invalid media row",
|
||||
[
|
||||
[
|
||||
"media-vectors.json",
|
||||
Buffer.from(
|
||||
JSON.stringify({ names: ["click"], dimensions: 1, rows: [{ ...row, id: "other" }] }),
|
||||
),
|
||||
],
|
||||
["media-vectors.bin", Buffer.alloc(4)],
|
||||
],
|
||||
],
|
||||
];
|
||||
|
||||
it.each(invalidPairs)("rejects pairs with %s", (_name, fetched) => {
|
||||
expect(vectorPairAgrees(fetched, "media-vectors")).toBe(false);
|
||||
});
|
||||
|
||||
it("accepts a matching media-vector pair", () => {
|
||||
const metadata = {
|
||||
names: ["click"],
|
||||
dimensions: LOCAL_MODEL_DIMENSIONS,
|
||||
rows: [row],
|
||||
};
|
||||
expect(
|
||||
vectorPairAgrees(
|
||||
[
|
||||
["media-vectors.json", Buffer.from(JSON.stringify(metadata))],
|
||||
["media-vectors.bin", Buffer.alloc(4 * LOCAL_MODEL_DIMENSIONS)],
|
||||
],
|
||||
"media-vectors",
|
||||
),
|
||||
).toBe(true);
|
||||
});
|
||||
});
|
||||
|
||||
@@ -12,6 +12,7 @@
|
||||
* being compared against.
|
||||
*/
|
||||
|
||||
// Vector metadata validation is intentionally defensive at this file boundary.
|
||||
import { existsSync, mkdirSync, readFileSync, writeFileSync } from "node:fs";
|
||||
import { homedir } from "node:os";
|
||||
import { join } from "node:path";
|
||||
@@ -29,11 +30,47 @@ interface LocalVectorMetadata {
|
||||
names: string[];
|
||||
dimensions: number;
|
||||
revision?: string;
|
||||
rows?: MediaVectorRow[];
|
||||
}
|
||||
|
||||
export interface MediaVectorRow {
|
||||
id: string;
|
||||
kind: string;
|
||||
title: string;
|
||||
description: string;
|
||||
tags: string[];
|
||||
file: string;
|
||||
duration?: number;
|
||||
dimensions?: { width: number; height: number };
|
||||
}
|
||||
|
||||
export function isMediaVectorRow(value: unknown): value is MediaVectorRow {
|
||||
if (!value || typeof value !== "object") return false;
|
||||
const row = value as Partial<MediaVectorRow>;
|
||||
return (
|
||||
typeof row.id === "string" &&
|
||||
typeof row.kind === "string" &&
|
||||
typeof row.title === "string" &&
|
||||
typeof row.description === "string" &&
|
||||
Array.isArray(row.tags) &&
|
||||
row.tags.every((tag) => typeof tag === "string") &&
|
||||
typeof row.file === "string" &&
|
||||
(row.duration === undefined ||
|
||||
(typeof row.duration === "number" && Number.isFinite(row.duration) && row.duration >= 0)) &&
|
||||
(row.dimensions === undefined ||
|
||||
(typeof row.dimensions === "object" &&
|
||||
row.dimensions !== null &&
|
||||
Number.isInteger(row.dimensions.width) &&
|
||||
Number.isInteger(row.dimensions.height) &&
|
||||
row.dimensions.width > 0 &&
|
||||
row.dimensions.height > 0))
|
||||
);
|
||||
}
|
||||
|
||||
export interface FetchLocalVectorOptions {
|
||||
directory?: string;
|
||||
expectedRevision?: string;
|
||||
artifactBasename?: "local-vectors" | "media-vectors";
|
||||
}
|
||||
|
||||
const CATALOG_ARTIFACT_TIMEOUT_MS = 30_000;
|
||||
@@ -63,16 +100,30 @@ function localVectorDirectory(): string {
|
||||
* contract. A pair that fails it is a truncated download or a different
|
||||
* model, never something worth caching.
|
||||
*/
|
||||
function vectorPairAgrees(fetched: Array<[string, Buffer]>): boolean {
|
||||
const meta = fetched.find(([file]) => file === "local-vectors.json")?.[1];
|
||||
const bin = fetched.find(([file]) => file === "local-vectors.bin")?.[1];
|
||||
export function vectorPairAgrees(
|
||||
fetched: Array<[string, Buffer]>,
|
||||
artifactBasename: "local-vectors" | "media-vectors",
|
||||
): boolean {
|
||||
const meta = fetched.find(([file]) => file === `${artifactBasename}.json`)?.[1];
|
||||
const bin = fetched.find(([file]) => file === `${artifactBasename}.bin`)?.[1];
|
||||
if (!meta || !bin) return false;
|
||||
try {
|
||||
const parsed = JSON.parse(meta.toString("utf-8")) as {
|
||||
names?: string[];
|
||||
dimensions?: number;
|
||||
rows?: Array<{ id?: string }>;
|
||||
};
|
||||
if (parsed.dimensions !== LOCAL_MODEL_DIMENSIONS) return false;
|
||||
if (
|
||||
artifactBasename === "media-vectors" &&
|
||||
(!parsed.rows ||
|
||||
parsed.rows.length !== (parsed.names?.length ?? -1) ||
|
||||
parsed.rows.some(
|
||||
(row, index) => !isMediaVectorRow(row) || row.id !== parsed.names?.[index],
|
||||
))
|
||||
) {
|
||||
return false;
|
||||
}
|
||||
return bin.byteLength === (parsed.names?.length ?? -1) * LOCAL_MODEL_DIMENSIONS * 4;
|
||||
} catch {
|
||||
return false;
|
||||
@@ -83,9 +134,10 @@ function vectorPairAgrees(fetched: Array<[string, Buffer]>): boolean {
|
||||
function vectorRevisionAgrees(
|
||||
fetched: Array<[string, Buffer]>,
|
||||
expectedRevision?: string,
|
||||
artifactBasename: "local-vectors" | "media-vectors" = "local-vectors",
|
||||
): boolean {
|
||||
if (expectedRevision === undefined) return true;
|
||||
const meta = fetched.find(([file]) => file === "local-vectors.json")?.[1];
|
||||
const meta = fetched.find(([file]) => file === `${artifactBasename}.json`)?.[1];
|
||||
if (!meta) return false;
|
||||
try {
|
||||
const parsed = JSON.parse(meta.toString("utf-8")) as { revision?: unknown };
|
||||
@@ -100,6 +152,7 @@ export async function fetchLocalVectors(
|
||||
options: FetchLocalVectorOptions = {},
|
||||
): Promise<boolean> {
|
||||
const directory = options.directory ?? localVectorDirectory();
|
||||
const artifactBasename = options.artifactBasename ?? "local-vectors";
|
||||
const base = registryBaseUrl.replace(/\/+$/, "");
|
||||
try {
|
||||
mkdirSync(directory, { recursive: true, mode: 0o700 });
|
||||
@@ -108,7 +161,7 @@ export async function fetchLocalVectors(
|
||||
// must leave the previous pair intact rather than pairing a new name list
|
||||
// with an old matrix, which loads as an error instead of as stale data.
|
||||
const fetched: Array<[string, Buffer]> = [];
|
||||
for (const file of ["local-vectors.json", "local-vectors.bin"] as const) {
|
||||
for (const file of [`${artifactBasename}.json`, `${artifactBasename}.bin`] as const) {
|
||||
const response = await fetch(`${base}/catalog-artifact/${file}`, {
|
||||
signal: AbortSignal.timeout(CATALOG_ARTIFACT_TIMEOUT_MS),
|
||||
});
|
||||
@@ -119,7 +172,10 @@ export async function fetchLocalVectors(
|
||||
// discovering the mismatch at load time leaves a cache that fails every
|
||||
// subsequent search until someone deletes it by hand, and it is the only
|
||||
// point where a truncated or wrong-model response can still be refused.
|
||||
if (!vectorPairAgrees(fetched) || !vectorRevisionAgrees(fetched, options.expectedRevision)) {
|
||||
if (
|
||||
!vectorPairAgrees(fetched, artifactBasename) ||
|
||||
!vectorRevisionAgrees(fetched, options.expectedRevision, artifactBasename)
|
||||
) {
|
||||
return false;
|
||||
}
|
||||
// 0o600: the cache is this user's, and the directory may be world-writable
|
||||
@@ -127,7 +183,9 @@ export async function fetchLocalVectors(
|
||||
for (const [file, bytes] of fetched) {
|
||||
writeFileSync(join(directory, file), bytes, { mode: 0o600 });
|
||||
}
|
||||
return hasLocalVectors(directory);
|
||||
return artifactBasename === "media-vectors"
|
||||
? hasMediaVectors(directory)
|
||||
: hasLocalVectors(directory);
|
||||
} catch {
|
||||
return false;
|
||||
}
|
||||
@@ -140,6 +198,13 @@ export function hasLocalVectors(directory = localVectorDirectory()): boolean {
|
||||
);
|
||||
}
|
||||
|
||||
function hasMediaVectors(directory = localVectorDirectory()): boolean {
|
||||
return (
|
||||
existsSync(join(directory, "media-vectors.bin")) &&
|
||||
existsSync(join(directory, "media-vectors.json"))
|
||||
);
|
||||
}
|
||||
|
||||
function loadLocalVectors(directory = localVectorDirectory()): LocalVectorSet {
|
||||
const meta = JSON.parse(
|
||||
readFileSync(join(directory, "local-vectors.json"), "utf-8"),
|
||||
@@ -161,6 +226,34 @@ function loadLocalVectors(directory = localVectorDirectory()): LocalVectorSet {
|
||||
return { names: meta.names, dimensions: meta.dimensions, vectors };
|
||||
}
|
||||
|
||||
function loadMediaVectors(directory = localVectorDirectory()): LocalVectorSet & {
|
||||
rows: MediaVectorRow[];
|
||||
} {
|
||||
const meta = JSON.parse(
|
||||
readFileSync(join(directory, "media-vectors.json"), "utf-8"),
|
||||
) as LocalVectorMetadata;
|
||||
const buffer = readFileSync(join(directory, "media-vectors.bin"));
|
||||
const vectors = new Float32Array(buffer.buffer, buffer.byteOffset, buffer.byteLength / 4);
|
||||
const rows = meta.rows ?? [];
|
||||
if (rows.length !== meta.names.length) {
|
||||
throw new Error(`media vectors hold ${rows.length} rows, expected ${meta.names.length}`);
|
||||
}
|
||||
if (rows.some((row, index) => !isMediaVectorRow(row) || row.id !== meta.names[index])) {
|
||||
throw new Error("media vector row order does not match media vector names");
|
||||
}
|
||||
if (vectors.length !== rows.length * meta.dimensions) {
|
||||
throw new Error(
|
||||
`media vectors hold ${vectors.length} floats, expected ${rows.length * meta.dimensions}`,
|
||||
);
|
||||
}
|
||||
if (meta.dimensions !== LOCAL_MODEL_DIMENSIONS) {
|
||||
throw new Error(
|
||||
`media vectors are ${meta.dimensions}-dimension, model produces ${LOCAL_MODEL_DIMENSIONS}`,
|
||||
);
|
||||
}
|
||||
return { names: meta.names, dimensions: meta.dimensions, vectors, rows };
|
||||
}
|
||||
|
||||
/**
|
||||
* The names the on-device index holds, without loading the vector matrix.
|
||||
*
|
||||
@@ -227,3 +320,23 @@ export async function localSemanticRanking(
|
||||
scored.sort((a, b) => b.score - a.score || b.name.localeCompare(a.name));
|
||||
return scored;
|
||||
}
|
||||
|
||||
export async function mediaSemanticRanking(
|
||||
query: string,
|
||||
directory = localVectorDirectory(),
|
||||
): Promise<Array<{ row: MediaVectorRow; score: number }> | null> {
|
||||
if (!isLocalModelReady() || !hasMediaVectors(directory)) return null;
|
||||
const set = loadMediaVectors(directory);
|
||||
const embedder = await loadLocalEmbedder();
|
||||
const [queryVector] = await embedder.embed([query], { isQuery: true });
|
||||
if (!queryVector) return null;
|
||||
return set.rows
|
||||
.map((row, index) => ({
|
||||
row,
|
||||
score: cosine(
|
||||
queryVector,
|
||||
Array.from(set.vectors.subarray(index * set.dimensions, (index + 1) * set.dimensions)),
|
||||
),
|
||||
}))
|
||||
.sort((a, b) => b.score - a.score || b.row.id.localeCompare(a.row.id));
|
||||
}
|
||||
|
||||
@@ -22,9 +22,9 @@ async function invalidInstallableMedia(entryName: string): Promise<string[]> {
|
||||
|
||||
for (const file of manifest.files) {
|
||||
if (file.type !== "hyperframes:snippet" || !file.path.endsWith(".html")) continue;
|
||||
const result = await lintHyperframeHtml(readFileSync(join(itemDir, file.path), "utf8"), {
|
||||
isSubComposition: true,
|
||||
});
|
||||
const html = readFileSync(join(itemDir, file.path), "utf8");
|
||||
if (!/<(?:audio|video)\b/i.test(html)) continue;
|
||||
const result = await lintHyperframeHtml(html, { isSubComposition: true });
|
||||
for (const finding of result.findings) {
|
||||
if (finding.code !== "media_missing_src") continue;
|
||||
invalidMedia.push(`${entryName}/${file.path}: ${finding.code}`);
|
||||
|
||||
@@ -17,6 +17,7 @@ export default defineConfig({
|
||||
runtimeVersion: "src/runtimeVersion.ts",
|
||||
renderSetupWorker: "src/renderSetupWorker.ts",
|
||||
shaderTransitionWorker: "../producer/src/services/shaderTransitionWorker.ts",
|
||||
"registry/localSemantic": "src/registry/localSemantic.ts",
|
||||
},
|
||||
format: ["esm"],
|
||||
outDir: "dist",
|
||||
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user