mirror of
https://github.com/heygen-com/hyperframes.git
synced 2026-11-15 22:08:03 +00:00
feat(cli): search the catalog by meaning, in three named tiers
Browsing the registry means matching names and tags, which fails whenever the author's wording differs from yours. "make the pace feel faster" finds nothing when the move is described as "velocity-driven blur". This ranks by meaning instead. Three tiers, and the command always says which one answered: words shared vocabulary, free, offline, no account on-device bge-small, free, offline, one opt-in download hosted Gemini, free for signed-in HeyGen users The tier is stated because a quietly worse answer looks exactly like a good one. --json carries it as a token alongside dropped, shown, total and top_score, so an agent reads provenance as data rather than matching English that is written to be reworded. Two consents, asked once each, and never conflated. Sending a query is a privacy question, so the prompt says the query is sent. Downloading a model is a disk and bandwidth question, so that prompt talks about size. Neither fires without a terminal: an unattended run sends nothing and downloads nothing unless a flag records that a person agreed. The catalog is derived from registry-item.json rather than from a separate document, so the set that is ranked and the set that can be installed are the same object by construction. Only the on-device vectors are committed; the hosted vectors are nine megabytes and belong on the server. top_score is reported and never acted on. A "nothing matched" threshold looked clean on long briefs and collapsed on the short queries people type: "a logo appears" scores 0.6181 and keyboard mash scores 0.6417, so any cut that catches the noise rejects the real query. The measurement is in the evals directory rather than in this branch. Not covered here. The published recall figures were measured against a separate hand-written document, not against registry text, so they should not be quoted for this catalog until re-measured. The offline tier needs a normal install: a single-file build cannot load the native ONNX runtime, which the command now reports instead of silently degrading. And the drop-detection path has never been observed firing outside its author's tests.
This commit is contained in:
Miguel Ángel
@@ -0,0 +1,138 @@
|
||||
/**
|
||||
* Publish the video-primitive catalog artifact.
|
||||
*
|
||||
* Usage:
|
||||
* tsx scripts/catalog/build-catalog-artifact.ts --shelf <path> --revision <sha> [--out <dir>]
|
||||
*
|
||||
* Requires OPENAI_API_KEY. Embedding 424 descriptions is a real, paid call, so
|
||||
* this is invoked deliberately rather than on every commit. The artifact is
|
||||
* verified after writing: a build that cannot be read back is a failed build.
|
||||
*/
|
||||
|
||||
import { mkdirSync, readFileSync, writeFileSync } from "node:fs";
|
||||
import { join } from "node:path";
|
||||
|
||||
import {
|
||||
buildArtifact,
|
||||
manifestBytes,
|
||||
movesMissingFromRegistry,
|
||||
verifyArtifact,
|
||||
type Embedder,
|
||||
} from "./catalog-artifact.js";
|
||||
|
||||
// Measured on the 405-brief gold set, not chosen from a leaderboard:
|
||||
// 3-large scores 60.5% recall@20 against 54.1% for 3-small, a 6.4 point gain.
|
||||
// Embedding the whole catalog costs a couple of cents, and per-query cost is
|
||||
// negligible, so the larger model is the better default by a wide margin.
|
||||
const EMBEDDING_MODEL = "text-embedding-3-large";
|
||||
const EMBEDDING_DIMENSION = 3072;
|
||||
const BATCH_SIZE = 100;
|
||||
const DEFAULT_REGISTRY_INDEX = "registry/registry.json";
|
||||
const DEFAULT_OUT = "registry/catalog-artifact";
|
||||
|
||||
/**
|
||||
* Read the names the registry can actually serve.
|
||||
*
|
||||
* Absent index means the caller opted out of the check rather than passed it,
|
||||
* so that case is reported by the caller instead of being treated as a pass.
|
||||
*/
|
||||
function registryNames(indexPath: string): string[] | undefined {
|
||||
try {
|
||||
const parsed = JSON.parse(readFileSync(indexPath, "utf8")) as {
|
||||
items?: { name?: string }[];
|
||||
};
|
||||
if (!Array.isArray(parsed.items)) return undefined;
|
||||
return parsed.items
|
||||
.map((item) => item.name)
|
||||
.filter((name): name is string => typeof name === "string");
|
||||
} catch {
|
||||
return undefined;
|
||||
}
|
||||
}
|
||||
|
||||
function arg(name: string): string | undefined {
|
||||
const index = process.argv.indexOf(`--${name}`);
|
||||
return index === -1 ? undefined : process.argv[index + 1];
|
||||
}
|
||||
|
||||
/** Batched so a 424-move shelf does not depend on one oversized request. */
|
||||
const openAiEmbedder: Embedder = async (texts) => {
|
||||
const key = process.env.OPENAI_API_KEY;
|
||||
if (!key) throw new Error("OPENAI_API_KEY is not set");
|
||||
|
||||
const vectors: number[][] = [];
|
||||
for (let start = 0; start < texts.length; start += BATCH_SIZE) {
|
||||
const batch = texts.slice(start, start + BATCH_SIZE);
|
||||
const response = await fetch("https://api.openai.com/v1/embeddings", {
|
||||
method: "POST",
|
||||
headers: { Authorization: `Bearer ${key}`, "Content-Type": "application/json" },
|
||||
body: JSON.stringify({ model: EMBEDDING_MODEL, input: batch }),
|
||||
});
|
||||
if (!response.ok) {
|
||||
throw new Error(`Embedding request failed: ${response.status} ${await response.text()}`);
|
||||
}
|
||||
const body = (await response.json()) as { data: { index: number; embedding: number[] }[] };
|
||||
// Order is documented as preserved, but index is returned explicitly, so
|
||||
// sort by it rather than trusting position.
|
||||
const ordered = [...body.data].sort((a, b) => a.index - b.index);
|
||||
if (ordered.length !== batch.length) {
|
||||
throw new Error(`Expected ${batch.length} embeddings, received ${ordered.length}`);
|
||||
}
|
||||
vectors.push(...ordered.map((item) => item.embedding));
|
||||
}
|
||||
return vectors;
|
||||
};
|
||||
|
||||
async function main(): Promise<void> {
|
||||
const shelfPath = arg("shelf");
|
||||
const revision = arg("revision");
|
||||
const out = arg("out") ?? DEFAULT_OUT;
|
||||
if (!shelfPath || !revision) {
|
||||
throw new Error("Both --shelf and --revision are required");
|
||||
}
|
||||
|
||||
const shelfText = readFileSync(shelfPath, "utf-8");
|
||||
// Only publish moves the registry can serve. The shelf doubles as a design
|
||||
// document, so it lists moves that were specified and never built.
|
||||
const installableNames = registryNames(arg("registry-index") ?? DEFAULT_REGISTRY_INDEX);
|
||||
const built = await buildArtifact({
|
||||
shelfText,
|
||||
...(installableNames ? { installableNames } : {}),
|
||||
sourceRevision: revision,
|
||||
embeddingModel: EMBEDDING_MODEL,
|
||||
embed: openAiEmbedder,
|
||||
expectedDimension: EMBEDDING_DIMENSION,
|
||||
});
|
||||
|
||||
// Verify before writing. Publishing something we cannot read back would put
|
||||
// the burden of discovering it on the consumer at request time.
|
||||
verifyArtifact(built);
|
||||
|
||||
// A move the registry cannot serve ranks well and then cannot be installed,
|
||||
// which looks like a bad search rather than a bad build. Reported, not
|
||||
// fatal: the shelf legitimately leads the registry while moves land.
|
||||
const known = registryNames(arg("registry-index") ?? DEFAULT_REGISTRY_INDEX);
|
||||
if (known === undefined) {
|
||||
console.warn("warning registry index unreadable; coverage not checked");
|
||||
} else {
|
||||
const missing = movesMissingFromRegistry(Object.keys(built.catalog), known);
|
||||
if (missing.length > 0) {
|
||||
console.warn(
|
||||
`warning ${missing.length} shelf moves have no registry item: ${missing.slice(0, 5).join(", ")}${missing.length > 5 ? ", ..." : ""}`,
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
mkdirSync(out, { recursive: true });
|
||||
writeFileSync(join(out, "catalog.json"), built.catalogBytes);
|
||||
writeFileSync(join(out, "vectors.json"), built.vectorsBytes);
|
||||
writeFileSync(join(out, "manifest.json"), manifestBytes(built.manifest));
|
||||
|
||||
console.log(`moves ${built.manifest.move_count}`);
|
||||
console.log(`model ${built.manifest.embedding_model}`);
|
||||
console.log(`revision ${built.manifest.source_revision}`);
|
||||
console.log(`version ${built.manifest.payload_sha256}`);
|
||||
console.log(`written ${out}`);
|
||||
}
|
||||
|
||||
await main();
|
||||
@@ -0,0 +1,75 @@
|
||||
/**
|
||||
* Embed the catalog with the on-device model so local search can rank by meaning.
|
||||
*
|
||||
* A second vector set, not a replacement. The hosted vectors are 1536-dimension
|
||||
* and measured; these are 384-dimension and free, and the two are not
|
||||
* interchangeable because vectors from different models cannot be compared.
|
||||
*
|
||||
* Usage:
|
||||
* tsx scripts/catalog/build-local-vectors.ts --artifact <dir>
|
||||
*
|
||||
* Writes `local-vectors.bin` (float32, row-major, names in manifest order) and
|
||||
* `local-vectors.json` (names, dimensions, and the catalog digest it was built
|
||||
* from). Splitting them keeps the payload small enough to ship: 424 moves at
|
||||
* 384 dimensions is about 650 KB as binary against several megabytes as JSON.
|
||||
*/
|
||||
|
||||
import { readFileSync, writeFileSync } from "node:fs";
|
||||
import { join } from "node:path";
|
||||
|
||||
import {
|
||||
LOCAL_MODEL_DIMENSIONS,
|
||||
LOCAL_MODEL_ID,
|
||||
} from "../../packages/cli/src/registry/localModel.js";
|
||||
import { loadLocalEmbedder } from "../../packages/cli/src/registry/localEmbedder.js";
|
||||
|
||||
const BATCH = 16;
|
||||
|
||||
function arg(name: string): string | undefined {
|
||||
const index = process.argv.indexOf(`--${name}`);
|
||||
return index === -1 ? undefined : process.argv[index + 1];
|
||||
}
|
||||
|
||||
async function main(): Promise<void> {
|
||||
const dir = arg("artifact");
|
||||
if (!dir) throw new Error("--artifact <dir> is required");
|
||||
|
||||
const catalog = JSON.parse(readFileSync(join(dir, "catalog.json"), "utf-8")) as Record<
|
||||
string,
|
||||
string
|
||||
>;
|
||||
const names = Object.keys(catalog).sort();
|
||||
const embedder = await loadLocalEmbedder();
|
||||
|
||||
const vectors: number[][] = [];
|
||||
for (let start = 0; start < names.length; start += BATCH) {
|
||||
const slice = names.slice(start, start + BATCH);
|
||||
// Passages carry no query instruction; only queries do.
|
||||
vectors.push(...(await embedder.embed(slice.map((name) => catalog[name] as string))));
|
||||
process.stdout.write(`\r embedded ${Math.min(start + BATCH, names.length)}/${names.length}`);
|
||||
}
|
||||
process.stdout.write("\n");
|
||||
|
||||
const flat = new Float32Array(names.length * LOCAL_MODEL_DIMENSIONS);
|
||||
vectors.forEach((vector, row) => {
|
||||
if (vector.length !== LOCAL_MODEL_DIMENSIONS) {
|
||||
throw new Error(`vector for ${names[row]} has ${vector.length} dimensions`);
|
||||
}
|
||||
flat.set(vector, row * LOCAL_MODEL_DIMENSIONS);
|
||||
});
|
||||
|
||||
writeFileSync(join(dir, "local-vectors.bin"), Buffer.from(flat.buffer));
|
||||
writeFileSync(
|
||||
join(dir, "local-vectors.json"),
|
||||
`${JSON.stringify({ model: LOCAL_MODEL_ID, dimensions: LOCAL_MODEL_DIMENSIONS, names }, null, 2)}\n`,
|
||||
);
|
||||
|
||||
const megabytes = (flat.byteLength / 1024 / 1024).toFixed(2);
|
||||
console.log(`moves ${names.length}`);
|
||||
console.log(`model ${LOCAL_MODEL_ID}`);
|
||||
console.log(`dimensions ${LOCAL_MODEL_DIMENSIONS}`);
|
||||
console.log(`payload ${megabytes} MB`);
|
||||
console.log(`written ${dir}`);
|
||||
}
|
||||
|
||||
await main();
|
||||
@@ -0,0 +1,227 @@
|
||||
import { describe, expect, it } from "vitest";
|
||||
import { readFileSync } from "node:fs";
|
||||
import { join } from "node:path";
|
||||
|
||||
import {
|
||||
buildArtifact,
|
||||
MANIFEST_SCHEMA_VERSION,
|
||||
movesMissingFromRegistry,
|
||||
parseShelf,
|
||||
payloadDigest,
|
||||
sortedNames,
|
||||
verifyArtifact,
|
||||
type Embedder,
|
||||
} from "./catalog-artifact.js";
|
||||
|
||||
const REVISION = "00dda8f35dac0b89defe438d465cfe3d9d941b4f";
|
||||
const MODEL = "text-embedding-3-small";
|
||||
|
||||
const SHELF = `# Video primitive shelf
|
||||
|
||||
Preamble that is not an entry.
|
||||
|
||||
### beta-move
|
||||
|
||||
group: Annotation.
|
||||
what: Second alphabetically, first in the document.
|
||||
use_when: A callout needs to land.
|
||||
avoid_when: The shot is already busy.
|
||||
notes: Not a retrieval field.
|
||||
|
||||
### alpha-move
|
||||
|
||||
group: Motion.
|
||||
what: First alphabetically, second in the document.
|
||||
use_when: Pace needs to lift.
|
||||
avoid_when: Motion would distract.
|
||||
`;
|
||||
|
||||
/** Deterministic stand-in so no test makes a paid call. */
|
||||
const fakeEmbed =
|
||||
(dimension = 4): Embedder =>
|
||||
async (texts) =>
|
||||
texts.map((text, index) => Array.from({ length: dimension }, (_, i) => (index + 1) / (i + 2)));
|
||||
|
||||
async function build(overrides: Partial<Parameters<typeof buildArtifact>[0]> = {}) {
|
||||
return buildArtifact({
|
||||
shelfText: SHELF,
|
||||
sourceRevision: REVISION,
|
||||
embeddingModel: MODEL,
|
||||
embed: fakeEmbed(),
|
||||
...overrides,
|
||||
});
|
||||
}
|
||||
|
||||
describe("shelf contract", () => {
|
||||
it("keys entries by move name and ignores the preamble", () => {
|
||||
const entries = parseShelf(SHELF);
|
||||
expect([...entries.keys()].sort()).toEqual(["alpha-move", "beta-move"]);
|
||||
expect(JSON.stringify([...entries.values()])).not.toContain("Preamble");
|
||||
});
|
||||
|
||||
it("excludes the move name from its own retrieval text", () => {
|
||||
// Including the name would let a move win on its own label, which is what
|
||||
// lexical ranking already does.
|
||||
for (const [name, text] of parseShelf(SHELF)) expect(text).not.toContain(name);
|
||||
});
|
||||
|
||||
it("keeps only the four evaluated fields", () => {
|
||||
expect(parseShelf(SHELF).get("beta-move")?.split("\n")).toEqual([
|
||||
"group: Annotation.",
|
||||
"what: Second alphabetically, first in the document.",
|
||||
"use_when: A callout needs to land.",
|
||||
"avoid_when: The shot is already busy.",
|
||||
]);
|
||||
});
|
||||
|
||||
it("orders embedding by name, not document order", () => {
|
||||
expect(sortedNames(parseShelf(SHELF))).toEqual(["alpha-move", "beta-move"]);
|
||||
});
|
||||
|
||||
it("rejects a duplicate move name", () => {
|
||||
expect(() => parseShelf(`${SHELF}\n### beta-move\n\ngroup: Repeat.\n`)).toThrow(
|
||||
/Duplicate move name/,
|
||||
);
|
||||
});
|
||||
|
||||
it("rejects a shelf with no entries", () => {
|
||||
expect(() => parseShelf("# Heading only\n\nProse.\n")).toThrow(/no entries/);
|
||||
});
|
||||
});
|
||||
|
||||
describe("build contract", () => {
|
||||
it("publishes descriptions and vectors covering the same moves", async () => {
|
||||
const { catalogBytes, vectorsBytes, manifest } = await build();
|
||||
const catalog = JSON.parse(catalogBytes.toString("utf-8"));
|
||||
const vectors = JSON.parse(vectorsBytes.toString("utf-8"));
|
||||
expect(Object.keys(catalog).sort()).toEqual(Object.keys(vectors).sort());
|
||||
expect(manifest.move_count).toBe(2);
|
||||
expect(manifest.embedding_model).toBe(MODEL);
|
||||
expect(manifest.schema_version).toBe(MANIFEST_SCHEMA_VERSION);
|
||||
});
|
||||
|
||||
it("rebuilds byte-identically from an unchanged shelf", async () => {
|
||||
const first = await build();
|
||||
const second = await build();
|
||||
expect(first.catalogBytes.equals(second.catalogBytes)).toBe(true);
|
||||
expect(first.vectorsBytes.equals(second.vectorsBytes)).toBe(true);
|
||||
expect(first.manifest.payload_sha256).toBe(second.manifest.payload_sha256);
|
||||
});
|
||||
|
||||
it("refuses a branch name in place of a resolved commit", async () => {
|
||||
await expect(build({ sourceRevision: "feat-video-primitives" })).rejects.toThrow(
|
||||
/resolved commit SHA/,
|
||||
);
|
||||
});
|
||||
|
||||
it("fails the build when the embedder returns the wrong count", async () => {
|
||||
const short: Embedder = async () => [[0.1, 0.2, 0.3, 0.4]];
|
||||
await expect(build({ embed: short })).rejects.toThrow(/Expected 2 vectors/);
|
||||
});
|
||||
|
||||
it("fails the build when the embedder throws, publishing nothing", async () => {
|
||||
const failing: Embedder = async () => {
|
||||
throw new Error("provider unavailable");
|
||||
};
|
||||
await expect(build({ embed: failing })).rejects.toThrow(/provider unavailable/);
|
||||
});
|
||||
|
||||
it("fails the build on an unexpected dimension", async () => {
|
||||
await expect(build({ expectedDimension: 1536 })).rejects.toThrow(/dimension 4, expected 1536/);
|
||||
});
|
||||
|
||||
it("fails the build on a non-finite value", async () => {
|
||||
const bad: Embedder = async (texts) => texts.map(() => [Number.NaN, 1, 1, 1]);
|
||||
await expect(build({ embed: bad })).rejects.toThrow(/non-finite/);
|
||||
});
|
||||
|
||||
it("fails the build on a zero vector", async () => {
|
||||
const zero: Embedder = async (texts) => texts.map(() => [0, 0, 0, 0]);
|
||||
await expect(build({ embed: zero })).rejects.toThrow(/zero vector/);
|
||||
});
|
||||
|
||||
it("fails the build on inconsistent vector widths", async () => {
|
||||
const ragged: Embedder = async (texts) =>
|
||||
texts.map((_, i) => (i === 0 ? [1, 1, 1, 1] : [1, 1]));
|
||||
await expect(build({ embed: ragged })).rejects.toThrow(/expected 4/);
|
||||
});
|
||||
});
|
||||
|
||||
describe("verify contract", () => {
|
||||
it("accepts a freshly built artifact", async () => {
|
||||
const built = await build();
|
||||
expect(() => verifyArtifact(built)).not.toThrow();
|
||||
});
|
||||
|
||||
it("rejects a mutated description", async () => {
|
||||
const built = await build();
|
||||
const mutated = Buffer.from(
|
||||
built.catalogBytes.toString("utf-8").replace("Motion.", "Tampered."),
|
||||
);
|
||||
expect(() => verifyArtifact({ ...built, catalogBytes: mutated })).toThrow(
|
||||
/does not match the manifest/,
|
||||
);
|
||||
});
|
||||
|
||||
it("rejects a mutated vector", async () => {
|
||||
const built = await build();
|
||||
const mutated = Buffer.from(built.vectorsBytes.toString("utf-8").replace("0.5", "0.6"));
|
||||
expect(() => verifyArtifact({ ...built, vectorsBytes: mutated })).toThrow(
|
||||
/does not match the manifest/,
|
||||
);
|
||||
});
|
||||
|
||||
it("rejects a manifest whose count disagrees with the payload", async () => {
|
||||
const built = await build();
|
||||
expect(() =>
|
||||
verifyArtifact({ ...built, manifest: { ...built.manifest, move_count: 99 } }),
|
||||
).toThrow(/declares 99/);
|
||||
});
|
||||
|
||||
it("rejects an unknown schema version", async () => {
|
||||
const built = await build();
|
||||
expect(() =>
|
||||
verifyArtifact({ ...built, manifest: { ...built.manifest, schema_version: 99 } }),
|
||||
).toThrow(/schema version/);
|
||||
});
|
||||
});
|
||||
|
||||
describe("consumer parity", () => {
|
||||
it("computes the digest the Python consumer computes", () => {
|
||||
// Pinned from the Python consumer for the same inputs. Both sides hash file
|
||||
// bytes as sha256(sha256(catalog) || sha256(vectors)), so a change to either
|
||||
// implementation fails here rather than producing an artifact the consumer
|
||||
// silently rejects at load time.
|
||||
const digest = payloadDigest(
|
||||
Buffer.from("catalog-bytes", "utf-8"),
|
||||
Buffer.from("vector-bytes", "utf-8"),
|
||||
);
|
||||
expect(digest).toBe("291c30dddc35c705a681081d9c2e2a96314bcccf0d024b77a3692e412adbf103");
|
||||
});
|
||||
|
||||
it("reproduces the evaluated shelf when one is available", () => {
|
||||
// Opt-in: needs the pinned shelf. Guards the ported contract against drift.
|
||||
const shelfPath = process.env.EVAL_SHELF_PATH;
|
||||
if (!shelfPath) return;
|
||||
const entries = parseShelf(readFileSync(join(shelfPath), "utf-8"));
|
||||
expect(entries.size).toBe(424);
|
||||
});
|
||||
});
|
||||
|
||||
describe("movesMissingFromRegistry", () => {
|
||||
it("names the shelf moves no registry item can serve", () => {
|
||||
// The failure this guards produced no error at all: the artifact shipped,
|
||||
// ranked these moves highest, and the client then had nothing to show.
|
||||
expect(movesMissingFromRegistry(["count-up", "number-wheel"], ["count-up"])).toEqual([
|
||||
"number-wheel",
|
||||
]);
|
||||
});
|
||||
|
||||
it("is empty when the registry covers the shelf", () => {
|
||||
expect(movesMissingFromRegistry(["count-up"], ["count-up", "badge-pop"])).toEqual([]);
|
||||
});
|
||||
|
||||
it("reports every move when the registry is empty", () => {
|
||||
expect(movesMissingFromRegistry(["a", "b"], [])).toEqual(["a", "b"]);
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,270 @@
|
||||
/**
|
||||
* Build the published video-primitive catalog artifact.
|
||||
*
|
||||
* HyperFrames owns the shelf, so HyperFrames owns this job. Descriptions and
|
||||
* their vectors publish together as one version, because a consumer that loads
|
||||
* a new description against an old vector produces a ranking that is wrong in a
|
||||
* way nothing alarms on.
|
||||
*
|
||||
* The retrieval-text contract is inherited from the evaluation and must not
|
||||
* drift: an entry starts at a `### ` heading, only `group`, `what`, `use_when`
|
||||
* and `avoid_when` form the body, the move name is deliberately excluded, and
|
||||
* names sort before embedding. Excluding the name keeps a move from winning on
|
||||
* its own label, which is what lexical ranking already does.
|
||||
*/
|
||||
|
||||
import { createHash } from "node:crypto";
|
||||
|
||||
export const RETRIEVAL_FIELDS = ["group", "what", "use_when", "avoid_when"] as const;
|
||||
export const MANIFEST_SCHEMA_VERSION = 1;
|
||||
|
||||
export interface CatalogManifest {
|
||||
schema_version: number;
|
||||
source_revision: string;
|
||||
shelf_sha256: string;
|
||||
embedding_model: string;
|
||||
move_count: number;
|
||||
payload_sha256: string;
|
||||
}
|
||||
|
||||
export interface BuiltArtifact {
|
||||
manifest: CatalogManifest;
|
||||
catalogBytes: Buffer;
|
||||
vectorsBytes: Buffer;
|
||||
}
|
||||
|
||||
/** Embed texts in the order given. Injected so tests never make a paid call. */
|
||||
export type Embedder = (texts: string[]) => Promise<number[][]>;
|
||||
|
||||
export function parseShelf(text: string): Map<string, string> {
|
||||
const entries = new Map<string, string>();
|
||||
const blocks = text.split(/^### /m).slice(1);
|
||||
for (const block of blocks) {
|
||||
const lines = block.split("\n");
|
||||
const name = (lines[0] ?? "").trim();
|
||||
if (!name) continue;
|
||||
if (entries.has(name)) throw new Error(`Duplicate move name in shelf: ${name}`);
|
||||
const body = lines
|
||||
.slice(1)
|
||||
.filter((line) => (RETRIEVAL_FIELDS as readonly string[]).includes(line.split(":")[0] ?? ""));
|
||||
entries.set(name, body.join("\n"));
|
||||
}
|
||||
if (entries.size === 0) throw new Error("Shelf contains no entries");
|
||||
return entries;
|
||||
}
|
||||
|
||||
/** Embedding order. Vector N corresponds to element N of this list. */
|
||||
/**
|
||||
* One registry item's retrieval text.
|
||||
*
|
||||
* Title, description and tags only. The name is deliberately excluded: it is
|
||||
* what the query is trying to find, and folding it into the text being matched
|
||||
* rewards items whose name happens to echo the query wording rather than items
|
||||
* that do what was asked.
|
||||
*/
|
||||
export function itemRetrievalText(item: {
|
||||
title?: string;
|
||||
description?: string;
|
||||
tags?: readonly string[];
|
||||
}): string {
|
||||
const parts = [item.title ?? "", item.description ?? "", (item.tags ?? []).join(" ")];
|
||||
return parts
|
||||
.map((part) => part.trim())
|
||||
.filter(Boolean)
|
||||
.join("\n");
|
||||
}
|
||||
|
||||
/**
|
||||
* Read every installable item into the same shape parseShelf produces.
|
||||
*
|
||||
* Items with no usable text are skipped rather than embedded empty: an
|
||||
* all-zero-signal entry still occupies a slot in every ranking.
|
||||
*/
|
||||
export function catalogFromRegistry(
|
||||
registryDir: string,
|
||||
read: (path: string) => string,
|
||||
listDirs: (path: string) => string[],
|
||||
): Map<string, string> {
|
||||
const catalog = new Map<string, string>();
|
||||
for (const type of ["blocks", "components"]) {
|
||||
let names: string[];
|
||||
try {
|
||||
names = listDirs(`${registryDir}/${type}`);
|
||||
} catch {
|
||||
continue;
|
||||
}
|
||||
for (const name of names.sort()) {
|
||||
let item: { title?: string; description?: string; tags?: string[] };
|
||||
try {
|
||||
item = JSON.parse(read(`${registryDir}/${type}/${name}/registry-item.json`)) as typeof item;
|
||||
} catch {
|
||||
continue;
|
||||
}
|
||||
const text = itemRetrievalText(item);
|
||||
if (text) catalog.set(name, text);
|
||||
}
|
||||
}
|
||||
return catalog;
|
||||
}
|
||||
|
||||
export function sortedNames(entries: Map<string, string>): string[] {
|
||||
return [...entries.keys()].sort();
|
||||
}
|
||||
|
||||
export function sha256Hex(data: Buffer | string): string {
|
||||
return createHash("sha256").update(data).digest("hex");
|
||||
}
|
||||
|
||||
/**
|
||||
* Digest the two published files as bytes.
|
||||
*
|
||||
* Deliberately over bytes rather than re-serialized structures. The consumer is
|
||||
* Python and this producer is TypeScript, and the two disagree on JSON number
|
||||
* formatting: a whole-valued float serializes as `1.0` in one and `1` in the
|
||||
* other. Hashing bytes removes the canonicalization question rather than
|
||||
* documenting it. The consumer computes this identically.
|
||||
*/
|
||||
export function payloadDigest(catalogBytes: Buffer, vectorsBytes: Buffer): string {
|
||||
const digest = createHash("sha256");
|
||||
digest.update(createHash("sha256").update(catalogBytes).digest());
|
||||
digest.update(createHash("sha256").update(vectorsBytes).digest());
|
||||
return digest.digest("hex");
|
||||
}
|
||||
|
||||
/** Stable serialization so an unchanged shelf rebuilds byte-identically. */
|
||||
function serialize(record: Record<string, unknown>): Buffer {
|
||||
const sorted: Record<string, unknown> = {};
|
||||
for (const key of Object.keys(record).sort()) sorted[key] = record[key];
|
||||
return Buffer.from(`${JSON.stringify(sorted, null, 2)}\n`, "utf-8");
|
||||
}
|
||||
|
||||
export async function buildArtifact(options: {
|
||||
shelfText: string;
|
||||
sourceRevision: string;
|
||||
embeddingModel: string;
|
||||
embed: Embedder;
|
||||
expectedDimension?: number;
|
||||
/**
|
||||
* Names the registry can serve. When given, shelf moves absent from it are
|
||||
* left out of the artifact: a move that ranks and cannot be installed is
|
||||
* worse than one that never appears, because it occupies a top slot.
|
||||
*/
|
||||
installableNames?: readonly string[];
|
||||
}): Promise<BuiltArtifact> {
|
||||
const { shelfText, sourceRevision, embeddingModel, embed, expectedDimension } = options;
|
||||
if (!/^[0-9a-f]{40}$/i.test(sourceRevision)) {
|
||||
throw new Error(`source_revision must be a resolved commit SHA, got ${sourceRevision}`);
|
||||
}
|
||||
|
||||
const entries = parseShelf(shelfText);
|
||||
if (options.installableNames) {
|
||||
for (const name of movesMissingFromRegistry([...entries.keys()], options.installableNames)) {
|
||||
entries.delete(name);
|
||||
}
|
||||
}
|
||||
const names = sortedNames(entries);
|
||||
const vectors = await embed(names.map((name) => entries.get(name) as string));
|
||||
|
||||
// Every validation runs before anything is written. A build that emits
|
||||
// descriptions and then fails to embed would publish exactly the half-artifact
|
||||
// the publish-together rule exists to prevent.
|
||||
if (vectors.length !== names.length) {
|
||||
throw new Error(`Expected ${names.length} vectors, embedder returned ${vectors.length}`);
|
||||
}
|
||||
const width = vectors[0]?.length ?? 0;
|
||||
if (width === 0) throw new Error("Embedder returned empty vectors");
|
||||
if (expectedDimension !== undefined && width !== expectedDimension) {
|
||||
throw new Error(`Vectors have dimension ${width}, expected ${expectedDimension}`);
|
||||
}
|
||||
vectors.forEach((vector, index) => {
|
||||
if (vector.length !== width) {
|
||||
throw new Error(
|
||||
`Vector for ${names[index]} has dimension ${vector.length}, expected ${width}`,
|
||||
);
|
||||
}
|
||||
if (!vector.every((value) => Number.isFinite(value))) {
|
||||
throw new Error(`Vector for ${names[index]} contains a non-finite value`);
|
||||
}
|
||||
if (!vector.some((value) => value !== 0)) {
|
||||
throw new Error(`Vector for ${names[index]} is a zero vector`);
|
||||
}
|
||||
});
|
||||
|
||||
const catalog: Record<string, string> = {};
|
||||
const vectorMap: Record<string, number[]> = {};
|
||||
names.forEach((name, index) => {
|
||||
catalog[name] = entries.get(name) as string;
|
||||
vectorMap[name] = vectors[index] as number[];
|
||||
});
|
||||
|
||||
const catalogBytes = serialize(catalog);
|
||||
const vectorsBytes = serialize(vectorMap);
|
||||
|
||||
return {
|
||||
catalogBytes,
|
||||
vectorsBytes,
|
||||
manifest: {
|
||||
schema_version: MANIFEST_SCHEMA_VERSION,
|
||||
source_revision: sourceRevision,
|
||||
shelf_sha256: sha256Hex(shelfText),
|
||||
embedding_model: embeddingModel,
|
||||
move_count: names.length,
|
||||
payload_sha256: payloadDigest(catalogBytes, vectorsBytes),
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Shelf moves with no matching registry item.
|
||||
*
|
||||
* Ranking is published separately from the items themselves, so the two drift.
|
||||
* A move only present in the shelf ranks well and then cannot be shown or
|
||||
* installed, which reads to the user as a bad search rather than a bad build.
|
||||
*/
|
||||
export function movesMissingFromRegistry(
|
||||
shelfNames: readonly string[],
|
||||
registryNames: readonly string[],
|
||||
): string[] {
|
||||
const known = new Set(registryNames);
|
||||
return shelfNames.filter((name) => !known.has(name));
|
||||
}
|
||||
|
||||
export function manifestBytes(manifest: CatalogManifest): Buffer {
|
||||
return serialize(manifest as unknown as Record<string, unknown>);
|
||||
}
|
||||
|
||||
/** Recompute the published digest and compare it to what the manifest claims. */
|
||||
export function verifyArtifact(input: {
|
||||
manifest: CatalogManifest;
|
||||
catalogBytes: Buffer;
|
||||
vectorsBytes: Buffer;
|
||||
}): void {
|
||||
const { manifest, catalogBytes, vectorsBytes } = input;
|
||||
if (manifest.schema_version !== MANIFEST_SCHEMA_VERSION) {
|
||||
throw new Error(
|
||||
`Manifest schema version ${manifest.schema_version} is not ${MANIFEST_SCHEMA_VERSION}`,
|
||||
);
|
||||
}
|
||||
const catalog = JSON.parse(catalogBytes.toString("utf-8")) as Record<string, string>;
|
||||
const vectors = JSON.parse(vectorsBytes.toString("utf-8")) as Record<string, number[]>;
|
||||
|
||||
const names = Object.keys(catalog);
|
||||
if (names.length !== manifest.move_count) {
|
||||
throw new Error(
|
||||
`Artifact holds ${names.length} moves but the manifest declares ${manifest.move_count}`,
|
||||
);
|
||||
}
|
||||
const missing = names.filter((name) => !(name in vectors));
|
||||
if (missing.length > 0) {
|
||||
throw new Error(
|
||||
`${missing.length} moves have a description but no vector, first is ${missing[0]}`,
|
||||
);
|
||||
}
|
||||
|
||||
const actual = payloadDigest(catalogBytes, vectorsBytes);
|
||||
if (actual !== manifest.payload_sha256) {
|
||||
throw new Error(
|
||||
`Payload digest ${actual} does not match the manifest's ${manifest.payload_sha256}`,
|
||||
);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,89 @@
|
||||
#!/usr/bin/env node
|
||||
/**
|
||||
* Fail if this branch would delete files that exist on the base.
|
||||
*
|
||||
* Written after a scare that turned out to be a measurement error, which is
|
||||
* the reason it uses the three-dot form. Comparing tip to tip (two dots) on a
|
||||
* branch that is a month behind reports every file main added since the merge
|
||||
* base as a deletion: 1,284 of them, including an entire skills tree. None of
|
||||
* that is real. A merge keeps main's side, and a pull request shows the
|
||||
* three-dot diff, which reported zero.
|
||||
*
|
||||
* So this exists to catch deletions a branch genuinely proposes, and to make
|
||||
* the distinction hard to get wrong again. If it ever disagrees with a manual
|
||||
* git diff, check which form the manual one used before believing it.
|
||||
*
|
||||
* Renames are reported separately. In a name-only diff a rename is
|
||||
* indistinguishable from a deletion, so treating them alike would either mask
|
||||
* real loss or block every legitimate move.
|
||||
*
|
||||
* node scripts/check-no-main-deletions.mjs [--base origin/main]
|
||||
*/
|
||||
|
||||
import { execFileSync } from "node:child_process";
|
||||
|
||||
const BASE_FLAG = "--base";
|
||||
|
||||
export function parseBase(argv, fallback = "origin/main") {
|
||||
const index = argv.indexOf(BASE_FLAG);
|
||||
if (index === -1) return fallback;
|
||||
const value = argv[index + 1];
|
||||
if (!value || value.startsWith("-")) {
|
||||
throw new Error(`${BASE_FLAG} needs a ref, for example ${BASE_FLAG} origin/main`);
|
||||
}
|
||||
return value;
|
||||
}
|
||||
|
||||
/**
|
||||
* Split a `--name-status` diff into deletions and renames.
|
||||
*
|
||||
* Git reports a rename as `R<score>\told\tnew`. Reading only the first column
|
||||
* would file that under "deleted", which is the false alarm this guards
|
||||
* against being noisy enough to ignore.
|
||||
*/
|
||||
export function classify(nameStatus) {
|
||||
const deleted = [];
|
||||
const renamed = [];
|
||||
for (const line of nameStatus.split("\n")) {
|
||||
if (!line.trim()) continue;
|
||||
const [status, ...paths] = line.split("\t");
|
||||
if (status.startsWith("R")) renamed.push({ from: paths[0], to: paths[1] });
|
||||
else if (status === "D") deleted.push(paths[0]);
|
||||
}
|
||||
return { deleted, renamed };
|
||||
}
|
||||
|
||||
function main() {
|
||||
const base = parseBase(process.argv.slice(2));
|
||||
let diff;
|
||||
try {
|
||||
diff = execFileSync("git", ["diff", "--name-status", "-M", `${base}...HEAD`], {
|
||||
encoding: "utf8",
|
||||
});
|
||||
} catch (error) {
|
||||
// An unreachable base is not a pass. Reporting "no deletions" because the
|
||||
// ref was misspelled is the exact failure this exists to prevent.
|
||||
console.error(`cannot diff against ${base}: ${error.message.trim()}`);
|
||||
process.exit(2);
|
||||
}
|
||||
|
||||
const { deleted, renamed } = classify(diff);
|
||||
if (renamed.length > 0) {
|
||||
console.log(`${renamed.length} renamed (allowed):`);
|
||||
for (const { from, to } of renamed.slice(0, 10)) console.log(` ${from} -> ${to}`);
|
||||
if (renamed.length > 10) console.log(` … and ${renamed.length - 10} more`);
|
||||
}
|
||||
|
||||
if (deleted.length === 0) {
|
||||
console.log(`No files from ${base} are deleted by this branch.`);
|
||||
return;
|
||||
}
|
||||
|
||||
console.error(`This branch deletes ${deleted.length} files that exist on ${base}:`);
|
||||
for (const path of deleted.slice(0, 25)) console.error(` ${path}`);
|
||||
if (deleted.length > 25) console.error(` … and ${deleted.length - 25} more`);
|
||||
console.error("\nIf a deletion is intended, remove this check for that path deliberately.");
|
||||
process.exit(1);
|
||||
}
|
||||
|
||||
if (import.meta.url === `file://${process.argv[1]}`) main();
|
||||
@@ -0,0 +1,42 @@
|
||||
import { strict as assert } from "node:assert";
|
||||
import { test } from "node:test";
|
||||
|
||||
import { classify, parseBase } from "./check-no-main-deletions.mjs";
|
||||
|
||||
test("a branch that only adds reports nothing", () => {
|
||||
const { deleted, renamed } = classify("A\tpackages/cli/src/new.ts\nM\tpackages/cli/src/old.ts\n");
|
||||
assert.deepEqual(deleted, []);
|
||||
assert.deepEqual(renamed, []);
|
||||
});
|
||||
|
||||
test("a deleted file is named", () => {
|
||||
const { deleted } = classify("D\t.agents/skills/README.md\nA\tsrc/new.ts\n");
|
||||
assert.deepEqual(deleted, [".agents/skills/README.md"]);
|
||||
});
|
||||
|
||||
test("a rename is not reported as a deletion", () => {
|
||||
// The false alarm worth avoiding: in a name-only diff a rename looks exactly
|
||||
// like loss, and a check that cried wolf on every move would be turned off.
|
||||
const { deleted, renamed } = classify("R096\tsrc/old/name.ts\tsrc/new/name.ts\n");
|
||||
assert.deepEqual(deleted, []);
|
||||
assert.deepEqual(renamed, [{ from: "src/old/name.ts", to: "src/new/name.ts" }]);
|
||||
});
|
||||
|
||||
test("deletions and renames are separated in one diff", () => {
|
||||
const { deleted, renamed } = classify(
|
||||
"D\tdocs/gone.md\nR100\ta.ts\tb.ts\nM\tc.ts\nD\tdocs/also-gone.md\n",
|
||||
);
|
||||
assert.deepEqual(deleted, ["docs/gone.md", "docs/also-gone.md"]);
|
||||
assert.equal(renamed.length, 1);
|
||||
});
|
||||
|
||||
test("the base ref defaults, and an explicit one is honoured", () => {
|
||||
assert.equal(parseBase([]), "origin/main");
|
||||
assert.equal(parseBase(["--base", "origin/release"]), "origin/release");
|
||||
});
|
||||
|
||||
test("a --base with no value fails rather than silently defaulting", () => {
|
||||
// Silently falling back would diff against the wrong ref and report a pass.
|
||||
assert.throws(() => parseBase(["--base"]), /needs a ref/);
|
||||
assert.throws(() => parseBase(["--base", "--other"]), /needs a ref/);
|
||||
});
|
||||
Reference in New Issue
Block a user