hyperframes/packages/cli/src/commands/catalog.test.ts

import { beforeEach, describe, expect, it, vi } from "vitest";

import { countUnindexed, pickByName, searchMissCommand } from "./catalog.js";

/** The whole registry, which is what "in this registry" has to be measured against. */
const registryNames = new Set(["fade-through", "whip-pan", "count-up"]);
const item = (name: string): { name: string } => ({ name });

describe("pickByName", () => {
  it("counts only the ranked names this registry has no item for", () => {
    const { ranked, missing } = pickByName(
      [item("fade-through"), item("whip-pan"), item("count-up")],
      ["whip-pan", "fade-through", "accordion", "alert-dialog"],
      registryNames,
    );

    expect(ranked.map((entry) => entry.name)).toEqual(["whip-pan", "fade-through"]);
    // accordion and alert-dialog are in the ranking artifact and nowhere in the
    // registry: a real skew between two separately published generations.
    expect(missing).toBe(2);
  });

  it("does not count moves the user's own filter removed", () => {
    // `items` is what survived --type/--tag; the registry still has the rest.
    const { ranked, missing } = pickByName(
      [item("fade-through")],
      ["whip-pan", "fade-through", "count-up", "accordion"],
      registryNames,
    );

    expect(ranked.map((entry) => entry.name)).toEqual(["fade-through"]);
    // whip-pan and count-up are installable, just filtered out. Only accordion
    // is genuinely absent, and filtering must not inflate that number.
    expect(missing).toBe(1);
  });

  it("reports nothing missing when the whole ranking is installable", () => {
    const { missing } = pickByName(
      [item("fade-through")],
      ["fade-through", "whip-pan"],
      registryNames,
    );

    expect(missing).toBe(0);
  });
});

describe("countUnindexed", () => {
  it("counts the registry moves the on-device index holds no vector for", () => {
    // The move published after the artifact was fetched. Meaning search cannot
    // rank it at all, which is the failure this number exists to expose.
    expect(countUnindexed(registryNames, ["fade-through"])).toBe(2);
  });

  it("reports nothing when the index covers the registry", () => {
    expect(countUnindexed(registryNames, ["count-up", "whip-pan", "fade-through"])).toBe(0);
  });

  it("does not let names the registry dropped paper over a gap", () => {
    // The artifact holds two names this registry cannot install and is missing
    // two it can. Comparing sizes rather than membership would call that even.
    expect(countUnindexed(registryNames, ["fade-through", "accordion", "alert-dialog"])).toBe(2);
  });
});

// ── The command envelope ────────────────────────────────────────────────────
// The JSON envelope is the surface an agent reads, so under-coverage and the
// score are pinned where they are actually published rather than only at the
// helper that computes them.

const state = vi.hoisted(() => ({
  registry: [] as Array<{ name: string; type: string; tags?: string[] }>,
  artifactRevision: "revision-current",
  cachedVectorRevision: "revision-current",
  vectorFetches: 0,
  vectorFetchSucceeds: true,
  ranking: null as Array<{ name: string; score: number }> | null,
  rankingError: null as Error | null,
  indexed: [] as string[],
  // The consent path. Static stubs could not reach it: with the status pinned
  // to "ready" the prompt never fires, so the answer was never a variable and
  // the decline branch was never executed by any test.
  modelStatus: "ready" as "ready" | "not-asked" | "declined" | "unavailable",
  confirmAnswer: true as boolean,
  consentRecorded: [] as boolean[],
  downloads: 0,
  runtimeAvailable: true,
}));

vi.mock("../registry/resolver.js", () => ({
  loadAllItems: async (entries: Array<{ name: string; type: string; tags?: string[] }>) =>
    entries.map((entry) => ({
      name: entry.name,
      type: entry.type,
      title: entry.name,
      description: `${entry.name} description`,
      tags: entry.tags ?? [],
    })),
}));

vi.mock("../registry/remote.js", () => ({
  fetchRegistryManifest: async () => ({
    items: state.registry,
    catalogArtifact: { revision: state.artifactRevision },
  }),
}));

vi.mock("@clack/prompts", () => ({
  confirm: async () => state.confirmAnswer,
  isCancel: (value: unknown) => value === null,
}));

vi.mock("../registry/localModel.js", () => ({
  // "ready" is a user who opted into the on-device tier at some point. Every
  // later search takes that tier with no flag, which is how a frozen artifact
  // goes on answering forever. "not-asked" is the first run, the one that asks.
  // Recording an answer is what stops the CLI asking again, so the stub has to
  // move with it. Pinned to "not-asked" the second offer later in the run also
  // fires, and the double prompt looks like a product bug rather than a stub
  // that does not model the contract.
  localModelStatus: () => {
    const answer = state.consentRecorded.at(-1);
    return {
      status: answer === false ? "declined" : answer === true ? "ready" : state.modelStatus,
    };
  },
  ensureLocalModel: async () => {
    state.downloads += 1;
    if (state.modelStatus === "unavailable") state.modelStatus = "ready";
    return true;
  },
  recordLocalModelConsent: (enabled: boolean) => {
    state.consentRecorded.push(enabled);
  },
  downloadOfferMessage: () => "offer",
  nonInteractiveConsentMessage: () => "consent",
}));

vi.mock("../registry/localEmbedder.js", () => ({
  // The native runtime is present in these tests. Left unmocked it answers
  // false under vitest, and every accepted offer returns at the runtime guard
  // before it can download, which looks like the download being skipped.
  localRuntimeAvailable: async () => state.runtimeAvailable,
}));

vi.mock("../registry/localSemantic.js", () => ({
  localSemanticRanking: async () => {
    if (state.rankingError) throw state.rankingError;
    return state.ranking;
  },
  localVectorNames: () => state.indexed,
  cachedLocalVectorRevision: () => state.cachedVectorRevision,
  hasLocalVectors: () => true,
  fetchLocalVectors: async (_registry: string, options: { expectedRevision?: string } = {}) => {
    state.vectorFetches += 1;
    if (state.vectorFetchSucceeds && options.expectedRevision !== undefined) {
      state.cachedVectorRevision = options.expectedRevision;
    }
    return state.vectorFetchSucceeds;
  },
}));

const block = (name: string, tags?: string[]): { name: string; type: string; tags?: string[] } => ({
  name,
  type: "hyperframes:block",
  tags,
});
const component = (name: string): { name: string; type: string } => ({
  name,
  type: "hyperframes:component",
});

interface Envelope {
  tier: string;
  dropped: number;
  unindexed: number;
  top_score?: number;
  shown: number;
  warnings?: string[];
  // Not optional: both envelope-shaped emit sites are inside `if (query)`
  // branches and both set it, so a search envelope without it is a bug rather
  // than a shape the caller has to handle.
  report_gap: string;
}

async function runCatalog(args: Record<string, unknown>): Promise<string> {
  const command = (await import("./catalog.js")).default as unknown as {
    run: (context: { args: Record<string, unknown> }) => Promise<void>;
  };
  const lines: string[] = [];
  const log = vi.spyOn(console, "log").mockImplementation((...parts: unknown[]) => {
    lines.push(parts.map(String).join(" "));
  });
  try {
    await command.run({ args });
  } finally {
    log.mockRestore();
  }
  // Colour is decoration; assertions are about the words. The escape byte is
  // built rather than written: as a literal or as \u001B it is a control
  // character in the source, which the lint rules reject either way.
  const ansi = new RegExp(`${String.fromCharCode(27)}\\[[0-9;]*m`, "g");
  return lines.join("\n").replace(ansi, "");
}

async function runEnvelope(args: Record<string, unknown>): Promise<Envelope> {
  return JSON.parse(await runCatalog({ json: true, ...args })) as Envelope;
}

beforeEach(() => {
  state.modelStatus = "ready";
  state.artifactRevision = "revision-current";
  state.cachedVectorRevision = "revision-current";
  state.vectorFetches = 0;
  state.vectorFetchSucceeds = true;
  state.rankingError = null;
  state.confirmAnswer = true;
  state.consentRecorded = [];
  state.downloads = 0;
  state.runtimeAvailable = true;
  state.registry = [block("count-up"), block("fade-through"), component("whip-pan")];
  state.indexed = ["count-up", "fade-through", "whip-pan"];
  state.ranking = [
    { name: "count-up", score: 0.71 },
    { name: "whip-pan", score: 0.42 },
    { name: "fade-through", score: 0.31 },
  ];
});

describe("catalog --json meaning search", () => {
  it("reports the registry moves meaning search cannot see", async () => {
    // Published after the user's artifact was fetched: in the registry, absent
    // from the index, and therefore unreturnable by any query.
    state.registry.push(block("split-screen"));

    const envelope = await runEnvelope({ query: "make a number count up" });

    expect(envelope.tier).toBe("on-device");
    expect(envelope.unindexed).toBe(1);
  });

  it("reports nothing unindexed when the artifact matches the registry", async () => {
    const envelope = await runEnvelope({ query: "make a number count up" });

    expect(envelope.unindexed).toBe(0);
  });

  it("measures unindexed against the unfiltered registry under --type", async () => {
    // The missing move is a component; the user asked for blocks. Their filter
    // is not the index being stale, and it must not hide a stale index either.
    state.registry.push(component("push-in"));

    const envelope = await runEnvelope({ query: "make a number count up", type: "block" });

    expect(envelope.shown).toBe(2);
    expect(envelope.unindexed).toBe(1);
  });

  it("keeps dropped counted against the unfiltered registry under --type", async () => {
    // accordion is the only name the registry genuinely lacks. whip-pan is
    // installable and merely filtered out, so it is not a drop.
    state.ranking = [
      { name: "count-up", score: 0.71 },
      { name: "whip-pan", score: 0.62 },
      { name: "accordion", score: 0.55 },
      { name: "fade-through", score: 0.31 },
    ];

    const envelope = await runEnvelope({ query: "make a number count up", type: "block" });

    expect(envelope.dropped).toBe(1);
    expect(envelope.shown).toBe(2);
  });

  it("carries the score of the best result it actually showed", async () => {
    // The top-ranked name is not installable here, so reporting the ranking's
    // own head would describe a row the caller never received.
    state.ranking = [
      { name: "accordion", score: 0.93 },
      { name: "count-up", score: 0.71 },
      { name: "whip-pan", score: 0.42 },
      { name: "fade-through", score: 0.31 },
    ];

    const envelope = await runEnvelope({ query: "make a number count up" });

    expect(envelope.top_score).toBeCloseTo(0.71);
  });

  it("omits the score on the word tier, whose scale is not the same one", async () => {
    state.ranking = null;

    const envelope = await runEnvelope({ query: "count up" });

    expect(envelope.tier).toBe("words");
    expect(envelope.top_score).toBeUndefined();
    // Word matching ranks the live registry listing, so it is never stale.
    expect(envelope.unindexed).toBe(0);
  });

  it("finds registry tags on the word tier", async () => {
    state.modelStatus = "declined";
    state.ranking = null;
    state.registry = [block("fade-through", ["transition"]), block("count-up", ["number"])];

    const envelope = await runEnvelope({ query: "transition" });

    expect(envelope.tier).toBe("words");
    expect(envelope.shown).toBe(1);
  });

  it("carries an on-device runtime failure into the JSON envelope", async () => {
    state.rankingError = new Error("model could not load");

    const envelope = await runEnvelope({ query: "count up" });

    expect(envelope.tier).toBe("words");
    expect(envelope.warnings).toEqual(["on-device search did not run: model could not load"]);
  });

  it("refreshes a changed vector revision under existing consent", async () => {
    state.cachedVectorRevision = "revision-previous";

    const envelope = await runEnvelope({ query: "count up" });

    expect(envelope.tier).toBe("on-device");
    expect(state.vectorFetches).toBe(1);
    expect(state.cachedVectorRevision).toBe("revision-current");
    expect(state.consentRecorded).toEqual([]);
  });

  it("replaces a changed model revision under existing consent", async () => {
    state.modelStatus = "unavailable";

    const envelope = await runEnvelope({ query: "count up" });

    expect(envelope.tier).toBe("on-device");
    expect(state.downloads).toBe(1);
    expect(state.consentRecorded).toEqual([]);
  });

  it("keeps the previous vectors and reports a failed routine refresh", async () => {
    state.cachedVectorRevision = "revision-previous";
    state.vectorFetchSucceeds = false;

    const envelope = await runEnvelope({ query: "count up" });

    expect(envelope.tier).toBe("on-device");
    expect(state.cachedVectorRevision).toBe("revision-previous");
    expect(envelope.warnings).toEqual([
      "on-device search is using the previous catalog vectors because the update failed",
    ]);
  });

  it("hands back the gap-report command even when the search found things", async () => {
    // The reports we actually want come from searches that returned plausible
    // items where none of them did the job. If the command only appeared on
    // zero results it would be absent from every case worth reporting.
    const envelope = await runEnvelope({ query: "make a number count up" });

    expect(envelope.shown).toBeGreaterThan(0);
    expect(envelope.report_gap).toBe(
      'npx hyperframes feedback --search-miss "make a number count up" ' +
        '--wanted "<the move you needed>" --tier on-device',
    );
  });

  it("names the tier that actually answered in the gap-report command", async () => {
    state.modelStatus = "declined";
    state.ranking = null;

    const envelope = await runEnvelope({ query: "count up" });

    expect(envelope.tier).toBe("words");
    expect(envelope.report_gap).toContain("--tier words");
  });
});

describe("a query with no searchable words", () => {
  // Runs the command capturing stderr, and reports the exit code the CLI would
  // have used. finishCommand throws a signal rather than calling process.exit.
  async function runForExit(
    args: Record<string, unknown>,
  ): Promise<{ exitCode: number; err: string }> {
    const command = (await import("./catalog.js")).default as unknown as {
      run: (context: { args: Record<string, unknown> }) => Promise<void>;
    };
    const lines: string[] = [];
    const capture = (...parts: unknown[]): void => {
      lines.push(parts.map(String).join(" "));
    };
    const log = vi.spyOn(console, "log").mockImplementation(capture);
    const error = vi.spyOn(console, "error").mockImplementation(capture);
    let exitCode = 0;
    try {
      await command.run({ args });
    } catch (thrown) {
      const signal = thrown as { result?: { exitCode?: number }; exitCode?: number };
      exitCode = signal.result?.exitCode ?? signal.exitCode ?? -1;
    } finally {
      log.mockRestore();
      error.mockRestore();
    }
    // eslint-disable-next-line no-control-regex
    const esc = String.fromCharCode(27);
    return { exitCode, err: lines.join("\n").split(`${esc}[`).join("").replace(/\d+m/g, "") };
  }

  it("exits non-zero, because it is bad input rather than an empty shelf", async () => {
    // The flag is word-tier only, so the stubbed on-device ranker has to be off
    // or it answers with hits and the branch never runs.
    state.modelStatus = "declined";
    state.ranking = null;

    const { exitCode } = await runForExit({ query: "実写写真のみ 9:16 生活ハック" });

    // An agent that only reads the exit code would otherwise take "success, no
    // results" at face value and hand-author a move already in the registry.
    expect(exitCode).toBe(1);
  });

  it("says to search in English and does not blame the catalog", async () => {
    state.modelStatus = "declined";
    state.ranking = null;

    const { err } = await runForExit({ query: "実写写真のみ 9:16 生活ハック" });

    expect(err).toContain("No searchable words in query");
    expect(err).toContain("Search in English");
    expect(err).toContain("not a gap in");
    // The gap channel must not be offered: nothing was searched, so a report
    // here is noise in the one signal that tells us what to build.
    expect(err).not.toContain("--search-miss");
  });

  it("leaves a genuine empty result exiting zero", async () => {
    state.ranking = [];
    state.modelStatus = "declined";

    const { exitCode, err } = await runForExit({ query: "quantum entanglement reactor" });

    expect(exitCode).toBe(0);
    expect(err).toContain("No items match");
  });
});

describe("searchMissCommand", () => {
  it("keeps a non-ASCII query intact", () => {
    // Half of the gap reports received so far were CJK. A query mangled on the
    // way into the command is a report nobody can act on.
    expect(searchMissCommand("実写写真のみ 9:16 生活ハック", "on-device")).toContain(
      '--search-miss "実写写真のみ 9:16 生活ハック"',
    );
  });

  it("escapes shell metacharacters so the printed line is safe to paste", () => {
    const cmd = searchMissCommand('a "quoted" $VAR `sub` \\ thing', "words");

    expect(cmd).toContain('--search-miss "a \\"quoted\\" \\$VAR \\`sub\\` \\\\ thing"');
  });
});

describe("catalog meaning search, on a terminal", () => {
  it("says how much is missing and what to run about it", async () => {
    state.registry.push(block("split-screen"));

    const output = await runCatalog({ query: "make a number count up" });

    expect(output).toContain("1 of 4 moves are missing from the on-device index");
    expect(output).toContain("Re-run with --on-device to refresh it.");
  });

  it("says nothing when the index covers the registry", async () => {
    const output = await runCatalog({ query: "make a number count up" });

    expect(output).not.toContain("missing from the on-device index");
  });

  it("offers the gap report on the word tier, not just on-device", async () => {
    // The tier that answers almost every real search, because on-device needs
    // a consented download. Gating the nudge on on-device left it unprinted in
    // the only case that occurs, which is how the gap channel stayed silent.
    state.modelStatus = "declined";
    state.ranking = null;

    const output = await runCatalog({ query: "count up" });

    expect(output).toContain("None of these do it?");
    expect(output).toContain("--tier words");
  });

  it("offers the gap report on the on-device tier too", async () => {
    const output = await runCatalog({ query: "make a number count up" });

    expect(output).toContain("None of these do it?");
    expect(output).toContain("--tier on-device");
  });
});

describe("the on-device download offer", () => {
  // The offer only exists for someone who can answer it. Off a terminal the
  // caller must add --yes explicitly, so a test that forgets the terminal
  // never reaches the prompt and passes for the wrong reason.
  const asATerminal = async (run: () => Promise<string>): Promise<string> => {
    const descriptor = Object.getOwnPropertyDescriptor(process.stdout, "isTTY");
    Object.defineProperty(process.stdout, "isTTY", { value: true, configurable: true });
    try {
      return await run();
    } finally {
      if (descriptor) Object.defineProperty(process.stdout, "isTTY", descriptor);
      else delete (process.stdout as unknown as { isTTY?: boolean }).isTTY;
    }
  };

  it("downloads nothing and records no consent when the offer is declined", async () => {
    // The whole point of asking. Nothing below this line may fetch 32 MB.
    state.modelStatus = "not-asked";
    state.confirmAnswer = false;

    const output = await asATerminal(() =>
      runCatalog({ query: "make a number count up", "on-device": true }),
    );

    expect(state.downloads).toBe(0);
    expect(state.consentRecorded).toEqual([false]);
    expect(output).not.toContain("offer");
  });

  it("downloads once when the offer is accepted", async () => {
    state.modelStatus = "not-asked";
    state.confirmAnswer = true;

    await asATerminal(() => runCatalog({ query: "make a number count up", "on-device": true }));

    expect(state.downloads).toBe(1);
    expect(state.consentRecorded).toEqual([true]);
  });

  it("keeps a decline sticky until explicit --yes consent", async () => {
    state.modelStatus = "not-asked";
    state.confirmAnswer = false;

    await asATerminal(() => runCatalog({ query: "count up", "on-device": true }));
    state.confirmAnswer = true;
    await asATerminal(() => runCatalog({ query: "count up", "on-device": true }));

    expect(state.downloads).toBe(0);
    expect(state.consentRecorded).toEqual([false]);

    await asATerminal(() => runCatalog({ query: "count up", "on-device": true, yes: true }));
    expect(state.downloads).toBe(1);
    expect(state.consentRecorded).toEqual([false, true]);
  });

  it("does not treat non-interactive output as download consent", async () => {
    state.modelStatus = "not-asked";

    await runEnvelope({ query: "count up", "on-device": true });

    expect(state.downloads).toBe(0);
    expect(state.consentRecorded).toEqual([]);
  });
});