Files
hyperframes/packages/cli/src/utils/command-failure-tracking.test.ts
T
Miguel Ángel 5f6ced116d feat(cli): report command failure reasons to telemetry (de-blind browser/info) (#1484)
Observability showed `browser` (~75% fail, ~1.3k users/day) and `info` (~60%
fail) failing at high rates with no captured reason — only
`cli_command_result success=false`. citty's `runMain` catches a command's
thrown error and `process.exit(1)`s without re-throwing, so a thrown failure
never reached the existing `cli_error` telemetry (which only fired from the
uncaughtException / unhandledRejection handlers).

Wrap every command's `run()` at the dispatch boundary (cli.ts) so a thrown
failure reports its reason via `cli_error` (kind=command_error) before being
re-thrown unchanged — citty's print + exit-1 behavior is preserved. This
de-blinds every throw-style command at once: `browser ensure` (Chrome
download), `tts`, `inspect`, `render`, etc.

Paths that bypass the wrapper are handled inline:
- `browser` self-exits (`path` download failure, unknown subcommand) — report
  inline; the ARM64 `ensure` branch previously swallowed a failed install and
  returned success, now reports and exits 1.
- `resolveProject()` self-exits on InvalidProjectError (the dominant `info`
  failure — run outside a project) — report inline before exit.

Hardening:
- PII: `trackCliError` now redacts error_message + stack_trace via
  redactTelemetryString (matching render_* events) — CLI errors and stacks
  carry absolute install paths / cache dirs / user args.
- Race: the wrapper awaits an on-demand telemetry import before re-throwing, so
  a command that fails before the lazy telemetry import settles still reports
  (a telemetry failure is swallowed and never masks the real error).

Pure helpers in utils/command-failure-tracking.ts with unit tests for the
throw / success / no-run / onFailure-rejection cases, the reporter wiring, and
trackCliError redaction. CommandDef<any> mirrors citty's SubCommandsDef.

Known scope: commands that print + `process.exit(1)` on their own validation
paths (tts/validate/lint argument errors) remain wrapper-blind — follow-up.
2026-06-16 01:39:34 -04:00

82 lines
2.8 KiB
TypeScript

import { beforeEach, describe, expect, it, vi } from "vitest";
import type { CommandDef } from "citty";
const trackCommandFailure = vi.fn();
vi.mock("../telemetry/events.js", () => ({
trackCommandFailure: (...args: unknown[]) => trackCommandFailure(...args),
}));
const { trackCommandFailures, reportCommandFailure } =
await import("./command-failure-tracking.js");
function defineRun(run: CommandDef["run"]): CommandDef {
return { meta: { name: "test" }, run };
}
describe("trackCommandFailures", () => {
it("reports the error and re-throws when run() rejects", async () => {
const onFailure = vi.fn();
const boom = new Error("ffmpeg not found");
const wrapped = trackCommandFailures(
() => Promise.resolve(defineRun(() => Promise.reject(boom))),
onFailure,
);
const cmd = await wrapped();
await expect((cmd.run as () => Promise<unknown>)()).rejects.toBe(boom);
expect(onFailure).toHaveBeenCalledWith(boom);
});
it("does not report when run() succeeds, and returns its value", async () => {
const onFailure = vi.fn();
const wrapped = trackCommandFailures(
() => Promise.resolve(defineRun(() => Promise.resolve("ok" as unknown as void))),
onFailure,
);
const cmd = await wrapped();
await expect((cmd.run as () => Promise<unknown>)()).resolves.toBe("ok");
expect(onFailure).not.toHaveBeenCalled();
});
it("passes through a command with no run() untouched", async () => {
const onFailure = vi.fn();
const parent: CommandDef = { meta: { name: "parent" } };
const wrapped = trackCommandFailures(() => Promise.resolve(parent), onFailure);
const cmd = await wrapped();
expect(cmd).toBe(parent);
expect(onFailure).not.toHaveBeenCalled();
});
it("awaits onFailure and re-throws the ORIGINAL error even if onFailure rejects", async () => {
const boom = new Error("original failure");
const wrapped = trackCommandFailures(
() => Promise.resolve(defineRun(() => Promise.reject(boom))),
() => Promise.reject(new Error("telemetry is down")),
);
const cmd = await wrapped();
await expect((cmd.run as () => Promise<unknown>)()).rejects.toBe(boom);
});
});
describe("reportCommandFailure", () => {
beforeEach(() => {
trackCommandFailure.mockReset();
});
it("forwards the command and error to trackCommandFailure", async () => {
const err = new Error("ENOENT /Users/me/project/index.html");
await reportCommandFailure("info", err);
expect(trackCommandFailure).toHaveBeenCalledWith("info", err);
});
it("never throws when the telemetry call throws", async () => {
trackCommandFailure.mockImplementationOnce(() => {
throw new Error("telemetry blew up");
});
await expect(reportCommandFailure("browser", new Error("x"))).resolves.toBeUndefined();
});
});