feat(cli/telemetry): surface unrecognized agents in the agent_runtime=null bucket (#1978)

* feat(cli/telemetry): surface unrecognized agents in the agent_runtime=null bucket

agent_runtime is a closed allowlist: an agent we have no rule for collapses
to null with no trace of what it was, so ~18% of CLI users are unattributable
and new agents stay invisible until reverse-engineered by hand.

Add detectAgentHints(), a self-populating residual signal computed only for
the null bucket (gated off classified events):
- agent_hint: value of AGENT / AI_AGENT (the emerging self-identification
  convention; Crush and Goose set AGENT=<name>) — names agents the allowlist
  misses.
- term_program: raw TERM_PROGRAM (editor name) — catches the IDE-terminal
  class the same way the cursor/windsurf rules do.
- agent_env_hints: sorted, comma-joined "agent-ish" env-var KEY names present
  but matched by no vendor rule — a fingerprint that clusters by agent.

Privacy stays consistent with the existing "never read secret-shaped values"
stance: agent_env_hints emits key names only; the three value-reads are vars
whose sole purpose is non-secret identification, each passed through a strict
short-slug allowlist so anything long/spaced/secret-shaped is dropped.

Breaking down agent_hint / agent_env_hints filtered to agent_runtime IS NULL
AND is_tty=false gives a ranked leaderboard of new agents to promote into
VENDOR_RULES.

Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>

* fix(cli/telemetry): guard agent_hint/term_program against short credential-shaped values

Review feedback (Magi, #1978): the short-slug allowlist in sanitizeHint() still
accepted short credential-shaped values (AGENT=sk-ant-api03,
AGENT=AKIAIOSFODNN7EXAMPLE, AGENT=github_pat_abc), so the "never emit a secret"
claim wasn't actually enforced — only overlong values were dropped.

Add a credential-shape guard on top of the slug allowlist:
- known token/credential prefixes (sk-, ghp_, github_pat_, akia, ya29, ...)
- any unbroken alphanumeric run >= 16 chars (key bodies, hex, base64-ish),
  while agent names segment on _/-/. and keep each run short.

Replace the single overlong-value test with the short credential shapes from the
review (parametrized) plus a positive case (gemini_managed_agent) proving real
multi-segment names still pass.

Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>

---------

Co-authored-by: Claude Opus 4.8 (1M context) <noreply@anthropic.com>

packages/cli/src/telemetry/agent_runtime.test.ts

@@ -330,6 +330,83 @@ describe("detectAgentRuntime — Windsurf / Cline / Gemini CLI / Crush", () => {
   });
 });
 
+describe("detectAgentHints — new-agent discovery signals", () => {
+  const savedEnv = { ...process.env };
+  beforeEach(() => {
+    stripVendorEnv();
+    // stripVendorEnv clears TERM_PROGRAM; also clear the value-captured generics
+    // and any hint-shaped keys a test sets so assertions stay deterministic.
+    delete process.env["AGENT"];
+    delete process.env["AI_AGENT"];
+  });
+  afterEach(() => {
+    process.env = { ...savedEnv };
+  });
+
+  it("reads AGENT as the agent_hint (self-identification convention), lowercased", async () => {
+    process.env["AGENT"] = "Crush";
+    const { detectAgentHints } = await import("./agent_runtime.js");
+    expect(detectAgentHints().agent_hint).toBe("crush");
+  });
+
+  it("falls back to AI_AGENT when AGENT is unset", async () => {
+    process.env["AI_AGENT"] = "goose";
+    const { detectAgentHints } = await import("./agent_runtime.js");
+    expect(detectAgentHints().agent_hint).toBe("goose");
+  });
+
+  it("drops an overlong secret-looking AGENT value rather than leaking it", async () => {
+    process.env["AGENT"] = "sk-ant-api03-THIS-IS-A-LONG-SECRET-LOOKING-VALUE-xyz";
+    const { detectAgentHints } = await import("./agent_runtime.js");
+    expect(detectAgentHints().agent_hint).toBeNull();
+  });
+
+  // The short-slug allowlist alone accepts these; the credential-shape guard
+  // (prefixes + long alnum runs) is what actually enforces the privacy claim.
+  it.each([
+    ["sk-ant-api03", "token prefix"],
+    ["AKIAIOSFODNN7EXAMPLE", "AWS access key id (prefix + long run)"],
+    ["github_pat_abc", "GitHub PAT prefix"],
+    ["ghp_0123456789abcdef", "GitHub token prefix"],
+    ["ya29.a0veryrealtoken", "Google OAuth prefix"],
+    ["deadbeefdeadbeef01", "18-char unbroken token body"],
+  ])("drops short credential-shaped AGENT value %s (%s)", async (value) => {
+    process.env["AGENT"] = value;
+    const { detectAgentHints } = await import("./agent_runtime.js");
+    expect(detectAgentHints().agent_hint).toBeNull();
+  });
+
+  it("still captures a real multi-segment agent name (no over-rejection)", async () => {
+    process.env["AGENT"] = "gemini_managed_agent";
+    const { detectAgentHints } = await import("./agent_runtime.js");
+    expect(detectAgentHints().agent_hint).toBe("gemini_managed_agent");
+  });
+
+  it("captures TERM_PROGRAM as the editor/terminal hint", async () => {
+    process.env["TERM_PROGRAM"] = "zed";
+    const { detectAgentHints } = await import("./agent_runtime.js");
+    expect(detectAgentHints().term_program).toBe("zed");
+  });
+
+  it("surfaces an unknown agent-ish env KEY in agent_env_hints", async () => {
+    process.env["FOO_AGENT_SESSION_ID"] = "whatever-value";
+    const { detectAgentHints } = await import("./agent_runtime.js");
+    expect(detectAgentHints().agent_env_hints).toContain("FOO_AGENT_SESSION_ID");
+  });
+
+  it("excludes SSH/GPG agent false-friends from agent_env_hints", async () => {
+    process.env["SSH_AGENT_PID"] = "12345";
+    const { detectAgentHints } = await import("./agent_runtime.js");
+    expect(detectAgentHints().agent_env_hints ?? "").not.toContain("SSH_AGENT");
+  });
+
+  it("does NOT match ENCODING keys (PYTHONIOENCODING) — no bare CODING token", async () => {
+    process.env["PYTHONIOENCODING"] = "utf-8";
+    const { detectAgentHints } = await import("./agent_runtime.js");
+    expect(detectAgentHints().agent_env_hints ?? "").not.toContain("ENCODING");
+  });
+});
+
 describe("detectSandboxRuntime — file-system path", () => {
   beforeEach(() => {
     vi.resetModules();

packages/cli/src/telemetry/agent_runtime.ts

@@ -244,6 +244,131 @@ export function detectAgentRuntime(): AgentRuntime {
   return null;
 }
 
+// ---------------------------------------------------------------------------
+// New-agent discovery signals.
+//
+// VENDOR_RULES is a CLOSED allowlist: an agent we haven't written a rule for
+// collapses to agent_runtime=null, leaving no trace of what it was. That makes
+// the null bucket un-attributable — new agents stay invisible until someone
+// reverse-engineers their marker by hand (which is how every rule above was
+// derived).
+//
+// detectAgentHints() adds a self-populating residual signal for exactly that
+// null bucket, so an unrecognized agent surfaces on its own in analytics and
+// can be promoted to a real VENDOR_RULE later. Callers should only emit these
+// when detectAgentRuntime() returns null — a classified event needs no hint.
+//
+// Privacy — consistent with the "never read secret-shaped values" stance above:
+//   - agent_env_hints emits KEY NAMES only, never values.
+//   - agent_hint / term_program read the VALUE of three vars whose sole purpose
+//     is non-secret self-identification: the emerging AGENT / AI_AGENT
+//     agent-name convention (e.g. Crush and Goose set AGENT=<name>) and
+//     TERM_PROGRAM's editor name (how the cursor/windsurf rules already work).
+//     Each value is passed through a strict short-slug allowlist, so anything
+//     long, spaced, or secret-shaped is dropped to null.
+// ---------------------------------------------------------------------------
+
+export interface AgentHints {
+  /**
+   * Best-effort agent name from a self-identifying env-var value: AGENT, else
+   * AI_AGENT. null when neither is set or the value isn't a short safe slug.
+   */
+  agent_hint: string | null;
+  /**
+   * Raw TERM_PROGRAM value (editor/terminal name) — surfaces IDE-terminal
+   * agents not yet covered by the cursor/windsurf rules. Noisy (also set by
+   * plain human terminals); read alongside is_tty=false. null when unset/unsafe.
+   */
+  term_program: string | null;
+  /**
+   * Sorted, comma-joined list of "agent-ish" env-var KEY names present but
+   * matched by no vendor rule — a compact fingerprint that clusters by agent.
+   * KEYS only, never values. null when none are present.
+   */
+  agent_env_hints: string | null;
+}
+
+// Self-identifying values are agent/editor NAMES by convention — short slugs.
+// Anything longer, spaced, or secret-shaped falls outside this and is dropped.
+const SAFE_HINT = /^[a-z0-9_.-]{1,32}$/;
+
+// The slug allowlist alone still accepts SHORT credential-shaped values
+// (AGENT=sk-ant-api03, AGENT=AKIAIOSFODNN7EXAMPLE, AGENT=github_pat_abc), so
+// two extra guards enforce the "never emit a secret" boundary this PR relies on:
+//   1. known credential/token prefixes (compared lowercased), and
+//   2. any unbroken alphanumeric run >= 16 chars — the shape of key bodies,
+//      hex digests, and base64-ish tokens (agent names segment on _/-/. and
+//      keep each run short).
+const CREDENTIAL_PREFIXES = [
+  "sk-",
+  "sk_",
+  "pk-",
+  "pplx-",
+  "ghp_",
+  "gho_",
+  "ghu_",
+  "ghs_",
+  "ghr_",
+  "github_pat_",
+  "glpat-",
+  "gsk_",
+  "xox",
+  "akia",
+  "asia",
+  "aiza",
+  "ya29",
+  "hf_",
+  "r8_",
+];
+const LONG_ALNUM_RUN = /[a-z0-9]{16,}/;
+
+function looksLikeCredential(v: string): boolean {
+  if (CREDENTIAL_PREFIXES.some((p) => v.startsWith(p))) return true;
+  return LONG_ALNUM_RUN.test(v);
+}
+
+function sanitizeHint(value: string | undefined): string | null {
+  if (typeof value !== "string") return null;
+  const v = value.trim().toLowerCase();
+  if (!SAFE_HINT.test(v)) return null;
+  if (looksLikeCredential(v)) return null;
+  return v;
+}
+
+// A key (uppercased) looks like a coding-agent marker. Excludes the two
+// value-captured generics (read via agent_hint) and the SSH/GPG "agent" false
+// friends, which are credential agents, not coding agents. No bare `CODING`
+// token — it substring-matches `ENCODING` (e.g. PYTHONIOENCODING) and real
+// coding-agent keys already match via `AGENT` (e.g. PI_CODING_AGENT).
+const HINT_KEY_PATTERN = /AGENT|ASSISTANT|COPILOT|CODEX|CLAUDE|LLM|_THREAD_ID$|_SESSION_ID$/;
+
+function isDiscoveryHintKey(upperKey: string): boolean {
+  if (upperKey === "AGENT" || upperKey === "AI_AGENT") return false;
+  if (upperKey.startsWith("SSH_") || upperKey.startsWith("GPG_")) return false;
+  return HINT_KEY_PATTERN.test(upperKey);
+}
+
+/**
+ * Residual discovery signals for the agent_runtime=null bucket. See the section
+ * comment above for intent and the privacy contract. Pure over process.env.
+ */
+export function detectAgentHints(): AgentHints {
+  const env = process.env;
+  const agent_hint = sanitizeHint(env["AGENT"]) ?? sanitizeHint(env["AI_AGENT"]);
+  const term_program = sanitizeHint(env["TERM_PROGRAM"]);
+
+  const keys = new Set<string>();
+  for (const key of Object.keys(env)) {
+    if (key.length > 64) continue;
+    const upper = key.toUpperCase();
+    if (isDiscoveryHintKey(upper)) keys.add(upper);
+  }
+  const sorted = [...keys].sort();
+  const agent_env_hints = sorted.length ? sorted.slice(0, 16).join(",") : null;
+
+  return { agent_hint, term_program, agent_env_hints };
+}
+
 // ---------------------------------------------------------------------------
 // Sandbox runtime detectors — one per runtime, kept small and side-effect-free.
 // ---------------------------------------------------------------------------

packages/cli/src/telemetry/client.ts

@@ -92,6 +92,10 @@ export function trackEvent(
       is_tty: sys.is_tty,
       sandbox_runtime: sys.sandbox_runtime ?? undefined,
       agent_runtime: sys.agent_runtime ?? undefined,
+      // New-agent discovery signals — populated only when agent_runtime is null.
+      agent_hint: sys.agent_hint ?? undefined,
+      term_program: sys.term_program ?? undefined,
+      agent_env_hints: sys.agent_env_hints ?? undefined,
     },
     timestamp: new Date().toISOString(),
   });

packages/cli/src/telemetry/system.ts

@@ -4,6 +4,7 @@ import { execSync } from "node:child_process";
 import { getSystemTotalMb } from "@hyperframes/engine";
 import {
   detectAgentRuntime,
+  detectAgentHints,
   detectSandboxRuntime,
   type AgentRuntime,
   type SandboxRuntime,
@@ -49,6 +50,16 @@ export interface SystemMeta {
    * null when no agent is detected.
    */
   agent_runtime: AgentRuntime;
+  /**
+   * New-agent discovery signals for the agent_runtime=null bucket, so an agent
+   * we have no rule for surfaces on its own instead of vanishing into null.
+   * All three are null on a classified event (agent_runtime != null) and on a
+   * plain shell with no markers. See `detectAgentHints` in agent_runtime.ts for
+   * the fields and the privacy contract.
+   */
+  agent_hint: string | null;
+  term_program: string | null;
+  agent_env_hints: string | null;
 }
 
 let cached: SystemMeta | null = null;
@@ -63,6 +74,14 @@ export function getSystemMeta(): SystemMeta {
   const cpuInfo = cpus();
   const firstCpu = cpuInfo[0] ?? null;
 
+  // Only compute discovery hints for the unclassified bucket — a known agent
+  // needs no hint, and gating keeps them off the ~80%+ of classified events.
+  const agent_runtime = detectAgentRuntime();
+  const hints =
+    agent_runtime === null
+      ? detectAgentHints()
+      : { agent_hint: null, term_program: null, agent_env_hints: null };
+
   cached = {
     os_release: release(),
     cpu_count: cpuInfo.length,
@@ -75,7 +94,10 @@ export function getSystemMeta(): SystemMeta {
     is_wsl: detectWSL(),
     is_tty: Boolean(process.stdout?.isTTY),
     sandbox_runtime: detectSandboxRuntime(),
-    agent_runtime: detectAgentRuntime(),
+    agent_runtime,
+    agent_hint: hints.agent_hint,
+    term_program: hints.term_program,
+    agent_env_hints: hints.agent_env_hints,
   };
   return cached;
 }