From 232d591479448233a9a54097da0963fc1034e450 Mon Sep 17 00:00:00 2001 From: James Russo Date: Sun, 5 Jul 2026 22:31:13 -0700 Subject: [PATCH] feat(cli/telemetry): surface unrecognized agents in the agent_runtime=null bucket (#1978) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * feat(cli/telemetry): surface unrecognized agents in the agent_runtime=null bucket agent_runtime is a closed allowlist: an agent we have no rule for collapses to null with no trace of what it was, so ~18% of CLI users are unattributable and new agents stay invisible until reverse-engineered by hand. Add detectAgentHints(), a self-populating residual signal computed only for the null bucket (gated off classified events): - agent_hint: value of AGENT / AI_AGENT (the emerging self-identification convention; Crush and Goose set AGENT=) — names agents the allowlist misses. - term_program: raw TERM_PROGRAM (editor name) — catches the IDE-terminal class the same way the cursor/windsurf rules do. - agent_env_hints: sorted, comma-joined "agent-ish" env-var KEY names present but matched by no vendor rule — a fingerprint that clusters by agent. Privacy stays consistent with the existing "never read secret-shaped values" stance: agent_env_hints emits key names only; the three value-reads are vars whose sole purpose is non-secret identification, each passed through a strict short-slug allowlist so anything long/spaced/secret-shaped is dropped. Breaking down agent_hint / agent_env_hints filtered to agent_runtime IS NULL AND is_tty=false gives a ranked leaderboard of new agents to promote into VENDOR_RULES. Co-Authored-By: Claude Opus 4.8 (1M context) * fix(cli/telemetry): guard agent_hint/term_program against short credential-shaped values Review feedback (Magi, #1978): the short-slug allowlist in sanitizeHint() still accepted short credential-shaped values (AGENT=sk-ant-api03, AGENT=AKIAIOSFODNN7EXAMPLE, AGENT=github_pat_abc), so the "never emit a secret" claim wasn't actually enforced — only overlong values were dropped. Add a credential-shape guard on top of the slug allowlist: - known token/credential prefixes (sk-, ghp_, github_pat_, akia, ya29, ...) - any unbroken alphanumeric run >= 16 chars (key bodies, hex, base64-ish), while agent names segment on _/-/. and keep each run short. Replace the single overlong-value test with the short credential shapes from the review (parametrized) plus a positive case (gemini_managed_agent) proving real multi-segment names still pass. Co-Authored-By: Claude Opus 4.8 (1M context) --------- Co-authored-by: Claude Opus 4.8 (1M context) --- .../cli/src/telemetry/agent_runtime.test.ts | 77 +++++++++++ packages/cli/src/telemetry/agent_runtime.ts | 125 ++++++++++++++++++ packages/cli/src/telemetry/client.ts | 4 + packages/cli/src/telemetry/system.ts | 24 +++- 4 files changed, 229 insertions(+), 1 deletion(-) diff --git a/packages/cli/src/telemetry/agent_runtime.test.ts b/packages/cli/src/telemetry/agent_runtime.test.ts index 036f3ff3f..3d46b1227 100644 --- a/packages/cli/src/telemetry/agent_runtime.test.ts +++ b/packages/cli/src/telemetry/agent_runtime.test.ts @@ -330,6 +330,83 @@ describe("detectAgentRuntime — Windsurf / Cline / Gemini CLI / Crush", () => { }); }); +describe("detectAgentHints — new-agent discovery signals", () => { + const savedEnv = { ...process.env }; + beforeEach(() => { + stripVendorEnv(); + // stripVendorEnv clears TERM_PROGRAM; also clear the value-captured generics + // and any hint-shaped keys a test sets so assertions stay deterministic. + delete process.env["AGENT"]; + delete process.env["AI_AGENT"]; + }); + afterEach(() => { + process.env = { ...savedEnv }; + }); + + it("reads AGENT as the agent_hint (self-identification convention), lowercased", async () => { + process.env["AGENT"] = "Crush"; + const { detectAgentHints } = await import("./agent_runtime.js"); + expect(detectAgentHints().agent_hint).toBe("crush"); + }); + + it("falls back to AI_AGENT when AGENT is unset", async () => { + process.env["AI_AGENT"] = "goose"; + const { detectAgentHints } = await import("./agent_runtime.js"); + expect(detectAgentHints().agent_hint).toBe("goose"); + }); + + it("drops an overlong secret-looking AGENT value rather than leaking it", async () => { + process.env["AGENT"] = "sk-ant-api03-THIS-IS-A-LONG-SECRET-LOOKING-VALUE-xyz"; + const { detectAgentHints } = await import("./agent_runtime.js"); + expect(detectAgentHints().agent_hint).toBeNull(); + }); + + // The short-slug allowlist alone accepts these; the credential-shape guard + // (prefixes + long alnum runs) is what actually enforces the privacy claim. + it.each([ + ["sk-ant-api03", "token prefix"], + ["AKIAIOSFODNN7EXAMPLE", "AWS access key id (prefix + long run)"], + ["github_pat_abc", "GitHub PAT prefix"], + ["ghp_0123456789abcdef", "GitHub token prefix"], + ["ya29.a0veryrealtoken", "Google OAuth prefix"], + ["deadbeefdeadbeef01", "18-char unbroken token body"], + ])("drops short credential-shaped AGENT value %s (%s)", async (value) => { + process.env["AGENT"] = value; + const { detectAgentHints } = await import("./agent_runtime.js"); + expect(detectAgentHints().agent_hint).toBeNull(); + }); + + it("still captures a real multi-segment agent name (no over-rejection)", async () => { + process.env["AGENT"] = "gemini_managed_agent"; + const { detectAgentHints } = await import("./agent_runtime.js"); + expect(detectAgentHints().agent_hint).toBe("gemini_managed_agent"); + }); + + it("captures TERM_PROGRAM as the editor/terminal hint", async () => { + process.env["TERM_PROGRAM"] = "zed"; + const { detectAgentHints } = await import("./agent_runtime.js"); + expect(detectAgentHints().term_program).toBe("zed"); + }); + + it("surfaces an unknown agent-ish env KEY in agent_env_hints", async () => { + process.env["FOO_AGENT_SESSION_ID"] = "whatever-value"; + const { detectAgentHints } = await import("./agent_runtime.js"); + expect(detectAgentHints().agent_env_hints).toContain("FOO_AGENT_SESSION_ID"); + }); + + it("excludes SSH/GPG agent false-friends from agent_env_hints", async () => { + process.env["SSH_AGENT_PID"] = "12345"; + const { detectAgentHints } = await import("./agent_runtime.js"); + expect(detectAgentHints().agent_env_hints ?? "").not.toContain("SSH_AGENT"); + }); + + it("does NOT match ENCODING keys (PYTHONIOENCODING) — no bare CODING token", async () => { + process.env["PYTHONIOENCODING"] = "utf-8"; + const { detectAgentHints } = await import("./agent_runtime.js"); + expect(detectAgentHints().agent_env_hints ?? "").not.toContain("ENCODING"); + }); +}); + describe("detectSandboxRuntime — file-system path", () => { beforeEach(() => { vi.resetModules(); diff --git a/packages/cli/src/telemetry/agent_runtime.ts b/packages/cli/src/telemetry/agent_runtime.ts index c0f2bbb5c..a7b21367e 100644 --- a/packages/cli/src/telemetry/agent_runtime.ts +++ b/packages/cli/src/telemetry/agent_runtime.ts @@ -244,6 +244,131 @@ export function detectAgentRuntime(): AgentRuntime { return null; } +// --------------------------------------------------------------------------- +// New-agent discovery signals. +// +// VENDOR_RULES is a CLOSED allowlist: an agent we haven't written a rule for +// collapses to agent_runtime=null, leaving no trace of what it was. That makes +// the null bucket un-attributable — new agents stay invisible until someone +// reverse-engineers their marker by hand (which is how every rule above was +// derived). +// +// detectAgentHints() adds a self-populating residual signal for exactly that +// null bucket, so an unrecognized agent surfaces on its own in analytics and +// can be promoted to a real VENDOR_RULE later. Callers should only emit these +// when detectAgentRuntime() returns null — a classified event needs no hint. +// +// Privacy — consistent with the "never read secret-shaped values" stance above: +// - agent_env_hints emits KEY NAMES only, never values. +// - agent_hint / term_program read the VALUE of three vars whose sole purpose +// is non-secret self-identification: the emerging AGENT / AI_AGENT +// agent-name convention (e.g. Crush and Goose set AGENT=) and +// TERM_PROGRAM's editor name (how the cursor/windsurf rules already work). +// Each value is passed through a strict short-slug allowlist, so anything +// long, spaced, or secret-shaped is dropped to null. +// --------------------------------------------------------------------------- + +export interface AgentHints { + /** + * Best-effort agent name from a self-identifying env-var value: AGENT, else + * AI_AGENT. null when neither is set or the value isn't a short safe slug. + */ + agent_hint: string | null; + /** + * Raw TERM_PROGRAM value (editor/terminal name) — surfaces IDE-terminal + * agents not yet covered by the cursor/windsurf rules. Noisy (also set by + * plain human terminals); read alongside is_tty=false. null when unset/unsafe. + */ + term_program: string | null; + /** + * Sorted, comma-joined list of "agent-ish" env-var KEY names present but + * matched by no vendor rule — a compact fingerprint that clusters by agent. + * KEYS only, never values. null when none are present. + */ + agent_env_hints: string | null; +} + +// Self-identifying values are agent/editor NAMES by convention — short slugs. +// Anything longer, spaced, or secret-shaped falls outside this and is dropped. +const SAFE_HINT = /^[a-z0-9_.-]{1,32}$/; + +// The slug allowlist alone still accepts SHORT credential-shaped values +// (AGENT=sk-ant-api03, AGENT=AKIAIOSFODNN7EXAMPLE, AGENT=github_pat_abc), so +// two extra guards enforce the "never emit a secret" boundary this PR relies on: +// 1. known credential/token prefixes (compared lowercased), and +// 2. any unbroken alphanumeric run >= 16 chars — the shape of key bodies, +// hex digests, and base64-ish tokens (agent names segment on _/-/. and +// keep each run short). +const CREDENTIAL_PREFIXES = [ + "sk-", + "sk_", + "pk-", + "pplx-", + "ghp_", + "gho_", + "ghu_", + "ghs_", + "ghr_", + "github_pat_", + "glpat-", + "gsk_", + "xox", + "akia", + "asia", + "aiza", + "ya29", + "hf_", + "r8_", +]; +const LONG_ALNUM_RUN = /[a-z0-9]{16,}/; + +function looksLikeCredential(v: string): boolean { + if (CREDENTIAL_PREFIXES.some((p) => v.startsWith(p))) return true; + return LONG_ALNUM_RUN.test(v); +} + +function sanitizeHint(value: string | undefined): string | null { + if (typeof value !== "string") return null; + const v = value.trim().toLowerCase(); + if (!SAFE_HINT.test(v)) return null; + if (looksLikeCredential(v)) return null; + return v; +} + +// A key (uppercased) looks like a coding-agent marker. Excludes the two +// value-captured generics (read via agent_hint) and the SSH/GPG "agent" false +// friends, which are credential agents, not coding agents. No bare `CODING` +// token — it substring-matches `ENCODING` (e.g. PYTHONIOENCODING) and real +// coding-agent keys already match via `AGENT` (e.g. PI_CODING_AGENT). +const HINT_KEY_PATTERN = /AGENT|ASSISTANT|COPILOT|CODEX|CLAUDE|LLM|_THREAD_ID$|_SESSION_ID$/; + +function isDiscoveryHintKey(upperKey: string): boolean { + if (upperKey === "AGENT" || upperKey === "AI_AGENT") return false; + if (upperKey.startsWith("SSH_") || upperKey.startsWith("GPG_")) return false; + return HINT_KEY_PATTERN.test(upperKey); +} + +/** + * Residual discovery signals for the agent_runtime=null bucket. See the section + * comment above for intent and the privacy contract. Pure over process.env. + */ +export function detectAgentHints(): AgentHints { + const env = process.env; + const agent_hint = sanitizeHint(env["AGENT"]) ?? sanitizeHint(env["AI_AGENT"]); + const term_program = sanitizeHint(env["TERM_PROGRAM"]); + + const keys = new Set(); + for (const key of Object.keys(env)) { + if (key.length > 64) continue; + const upper = key.toUpperCase(); + if (isDiscoveryHintKey(upper)) keys.add(upper); + } + const sorted = [...keys].sort(); + const agent_env_hints = sorted.length ? sorted.slice(0, 16).join(",") : null; + + return { agent_hint, term_program, agent_env_hints }; +} + // --------------------------------------------------------------------------- // Sandbox runtime detectors — one per runtime, kept small and side-effect-free. // --------------------------------------------------------------------------- diff --git a/packages/cli/src/telemetry/client.ts b/packages/cli/src/telemetry/client.ts index 44f10891a..e8630a3c6 100644 --- a/packages/cli/src/telemetry/client.ts +++ b/packages/cli/src/telemetry/client.ts @@ -92,6 +92,10 @@ export function trackEvent( is_tty: sys.is_tty, sandbox_runtime: sys.sandbox_runtime ?? undefined, agent_runtime: sys.agent_runtime ?? undefined, + // New-agent discovery signals — populated only when agent_runtime is null. + agent_hint: sys.agent_hint ?? undefined, + term_program: sys.term_program ?? undefined, + agent_env_hints: sys.agent_env_hints ?? undefined, }, timestamp: new Date().toISOString(), }); diff --git a/packages/cli/src/telemetry/system.ts b/packages/cli/src/telemetry/system.ts index 04a80cd25..d8d9bacf5 100644 --- a/packages/cli/src/telemetry/system.ts +++ b/packages/cli/src/telemetry/system.ts @@ -4,6 +4,7 @@ import { execSync } from "node:child_process"; import { getSystemTotalMb } from "@hyperframes/engine"; import { detectAgentRuntime, + detectAgentHints, detectSandboxRuntime, type AgentRuntime, type SandboxRuntime, @@ -49,6 +50,16 @@ export interface SystemMeta { * null when no agent is detected. */ agent_runtime: AgentRuntime; + /** + * New-agent discovery signals for the agent_runtime=null bucket, so an agent + * we have no rule for surfaces on its own instead of vanishing into null. + * All three are null on a classified event (agent_runtime != null) and on a + * plain shell with no markers. See `detectAgentHints` in agent_runtime.ts for + * the fields and the privacy contract. + */ + agent_hint: string | null; + term_program: string | null; + agent_env_hints: string | null; } let cached: SystemMeta | null = null; @@ -63,6 +74,14 @@ export function getSystemMeta(): SystemMeta { const cpuInfo = cpus(); const firstCpu = cpuInfo[0] ?? null; + // Only compute discovery hints for the unclassified bucket — a known agent + // needs no hint, and gating keeps them off the ~80%+ of classified events. + const agent_runtime = detectAgentRuntime(); + const hints = + agent_runtime === null + ? detectAgentHints() + : { agent_hint: null, term_program: null, agent_env_hints: null }; + cached = { os_release: release(), cpu_count: cpuInfo.length, @@ -75,7 +94,10 @@ export function getSystemMeta(): SystemMeta { is_wsl: detectWSL(), is_tty: Boolean(process.stdout?.isTTY), sandbox_runtime: detectSandboxRuntime(), - agent_runtime: detectAgentRuntime(), + agent_runtime, + agent_hint: hints.agent_hint, + term_program: hints.term_program, + agent_env_hints: hints.agent_env_hints, }; return cached; }