feat(cli): classify identity persistence on every telemetry event (#3065)

* feat(cli): classify identity persistence on every telemetry event

Install-grain metrics currently trust every anonymousId equally, but
ephemeral/isolated-HOME workloads mint a fresh id per run — one machine
produced 2,956 rotating render identities since Jul 30 (94.4% seen on a
single render command), inflating acquisition and diluting per-install
penetration while looking like real product usage.

Every event now carries:

- identity_persistence: durable (id loaded from a preexisting config —
  proven to survive a process boundary) | unknown (minted+persisted this
  run; an ephemeral HOME is indistinguishable from a genuine first run
  from inside one process) | process_only (persist failed). Sticky per
  process so a fresh install re-reading its own write cannot self-promote.
- config_write_outcome: ok | ok_unmirrored | failed for the identity-
  establishing write; absent when the id came from disk.
- invocation_id: random uuid per CLI process, so one invocation's events
  group even when the install identity is untrustworthy (unlike run_id,
  which needs an orchestrator to set HYPERFRAMES_RUN_ID).

Install metrics can then count only durable identities, and a daily
churn monitor can alert on the unknown share.

* fix(cli): require the anonymousId to come off disk before classifying durable

Review finding: materializeConfig mints a replacement anonymousId when a
hand-edited/image-baked config lacks one. That replacement only reaches
disk when the bucket-seed backfill happens to write; with a seed present
the read path performs no write at all, so the install re-mints a fresh
id every run while the unconditional durable branch stamped each of them
with the one label durable-only counting is allowed to trust.

durable now requires parseNonEmptyString(parsed.anonymousId): a minted
replacement classifies like a fresh mint — by the backfill write outcome
when that path runs (unknown/process_only), and process_only on the
no-write path where the id provably dies with the process. Two tests pin
both shapes.

packages/cli/src/telemetry/client.postureRefresh.test.ts

@@ -16,6 +16,8 @@ const configState = { telemetryEnabled: true };
 vi.mock("./config.js", () => ({
   readConfig: () => ({ anonymousId: "anon-1", telemetryEnabled: configState.telemetryEnabled }),
   writeConfig: () => {},
+  getIdentityPersistence: () => "durable",
+  getIdentityWriteOutcome: () => undefined,
 }));
 vi.mock("../utils/env.js", () => ({ isDevMode: () => false }));
 vi.mock("./canary.js", () => ({ canaryEventProperties: () => ({}) }));

packages/cli/src/telemetry/client.test.ts

@@ -10,6 +10,8 @@ vi.stubEnv("DO_NOT_TRACK", "");
 vi.mock("./config.js", () => ({
   readConfig: () => ({ anonymousId: "anon-test-123", telemetryEnabled: true }),
   writeConfig: () => {},
+  getIdentityPersistence: () => "durable",
+  getIdentityWriteOutcome: () => undefined,
 }));
 
 // shouldTrack() short-circuits in dev mode — force production behavior.

packages/cli/src/telemetry/client.ts

@@ -1,4 +1,10 @@
-import { readConfig, writeConfig } from "./config.js";
+import {
+  getIdentityPersistence,
+  getIdentityWriteOutcome,
+  readConfig,
+  writeConfig,
+} from "./config.js";
+import { getInvocationId } from "./runId.js";
 import { VERSION } from "../version.js";
 import { c } from "../ui/colors.js";
 import { diag } from "../ui/diagnostics.js";
@@ -101,6 +107,19 @@ export function trackEvent(
       // could not read. Without it a partial disk write is indistinguishable
       // from a genuinely fresh install. Absent in the normal case.
       install_state_file_corrupt: readConfig().stateFileCorrupt,
+      // Whether this process's anonymousId can be trusted to survive to the
+      // next run: `durable` (loaded from a preexisting config), `unknown`
+      // (minted+persisted this run — an ephemeral HOME is indistinguishable
+      // from a genuine first run), `process_only` (persist failed). Install-
+      // grain metrics should count only durable identities; the identity-
+      // churn workloads (fresh id per run) are never durable.
+      identity_persistence: getIdentityPersistence(),
+      // Outcome of the identity-establishing config write; absent when the
+      // identity came from disk and nothing needed writing.
+      config_write_outcome: getIdentityWriteOutcome(),
+      // Groups one invocation's events even when the install identity is
+      // untrustworthy. Always present, unlike the orchestrator-set run_id.
+      invocation_id: getInvocationId(),
       // Canary assignments as `$feature/canary-<name>` — PostHog's native flag
       // property shape, so breakdowns and experiment analysis work on a canary
       // with nothing configured server-side. On EVERY event, not just renders:

packages/cli/src/telemetry/config.test.ts

@@ -699,3 +699,92 @@ describe("an unwritable config dir must not re-roll the seed", () => {
     });
   });
 });
+
+describe("identity-persistence classification (sticky per process)", () => {
+  let readConfig: typeof import("./config.js").readConfig;
+  let readConfigFresh: typeof import("./config.js").readConfigFresh;
+  let getIdentityPersistence: typeof import("./config.js").getIdentityPersistence;
+  let getIdentityWriteOutcome: typeof import("./config.js").getIdentityWriteOutcome;
+  let CONFIG_PATH: typeof import("./config.js").CONFIG_PATH;
+
+  beforeEach(async () => {
+    fsState.files.clear();
+    policyState.runtimeOverride = null;
+    vi.resetModules();
+    ({ readConfig, readConfigFresh, getIdentityPersistence, getIdentityWriteOutcome, CONFIG_PATH } =
+      await import("./config.js"));
+  });
+
+  it("classifies a fresh mint whose write landed as unknown, never durable", () => {
+    readConfig();
+    expect(getIdentityPersistence()).toBe("unknown");
+    expect(getIdentityWriteOutcome()).toBe("ok");
+  });
+
+  it("classifies an id loaded from a preexisting config as durable, with no write outcome", () => {
+    fsState.files.set(
+      CONFIG_PATH,
+      JSON.stringify({ telemetryEnabled: true, anonymousId: "prior-id", bucketSeed: "seed" }),
+    );
+    readConfig();
+    expect(getIdentityPersistence()).toBe("durable");
+    expect(getIdentityWriteOutcome()).toBeUndefined();
+  });
+
+  it("classifies a fresh mint whose write failed as process_only", async () => {
+    const fs = await import("node:fs");
+    vi.mocked(fs.writeFileSync).mockImplementation(() => {
+      throw new Error("EACCES: permission denied");
+    });
+    const warn = vi.spyOn(console, "warn").mockImplementation(() => {});
+
+    readConfig();
+    expect(getIdentityPersistence()).toBe("process_only");
+    expect(getIdentityWriteOutcome()).toBe("failed");
+
+    warn.mockRestore();
+    vi.mocked(fs.writeFileSync).mockImplementation((path, content) => {
+      fsState.files.set(String(path), String(content));
+    });
+  });
+
+  it("does not self-promote to durable when a fresh-install process re-reads its own write", () => {
+    readConfig();
+    expect(fsState.files.has(CONFIG_PATH)).toBe(true);
+    // The file now exists on disk; a cache-bypassing re-read hits the
+    // existing-file path — the ephemeral-HOME churn signature.
+    readConfigFresh();
+    expect(getIdentityPersistence()).toBe("unknown");
+  });
+
+  it("does not label a replacement id minted at read time as durable (seed present, no write)", () => {
+    // Hand-edited / image-baked config: file exists with a seed but NO
+    // anonymousId. materializeConfig mints a replacement, nothing persists it
+    // (the seed suppresses the backfill write) — so the install re-mints
+    // every run. Labelling that durable would dress the churn signature in
+    // the one trustworthy label (review finding).
+    fsState.files.set(
+      CONFIG_PATH,
+      JSON.stringify({ telemetryEnabled: true, bucketSeed: "baked-seed" }),
+    );
+    readConfig();
+    expect(getIdentityPersistence()).toBe("process_only");
+  });
+
+  it("classifies a replacement id carried to disk by the seed backfill by its write outcome", () => {
+    // Same hand-edited shape but ALSO missing the seed: the backfill write
+    // persists the whole config, replacement id included — a fresh mint in
+    // all but name, so it classifies like one (unknown, never durable).
+    fsState.files.set(CONFIG_PATH, JSON.stringify({ telemetryEnabled: true }));
+    readConfig();
+    expect(getIdentityPersistence()).toBe("unknown");
+    expect(getIdentityWriteOutcome()).toBe("ok");
+  });
+
+  it("classifies a corrupt-config recovery mint by its write outcome, not as durable", () => {
+    fsState.files.set(CONFIG_PATH, "{not json");
+    readConfig();
+    expect(getIdentityPersistence()).toBe("unknown");
+    expect(getIdentityWriteOutcome()).toBe("ok");
+  });
+});

packages/cli/src/telemetry/config.ts

@@ -147,11 +147,12 @@ function warnSeedBackfillFailed(error: string | undefined): void {
  * is exactly when that happens, so it says so once rather than failing
  * invisibly.
  */
-function backfillBucketSeed(config: HyperframesConfig): void {
+function backfillBucketSeed(config: HyperframesConfig): ConfigWriteResult {
   const recorded = readInstallState();
   config.bucketSeed = (isInstallState(recorded) ? recorded.bucketSeed : undefined) ?? randomUUID();
   const write = writeConfigWithResult(config);
   if (!write.ok) warnSeedBackfillFailed(write.error);
+  return write;
 }
 
 // ONLY the positive is cached. The latch is monotonic across processes in one
@@ -350,6 +351,7 @@ function mintAndCacheConfig(): HyperframesConfig {
   const config = mintConfig();
   const write = writeConfigWithResult(config);
   if (!write.ok) warnSeedBackfillFailed(write.error);
+  classifyIdentity(write.ok ? "unknown" : "process_only", writeOutcomeOf(write));
   cachedConfig = { ...config };
   return { ...config };
 }
@@ -535,6 +537,60 @@ const DEFAULT_CONFIG: HyperframesConfig = {
 
 let cachedConfig: HyperframesConfig | null = null;
 
+// ---------------------------------------------------------------------------
+// Identity-persistence classification — one sticky verdict per process.
+//
+// Install-grain metrics need to know whether this process's anonymousId can
+// be trusted to survive to the next run. Three-way, because from inside a
+// single process durability is not always provable:
+//
+//   durable       — the id was LOADED from a preexisting config file: it has
+//                   already survived at least one process boundary.
+//   unknown       — the id was minted this run and the write landed. An
+//                   ephemeral/isolated HOME (the identity-churn workloads:
+//                   fresh id per run, install_predecessor_found=false every
+//                   time) looks IDENTICAL to a genuine first run from in
+//                   here, so this cannot be promoted to durable.
+//   process_only  — minted this run and the write failed (read-only mount,
+//                   full disk): the id dies with this process, guaranteed.
+//
+// The verdict is sticky: a fresh-install process that later re-reads its own
+// just-written file must not upgrade itself to durable.
+// ---------------------------------------------------------------------------
+
+export type IdentityPersistence = "durable" | "process_only" | "unknown";
+/** `ok_unmirrored`: config.json landed but the install-state mirror did not. */
+export type IdentityWriteOutcome = "ok" | "ok_unmirrored" | "failed";
+
+let identityPersistence: IdentityPersistence | undefined;
+let identityWriteOutcome: IdentityWriteOutcome | undefined;
+
+function classifyIdentity(persistence: IdentityPersistence, outcome?: IdentityWriteOutcome): void {
+  if (identityPersistence !== undefined) return;
+  identityPersistence = persistence;
+  identityWriteOutcome = outcome;
+}
+
+function writeOutcomeOf(write: ConfigWriteResult): IdentityWriteOutcome {
+  if (!write.ok) return "failed";
+  return write.mirrored === false ? "ok_unmirrored" : "ok";
+}
+
+/** The process's sticky identity-persistence verdict (classifies on demand). */
+export function getIdentityPersistence(): IdentityPersistence {
+  readConfig();
+  return identityPersistence ?? "unknown";
+}
+
+/**
+ * Outcome of the identity-establishing config write. Absent when the identity
+ * came from disk and nothing needed writing (the `durable` case).
+ */
+export function getIdentityWriteOutcome(): IdentityWriteOutcome | undefined {
+  readConfig();
+  return identityWriteOutcome;
+}
+
 /** A non-empty string, or undefined — hand-edited configs can carry anything. */
 function parseNonEmptyString(value: unknown): string | undefined {
   return typeof value === "string" && value.length > 0 ? value : undefined;
@@ -637,11 +693,25 @@ export function readConfig(): HyperframesConfig {
 
     const config = materializeConfig(parsed);
 
+    // `durable` requires the id to have actually COME OFF DISK — a file that
+    // predates this process proves cross-run persistence. materializeConfig
+    // mints a REPLACEMENT id when the parsed file lacks one (hand-edited /
+    // image-baked configs), and that replacement only reaches disk if the
+    // bucket-seed backfill below happens to write; labelling it durable would
+    // dress the exact churn signature this field exists to catch in the one
+    // trustworthy label (review finding). Sticky either way, so a
+    // fresh-install process re-reading its own write cannot self-promote.
+    const idFromDisk = parseNonEmptyString(parsed.anonymousId) !== undefined;
+
     // One-time backfill for configs predating the bucket seed: prefer the
     // recorded seed if a previous install already wrote one, else mint.
     // Persisted immediately — an unpersisted seed would re-roll every process.
     if (config.bucketSeed === undefined) {
-      backfillBucketSeed(config);
+      const write = backfillBucketSeed(config);
+      // The backfill write carries any replacement id to disk, so a minted id
+      // classifies exactly like a fresh mint: by whether the write landed.
+      if (idFromDisk) classifyIdentity("durable");
+      else classifyIdentity(write.ok ? "unknown" : "process_only", writeOutcomeOf(write));
       // Cache even if the write failed, so the seed is at least stable for
       // the life of this process (a re-roll per readConfigFresh would flip
       // cohorts mid-session).
@@ -649,6 +719,10 @@ export function readConfig(): HyperframesConfig {
       return { ...config };
     }
 
+    // No write happens on this path: a replacement id lives only in this
+    // process, guaranteed — the definition of process_only.
+    classifyIdentity(idFromDisk ? "durable" : "process_only");
+
     cachedConfig = config;
     return { ...config };
   } catch {
@@ -658,7 +732,8 @@ export function readConfig(): HyperframesConfig {
     // breaker survives config corruption too — but fail closed for the
     // privacy control: recovery must never silently turn telemetry back on.
     const config = { ...mintConfig(), telemetryEnabled: false };
-    writeConfig(config);
+    const write = writeConfigWithResult(config);
+    classifyIdentity(write.ok ? "unknown" : "process_only", writeOutcomeOf(write));
     return config;
   }
 }

packages/cli/src/telemetry/runId.ts

@@ -1,3 +1,5 @@
+import { randomUUID } from "node:crypto";
+
 let resolved = false;
 let runId: string | undefined;
 
@@ -10,3 +12,19 @@ export function getRunId(): string | undefined {
 
   return runId;
 }
+
+// ---------------------------------------------------------------------------
+// Invocation id — a random uuid minted once per CLI process, present on every
+// event that process emits. Unlike run_id (set only when an orchestrator
+// exports HYPERFRAMES_RUN_ID), it needs no environment plumbing: it exists so
+// the events of ONE invocation can be grouped even when the install identity
+// is untrustworthy (identity_persistence != durable, e.g. an ephemeral HOME
+// minting a fresh anonymousId per run).
+// ---------------------------------------------------------------------------
+
+let invocationId: string | undefined;
+
+export function getInvocationId(): string {
+  invocationId ??= randomUUID();
+  return invocationId;
+}