Files
hyperframes/packages/cli/src/telemetry/client.test.ts
T
WaterrrForever 3b65321a20 feat(cli): classify identity persistence on every telemetry event (#3065)
* feat(cli): classify identity persistence on every telemetry event

Install-grain metrics currently trust every anonymousId equally, but
ephemeral/isolated-HOME workloads mint a fresh id per run — one machine
produced 2,956 rotating render identities since Jul 30 (94.4% seen on a
single render command), inflating acquisition and diluting per-install
penetration while looking like real product usage.

Every event now carries:

- identity_persistence: durable (id loaded from a preexisting config —
  proven to survive a process boundary) | unknown (minted+persisted this
  run; an ephemeral HOME is indistinguishable from a genuine first run
  from inside one process) | process_only (persist failed). Sticky per
  process so a fresh install re-reading its own write cannot self-promote.
- config_write_outcome: ok | ok_unmirrored | failed for the identity-
  establishing write; absent when the id came from disk.
- invocation_id: random uuid per CLI process, so one invocation's events
  group even when the install identity is untrustworthy (unlike run_id,
  which needs an orchestrator to set HYPERFRAMES_RUN_ID).

Install metrics can then count only durable identities, and a daily
churn monitor can alert on the unknown share.

* fix(cli): require the anonymousId to come off disk before classifying durable

Review finding: materializeConfig mints a replacement anonymousId when a
hand-edited/image-baked config lacks one. That replacement only reaches
disk when the bucket-seed backfill happens to write; with a seed present
the read path performs no write at all, so the install re-mints a fresh
id every run while the unconditional durable branch stamped each of them
with the one label durable-only counting is allowed to trust.

durable now requires parseNonEmptyString(parsed.anonymousId): a minted
replacement classifies like a fresh mint — by the backfill write outcome
when that path runs (unknown/process_only), and process_only on the
no-write path where the id provably dies with the process. Two tests pin
both shapes.
2026-08-06 22:19:14 +08:00

191 lines
7.0 KiB
TypeScript

import { describe, expect, it, vi, beforeEach } from "vitest";
// CI exports HYPERFRAMES_NO_TELEMETRY=1 (and users may set DO_NOT_TRACK), which
// makes shouldTrack() short-circuit — and it caches that decision for the module's
// lifetime. Clear both BEFORE the module under test is imported / first tracks.
vi.stubEnv("HYPERFRAMES_NO_TELEMETRY", "");
vi.stubEnv("DO_NOT_TRACK", "");
// Pin config so the queue never touches disk and telemetry is enabled.
vi.mock("./config.js", () => ({
readConfig: () => ({ anonymousId: "anon-test-123", telemetryEnabled: true }),
writeConfig: () => {},
getIdentityPersistence: () => "durable",
getIdentityWriteOutcome: () => undefined,
}));
// shouldTrack() short-circuits in dev mode — force production behavior.
vi.mock("../utils/env.js", () => ({
isDevMode: () => false,
}));
// Canary enrolment is registry-driven and will change as rollouts ramp; stub
// it so this asserts the WIRING (does every event carry the cohort?) rather
// than whichever canaries happen to be live today.
const canaryProps = vi.fn<() => Record<string, string>>(() => ({
"$feature/canary-feat-x": "true",
"$feature/canary-feat-y": "false",
}));
vi.mock("./canary.js", () => ({
canaryEventProperties: () => canaryProps(),
}));
// Intercept the exit-time child process so flushSync delivery is assertable.
const spawnMock = vi.fn(() => ({ unref: vi.fn() }));
vi.mock("node:child_process", () => ({
spawn: (...args: unknown[]) => spawnMock(...(args as [])),
}));
const { trackEvent, flush, flushSync } = await import("./client.js");
type Batch = { uuid: string; event: string }[];
function sentBatch(fetchMock: ReturnType<typeof vi.fn>, call = 0): Batch {
const init = fetchMock.mock.calls[call]?.[1] as { body: string } | undefined;
if (!init) throw new Error(`expected fetch call #${call} to have been made`);
return JSON.parse(init.body).batch;
}
/** Properties of the first event in the first delivered batch. */
function eventProps(fetchMock: ReturnType<typeof vi.fn>): Record<string, unknown> {
const init = fetchMock.mock.calls[0]?.[1] as { body: string } | undefined;
if (!init) throw new Error("expected a fetch call to have been made");
const parsed = JSON.parse(init.body) as {
batch: Array<{ properties?: Record<string, unknown> }>;
};
return parsed.batch[0]?.properties ?? {};
}
describe("telemetry queue delivery", () => {
beforeEach(async () => {
vi.unstubAllGlobals();
// Drain anything a previous test left behind.
vi.stubGlobal(
"fetch",
vi.fn(() => Promise.resolve(new Response(""))),
);
await flush();
vi.unstubAllGlobals();
});
it("forgets events only after the request completes, and stamps each with a uuid", async () => {
const fetchMock = vi.fn(() => Promise.resolve(new Response("")));
vi.stubGlobal("fetch", fetchMock);
trackEvent("render_complete", { quality: "draft" });
await flush();
expect(fetchMock).toHaveBeenCalledTimes(1);
const batch = sentBatch(fetchMock);
expect(batch).toHaveLength(1);
expect(batch[0]?.event).toBe("render_complete");
expect(batch[0]?.uuid).toMatch(/^[0-9a-f-]{36}$/);
// Delivered — a second flush has nothing to send.
await flush();
expect(fetchMock).toHaveBeenCalledTimes(1);
});
it("keeps events queued when the request fails, and re-sends them with the SAME uuid", async () => {
const failing = vi.fn(() => Promise.reject(new Error("network down")));
vi.stubGlobal("fetch", failing);
trackEvent("render_complete", { quality: "draft" });
await flush();
expect(failing).toHaveBeenCalledTimes(1);
// Queue survived the failed send — the retry carries the same event uuid,
// so PostHog would dedupe even if the first request had actually landed.
const succeeding = vi.fn(() => Promise.resolve(new Response("")));
vi.stubGlobal("fetch", succeeding);
await flush();
expect(succeeding).toHaveBeenCalledTimes(1);
const first = sentBatch(failing);
const retry = sentBatch(succeeding);
expect(retry).toHaveLength(1);
expect(retry[0]?.uuid).toBe(first[0]?.uuid);
await flush();
expect(succeeding).toHaveBeenCalledTimes(1);
});
it("does not drop events queued while a flush is in flight", async () => {
let resolveFetch: (r: Response) => void = () => {};
const gated = vi.fn(() => new Promise<Response>((res) => (resolveFetch = res)));
vi.stubGlobal("fetch", gated);
trackEvent("render_complete", { quality: "draft" });
const inFlight = flush();
trackEvent("cli_command_result", { command: "render" });
resolveFetch(new Response(""));
await inFlight;
// Only the snapshot was forgotten; the late event is still queued.
const succeeding = vi.fn(() => Promise.resolve(new Response("")));
vi.stubGlobal("fetch", succeeding);
await flush();
const batch = sentBatch(succeeding);
expect(batch).toHaveLength(1);
expect(batch[0]?.event).toBe("cli_command_result");
});
it("flushSync hands the queue to a detached child that carries the payload", async () => {
spawnMock.mockClear();
trackEvent("render_complete", { quality: "draft" });
flushSync();
// The child was spawned detached with the batch inlined into its -e script.
expect(spawnMock).toHaveBeenCalledTimes(1);
const [execPath, args, opts] = spawnMock.mock.calls[0] as unknown as [
string,
string[],
{ detached: boolean },
];
expect(execPath).toBe(process.execPath);
expect(args[0]).toBe("-e");
expect(args[1]).toContain("render_complete");
expect(args[1]).toMatch(/[0-9a-f-]{36}/); // event uuid rides along
expect(opts.detached).toBe(true);
// Queue handed to the child — nothing left for a regular flush.
const fetchMock = vi.fn(() => Promise.resolve(new Response("")));
vi.stubGlobal("fetch", fetchMock);
await flush();
expect(fetchMock).not.toHaveBeenCalled();
});
});
describe("canary cohort on every event", () => {
it("attaches canary assignments as PostHog flag properties", async () => {
canaryProps.mockReturnValue({
"$feature/canary-feat-x": "true",
"$feature/canary-feat-y": "false",
});
const fetchMock = vi.fn(async () => ({ ok: true, status: 200 }) as Response);
vi.stubGlobal("fetch", fetchMock);
trackEvent("cli_command", { command: "render" });
await flush();
const props = eventProps(fetchMock);
// Enrolled AND control are both emitted — absent would mean "this build
// predates the canary", a different fact from "not enrolled".
expect(props["$feature/canary-feat-x"]).toBe("true");
expect(props["$feature/canary-feat-y"]).toBe("false");
});
it("adds no canary properties when the registry is empty", async () => {
canaryProps.mockReturnValue({});
const fetchMock = vi.fn(async () => ({ ok: true, status: 200 }) as Response);
vi.stubGlobal("fetch", fetchMock);
trackEvent("cli_command", { command: "render" });
await flush();
expect(Object.keys(eventProps(fetchMock)).some((k) => k.startsWith("$feature/canary-"))).toBe(
false,
);
});
});