mirror of
https://github.com/heygen-com/hyperframes.git
synced 2026-11-15 13:58:20 +00:00
* feat(cli): classify identity persistence on every telemetry event Install-grain metrics currently trust every anonymousId equally, but ephemeral/isolated-HOME workloads mint a fresh id per run — one machine produced 2,956 rotating render identities since Jul 30 (94.4% seen on a single render command), inflating acquisition and diluting per-install penetration while looking like real product usage. Every event now carries: - identity_persistence: durable (id loaded from a preexisting config — proven to survive a process boundary) | unknown (minted+persisted this run; an ephemeral HOME is indistinguishable from a genuine first run from inside one process) | process_only (persist failed). Sticky per process so a fresh install re-reading its own write cannot self-promote. - config_write_outcome: ok | ok_unmirrored | failed for the identity- establishing write; absent when the id came from disk. - invocation_id: random uuid per CLI process, so one invocation's events group even when the install identity is untrustworthy (unlike run_id, which needs an orchestrator to set HYPERFRAMES_RUN_ID). Install metrics can then count only durable identities, and a daily churn monitor can alert on the unknown share. * fix(cli): require the anonymousId to come off disk before classifying durable Review finding: materializeConfig mints a replacement anonymousId when a hand-edited/image-baked config lacks one. That replacement only reaches disk when the bucket-seed backfill happens to write; with a seed present the read path performs no write at all, so the install re-mints a fresh id every run while the unconditional durable branch stamped each of them with the one label durable-only counting is allowed to trust. durable now requires parseNonEmptyString(parsed.anonymousId): a minted replacement classifies like a fresh mint — by the backfill write outcome when that path runs (unknown/process_only), and process_only on the no-write path where the id provably dies with the process. Two tests pin both shapes.
191 lines
7.0 KiB
TypeScript
191 lines
7.0 KiB
TypeScript
import { describe, expect, it, vi, beforeEach } from "vitest";
|
|
|
|
// CI exports HYPERFRAMES_NO_TELEMETRY=1 (and users may set DO_NOT_TRACK), which
|
|
// makes shouldTrack() short-circuit — and it caches that decision for the module's
|
|
// lifetime. Clear both BEFORE the module under test is imported / first tracks.
|
|
vi.stubEnv("HYPERFRAMES_NO_TELEMETRY", "");
|
|
vi.stubEnv("DO_NOT_TRACK", "");
|
|
|
|
// Pin config so the queue never touches disk and telemetry is enabled.
|
|
vi.mock("./config.js", () => ({
|
|
readConfig: () => ({ anonymousId: "anon-test-123", telemetryEnabled: true }),
|
|
writeConfig: () => {},
|
|
getIdentityPersistence: () => "durable",
|
|
getIdentityWriteOutcome: () => undefined,
|
|
}));
|
|
|
|
// shouldTrack() short-circuits in dev mode — force production behavior.
|
|
vi.mock("../utils/env.js", () => ({
|
|
isDevMode: () => false,
|
|
}));
|
|
|
|
// Canary enrolment is registry-driven and will change as rollouts ramp; stub
|
|
// it so this asserts the WIRING (does every event carry the cohort?) rather
|
|
// than whichever canaries happen to be live today.
|
|
const canaryProps = vi.fn<() => Record<string, string>>(() => ({
|
|
"$feature/canary-feat-x": "true",
|
|
"$feature/canary-feat-y": "false",
|
|
}));
|
|
vi.mock("./canary.js", () => ({
|
|
canaryEventProperties: () => canaryProps(),
|
|
}));
|
|
|
|
// Intercept the exit-time child process so flushSync delivery is assertable.
|
|
const spawnMock = vi.fn(() => ({ unref: vi.fn() }));
|
|
vi.mock("node:child_process", () => ({
|
|
spawn: (...args: unknown[]) => spawnMock(...(args as [])),
|
|
}));
|
|
|
|
const { trackEvent, flush, flushSync } = await import("./client.js");
|
|
|
|
type Batch = { uuid: string; event: string }[];
|
|
|
|
function sentBatch(fetchMock: ReturnType<typeof vi.fn>, call = 0): Batch {
|
|
const init = fetchMock.mock.calls[call]?.[1] as { body: string } | undefined;
|
|
if (!init) throw new Error(`expected fetch call #${call} to have been made`);
|
|
return JSON.parse(init.body).batch;
|
|
}
|
|
|
|
/** Properties of the first event in the first delivered batch. */
|
|
function eventProps(fetchMock: ReturnType<typeof vi.fn>): Record<string, unknown> {
|
|
const init = fetchMock.mock.calls[0]?.[1] as { body: string } | undefined;
|
|
if (!init) throw new Error("expected a fetch call to have been made");
|
|
const parsed = JSON.parse(init.body) as {
|
|
batch: Array<{ properties?: Record<string, unknown> }>;
|
|
};
|
|
return parsed.batch[0]?.properties ?? {};
|
|
}
|
|
|
|
describe("telemetry queue delivery", () => {
|
|
beforeEach(async () => {
|
|
vi.unstubAllGlobals();
|
|
// Drain anything a previous test left behind.
|
|
vi.stubGlobal(
|
|
"fetch",
|
|
vi.fn(() => Promise.resolve(new Response(""))),
|
|
);
|
|
await flush();
|
|
vi.unstubAllGlobals();
|
|
});
|
|
|
|
it("forgets events only after the request completes, and stamps each with a uuid", async () => {
|
|
const fetchMock = vi.fn(() => Promise.resolve(new Response("")));
|
|
vi.stubGlobal("fetch", fetchMock);
|
|
|
|
trackEvent("render_complete", { quality: "draft" });
|
|
await flush();
|
|
|
|
expect(fetchMock).toHaveBeenCalledTimes(1);
|
|
const batch = sentBatch(fetchMock);
|
|
expect(batch).toHaveLength(1);
|
|
expect(batch[0]?.event).toBe("render_complete");
|
|
expect(batch[0]?.uuid).toMatch(/^[0-9a-f-]{36}$/);
|
|
|
|
// Delivered — a second flush has nothing to send.
|
|
await flush();
|
|
expect(fetchMock).toHaveBeenCalledTimes(1);
|
|
});
|
|
|
|
it("keeps events queued when the request fails, and re-sends them with the SAME uuid", async () => {
|
|
const failing = vi.fn(() => Promise.reject(new Error("network down")));
|
|
vi.stubGlobal("fetch", failing);
|
|
|
|
trackEvent("render_complete", { quality: "draft" });
|
|
await flush();
|
|
expect(failing).toHaveBeenCalledTimes(1);
|
|
|
|
// Queue survived the failed send — the retry carries the same event uuid,
|
|
// so PostHog would dedupe even if the first request had actually landed.
|
|
const succeeding = vi.fn(() => Promise.resolve(new Response("")));
|
|
vi.stubGlobal("fetch", succeeding);
|
|
await flush();
|
|
|
|
expect(succeeding).toHaveBeenCalledTimes(1);
|
|
const first = sentBatch(failing);
|
|
const retry = sentBatch(succeeding);
|
|
expect(retry).toHaveLength(1);
|
|
expect(retry[0]?.uuid).toBe(first[0]?.uuid);
|
|
|
|
await flush();
|
|
expect(succeeding).toHaveBeenCalledTimes(1);
|
|
});
|
|
|
|
it("does not drop events queued while a flush is in flight", async () => {
|
|
let resolveFetch: (r: Response) => void = () => {};
|
|
const gated = vi.fn(() => new Promise<Response>((res) => (resolveFetch = res)));
|
|
vi.stubGlobal("fetch", gated);
|
|
|
|
trackEvent("render_complete", { quality: "draft" });
|
|
const inFlight = flush();
|
|
trackEvent("cli_command_result", { command: "render" });
|
|
resolveFetch(new Response(""));
|
|
await inFlight;
|
|
|
|
// Only the snapshot was forgotten; the late event is still queued.
|
|
const succeeding = vi.fn(() => Promise.resolve(new Response("")));
|
|
vi.stubGlobal("fetch", succeeding);
|
|
await flush();
|
|
const batch = sentBatch(succeeding);
|
|
expect(batch).toHaveLength(1);
|
|
expect(batch[0]?.event).toBe("cli_command_result");
|
|
});
|
|
|
|
it("flushSync hands the queue to a detached child that carries the payload", async () => {
|
|
spawnMock.mockClear();
|
|
trackEvent("render_complete", { quality: "draft" });
|
|
flushSync();
|
|
|
|
// The child was spawned detached with the batch inlined into its -e script.
|
|
expect(spawnMock).toHaveBeenCalledTimes(1);
|
|
const [execPath, args, opts] = spawnMock.mock.calls[0] as unknown as [
|
|
string,
|
|
string[],
|
|
{ detached: boolean },
|
|
];
|
|
expect(execPath).toBe(process.execPath);
|
|
expect(args[0]).toBe("-e");
|
|
expect(args[1]).toContain("render_complete");
|
|
expect(args[1]).toMatch(/[0-9a-f-]{36}/); // event uuid rides along
|
|
expect(opts.detached).toBe(true);
|
|
|
|
// Queue handed to the child — nothing left for a regular flush.
|
|
const fetchMock = vi.fn(() => Promise.resolve(new Response("")));
|
|
vi.stubGlobal("fetch", fetchMock);
|
|
await flush();
|
|
expect(fetchMock).not.toHaveBeenCalled();
|
|
});
|
|
});
|
|
|
|
describe("canary cohort on every event", () => {
|
|
it("attaches canary assignments as PostHog flag properties", async () => {
|
|
canaryProps.mockReturnValue({
|
|
"$feature/canary-feat-x": "true",
|
|
"$feature/canary-feat-y": "false",
|
|
});
|
|
const fetchMock = vi.fn(async () => ({ ok: true, status: 200 }) as Response);
|
|
vi.stubGlobal("fetch", fetchMock);
|
|
|
|
trackEvent("cli_command", { command: "render" });
|
|
await flush();
|
|
|
|
const props = eventProps(fetchMock);
|
|
// Enrolled AND control are both emitted — absent would mean "this build
|
|
// predates the canary", a different fact from "not enrolled".
|
|
expect(props["$feature/canary-feat-x"]).toBe("true");
|
|
expect(props["$feature/canary-feat-y"]).toBe("false");
|
|
});
|
|
|
|
it("adds no canary properties when the registry is empty", async () => {
|
|
canaryProps.mockReturnValue({});
|
|
const fetchMock = vi.fn(async () => ({ ok: true, status: 200 }) as Response);
|
|
vi.stubGlobal("fetch", fetchMock);
|
|
|
|
trackEvent("cli_command", { command: "render" });
|
|
await flush();
|
|
|
|
expect(Object.keys(eventProps(fetchMock)).some((k) => k.startsWith("$feature/canary-"))).toBe(
|
|
false,
|
|
);
|
|
});
|
|
});
|