From 8dd20ac2e8f8c082c69ac9a4be5f3df8b7690b2c Mon Sep 17 00:00:00 2001 From: Vance Ingalls Date: Tue, 28 Jul 2026 16:57:23 -0700 Subject: [PATCH] feat(cli): bucket canaries on a machine-lineage seed, not the telemetry id MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Cohort membership now survives a config wipe. Canaries bucket on a dedicated bucketSeed (fresh random UUID, distinct from anonymousId by design) that is mirrored write-once into the install-state file and inherited at mint: a wipe re-rolls the telemetry id but never the canary assignment. This removes cumulative-exposure drift for the recoverable churn bucket entirely — the residual drift comes only from fresh machines, containers, and genuinely new users — and keeps before/after comparisons valid across a reinstall. The seed is never emitted in telemetry (only the resulting true/false assignments are), so it does not link the old id to the new one server-side. The residual linker is the flag vector itself (k bits for k live canaries), documented as such. An explicit reset still works by deleting the state file, and the no-identity test now also asserts the seed differs from the anonymousId. Cross-surface coherence: the CLI's studio server injects the seed as window.__HF_CLI_BUCKET_SEED (same telemetry gate and script-escaping as the distinct id, and on the /api/telemetry-identity fallback), and the Studio binding buckets on it when present — without this the CLI would bucket on the seed while Studio bucketed on the distinct id, splitting one machine across cohorts (calibration check 4 would catch exactly this). Standalone Studio still buckets on its localStorage id: the browser has no second storage location, so that id doubles as the seed. Legacy configs are backfilled once (lineage seed if the state file has one, else minted) and persisted immediately — an unpersisted seed would re-roll cohorts every process. Safe to ship in the same release as the first canaries: no prior release emitted canary properties, so the bucketing-unit change is unobservable. Co-Authored-By: Claude Opus 5 (1M context) --- docs/contributing/canary-rollouts.mdx | 27 +++-- packages/cli/src/server/studioServer.ts | 11 +- .../cli/src/server/telemetryIdentity.test.ts | 8 ++ packages/cli/src/server/telemetryIdentity.ts | 36 +++++- packages/cli/src/telemetry/canary.test.ts | 40 ++++++- packages/cli/src/telemetry/canary.ts | 7 +- packages/cli/src/telemetry/config.test.ts | 78 ++++++++++++- packages/cli/src/telemetry/config.ts | 110 ++++++++++++++---- packages/studio/src/telemetry/canary.test.ts | 31 +++-- packages/studio/src/telemetry/canary.ts | 36 +++++- 10 files changed, 327 insertions(+), 57 deletions(-) diff --git a/docs/contributing/canary-rollouts.mdx b/docs/contributing/canary-rollouts.mdx index 0db264ee6..4b7344dac 100644 --- a/docs/contributing/canary-rollouts.mdx +++ b/docs/contributing/canary-rollouts.mdx @@ -167,18 +167,24 @@ experiment, and it survives uninstall), and account identity covers only accepted limit — but its worst consequence is mitigated, and its size is now directly measurable rather than inferred: +- **Cohorts survive the wipe.** Canaries bucket on a dedicated `bucketSeed` — + not the telemetry id — and the seed is inherited across config wipes via a + machine-local state file (write-once: the first install's seed is the + lineage's seed forever). A wipe re-rolls the telemetry id, never the canary + assignment. The seed is never emitted, so it does not link the old id to + the new one. - **A re-rolled install cannot re-enter a path that already failed on that - machine.** The circuit breaker's tripped state is mirrored to the - machine-local state file, so it survives the same config wipe that re-rolls - the cohort. + machine.** The circuit breaker's tripped state is mirrored to the same + state file. - **`install_predecessor_found`** on every event says whether this install's mint found a previous install's state marker. Its true-share splits check 2's drift into the recoverable part (config wiped, machine persisted) and the part no local mechanism can link (fresh machine, container, new user). -The calibration read is therefore two numbers — total drift, and the fraction -the rollover already covers — and canary window lengths get picked from the -residual, not the raw turnover. +With the seed carryover, check 2's residual drift comes from the +unrecoverable buckets only — fresh machines, containers, and genuinely new +users — and canary window lengths get picked from that residual, not the raw +turnover. ## Behaviour worth knowing @@ -186,11 +192,16 @@ residual, not the raw turnover. in the 10. Cohorts never reshuffle, so a before/after comparison stays valid across a ramp. -**Slices are independent per feature.** The bucket hashes `feature:installId`, -not the install id alone — two canaries at 10% select two different 10%s. If +**Slices are independent per feature.** The bucket hashes `feature:seed`, +not the seed alone — two canaries at 10% select two different 10%s. If they shared a slice, one unlucky cohort would receive every experiment at once and no two rollouts could be read apart. +**Cohorts are keyed to the machine, not the config.** The bucketing unit is a +dedicated seed inherited across config wipes (see the calibration section) — +distinct from the telemetry id, never emitted, and shared with a CLI-launched +Studio so both surfaces agree. + **It fails closed.** No install id, an unregistered name, or a CI machine all resolve to *not enrolled*. A canary exists to bound blast radius, so "we don't know who this is" must never mean "enrol everyone". CI is excluded diff --git a/packages/cli/src/server/studioServer.ts b/packages/cli/src/server/studioServer.ts index 358189ad6..9bf7e9742 100644 --- a/packages/cli/src/server/studioServer.ts +++ b/packages/cli/src/server/studioServer.ts @@ -16,7 +16,11 @@ import { loadRuntimeSourceSignature, } from "./runtimeSource.js"; import { VERSION as version } from "../version.js"; -import { buildStudioHeadScripts, resolveCliTelemetryDistinctId } from "./telemetryIdentity.js"; +import { + buildStudioHeadScripts, + resolveCliBucketSeed, + resolveCliTelemetryDistinctId, +} from "./telemetryIdentity.js"; import { emitStudioRenderComplete, emitStudioRenderError } from "./studioRenderTelemetry.js"; import { isDevMode } from "../utils/env.js"; import { @@ -652,7 +656,10 @@ export function createStudioServer(options: StudioServerOptions): StudioServer { // distinct id (no PII) so the browser session can join the CLI's PostHog // person, or `{ distinctId: null }` when CLI telemetry is disabled. app.get("/api/telemetry-identity", (c) => { - return c.json({ distinctId: resolveCliTelemetryDistinctId() }); + return c.json({ + distinctId: resolveCliTelemetryDistinctId(), + bucketSeed: resolveCliBucketSeed(), + }); }); app.get("/api/events", (c) => { diff --git a/packages/cli/src/server/telemetryIdentity.test.ts b/packages/cli/src/server/telemetryIdentity.test.ts index 9f779841f..cd5535bd2 100644 --- a/packages/cli/src/server/telemetryIdentity.test.ts +++ b/packages/cli/src/server/telemetryIdentity.test.ts @@ -66,6 +66,14 @@ describe("buildCliIdentityScript", () => { ); }); + it("also seeds window.__HF_CLI_BUCKET_SEED when the config carries a bucket seed", () => { + shouldTrack.mockReturnValue(true); + readConfig.mockReturnValue({ anonymousId: "machine-uuid", bucketSeed: "seed-uuid" }); + expect(buildCliIdentityScript()).toBe( + '', + ); + }); + it("emits an empty string when telemetry is disabled (nothing to seed)", () => { shouldTrack.mockReturnValue(false); expect(buildCliIdentityScript()).toBe(""); diff --git a/packages/cli/src/server/telemetryIdentity.ts b/packages/cli/src/server/telemetryIdentity.ts index f5d4d36df..9a24765e4 100644 --- a/packages/cli/src/server/telemetryIdentity.ts +++ b/packages/cli/src/server/telemetryIdentity.ts @@ -41,15 +41,39 @@ export function resolveCliTelemetryDistinctId(): string | null { * `url_hash` telemetry or browser history. Empty string when there's nothing to * seed (telemetry off / no id). */ +/** + * The CLI's canary bucket seed to hand to Studio, or null. Injected alongside + * the distinct id so a CLI-launched Studio buckets canaries on the SAME unit + * as the CLI — without it the two surfaces would agree only while the seed + * still equals whatever Studio falls back to, and a rollout spanning render + * and editor would split one user across cohorts. Same telemetry gate as the + * distinct id: seeding is part of the identity stitch, not a separate channel. + */ +export function resolveCliBucketSeed(): string | null { + try { + if (!telemetryShouldTrack()) return null; + const seed = readConfig().bucketSeed; + return typeof seed === "string" && seed.length > 0 ? seed : null; + } catch { + return null; + } +} + +// JSON.stringify does not escape "<" or "/". Escaping both means no +// "" (or " or open a new tag. (The values are +// randomUUID()s, so this is belt-and-suspenders.) +function encodeInlineScriptValue(value: string): string { + return JSON.stringify(value).replace(/" (or " or open a new tag. - const encoded = JSON.stringify(cliId).replace(/window.__HF_CLI_DISTINCT_ID=${encoded};`; + const parts = [`window.__HF_CLI_DISTINCT_ID=${encodeInlineScriptValue(cliId)};`]; + const seed = resolveCliBucketSeed(); + if (seed) parts.push(`window.__HF_CLI_BUCKET_SEED=${encodeInlineScriptValue(seed)};`); + return ``; } /** diff --git a/packages/cli/src/telemetry/canary.test.ts b/packages/cli/src/telemetry/canary.test.ts index 0d5043170..f1d8b8b10 100644 --- a/packages/cli/src/telemetry/canary.test.ts +++ b/packages/cli/src/telemetry/canary.test.ts @@ -1,10 +1,13 @@ import { describe, expect, it, vi, beforeEach } from "vitest"; -const configState = { anonymousId: "db0c1f4a-b95e-4c35-90c6-1a15bd76f717" }; +const configState: { anonymousId: string; bucketSeed: string | undefined } = { + anonymousId: "db0c1f4a-b95e-4c35-90c6-1a15bd76f717", + bucketSeed: undefined, +}; const systemState = { is_ci: false }; vi.mock("./config.js", () => ({ - readConfig: () => ({ anonymousId: configState.anonymousId }), + readConfig: () => ({ anonymousId: configState.anonymousId, bucketSeed: configState.bucketSeed }), })); vi.mock("./system.js", () => ({ getSystemMeta: () => ({ is_ci: systemState.is_ci }), @@ -60,11 +63,44 @@ const { isCanaryEnabled, resolveCanary, canaryEventProperties, __resetCanaryCach beforeEach(() => { __resetCanaryCacheForTests(); configState.anonymousId = "db0c1f4a-b95e-4c35-90c6-1a15bd76f717"; + configState.bucketSeed = undefined; systemState.is_ci = false; delete process.env.HF_CANARY_TEST_ALPHA; delete process.env.HF_CANARY_TEST_BETA; }); +describe("bucketing unit", () => { + it("buckets on the bucketSeed when present — the unit that survives config wipes", async () => { + const { evaluateCanary } = await import("@hyperframes/core/canary"); + configState.bucketSeed = "5f1c9d2e-0000-4000-8000-aaaaaaaaaaaa"; + const viaBinding = resolveCanary("test-alpha").bucket; + const bySeed = evaluateCanary({ + feature: "test-alpha", + unitId: configState.bucketSeed, + percentage: 100, + }).bucket; + const byId = evaluateCanary({ + feature: "test-alpha", + unitId: configState.anonymousId, + percentage: 100, + }).bucket; + expect(viaBinding).toBe(bySeed); + // Only meaningful if the two units actually bucket differently. + expect(bySeed).not.toBe(byId); + }); + + it("falls back to the anonymousId when no seed exists (failed legacy backfill)", async () => { + const { evaluateCanary } = await import("@hyperframes/core/canary"); + const viaBinding = resolveCanary("test-alpha").bucket; + const byId = evaluateCanary({ + feature: "test-alpha", + unitId: configState.anonymousId, + percentage: 100, + }).bucket; + expect(viaBinding).toBe(byId); + }); +}); + describe("CLI canary binding", () => { it("reads the percentage from the registry", () => { expect(isCanaryEnabled("test-alpha")).toBe(true); diff --git a/packages/cli/src/telemetry/canary.ts b/packages/cli/src/telemetry/canary.ts index 56d41ae7e..49d86918e 100644 --- a/packages/cli/src/telemetry/canary.ts +++ b/packages/cli/src/telemetry/canary.ts @@ -55,10 +55,15 @@ export function resolveCanary(name: string): CanaryDecision { if (cached) return cached; const definition = findCanary(name); + const config = readConfig(); const decision: CanaryDecision = definition ? evaluateCanary({ feature: definition.name, - unitId: readConfig().anonymousId, + // The bucket seed, NOT the anonymousId: the seed is inherited across + // config wipes via the install-state file, so the machine keeps its + // cohorts when the telemetry id re-rolls. Fallback covers only a + // failed backfill write on a legacy config. + unitId: config.bucketSeed ?? config.anonymousId, percentage: definition.percentage, override: parseCanaryOverride(process.env[canaryEnvVar(definition.name)]), // CI installs regenerate their config per run, so their ids are diff --git a/packages/cli/src/telemetry/config.test.ts b/packages/cli/src/telemetry/config.test.ts index 571ef736a..646284eda 100644 --- a/packages/cli/src/telemetry/config.test.ts +++ b/packages/cli/src/telemetry/config.test.ts @@ -204,11 +204,18 @@ describe("install-state rollover (breaker survives a config re-mint)", () => { expect(readConfigFresh().deParallelRouterTrialFired).toBe(true); }); - it("the state file holds no identity — only the marker timestamp and breaker fact", () => { + it("the state file holds no telemetry identity — marker, breaker fact, bucket seed only", () => { const config = readConfig(); config.deParallelRouterTrialFired = true; writeConfig(config); - expect(Object.keys(stateFile()).sort()).toEqual(["deParallelRouterTrialFired", "markerAt"]); + expect(Object.keys(stateFile()).sort()).toEqual([ + "bucketSeed", + "deParallelRouterTrialFired", + "markerAt", + ]); + // The seed is a separate random UUID, never the anonymousId — the old + // telemetry id must not survive a wipe in any form. + expect(config.bucketSeed).not.toBe(config.anonymousId); expect(JSON.stringify(stateFile())).not.toContain(config.anonymousId); }); @@ -292,3 +299,70 @@ describe("install-state rollover (breaker survives a config re-mint)", () => { expect(fsState.files.has(LEGACY_STATE_PATH)).toBe(false); }); }); + +describe("bucket-seed carryover (cohorts survive a config wipe)", () => { + let readConfig: typeof import("./config.js").readConfig; + let readConfigFresh: typeof import("./config.js").readConfigFresh; + let writeConfig: typeof import("./config.js").writeConfig; + let CONFIG_PATH: typeof import("./config.js").CONFIG_PATH; + let STATE_PATH: typeof import("./config.js").STATE_PATH; + + beforeEach(async () => { + fsState.files.clear(); + vi.resetModules(); + ({ readConfig, readConfigFresh, writeConfig, CONFIG_PATH, STATE_PATH } = + await import("./config.js")); + }); + + it("a fresh install mints a seed distinct from its anonymousId and mirrors it to the state file", () => { + const config = readConfig(); + expect(config.bucketSeed).toBeTruthy(); + expect(config.bucketSeed).not.toBe(config.anonymousId); + const state = JSON.parse(fsState.files.get(STATE_PATH) as string) as { bucketSeed?: string }; + expect(state.bucketSeed).toBe(config.bucketSeed); + }); + + it("the seed survives a config wipe — the machine keeps its cohorts, only the id re-rolls", () => { + const first = readConfig(); + fsState.files.delete(CONFIG_PATH); + const second = readConfigFresh(); + expect(second.bucketSeed).toBe(first.bucketSeed); + expect(second.anonymousId).not.toBe(first.anonymousId); + }); + + it("the seed survives config corruption via the same mint path", () => { + const first = readConfig(); + fsState.files.set(CONFIG_PATH, "{not valid json"); + expect(readConfigFresh().bucketSeed).toBe(first.bucketSeed); + }); + + it("the state-file seed is write-once — a later install never overwrites the lineage seed", () => { + const first = readConfig(); + fsState.files.delete(CONFIG_PATH); + const second = readConfigFresh(); + // Force more config writes from the second install; the state seed must not move. + second.commandCount = 7; + writeConfig(second); + const state = JSON.parse(fsState.files.get(STATE_PATH) as string) as { bucketSeed?: string }; + expect(state.bucketSeed).toBe(first.bucketSeed); + }); + + it("a legacy config without a seed is backfilled ONCE and stays stable across fresh reads", () => { + const base = readConfig(); + const { bucketSeed: _dropped, ...legacy } = base; + fsState.files.set(CONFIG_PATH, JSON.stringify(legacy)); + fsState.files.delete(STATE_PATH); // no lineage either — pure legacy machine + const first = readConfigFresh(); + expect(first.bucketSeed).toBeTruthy(); + const second = readConfigFresh(); + expect(second.bucketSeed).toBe(first.bucketSeed); // persisted, not re-rolled per read + }); + + it("a legacy config adopts the lineage seed from the state file when one exists", () => { + const base = readConfig(); // wrote the state file with a seed + const lineageSeed = base.bucketSeed; + const { bucketSeed: _dropped, ...legacy } = base; + fsState.files.set(CONFIG_PATH, JSON.stringify(legacy)); + expect(readConfigFresh().bucketSeed).toBe(lineageSeed); + }); +}); diff --git a/packages/cli/src/telemetry/config.ts b/packages/cli/src/telemetry/config.ts index 87831bec1..23c254268 100644 --- a/packages/cli/src/telemetry/config.ts +++ b/packages/cli/src/telemetry/config.ts @@ -56,6 +56,12 @@ interface InstallState { markerAt: string; /** Rolled-over circuit-breaker state — see HyperframesConfig's field. */ deParallelRouterTrialFired?: boolean; + /** + * The machine's canary bucketing seed — see HyperframesConfig's field. + * Write-once: the first install's seed is the lineage's seed forever, so a + * config wipe re-mints the telemetry id but NOT the canary cohorts. + */ + bucketSeed?: string; } /** Parse one state file; any parse/shape failure reads as absent. */ @@ -67,6 +73,10 @@ function parseInstallState(file: string): InstallState | null { return { markerAt: parsed.markerAt, deParallelRouterTrialFired: parsed.deParallelRouterTrialFired === true ? true : undefined, + bucketSeed: + typeof parsed.bucketSeed === "string" && parsed.bucketSeed.length > 0 + ? parsed.bucketSeed + : undefined, }; } catch { return null; @@ -136,18 +146,35 @@ function writeInstallState(next: InstallState): void { * no breaker write site can forget it. Never throws — same contract as the * rest of this file, telemetry must not break the CLI. */ +function sameInstallState(a: InstallState, b: InstallState): boolean { + return ( + a.deParallelRouterTrialFired === b.deParallelRouterTrialFired && a.bucketSeed === b.bucketSeed + ); +} + +/** Latching: once tripped (by any install in this machine's lineage), stays tripped. */ +function latchedFired(state: InstallState | null, config: HyperframesConfig): true | undefined { + return ( + state?.deParallelRouterTrialFired === true || + config.deParallelRouterTrialFired === true || + undefined + ); +} + /** What the state file should say after this config write; null = already correct. */ -function nextInstallState(state: InstallState | null, wantFired: boolean): InstallState | null { - const hadFired = state?.deParallelRouterTrialFired === true; - if (state !== null && (hadFired || !wantFired)) return null; - // Every path reaching here has hadFired === false (state is either null, or - // the guard above already returned when hadFired was true) — the field is - // simply wantFired, not a merge of the two (review nit, two independent - // reviewers). - return { +function nextInstallState( + state: InstallState | null, + config: HyperframesConfig, +): InstallState | null { + const next: InstallState = { markerAt: state?.markerAt ?? new Date().toISOString(), - deParallelRouterTrialFired: wantFired || undefined, + deParallelRouterTrialFired: latchedFired(state, config), + // Write-once: an existing lineage seed always wins, so cohorts stay + // anchored to the FIRST install in this config dir, not the latest one. + bucketSeed: state?.bucketSeed ?? config.bucketSeed, }; + if (state !== null && sameInstallState(state, next)) return null; + return next; } function syncInstallState(config: HyperframesConfig): void { @@ -155,7 +182,7 @@ function syncInstallState(config: HyperframesConfig): void { if (stateMarkerSynced && (stateFiredSynced || !wantFired)) return; try { const state = readInstallState(); - const next = nextInstallState(state, wantFired); + const next = nextInstallState(state, config); if (next !== null) writeInstallState(next); stateMarkerSynced = true; stateFiredSynced = wantFired || state?.deParallelRouterTrialFired === true; @@ -176,8 +203,11 @@ function mintConfig(): HyperframesConfig { anonymousId: randomUUID(), predecessorFound: state !== null, // The rollover itself: a breaker tripped by a previous install on this - // machine stays tripped for the new one. + // machine stays tripped for the new one, and the canary bucketing seed is + // inherited so the machine keeps its cohorts — a wipe re-rolls the + // telemetry id, never the canary assignment. deParallelRouterTrialFired: state?.deParallelRouterTrialFired === true ? true : undefined, + bucketSeed: state?.bucketSeed ?? randomUUID(), }; } @@ -264,6 +294,17 @@ export interface HyperframesConfig { * existed — a different fact from `false` (minted fresh, no predecessor). */ predecessorFound?: boolean; + /** + * The unit canary percentages bucket on — deliberately NOT the anonymousId. + * A fresh random UUID, mirrored write-once into the install-state file and + * inherited at mint, so a config wipe re-rolls the telemetry id but keeps + * the machine's canary cohorts: no cumulative-exposure drift from wipes, + * and before/after comparisons survive a reinstall. It is never emitted in + * telemetry (only the resulting true/false assignments are), so it does not + * link the old id to the new one server-side. Backfilled once for configs + * predating the field. + */ + bucketSeed?: string; /** * Ring of the last few local renders (newest last). `hyperframes feedback` * attaches these ids — which are the `render_job_id` / @@ -310,6 +351,26 @@ const DEFAULT_CONFIG: HyperframesConfig = { let cachedConfig: HyperframesConfig | null = null; +/** A non-empty string, or undefined — hand-edited configs can carry anything. */ +function parseNonEmptyString(value: unknown): string | undefined { + return typeof value === "string" && value.length > 0 ? value : undefined; +} + +/** Shape-validate the recent-renders ring from a parsed config. */ +function parseRecentRenders(value: unknown): RecentRenderRecord[] | undefined { + if (!Array.isArray(value)) return undefined; + return value + .filter( + (r): r is RecentRenderRecord => + typeof r === "object" && + r !== null && + typeof (r as RecentRenderRecord).id === "string" && + typeof (r as RecentRenderRecord).at === "string" && + typeof (r as RecentRenderRecord).ok === "boolean", + ) + .slice(-MAX_RECENT_RENDERS); +} + /** * Read the config file, creating it with defaults if it doesn't exist. * Returns a mutable copy — call `writeConfig()` to persist changes. @@ -356,20 +417,23 @@ export function readConfig(): HyperframesConfig { : undefined, predecessorFound: typeof parsed.predecessorFound === "boolean" ? parsed.predecessorFound : undefined, - recentRenders: Array.isArray(parsed.recentRenders) - ? parsed.recentRenders - .filter( - (r): r is RecentRenderRecord => - typeof r === "object" && - r !== null && - typeof (r as RecentRenderRecord).id === "string" && - typeof (r as RecentRenderRecord).at === "string" && - typeof (r as RecentRenderRecord).ok === "boolean", - ) - .slice(-MAX_RECENT_RENDERS) - : undefined, + bucketSeed: parseNonEmptyString(parsed.bucketSeed), + recentRenders: parseRecentRenders(parsed.recentRenders), }; + // One-time backfill for configs predating the bucket seed: prefer the + // lineage seed if a previous install already recorded one, else mint. + // Persisted immediately — an unpersisted seed would re-roll every process. + if (config.bucketSeed === undefined) { + config.bucketSeed = readInstallState()?.bucketSeed ?? randomUUID(); + writeConfig(config); + // Cache even if the write failed, so the seed is at least stable for + // the life of this process (a re-roll per readConfigFresh would flip + // cohorts mid-session). + cachedConfig = config; + return { ...config }; + } + cachedConfig = config; return { ...config }; } catch { diff --git a/packages/studio/src/telemetry/canary.test.ts b/packages/studio/src/telemetry/canary.test.ts index 5ae407721..bd51b998b 100644 --- a/packages/studio/src/telemetry/canary.test.ts +++ b/packages/studio/src/telemetry/canary.test.ts @@ -47,6 +47,7 @@ beforeEach(() => { sessionStorage.clear(); setSearch(""); delete window.__HF_CLI_DISTINCT_ID; + delete window.__HF_CLI_BUCKET_SEED; Object.defineProperty(navigator, "webdriver", { value: false, configurable: true }); __resetStudioCanaryCacheForTests(); __resetStudioDistinctIdForTests(); @@ -131,19 +132,35 @@ describe("automated browsers", () => { }); describe("cohort identity", () => { - it("buckets on the Studio distinct id unmodified — so a CLI-launched Studio shares the CLI's cohort", () => { - // distinctId.ts adopts window.__HF_CLI_DISTINCT_ID when the CLI launched - // Studio. This asserts the binding passes that id through untouched: if it - // prefixed or re-hashed it, the editor would land in a different cohort - // than the terminal for the same user, and a rollout spanning both would - // be incoherent. + it("buckets on the CLI's bucket seed when injected — the unit that survives config wipes", () => { + // The CLI buckets on its bucketSeed (inherited across config wipes via + // the install-state file), so a CLI-launched Studio must bucket on the + // SAME seed or the two surfaces would split one machine across cohorts. + const cliId = "db0c1f4a-b95e-4c35-90c6-1a15bd76f717"; + const cliSeed = "5f1c9d2e-0000-4000-8000-aaaaaaaaaaaa"; + window.__HF_CLI_DISTINCT_ID = cliId; + window.__HF_CLI_BUCKET_SEED = cliSeed; + __resetStudioDistinctIdForTests(); + __resetStudioCanaryCacheForTests(); + + // Telemetry identity still adopts the DISTINCT id — the seed only buckets. + expect(resolveStudioDistinctId()).toBe(cliId); + const viaBinding = resolveCanary("on-everywhere").bucket; + const bySeed = evaluateCanary({ + feature: "on-everywhere", + unitId: cliSeed, + percentage: 100, + }).bucket; + expect(viaBinding).toBe(bySeed); + }); + + it("buckets on the Studio distinct id when no seed is injected (standalone Studio)", () => { const cliId = "db0c1f4a-b95e-4c35-90c6-1a15bd76f717"; window.__HF_CLI_DISTINCT_ID = cliId; __resetStudioDistinctIdForTests(); __resetStudioCanaryCacheForTests(); expect(resolveStudioDistinctId()).toBe(cliId); - // 50% so the answer is id-dependent rather than trivially true. const viaBinding = resolveCanary("on-everywhere").bucket; const direct = evaluateCanary({ feature: "on-everywhere", diff --git a/packages/studio/src/telemetry/canary.ts b/packages/studio/src/telemetry/canary.ts index bf8b13905..af95db2b8 100644 --- a/packages/studio/src/telemetry/canary.ts +++ b/packages/studio/src/telemetry/canary.ts @@ -14,11 +14,11 @@ // // Three things differ from the CLI, each for a reason: // -// 1. UNIT ID — `resolveStudioDistinctId()` instead of the CLI's config file. -// That function already adopts `window.__HF_CLI_DISTINCT_ID` when the CLI -// launched Studio, so a CLI-launched Studio lands in the SAME cohort as the -// CLI itself: a rollout spanning render and editor is coherent for that -// user instead of enrolling their terminal but not their editor. +// 1. UNIT ID — the CLI's bucket seed (`window.__HF_CLI_BUCKET_SEED`) when the +// CLI launched Studio, so both surfaces bucket on the SAME unit and a +// rollout spanning render and editor is coherent for that user. Standalone +// Studio falls back to `resolveStudioDistinctId()` — the browser has no +// second storage location, so its localStorage id doubles as the seed. // // 2. OVERRIDE — there is no `process.env` in a page, so the override is a URL // query param mirrored into sessionStorage (see `readOverride`). @@ -48,6 +48,30 @@ export function canaryParamName(name: string): string { return `hf_canary_${name.toLowerCase().replace(/[^a-z0-9]+/g, "_")}`; } +// Injected by the CLI's studio server alongside __HF_CLI_DISTINCT_ID. +declare global { + interface Window { + __HF_CLI_BUCKET_SEED?: string; + } +} + +/** + * The bucketing unit. A CLI-launched Studio buckets on the CLI's SEED (which + * survives config wipes via the install-state file), not its distinct id — + * the CLI itself buckets on the seed, and the two surfaces must agree per + * machine. Standalone Studio falls back to its own distinct id: the browser + * has no second storage location, so localStorage IS both id and seed there. + */ +function resolveBucketUnit(): string { + try { + const seed = typeof window === "undefined" ? undefined : window.__HF_CLI_BUCKET_SEED; + if (typeof seed === "string" && seed.length > 0) return seed; + } catch { + /* fall through */ + } + return resolveStudioDistinctId(); +} + const STORAGE_PREFIX = "hyperframes-studio:canary:"; /** @@ -130,7 +154,7 @@ export function resolveCanary(name: string): CanaryDecision { const decision: CanaryDecision = definition ? evaluateCanary({ feature: definition.name, - unitId: resolveStudioDistinctId(), + unitId: resolveBucketUnit(), percentage: definition.percentage, override: readOverride(definition.name), exclude: isAutomatedBrowser(),