feat(core): percentage-based canary rollouts

Adds a reusable staged-rollout primitive so a change can ship to a stable
slice of installs instead of all-or-nothing.

The gap it fills: the repo carries ~49 HF_*/PRODUCER_* booleans and every one
is binary — a feature is either off (and therefore unexercised on real
traffic) or on for everyone (and therefore a fleet-wide bet). The
parallel-drawElement router sat in that gap for weeks: default-off produced
almost no signal, and flipping it default-on would have exposed 100% of
eligible installs at once.

Shape:

- packages/core/src/canary.ts — pure evaluator. No fs, no network, no
  `process`; the caller supplies the unit id and overrides, so it imports
  cleanly into the CLI, producer, engine, studio-server, the browser-side
  studio bundle and the embeddable player. FNV-1a rather than node:crypto for
  the same reason.
- packages/core/src/canaryRegistry.ts — every rollout in one table (name,
  percentage, owner, description, sunsetAfter), so "what is rolling out, to
  whom, owned by whom" is answerable without grepping 49 env vars.
- packages/cli/src/telemetry/canary.ts — supplies the three things only the
  CLI knows: anonymousId, the HF_CANARY_<FEATURE> override, and is_ci.
  Day-to-day API is `isCanaryEnabled("name")`.

Three properties the tests pin, because getting them wrong is subtle:

- Slices are INDEPENDENT per feature: the bucket hashes `feature:unitId`, not
  the id alone. Bucketing on the id would hand every concurrent experiment to
  the same unlucky cohort and make two rollouts unreadable apart.
- Ramping is INCLUSIVE: `bucket < percentage`, so widening 10 -> 25 keeps the
  original cohort and before/after comparisons survive the ramp.
- It fails CLOSED: no unit id, unknown name, or CI install means not enrolled.
  A canary exists to bound blast radius, so "we don't know who this is" must
  never mean "enrol everyone".

Registry entries also carry a sunset date, and a test fails once one is past
due — a canary that outlives its rollout is a permanent fork of the product
with none of the review a permanent fork would get.

Ships with de-parallel-router registered at 0%: inert, and ready to ramp in a
patch release once #2840 lands.

Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
This commit is contained in:
Vance Ingalls
2026-07-30 15:13:56 -07:00
co-authored by Claude Opus 5
parent fa564547dc
commit 3aea786687
6 changed files with 601 additions and 0 deletions
+113
View File
@@ -0,0 +1,113 @@
import { describe, expect, it, vi, beforeEach } from "vitest";
const configState = { anonymousId: "db0c1f4a-b95e-4c35-90c6-1a15bd76f717" };
const systemState = { is_ci: false };
vi.mock("./config.js", () => ({
readConfig: () => ({ anonymousId: configState.anonymousId }),
}));
vi.mock("./system.js", () => ({
getSystemMeta: () => ({ is_ci: systemState.is_ci }),
}));
// The registry is data; pin a known shape so these tests don't move when a
// real canary is added or ramped.
vi.mock("@hyperframes/core", async () => {
const actual = await vi.importActual<typeof import("@hyperframes/core")>("@hyperframes/core");
return {
...actual,
CANARIES: [
{
name: "test-alpha",
percentage: 100,
description: "always on",
owner: "t",
sunsetAfter: "2099-01-01",
},
{
name: "test-beta",
percentage: 0,
description: "always off",
owner: "t",
sunsetAfter: "2099-01-01",
},
],
findCanary: (n: string) =>
[
{
name: "test-alpha",
percentage: 100,
description: "",
owner: "t",
sunsetAfter: "2099-01-01",
},
{
name: "test-beta",
percentage: 0,
description: "",
owner: "t",
sunsetAfter: "2099-01-01",
},
].find((c) => c.name === n),
};
});
const { isCanaryEnabled, resolveCanary, activeCanaryNames, __resetCanaryCacheForTests } =
await import("./canary.js");
beforeEach(() => {
__resetCanaryCacheForTests();
configState.anonymousId = "db0c1f4a-b95e-4c35-90c6-1a15bd76f717";
systemState.is_ci = false;
delete process.env.HF_CANARY_TEST_ALPHA;
delete process.env.HF_CANARY_TEST_BETA;
});
describe("CLI canary binding", () => {
it("reads the percentage from the registry", () => {
expect(isCanaryEnabled("test-alpha")).toBe(true);
expect(isCanaryEnabled("test-beta")).toBe(false);
});
it("an unregistered name is off, not a throw — a typo must not break a render", () => {
expect(isCanaryEnabled("does-not-exist")).toBe(false);
expect(resolveCanary("does-not-exist").reason).toBe("out_of_cohort");
});
it("HF_CANARY_<FEATURE> overrides the registry in both directions", () => {
process.env.HF_CANARY_TEST_ALPHA = "off";
process.env.HF_CANARY_TEST_BETA = "on";
expect(resolveCanary("test-alpha")).toMatchObject({ enabled: false, reason: "forced_off" });
expect(resolveCanary("test-beta")).toMatchObject({ enabled: true, reason: "forced_on" });
});
it("excludes CI from percentage enrolment, but an override still reaches it", () => {
systemState.is_ci = true;
expect(resolveCanary("test-alpha")).toMatchObject({ enabled: false, reason: "excluded" });
__resetCanaryCacheForTests();
process.env.HF_CANARY_TEST_ALPHA = "on";
expect(resolveCanary("test-alpha")).toMatchObject({ enabled: true, reason: "forced_on" });
});
it("fails closed when the install has no anonymousId", () => {
configState.anonymousId = "";
expect(resolveCanary("test-alpha")).toMatchObject({ enabled: false, reason: "no_unit_id" });
});
it("memoizes so a decision cannot change mid-process", () => {
expect(isCanaryEnabled("test-beta")).toBe(false);
// A late env change must NOT flip a render that already started.
process.env.HF_CANARY_TEST_BETA = "on";
expect(isCanaryEnabled("test-beta")).toBe(false);
__resetCanaryCacheForTests();
expect(isCanaryEnabled("test-beta")).toBe(true);
});
it("reports enrolled canaries for telemetry, undefined when none", () => {
expect(activeCanaryNames()).toBe("test-alpha");
__resetCanaryCacheForTests();
process.env.HF_CANARY_TEST_ALPHA = "off";
expect(activeCanaryNames()).toBeUndefined();
});
});
+88
View File
@@ -0,0 +1,88 @@
/**
* CLI binding for the canary registry.
*
* `@hyperframes/core` owns the decision (pure, browser-safe, caller supplies
* everything). This file supplies the three things only the CLI knows: the
* install's stable id, the env override, and whether we're on CI.
*
* Using one, from anywhere in the CLI / producer call path:
*
* ```ts
* import { isCanaryEnabled } from "../telemetry/canary.js";
* if (isCanaryEnabled("de-parallel-router")) { ...ramped path... }
* ```
*
* That is the whole API. Percentage lives in the registry, not at the call
* site, so ramping is a one-line edit in a patch release and never touches
* the feature's own code.
*/
import {
CANARIES,
canaryEnvVar,
evaluateCanary,
findCanary,
parseCanaryOverride,
type CanaryDecision,
} from "@hyperframes/core";
import { readConfig } from "./config.js";
import { getSystemMeta } from "./system.js";
/**
* Decisions are memoized per process: a `--batch` run asks the same question
* once per row, and a canary must not change its mind mid-process — a render
* that starts enrolled has to finish enrolled, and its telemetry has to agree
* with what actually ran.
*/
const decisions = new Map<string, CanaryDecision>();
/** Test-only: drop memoized decisions so cases don't leak into each other. */
export function __resetCanaryCacheForTests(): void {
decisions.clear();
}
/**
* Full decision for a registered canary, including the reason — use this when
* you want to record WHY, not just whether.
*
* An unregistered name resolves to off rather than throwing: a canary is a
* rollout control, and a typo in one must never take down a render.
*/
export function resolveCanary(name: string): CanaryDecision {
const cached = decisions.get(name);
if (cached) return cached;
const definition = findCanary(name);
const decision: CanaryDecision = definition
? evaluateCanary({
feature: definition.name,
unitId: readConfig().anonymousId,
percentage: definition.percentage,
override: parseCanaryOverride(process.env[canaryEnvVar(definition.name)]),
// CI installs regenerate their config per run, so their ids are
// ephemeral — they would hop cohorts between runs, adding noise to the
// rollout signal while saying nothing about real users. An explicit
// override still gets through, which is how you test a canary in CI.
exclude: getSystemMeta().is_ci,
})
: { enabled: false, reason: "out_of_cohort" };
decisions.set(name, decision);
return decision;
}
/** Is this canary on for this install? The everyday call. */
export function isCanaryEnabled(name: string): boolean {
return resolveCanary(name).enabled;
}
/**
* Comma-joined names of the canaries this install is enrolled in, or
* undefined when none — attach to telemetry so every event can be segmented
* by cohort. One low-cardinality property beats a dynamic property per
* canary, and `contains` filtering works fine in PostHog.
*/
export function activeCanaryNames(): string | undefined {
const active = CANARIES.filter((c) => resolveCanary(c.name).enabled).map((c) => c.name);
return active.length > 0 ? active.join(",") : undefined;
}
+171
View File
@@ -0,0 +1,171 @@
import { describe, expect, it } from "vitest";
import { randomUUID } from "node:crypto";
import { canaryBucket, evaluateCanary, parseCanaryOverride, type CanaryInput } from "./canary.js";
import { CANARIES, canaryEnvVar, findCanary, overdueCanaries } from "./canaryRegistry.js";
const base = (over: Partial<CanaryInput> = {}): CanaryInput => ({
feature: "test-feature",
unitId: "db0c1f4a-b95e-4c35-90c6-1a15bd76f717",
percentage: 10,
...over,
});
/** A realistic population: install ids are v4 UUIDs (`randomUUID()`). */
function uuids(n: number): string[] {
return Array.from({ length: n }, () => randomUUID());
}
describe("evaluateCanary", () => {
it("is deterministic for the same feature + unit", () => {
const a = evaluateCanary(base());
const b = evaluateCanary(base());
expect(a).toEqual(b);
});
it("honours an explicit override in both directions, over any percentage", () => {
expect(evaluateCanary(base({ percentage: 0, override: true }))).toEqual({
enabled: true,
reason: "forced_on",
});
expect(evaluateCanary(base({ percentage: 100, override: false }))).toEqual({
enabled: false,
reason: "forced_off",
});
});
it("0% is off for everyone and 100% is on for everyone", () => {
for (const id of uuids(50)) {
expect(evaluateCanary(base({ unitId: id, percentage: 0 })).enabled).toBe(false);
expect(evaluateCanary(base({ unitId: id, percentage: 100 })).enabled).toBe(true);
}
});
it("fails closed without a unit id — unknown must never mean everyone", () => {
for (const id of [undefined, "", " "]) {
expect(evaluateCanary(base({ unitId: id, percentage: 100 }))).toEqual({
enabled: false,
reason: "no_unit_id",
});
}
});
it("excludes flagged units (CI) from percentage enrolment but not from an override", () => {
expect(evaluateCanary(base({ percentage: 100, exclude: true })).reason).toBe("excluded");
expect(evaluateCanary(base({ percentage: 100, exclude: true, override: true })).enabled).toBe(
true,
);
});
it("clamps out-of-range and fractional percentages", () => {
expect(evaluateCanary(base({ percentage: -5 })).enabled).toBe(false);
expect(evaluateCanary(base({ percentage: 999 })).enabled).toBe(true);
// 10.9 truncates to 10 — same cohort as an even 10, no surprise widening.
const ids = uuids(300);
const at10 = ids.filter((id) => evaluateCanary(base({ unitId: id, percentage: 10 })).enabled);
const at109 = ids.filter(
(id) => evaluateCanary(base({ unitId: id, percentage: 10.9 })).enabled,
);
expect(at109).toEqual(at10);
});
});
describe("cohort properties", () => {
it("ramping is INCLUSIVE — widening never drops an already-enrolled install", () => {
// If a ramp reshuffled the cohort, before/after comparisons across the
// ramp would be meaningless and some users would flap in and out.
const ids = uuids(500);
const enrolledAt = (pct: number) =>
new Set(ids.filter((id) => evaluateCanary(base({ unitId: id, percentage: pct })).enabled));
const p5 = enrolledAt(5);
const p25 = enrolledAt(25);
const p100 = enrolledAt(100);
for (const id of p5) expect(p25.has(id)).toBe(true);
for (const id of p25) expect(p100.has(id)).toBe(true);
expect(p25.size).toBeGreaterThan(p5.size);
});
it("different features select INDEPENDENT slices of the same population", () => {
// The whole reason the hash includes the feature name: bucketing on the
// unit id alone would hand every simultaneous experiment to one unlucky
// cohort, and make two rollouts impossible to read apart.
const ids = uuids(2000);
const a = new Set(
ids.filter((id) => evaluateCanary({ feature: "feat-a", unitId: id, percentage: 10 }).enabled),
);
const b = new Set(
ids.filter((id) => evaluateCanary({ feature: "feat-b", unitId: id, percentage: 10 }).enabled),
);
const overlap = [...a].filter((id) => b.has(id)).length;
// Independent 10% slices overlap ~1% of the population (~20 of 2000).
// Identical slices would overlap ~200. Assert well below that.
expect(overlap).toBeLessThan(70);
expect(a.size).toBeGreaterThan(0);
expect(b.size).toBeGreaterThan(0);
});
it("selects approximately the requested share of a UUID population", () => {
const ids = uuids(4000);
for (const pct of [5, 10, 25]) {
const hits = ids.filter(
(id) => evaluateCanary(base({ unitId: id, percentage: pct })).enabled,
).length;
const actual = (hits / ids.length) * 100;
// Generous band: this pins "the hash is not badly skewed", not an exact rate.
expect(actual).toBeGreaterThan(pct * 0.6);
expect(actual).toBeLessThan(pct * 1.4);
}
});
it("spreads buckets across the full 0-99 range", () => {
const seen = new Set(uuids(2000).map((id) => canaryBucket("spread", id)));
expect(seen.size).toBeGreaterThan(80);
});
});
describe("parseCanaryOverride", () => {
it("accepts the spellings people actually type", () => {
for (const v of ["1", "true", "TRUE", "on", "yes", " On "]) {
expect(parseCanaryOverride(v)).toBe(true);
}
for (const v of ["0", "false", "FALSE", "off", "no", " Off "]) {
expect(parseCanaryOverride(v)).toBe(false);
}
});
it("treats unset, empty and unrecognised values as 'no override'", () => {
// An exported-but-empty var must not force a feature on.
for (const v of [undefined, "", " ", "maybe"]) {
expect(parseCanaryOverride(v)).toBeUndefined();
}
});
});
describe("registry", () => {
it("has unique, kebab-case names", () => {
const names = CANARIES.map((c) => c.name);
expect(new Set(names).size).toBe(names.length);
for (const n of names) expect(n).toMatch(/^[a-z0-9]+(-[a-z0-9]+)*$/);
});
it("has in-range percentages and a parseable sunset date", () => {
for (const c of CANARIES) {
expect(c.percentage).toBeGreaterThanOrEqual(0);
expect(c.percentage).toBeLessThanOrEqual(100);
expect(Number.isNaN(Date.parse(`${c.sunsetAfter}T00:00:00Z`))).toBe(false);
expect(c.owner.length).toBeGreaterThan(0);
expect(c.description.length).toBeGreaterThan(0);
}
});
it("derives the override env var from the name", () => {
expect(canaryEnvVar("de-parallel-router")).toBe("HF_CANARY_DE_PARALLEL_ROUTER");
expect(findCanary("de-parallel-router")?.name).toBe("de-parallel-router");
expect(findCanary("nope")).toBeUndefined();
});
it("no canary is past its sunset date", () => {
// Fails the suite when a rollout has been left half-finished. Either take
// it to 100 and delete the entry, or move the date deliberately.
expect(overdueCanaries()).toEqual([]);
});
});
+136
View File
@@ -0,0 +1,136 @@
/**
* Canary rollouts — ship a change to a stable slice of installs instead of
* all-or-nothing.
*
* The problem this solves: the repo has ~49 `HF_*` / `PRODUCER_*` boolean
* toggles, and every one of them is binary. A change is either off (and
* therefore untested on real traffic) or on for everyone (and therefore a
* fleet-wide bet). The parallel-drawElement router spent weeks in that gap:
* default-off collected almost no signal, and flipping it default-on exposed
* 100% of eligible installs at once. A percentage slice is the missing rung.
*
* Design notes worth knowing before you add one:
*
* - **Pure and universal.** No fs, no network, no `process` — the caller
* supplies the unit id and the overrides. That keeps this importable from
* the CLI, the producer, the engine, studio-server, the browser-side studio
* bundle, and the embeddable player alike.
*
* - **Independent slices.** The bucket is a hash of `feature:unitId`, NOT of
* `unitId` alone. If every canary bucketed on the id by itself, they would
* all select the SAME installs — one unlucky cohort would receive every
* experiment simultaneously, and no two rollouts could be read
* independently.
*
* - **Ramping is inclusive.** `bucket < percentage` means widening 10 → 25
* keeps every install that was already at 10. Cohorts never reshuffle, so
* before/after comparisons stay valid across a ramp.
*
* - **Stable per install, for the life of the install.** The same id and
* feature always resolve the same way, with no persisted state to keep in
* sync and nothing to look up at runtime.
*/
/** Why a canary resolved the way it did. Attach to telemetry — a rollout you
* can't segment by enrolment reason is a rollout you can't debug. */
export type CanaryReason =
| "forced_on"
| "forced_off"
| "in_cohort"
| "out_of_cohort"
| "no_unit_id"
| "excluded";
export interface CanaryDecision {
enabled: boolean;
reason: CanaryReason;
/** 0-99 slot this unit landed in for this feature; undefined when not computed. */
bucket?: number;
}
export interface CanaryInput {
/** Registry key, e.g. "de-parallel-router". Part of the hash, so each feature gets its own slice. */
feature: string;
/** Stable per-install id — the CLI's telemetry `anonymousId`. Missing/blank fails closed. */
unitId: string | undefined;
/** 0 = off for everyone, 100 = on for everyone. Values outside 0-100 are clamped. */
percentage: number;
/**
* Explicit override, both directions — support escalations, dogfooding, a
* bisect, or a panic-off. Always wins over the percentage.
*/
override?: boolean | undefined;
/**
* Exclude this unit from percentage-based enrolment (an explicit override
* still applies). Callers pass `isCI` here: CI installs regenerate their
* config constantly, so their ids are ephemeral — they would hop cohorts
* between runs, adding noise to the rollout signal while telling you
* nothing about real users.
*/
exclude?: boolean | undefined;
}
/**
* FNV-1a (32-bit). Chosen over `node:crypto` deliberately: this module has to
* run in the browser-side studio bundle and the embeddable player too, and a
* six-line hash beats shipping a polyfill or maintaining two code paths.
* Distribution is uniform enough for bucketing (pinned by a test).
*/
function fnv1a32(input: string): number {
let hash = 0x811c9dc5;
for (let i = 0; i < input.length; i++) {
hash ^= input.charCodeAt(i);
// hash * 16777619, kept in 32-bit unsigned range without Math.imul overflow
hash = (hash + ((hash << 1) + (hash << 4) + (hash << 7) + (hash << 8) + (hash << 24))) >>> 0;
}
return hash >>> 0;
}
/** The 0-99 slot a unit occupies for a given feature. Exported for tests and diagnostics. */
export function canaryBucket(feature: string, unitId: string): number {
return fnv1a32(`${feature}:${unitId}`) % 100;
}
/**
* Resolve whether a feature is on for this unit.
*
* Fails closed on a missing id: the canary exists to bound blast radius, so
* "we don't know who this is" must mean "not enrolled", never "enrol
* everyone".
*/
export function evaluateCanary(input: CanaryInput): CanaryDecision {
if (input.override === true) return { enabled: true, reason: "forced_on" };
if (input.override === false) return { enabled: false, reason: "forced_off" };
const pct = Math.max(0, Math.min(100, Math.trunc(input.percentage)));
if (pct <= 0) return { enabled: false, reason: "out_of_cohort" };
if (input.exclude) return { enabled: false, reason: "excluded" };
const unitId = input.unitId?.trim();
if (!unitId) return { enabled: false, reason: "no_unit_id" };
if (pct >= 100)
return { enabled: true, reason: "in_cohort", bucket: canaryBucket(input.feature, unitId) };
const bucket = canaryBucket(input.feature, unitId);
return bucket < pct
? { enabled: true, reason: "in_cohort", bucket }
: { enabled: false, reason: "out_of_cohort", bucket };
}
/**
* Parse a canary override from an env-var value.
*
* Accepts the spellings people actually type. Returns undefined for
* unset/empty so the percentage decides — matching how the existing HF_*
* knobs treat a set-but-empty var, and avoiding the failure mode where an
* exported-but-empty variable silently forces a feature on.
*/
export function parseCanaryOverride(raw: string | undefined): boolean | undefined {
const v = raw?.trim().toLowerCase();
if (v === undefined || v === "") return undefined;
if (v === "1" || v === "true" || v === "on" || v === "yes") return true;
if (v === "0" || v === "false" || v === "off" || v === "no") return false;
return undefined;
}
+77
View File
@@ -0,0 +1,77 @@
/**
* The canary registry — every staged rollout in the product, in one file.
*
* Why a registry rather than a percentage inlined at each call site: the repo
* already carries ~49 loose `HF_*` / `PRODUCER_*` toggles with no index, so
* nobody can answer "what is currently rolling out, to how many people, and
* who owns it" without grepping. One table fixes that, and gives the
* telemetry and `doctor` surfaces something to enumerate.
*
* ## Adding one
*
* 1. Add an entry below. Start at `percentage: 0` and merge that — a canary
* at 0 is dead code you can land safely and ramp without a code review.
* 2. Read it at the decision point via the surface's binding (in the CLI,
* `isCanaryEnabled("your-feature")`).
* 3. Ramp by editing `percentage` in a patch release: 0 → 5 → 25 → 100.
* Widening is inclusive, so the earlier cohort stays enrolled and the
* before/after comparison survives the ramp.
* 4. At 100 and holding, DELETE the entry and the branch it guarded. That is
* the point of `sunsetAfter`.
*
* ## Overriding
*
* `HF_CANARY_<FEATURE>` with the feature name upper-snake-cased, e.g.
* `HF_CANARY_DE_PARALLEL_ROUTER=on` (also: off/true/false/1/0/yes/no).
* An override always wins over the percentage, in both directions.
*/
export interface CanaryDefinition {
/** Registry key. Kebab-case; also the hash input, so renaming reshuffles the cohort. */
name: string;
/** 0-100. Start at 0, ramp in patch releases. */
percentage: number;
/** What turning this on actually changes, in one line. */
description: string;
/** Who to ask. */
owner: string;
/**
* ISO date after which this canary is overdue for removal. A canary that
* outlives its rollout is a permanent fork of the product with none of the
* review a permanent fork would have received. `assertNoOverdueCanaries`
* turns the date into a failing test rather than a good intention.
*/
sunsetAfter: string;
}
export const CANARIES: readonly CanaryDefinition[] = [
{
name: "de-parallel-router",
percentage: 0,
description:
"Route auto multi-worker renders to verified parallel drawElement streaming (HF_DE_PARALLEL_ROUTER). Ramp only alongside the per-install circuit breaker.",
owner: "vance",
sunsetAfter: "2026-10-01",
},
] as const;
export function findCanary(name: string): CanaryDefinition | undefined {
return CANARIES.find((c) => c.name === name);
}
/** Env-var name for a feature's manual override: `de-parallel-router` → `HF_CANARY_DE_PARALLEL_ROUTER`. */
export function canaryEnvVar(name: string): string {
return `HF_CANARY_${name.toUpperCase().replace(/[^A-Z0-9]+/g, "_")}`;
}
/**
* Names of canaries whose sunset date has passed — either finish the rollout
* and delete the entry, or push the date with a reason. Exposed as a function
* (not a lint rule) so the check runs in the normal test suite.
*/
export function overdueCanaries(now: Date = new Date()): string[] {
return CANARIES.filter((c) => {
const sunset = Date.parse(`${c.sunsetAfter}T00:00:00Z`);
return Number.isFinite(sunset) && now.getTime() > sunset;
}).map((c) => c.name);
}
+16
View File
@@ -329,3 +329,19 @@ export {
isBlockItem, isBlockItem,
isComponentItem, isComponentItem,
} from "./registry/index.js"; } from "./registry/index.js";
export {
canaryBucket,
evaluateCanary,
parseCanaryOverride,
type CanaryDecision,
type CanaryInput,
type CanaryReason,
} from "./canary.js";
export {
CANARIES,
canaryEnvVar,
findCanary,
overdueCanaries,
type CanaryDefinition,
} from "./canaryRegistry.js";