mirror of
https://github.com/heygen-com/hyperframes.git
synced 2026-09-10 22:20:14 +00:00
feat(core,cli): emit the canary decision reason alongside the assignment
The calibration contract deferred this until the stability check came back dirty. It did: the first fleet read found 304 installs (1.08%) reporting both values for a canary whose percentage never moved, and the genuinely anomalous ones could not be separated from a developer toggling HF_CANARY_*, because the assignment alone is identical in both cases. resolveCanary has always computed the reason and canaryEventProperties dropped it. Now every canary emits canary_reason_<name> beside its assignment. Deliberately outside the $feature/ namespace: PostHog treats those as flag values, and a non-boolean there would corrupt the flag's own breakdowns. Two of the six wire values are immediately useful beyond override attribution. 'excluded' identifies CI installs, which today have to be dropped by joining on is_ci — conflating them with out_of_cohort is what made the first accuracy read look like a significant failure (9.22% against a 10% target) when it was not. 'no_unit_id' surfaces the fails-closed corner. The reason is optional on the core helper so existing callers are unaffected. Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
This commit is contained in:
co-authored by
Claude Opus 5
parent
fc0298de4a
commit
076657a639
@@ -1,7 +1,12 @@
|
||||
import { describe, expect, it } from "vitest";
|
||||
import { canaryBucket, evaluateCanary, parseCanaryOverride, type CanaryInput } from "./canary.js";
|
||||
import { CANARIES, canaryEnvVar, findCanary, overdueCanaries } from "./canaryRegistry.js";
|
||||
import { CANARY_FEATURE_PREFIX, canaryFeatureKey, canaryFeatureProperties } from "./canary.js";
|
||||
import {
|
||||
CANARY_FEATURE_PREFIX,
|
||||
canaryFeatureKey,
|
||||
canaryFeatureProperties,
|
||||
canaryReasonKey,
|
||||
} from "./canary.js";
|
||||
|
||||
const base = (over: Partial<CanaryInput> = {}): CanaryInput => ({
|
||||
feature: "test-feature",
|
||||
@@ -371,3 +376,56 @@ describe("PostHog flag-shaped properties", () => {
|
||||
expect(canaryFeatureProperties([])).toEqual({});
|
||||
});
|
||||
});
|
||||
|
||||
// The attribution property. Without it, an install reporting both "true" and
|
||||
// "false" for a canary whose percentage never moved is indistinguishable from
|
||||
// a developer toggling HF_CANARY_*. The first calibration read hit exactly
|
||||
// that: 304 installs reported both values and the anomalous ones could not be
|
||||
// separated from deliberate overrides.
|
||||
describe("canary reason property", () => {
|
||||
it("rides alongside the assignment, outside the $feature namespace", () => {
|
||||
const props = canaryFeatureProperties([
|
||||
{ name: "de-parallel-router", enabled: true, reason: "in_cohort" },
|
||||
]);
|
||||
expect(props["$feature/canary-de-parallel-router"]).toBe("true");
|
||||
expect(props["canary_reason_de_parallel_router"]).toBe("in_cohort");
|
||||
});
|
||||
|
||||
// A non-boolean under `$feature/` would corrupt the flag's own breakdowns,
|
||||
// which is the whole reason the reason gets its own key.
|
||||
it("never puts a reason inside the flag namespace", () => {
|
||||
const props = canaryFeatureProperties([{ name: "x", enabled: false, reason: "forced_off" }]);
|
||||
for (const [key, value] of Object.entries(props)) {
|
||||
if (key.startsWith(CANARY_FEATURE_PREFIX)) {
|
||||
expect(value).toMatch(/^(true|false)$/);
|
||||
}
|
||||
}
|
||||
});
|
||||
|
||||
it("separates a forced override from a genuine cohort roll at the same value", () => {
|
||||
const forced = canaryFeatureProperties([{ name: "f", enabled: true, reason: "forced_on" }]);
|
||||
const rolled = canaryFeatureProperties([{ name: "f", enabled: true, reason: "in_cohort" }]);
|
||||
// Identical assignment — only the reason tells them apart. This is the
|
||||
// distinction the calibration read could not make.
|
||||
expect(forced["$feature/canary-f"]).toBe(rolled["$feature/canary-f"]);
|
||||
expect(forced["canary_reason_f"]).not.toBe(rolled["canary_reason_f"]);
|
||||
});
|
||||
|
||||
it("emits `excluded` for CI, which replaces joining on is_ci", () => {
|
||||
const props = canaryFeatureProperties([{ name: "c", enabled: false, reason: "excluded" }]);
|
||||
// `excluded` and `out_of_cohort` are both enabled:false but mean different
|
||||
// things — CI was never bucketed, the other lost the roll. Counting them
|
||||
// together is what biased the first accuracy read low.
|
||||
expect(props["canary_reason_c"]).toBe("excluded");
|
||||
});
|
||||
|
||||
it("omits the reason key when no reason is supplied", () => {
|
||||
const props = canaryFeatureProperties([{ name: "n", enabled: true }]);
|
||||
expect(props["$feature/canary-n"]).toBe("true");
|
||||
expect(props).not.toHaveProperty("canary_reason_n");
|
||||
});
|
||||
|
||||
it("sanitizes the name into a property-safe key", () => {
|
||||
expect(canaryReasonKey("de-parallel-router")).toBe("canary_reason_de_parallel_router");
|
||||
});
|
||||
});
|
||||
|
||||
Reference in New Issue
Block a user