mirror of
https://github.com/heygen-com/hyperframes.git
synced 2026-09-11 14:50:02 +00:00
fix(cli,studio): close the four R3 blocking gaps
P1 — SPA route bypassed the DNS-rebinding guard. Guarding only
/api/telemetry-identity left the catch-all as an open side door: a
rebound origin could fetch `/` and read __HF_CLI_DISTINCT_ID and
__HF_CLI_BUCKET_SEED straight out of the returned HTML. The SPA response
now applies the same isLoopbackHost() check; an untrusted Host still gets
a working Studio, just with no identity, seed or decisions injected.
Route-level regression added.
P1 — a CLI cohort roll could override Studio's own opt-out.
decideStudioCanary() adopted the injected decision before checking
isOptedOut(), so CLI-telemetry-on plus Studio-opted-out still enrolled
Studio. A bare boolean could not express the difference between a
deliberate override and an ordinary cohort roll, so the injected map now
carries provenance ({ enabled, forced }). Forced wins outright — it is
the documented escalation channel and must behave the same on both
surfaces — while a percentage roll now loses to this profile's opt-out.
Full interaction matrix tested.
P1 — the legacy studio:* path sat outside both contracts.
utils/studioTelemetry.ts shipped its own opt-out key and its own send
loop, so the documented hyperframes-studio:telemetryDisabled did not
silence it and its events carried no cohort assignment. It now honours
both keys (the legacy one stays, so nobody already opted out is quietly
re-enabled) and mixes in canaryEventProperties(), making "every
telemetry event carries the assignment" actually true.
P2 — partial salvage could drop a tripped breaker.
salvageInstallState() discarded the whole record when markerAt and
bucketSeed were both unusable, taking deParallelRouterTrialFired with it
and re-enrolling a machine whose router already failed. All three fields
are now independently salvageable.
Docs: canary-rollouts.mdx said "disabling telemetry disables the
reporting, not the enrolment" — exactly backwards since the opt-out gate
landed. Corrected; checked for other copies, none.
Tests: 13 new (4 opt-out precedence, 4 legacy-path opt-out and canary
props, 3 route-level host guard, 2 breaker salvage). Fault injection:
each of the four fixes reverted independently fails its own tests
(2 CLI + 1 Studio + 2 Studio).
Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
This commit is contained in:
co-authored by
Claude Opus 5
parent
54534d53c2
commit
f81ab0162e
@@ -80,7 +80,9 @@ tooling, so whoever operates the telemetry backend can split any metric by
|
||||
cohort with **nothing configured server-side** — while the decision itself
|
||||
still happens locally and offline, which the render path requires.
|
||||
(Assignments ride the same anonymous, opt-out telemetry pipeline as every
|
||||
other event; disabling telemetry disables the reporting, not the enrolment.)
|
||||
other event — and disabling telemetry disables the **enrolment**, not just the
|
||||
reporting: an opted-out install is never bucketed at all. See the note at the
|
||||
top of this page.)
|
||||
|
||||
Two details worth knowing:
|
||||
|
||||
|
||||
@@ -57,3 +57,57 @@ describe("createStudioServer autoProxy plumbing", () => {
|
||||
expect(server.adapter.autoProxy).toBe(true);
|
||||
});
|
||||
});
|
||||
|
||||
describe("host guarding on identity-bearing responses", () => {
|
||||
const dirs: string[] = [];
|
||||
let server: StudioServer | undefined;
|
||||
|
||||
function tmpProject(): string {
|
||||
const dir = mkdtempSync(join(tmpdir(), "hf-studio-host-test-"));
|
||||
dirs.push(dir);
|
||||
return dir;
|
||||
}
|
||||
|
||||
afterEach(() => {
|
||||
server?.watcher.close();
|
||||
server = undefined;
|
||||
for (const dir of dirs.splice(0)) rmSync(dir, { recursive: true, force: true });
|
||||
});
|
||||
|
||||
// A rebound origin can point its own hostname at 127.0.0.1 and read
|
||||
// responses as same-origin. Guarding only /api/telemetry-identity left the
|
||||
// SPA route as an open side door: fetching `/` returned the same distinct
|
||||
// id and bucket seed inline in the HTML.
|
||||
it("omits identity injection from the SPA response for a hostile Host", async () => {
|
||||
server = createStudioServer({ projectDir: tmpProject() });
|
||||
const res = await server.app.request("/", { headers: { host: "evil.example.com" } });
|
||||
const html = await res.text();
|
||||
expect(html).not.toContain("__HF_CLI_DISTINCT_ID");
|
||||
expect(html).not.toContain("__HF_CLI_BUCKET_SEED");
|
||||
expect(html).not.toContain("__HF_CLI_CANARY_DECISIONS");
|
||||
// Studio still loads — only the identity block is withheld. (The env
|
||||
// script is empty here: it only emits with VITE_STUDIO_* vars set.)
|
||||
expect(res.status).toBe(200);
|
||||
expect(html).toContain("<head>");
|
||||
});
|
||||
|
||||
it("refuses the identity endpoint for a hostile Host", async () => {
|
||||
server = createStudioServer({ projectDir: tmpProject() });
|
||||
const res = await server.app.request("/api/telemetry-identity", {
|
||||
headers: { host: "evil.example.com" },
|
||||
});
|
||||
expect(res.status).toBe(403);
|
||||
expect(await res.text()).not.toContain('distinctId":"');
|
||||
});
|
||||
|
||||
it("serves the identity endpoint on a loopback Host", async () => {
|
||||
server = createStudioServer({ projectDir: tmpProject() });
|
||||
const res = await server.app.request("/api/telemetry-identity", {
|
||||
headers: { host: "127.0.0.1:5173" },
|
||||
});
|
||||
expect(res.status).toBe(200);
|
||||
// The seed is no longer served here at all — Studio gets decisions
|
||||
// injected instead, so nothing needs it over HTTP.
|
||||
expect(Object.keys((await res.json()) as object)).toEqual(["distinctId"]);
|
||||
});
|
||||
});
|
||||
|
||||
@@ -814,7 +814,17 @@ export function createStudioServer(options: StudioServerOptions): StudioServer {
|
||||
// Inject before the studio bundle runs. Identity script first (see
|
||||
// buildStudioHeadScripts) so the CLI distinct id is on `window` by the time
|
||||
// telemetry init reads it.
|
||||
const headScript = buildStudioHeadScripts(buildRuntimeEnvScript());
|
||||
//
|
||||
// Host-guarded for the same reason /api/telemetry-identity is, and it has
|
||||
// to be checked HERE too: guarding only the endpoint leaves this route as
|
||||
// an open side door, since a rebound origin can simply fetch `/` and read
|
||||
// the same distinct id and seed out of the returned HTML. Untrusted Host
|
||||
// still gets a working Studio — it just gets the env script alone, with no
|
||||
// identity, no seed, and no canary decisions.
|
||||
const trustedHost = isLoopbackHost(c.req.header("host"));
|
||||
const headScript = trustedHost
|
||||
? buildStudioHeadScripts(buildRuntimeEnvScript())
|
||||
: buildRuntimeEnvScript();
|
||||
if (headScript) {
|
||||
html = html.replace("<head>", `<head>${headScript}`);
|
||||
}
|
||||
|
||||
@@ -8,7 +8,7 @@ const shouldTrack = vi.fn();
|
||||
const readConfig = vi.fn();
|
||||
// Pinned rather than using the real registry, so these string assertions
|
||||
// don't move every time a canary is added, ramped, or retired.
|
||||
const canaryDecisions = vi.fn<() => Record<string, boolean>>();
|
||||
const canaryDecisions = vi.fn<() => Record<string, { enabled: boolean; forced: boolean }>>();
|
||||
|
||||
vi.mock("../telemetry/client.js", () => ({
|
||||
shouldTrack: (...args: unknown[]) => shouldTrack(...args),
|
||||
@@ -99,10 +99,11 @@ describe("buildCliIdentityScript", () => {
|
||||
// stops Studio evaluating independently and enrolling anyway.
|
||||
it("still publishes canary decisions when telemetry is off, but no identity", () => {
|
||||
shouldTrack.mockReturnValue(false);
|
||||
canaryDecisions.mockReturnValue({ "de-parallel-router": false });
|
||||
canaryDecisions.mockReturnValue({ "de-parallel-router": { enabled: false, forced: false } });
|
||||
const script = buildCliIdentityScript();
|
||||
expect(script).toBe(
|
||||
'<script>window.__HF_CLI_CANARY_DECISIONS={"de-parallel-router":false};</script>',
|
||||
"<script>window.__HF_CLI_CANARY_DECISIONS=" +
|
||||
'{"de-parallel-router":{"enabled":false,"forced":false}};</script>',
|
||||
);
|
||||
expect(script).not.toContain("__HF_CLI_DISTINCT_ID");
|
||||
expect(script).not.toContain("__HF_CLI_BUCKET_SEED");
|
||||
@@ -111,17 +112,20 @@ describe("buildCliIdentityScript", () => {
|
||||
it("publishes decisions alongside the identity when telemetry is on", () => {
|
||||
shouldTrack.mockReturnValue(true);
|
||||
readConfig.mockReturnValue({ anonymousId: "machine-uuid", bucketSeed: "seed-uuid" });
|
||||
canaryDecisions.mockReturnValue({ "de-parallel-router": true });
|
||||
canaryDecisions.mockReturnValue({ "de-parallel-router": { enabled: true, forced: true } });
|
||||
expect(buildCliIdentityScript()).toBe(
|
||||
'<script>window.__HF_CLI_DISTINCT_ID="machine-uuid";' +
|
||||
'window.__HF_CLI_BUCKET_SEED="seed-uuid";' +
|
||||
'window.__HF_CLI_CANARY_DECISIONS={"de-parallel-router":true};</script>',
|
||||
"window.__HF_CLI_CANARY_DECISIONS=" +
|
||||
'{"de-parallel-router":{"enabled":true,"forced":true}};</script>',
|
||||
);
|
||||
});
|
||||
|
||||
it("escapes a canary name that tries to close the script tag", () => {
|
||||
shouldTrack.mockReturnValue(false);
|
||||
canaryDecisions.mockReturnValue({ "</script><script>alert(1)": true });
|
||||
canaryDecisions.mockReturnValue({
|
||||
"</script><script>alert(1)": { enabled: true, forced: false },
|
||||
});
|
||||
const script = buildCliIdentityScript();
|
||||
expect(script).not.toContain("</script><script>alert(1)");
|
||||
expect(script).toContain("__HF_CLI_CANARY_DECISIONS");
|
||||
|
||||
@@ -18,7 +18,7 @@
|
||||
|
||||
import { readConfig } from "../telemetry/config.js";
|
||||
import { shouldTrack as telemetryShouldTrack } from "../telemetry/client.js";
|
||||
import { canaryDecisionsForStudio } from "../telemetry/canary.js";
|
||||
import { canaryDecisionsForStudio, type CliCanaryDecision } from "../telemetry/canary.js";
|
||||
|
||||
/**
|
||||
* The CLI's anonymous distinct id to hand to Studio, or null when CLI telemetry
|
||||
@@ -102,7 +102,7 @@ function encodeInlineScriptJson(value: unknown): string {
|
||||
* The CLI's resolved canary decisions for a launched Studio, or null when
|
||||
* there are none to publish. Fail-silent, like everything else here.
|
||||
*/
|
||||
function resolveCliCanaryDecisions(): Record<string, boolean> | null {
|
||||
function resolveCliCanaryDecisions(): Record<string, CliCanaryDecision> | null {
|
||||
try {
|
||||
const decisions = canaryDecisionsForStudio();
|
||||
return Object.keys(decisions).length > 0 ? decisions : null;
|
||||
|
||||
@@ -118,6 +118,15 @@ export function isCanaryEnabled(name: string): boolean {
|
||||
return resolveCanary(name).enabled;
|
||||
}
|
||||
|
||||
/**
|
||||
* One canary's outcome as handed to Studio: the answer, plus whether it came
|
||||
* from an explicit override rather than a percentage roll.
|
||||
*/
|
||||
export interface CliCanaryDecision {
|
||||
enabled: boolean;
|
||||
forced: boolean;
|
||||
}
|
||||
|
||||
/**
|
||||
* Every registered canary's resolved on/off for this process, for handing to
|
||||
* a CLI-launched Studio.
|
||||
@@ -138,9 +147,20 @@ export function isCanaryEnabled(name: string): boolean {
|
||||
* impossible: one evaluation, two surfaces. It is also strictly less to
|
||||
* expose — booleans about features, not the seed the buckets derive from.
|
||||
*/
|
||||
export function canaryDecisionsForStudio(): Record<string, boolean> {
|
||||
const out: Record<string, boolean> = {};
|
||||
for (const canary of CANARIES) out[canary.name] = resolveCanary(canary.name).enabled;
|
||||
export function canaryDecisionsForStudio(): Record<string, CliCanaryDecision> {
|
||||
const out: Record<string, CliCanaryDecision> = {};
|
||||
for (const canary of CANARIES) {
|
||||
const { enabled, reason } = resolveCanary(canary.name);
|
||||
out[canary.name] = {
|
||||
enabled,
|
||||
// Provenance, not decoration. Studio has its own telemetry opt-out that
|
||||
// the CLI cannot see, and it must be able to refuse ordinary cohort
|
||||
// enrolment while still honouring a deliberate `HF_CANARY_*` override.
|
||||
// A bare boolean cannot express that difference, so Studio would have
|
||||
// had to either ignore its own opt-out or drop the override.
|
||||
forced: reason === "forced_on" || reason === "forced_off",
|
||||
};
|
||||
}
|
||||
return out;
|
||||
}
|
||||
|
||||
|
||||
@@ -465,3 +465,32 @@ describe("seed backfill write failure is surfaced", () => {
|
||||
});
|
||||
});
|
||||
});
|
||||
|
||||
describe("a tripped breaker survives even total marker+seed corruption", () => {
|
||||
let readConfig: typeof import("./config.js").readConfig;
|
||||
let STATE_PATH: typeof import("./config.js").STATE_PATH;
|
||||
|
||||
beforeEach(async () => {
|
||||
fsState.files.clear();
|
||||
vi.resetModules();
|
||||
({ readConfig, STATE_PATH } = await import("./config.js"));
|
||||
});
|
||||
|
||||
// Dropping the latch re-enrols a machine into an experimental path that
|
||||
// already FAILED on it — the one outcome install-state exists to prevent.
|
||||
// So the tripped bit is salvageable independently of the other two fields.
|
||||
it("keeps deParallelRouterTrialFired when markerAt and bucketSeed are both unusable", () => {
|
||||
fsState.files.set(
|
||||
STATE_PATH,
|
||||
JSON.stringify({ markerAt: 42, bucketSeed: "", deParallelRouterTrialFired: true }),
|
||||
);
|
||||
const config = readConfig();
|
||||
expect(config.deParallelRouterTrialFired).toBe(true);
|
||||
expect(config.predecessorFound).toBe(true);
|
||||
});
|
||||
|
||||
it("still reports corrupt when none of the three fields survive", () => {
|
||||
fsState.files.set(STATE_PATH, JSON.stringify({ markerAt: 42, bucketSeed: "" }));
|
||||
expect(readConfig().stateFileCorrupt).toBe(true);
|
||||
});
|
||||
});
|
||||
|
||||
@@ -89,13 +89,15 @@ type InstallStateMiss = "absent" | "corrupt";
|
||||
function salvageInstallState(parsed: Partial<InstallState>): InstallState | InstallStateMiss {
|
||||
const markerAt = typeof parsed.markerAt === "string" ? parsed.markerAt : undefined;
|
||||
const bucketSeed = parseNonEmptyString(parsed.bucketSeed);
|
||||
// Nothing salvageable in the record at all — treat as corrupt rather than
|
||||
// inventing a marker, so the miss is still counted as "we knew this machine"
|
||||
// instead of masquerading as a fresh install.
|
||||
if (markerAt === undefined && bucketSeed === undefined) return "corrupt";
|
||||
const fired = parsed.deParallelRouterTrialFired === true ? true : undefined;
|
||||
// Each of the three is independently salvageable, and the tripped breaker
|
||||
// most of all: dropping it re-enrols a machine into an experimental path
|
||||
// that already FAILED there, which is the one outcome this file exists to
|
||||
// prevent. `markerAt` is only a timestamp and can be restamped.
|
||||
if (markerAt === undefined && bucketSeed === undefined && fired === undefined) return "corrupt";
|
||||
return {
|
||||
markerAt: markerAt ?? new Date().toISOString(),
|
||||
deParallelRouterTrialFired: parsed.deParallelRouterTrialFired === true ? true : undefined,
|
||||
deParallelRouterTrialFired: fired,
|
||||
bucketSeed,
|
||||
};
|
||||
}
|
||||
|
||||
@@ -228,6 +228,8 @@ describe("telemetry opt-out is canary opt-out", () => {
|
||||
|
||||
describe("CLI-launched Studio adopts the CLI's decisions", () => {
|
||||
const OPT_OUT_KEY = "hyperframes-studio:telemetryDisabled";
|
||||
const cohort = (enabled: boolean) => ({ enabled, forced: false });
|
||||
const forced = (enabled: boolean) => ({ enabled, forced: true });
|
||||
|
||||
afterEach(() => {
|
||||
delete window.__HF_CLI_CANARY_DECISIONS;
|
||||
@@ -238,40 +240,72 @@ describe("CLI-launched Studio adopts the CLI's decisions", () => {
|
||||
// flag it cannot see — left to itself it would evaluate and could enrol.
|
||||
it("stays off when the CLI opted out, even though Studio's own flag is unset", () => {
|
||||
expect(localStorage.getItem(OPT_OUT_KEY)).toBeNull();
|
||||
window.__HF_CLI_CANARY_DECISIONS = { "on-everywhere": false };
|
||||
window.__HF_CLI_CANARY_DECISIONS = { "on-everywhere": cohort(false) };
|
||||
expect(resolveCanary("on-everywhere").enabled).toBe(false);
|
||||
});
|
||||
|
||||
// HF_CANARY_* never crosses into the browser, so before this the CLI was
|
||||
// forced on and Studio silently guessed from the percentage.
|
||||
it("turns on when the CLI forced it on, with no URL param present", () => {
|
||||
window.__HF_CLI_CANARY_DECISIONS = { "off-everywhere": true };
|
||||
window.__HF_CLI_CANARY_DECISIONS = { "off-everywhere": forced(true) };
|
||||
expect(resolveCanary("off-everywhere").enabled).toBe(true);
|
||||
});
|
||||
|
||||
it("beats a contradicting URL override — one render must not run half-enrolled", () => {
|
||||
window.__HF_CLI_CANARY_DECISIONS = { "on-everywhere": false };
|
||||
window.__HF_CLI_CANARY_DECISIONS = { "on-everywhere": forced(false) };
|
||||
setSearch("?hf_canary_on_everywhere=on");
|
||||
expect(resolveCanary("on-everywhere").enabled).toBe(false);
|
||||
});
|
||||
|
||||
it("beats the seed-derived bucket", () => {
|
||||
window.__HF_CLI_BUCKET_SEED = "5f1c9d2e-0000-4000-8000-aaaaaaaaaaaa";
|
||||
window.__HF_CLI_CANARY_DECISIONS = { "on-everywhere": false };
|
||||
window.__HF_CLI_CANARY_DECISIONS = { "on-everywhere": cohort(false) };
|
||||
expect(resolveCanary("on-everywhere").enabled).toBe(false);
|
||||
});
|
||||
|
||||
it("falls back to local evaluation for a canary the CLI did not publish", () => {
|
||||
window.__HF_CLI_CANARY_DECISIONS = { "off-everywhere": true };
|
||||
window.__HF_CLI_CANARY_DECISIONS = { "off-everywhere": cohort(true) };
|
||||
expect(resolveCanary("on-everywhere").enabled).toBe(true);
|
||||
});
|
||||
|
||||
it("ignores a non-boolean value rather than trusting it", () => {
|
||||
window.__HF_CLI_CANARY_DECISIONS = { "on-everywhere": "false" } as unknown as Record<
|
||||
string,
|
||||
boolean
|
||||
>;
|
||||
it("ignores a malformed entry rather than trusting it", () => {
|
||||
window.__HF_CLI_CANARY_DECISIONS = {
|
||||
"on-everywhere": { enabled: "false" },
|
||||
} as unknown as Record<string, { enabled?: boolean; forced?: boolean }>;
|
||||
// Falls through to local evaluation: on-everywhere is at 100%.
|
||||
expect(resolveCanary("on-everywhere").enabled).toBe(true);
|
||||
});
|
||||
|
||||
// Miguel's P1: a percentage roll from the CLI must NOT be able to enrol a
|
||||
// browser profile that opted out. The two surfaces have independent
|
||||
// opt-outs, and CLI telemetry being on says nothing about this profile.
|
||||
describe("precedence against Studio's own opt-out", () => {
|
||||
beforeEach(() => {
|
||||
localStorage.setItem(OPT_OUT_KEY, "1");
|
||||
});
|
||||
|
||||
it("refuses a CLI COHORT enrolment when this profile opted out", () => {
|
||||
window.__HF_CLI_CANARY_DECISIONS = { "off-everywhere": cohort(true) };
|
||||
expect(resolveCanary("off-everywhere")).toEqual({
|
||||
enabled: false,
|
||||
reason: "telemetry_opt_out",
|
||||
});
|
||||
});
|
||||
|
||||
it("honours a CLI FORCED enrolment even when this profile opted out", () => {
|
||||
// An explicit HF_CANARY_* override is a deliberate operator choice —
|
||||
// the documented escalation channel, same as a local URL override.
|
||||
window.__HF_CLI_CANARY_DECISIONS = { "off-everywhere": forced(true) };
|
||||
expect(resolveCanary("off-everywhere")).toEqual({ enabled: true, reason: "forced_on" });
|
||||
});
|
||||
|
||||
it("honours a CLI forced-OFF when this profile opted out", () => {
|
||||
window.__HF_CLI_CANARY_DECISIONS = { "on-everywhere": forced(false) };
|
||||
expect(resolveCanary("on-everywhere")).toEqual({ enabled: false, reason: "forced_off" });
|
||||
});
|
||||
|
||||
it("still refuses cohort enrolment with no CLI decision at all", () => {
|
||||
expect(resolveCanary("on-everywhere").reason).toBe("telemetry_opt_out");
|
||||
});
|
||||
});
|
||||
});
|
||||
|
||||
@@ -53,8 +53,12 @@ export function canaryParamName(name: string): string {
|
||||
declare global {
|
||||
interface Window {
|
||||
__HF_CLI_BUCKET_SEED?: string;
|
||||
/** Resolved on/off per canary from the launching CLI. Authoritative. */
|
||||
__HF_CLI_CANARY_DECISIONS?: Record<string, boolean>;
|
||||
/**
|
||||
* Resolved per-canary outcome from the launching CLI. `forced` marks an
|
||||
* explicit `HF_CANARY_*` override as opposed to a percentage roll — the
|
||||
* two get different precedence against Studio's own opt-out.
|
||||
*/
|
||||
__HF_CLI_CANARY_DECISIONS?: Record<string, { enabled?: boolean; forced?: boolean }>;
|
||||
}
|
||||
}
|
||||
|
||||
@@ -74,11 +78,12 @@ declare global {
|
||||
* In all three the CLI has already decided, and one render spanning both
|
||||
* surfaces must not run half-enrolled.
|
||||
*/
|
||||
function cliDecision(name: string): boolean | undefined {
|
||||
function cliDecision(name: string): { enabled: boolean; forced: boolean } | undefined {
|
||||
try {
|
||||
if (typeof window === "undefined") return undefined;
|
||||
const decision = window.__HF_CLI_CANARY_DECISIONS?.[name];
|
||||
return typeof decision === "boolean" ? decision : undefined;
|
||||
if (typeof decision?.enabled !== "boolean") return undefined;
|
||||
return { enabled: decision.enabled, forced: decision.forced === true };
|
||||
} catch {
|
||||
return undefined;
|
||||
}
|
||||
@@ -171,27 +176,44 @@ export function __resetStudioCanaryCacheForTests(): void {
|
||||
}
|
||||
|
||||
/** The uncached decision. Split out so `resolveCanary` is purely the memo. */
|
||||
const forcedOutcome = (enabled: boolean): CanaryDecision => ({
|
||||
enabled,
|
||||
reason: enabled ? "forced_on" : "forced_off",
|
||||
});
|
||||
|
||||
const cohortOutcome = (enabled: boolean): CanaryDecision => ({
|
||||
enabled,
|
||||
reason: enabled ? "in_cohort" : "out_of_cohort",
|
||||
});
|
||||
|
||||
/**
|
||||
* Precedence, highest first:
|
||||
*
|
||||
* 1. An explicit `HF_CANARY_*` from the launching CLI. A deliberate operator
|
||||
* choice — the documented escalation channel (dogfooding, bisect,
|
||||
* panic-off) — so it wins outright, including over this profile's
|
||||
* opt-out, exactly as a local URL override does.
|
||||
* 2. A local URL / sessionStorage override, same reasoning.
|
||||
* 3. This profile's telemetry opt-out. Checked BEFORE any percentage-derived
|
||||
* CLI decision: the two surfaces have independent opt-outs, and CLI
|
||||
* telemetry being on says nothing about whether THIS browser profile
|
||||
* agreed to be measured. A cohort roll must never enrol an opted-out
|
||||
* profile.
|
||||
* 4. The CLI's percentage decision — authoritative, since it already applied
|
||||
* the shared seed and re-deriving is what let the surfaces disagree.
|
||||
* 5. Local evaluation (standalone Studio, or a canary the CLI didn't publish).
|
||||
*/
|
||||
function decideStudioCanary(name: string): CanaryDecision {
|
||||
const definition = findCanary(name);
|
||||
if (!definition) return { enabled: false, reason: "out_of_cohort" };
|
||||
|
||||
// The launching CLI's decision, when there is one, is the whole answer:
|
||||
// it already applied telemetry opt-out, HF_CANARY_* overrides and the
|
||||
// percentage against the shared seed. Re-deriving here is what let the two
|
||||
// surfaces disagree on the same render.
|
||||
const fromCli = cliDecision(definition.name);
|
||||
if (fromCli !== undefined) {
|
||||
return { enabled: fromCli, reason: fromCli ? "forced_on" : "forced_off" };
|
||||
}
|
||||
if (fromCli?.forced) return forcedOutcome(fromCli.enabled);
|
||||
|
||||
const override = readOverride(definition.name);
|
||||
// Standalone Studio. Opting out of telemetry opts you out of canaries —
|
||||
// same rule as the CLI (see packages/cli/src/telemetry/canary.ts). A
|
||||
// profile that sends nothing can't be compared against anyone, so
|
||||
// enrolling it changes that user's code path for no signal. An explicit
|
||||
// override still wins: a deliberate local choice, not silent enrolment.
|
||||
if (override === undefined && isOptedOut()) {
|
||||
return { enabled: false, reason: "telemetry_opt_out" };
|
||||
if (override === undefined) {
|
||||
if (isOptedOut()) return { enabled: false, reason: "telemetry_opt_out" };
|
||||
if (fromCli !== undefined) return cohortOutcome(fromCli.enabled);
|
||||
}
|
||||
|
||||
return evaluateCanary({
|
||||
|
||||
@@ -0,0 +1,74 @@
|
||||
// @vitest-environment happy-dom
|
||||
import { afterEach, beforeEach, describe, expect, it, vi } from "vitest";
|
||||
|
||||
// The `studio:*` path predates telemetry/config.ts and shipped its own
|
||||
// opt-out key and its own send loop, so it sat outside both contracts the
|
||||
// canary work established: the documented opt-out did not silence it, and its
|
||||
// events carried no cohort assignment. These pin both.
|
||||
|
||||
const DOCUMENTED_OPT_OUT = "hyperframes-studio:telemetryDisabled";
|
||||
const LEGACY_OPT_OUT = "hf-studio-telemetry-opt-out";
|
||||
|
||||
vi.mock("../telemetry/canary", () => ({
|
||||
canaryEventProperties: () => ({ "$feature/canary-test-one": "true" }),
|
||||
}));
|
||||
|
||||
describe("studioTelemetry — shared opt-out and canary properties", () => {
|
||||
let trackStudioEvent: typeof import("./studioTelemetry").trackStudioEvent;
|
||||
let fetchMock: ReturnType<typeof vi.fn>;
|
||||
|
||||
beforeEach(async () => {
|
||||
localStorage.clear();
|
||||
vi.resetModules();
|
||||
vi.useFakeTimers();
|
||||
fetchMock = vi.fn(() => Promise.resolve({ ok: true } as Response));
|
||||
vi.stubGlobal("fetch", fetchMock);
|
||||
({ trackStudioEvent } = await import("./studioTelemetry"));
|
||||
});
|
||||
|
||||
afterEach(() => {
|
||||
vi.useRealTimers();
|
||||
vi.unstubAllGlobals();
|
||||
});
|
||||
|
||||
/** Drain the queue and return the events the batch would have sent. */
|
||||
async function sentEvents(): Promise<Array<Record<string, unknown>>> {
|
||||
await vi.runOnlyPendingTimersAsync();
|
||||
if (fetchMock.mock.calls.length === 0) return [];
|
||||
const body = fetchMock.mock.calls[0]?.[1] as { body?: string } | undefined;
|
||||
const parsed = JSON.parse(body?.body ?? "{}") as { batch?: Array<Record<string, unknown>> };
|
||||
return parsed.batch ?? [];
|
||||
}
|
||||
|
||||
it("honours the documented opt-out key", async () => {
|
||||
// Previously only the legacy key was checked, so a user who opted out the
|
||||
// documented way kept emitting every `studio:*` event.
|
||||
localStorage.setItem(DOCUMENTED_OPT_OUT, "1");
|
||||
trackStudioEvent("thing_happened");
|
||||
expect(await sentEvents()).toHaveLength(0);
|
||||
});
|
||||
|
||||
it("still honours the legacy opt-out key", async () => {
|
||||
// Anyone already opted out this way must not be quietly re-enabled.
|
||||
localStorage.setItem(LEGACY_OPT_OUT, "1");
|
||||
trackStudioEvent("thing_happened");
|
||||
expect(await sentEvents()).toHaveLength(0);
|
||||
});
|
||||
|
||||
it("attaches canary assignments to every event", async () => {
|
||||
trackStudioEvent("thing_happened");
|
||||
const events = await sentEvents();
|
||||
expect(events).toHaveLength(1);
|
||||
expect(events[0]?.["properties"]).toMatchObject({
|
||||
"$feature/canary-test-one": "true",
|
||||
});
|
||||
});
|
||||
|
||||
it("lets an explicit property win over the canary mixin", async () => {
|
||||
trackStudioEvent("thing_happened", { "$feature/canary-test-one": "false" });
|
||||
const events = await sentEvents();
|
||||
expect(events[0]?.["properties"]).toMatchObject({
|
||||
"$feature/canary-test-one": "false",
|
||||
});
|
||||
});
|
||||
});
|
||||
@@ -1,4 +1,6 @@
|
||||
import { resolveStudioDistinctId } from "../telemetry/distinctId";
|
||||
import { isOptedOut } from "../telemetry/config";
|
||||
import { canaryEventProperties } from "../telemetry/canary";
|
||||
|
||||
// PostHog public ingest key — write-only, safe to ship in the client bundle
|
||||
const POSTHOG_API_KEY = "phc_zjjbX0PnWxERXrMHhkEJWj9A9BhGVLRReICgsfTMmpx";
|
||||
@@ -26,7 +28,17 @@ function getDistinctId(): string {
|
||||
return resolveStudioDistinctId();
|
||||
}
|
||||
|
||||
/**
|
||||
* Honours BOTH opt-out keys.
|
||||
*
|
||||
* This path predates telemetry/config.ts and shipped its own key, so the
|
||||
* documented `hyperframes-studio:telemetryDisabled` was silently ignored here
|
||||
* — `studio:*` events kept flowing for anyone who opted out the documented
|
||||
* way. The legacy key stays honoured so nobody who already opted out gets
|
||||
* quietly re-enabled by this fix.
|
||||
*/
|
||||
function isEnabled(): boolean {
|
||||
if (isOptedOut()) return false;
|
||||
try {
|
||||
return localStorage.getItem("hf-studio-telemetry-opt-out") !== "1";
|
||||
} catch {
|
||||
@@ -56,7 +68,10 @@ export function trackStudioEvent(event: string, properties: EventProperties = {}
|
||||
|
||||
queue.push({
|
||||
event: `studio:${event}`,
|
||||
properties: { ...getSessionProperties(), ...properties },
|
||||
// Canary assignments on every event, matching the CLI and the newer
|
||||
// studio client — "every telemetry event carries the assignment" has to
|
||||
// include this path or a cohort breakdown silently omits `studio:*`.
|
||||
properties: { ...getSessionProperties(), ...canaryEventProperties(), ...properties },
|
||||
timestamp: new Date().toISOString(),
|
||||
});
|
||||
|
||||
|
||||
Reference in New Issue
Block a user