mirror of
https://github.com/heygen-com/hyperframes.git
synced 2026-09-11 14:50:02 +00:00
fix(cli,studio): close the four R3 blocking gaps
P1 — SPA route bypassed the DNS-rebinding guard. Guarding only
/api/telemetry-identity left the catch-all as an open side door: a
rebound origin could fetch `/` and read __HF_CLI_DISTINCT_ID and
__HF_CLI_BUCKET_SEED straight out of the returned HTML. The SPA response
now applies the same isLoopbackHost() check; an untrusted Host still gets
a working Studio, just with no identity, seed or decisions injected.
Route-level regression added.
P1 — a CLI cohort roll could override Studio's own opt-out.
decideStudioCanary() adopted the injected decision before checking
isOptedOut(), so CLI-telemetry-on plus Studio-opted-out still enrolled
Studio. A bare boolean could not express the difference between a
deliberate override and an ordinary cohort roll, so the injected map now
carries provenance ({ enabled, forced }). Forced wins outright — it is
the documented escalation channel and must behave the same on both
surfaces — while a percentage roll now loses to this profile's opt-out.
Full interaction matrix tested.
P1 — the legacy studio:* path sat outside both contracts.
utils/studioTelemetry.ts shipped its own opt-out key and its own send
loop, so the documented hyperframes-studio:telemetryDisabled did not
silence it and its events carried no cohort assignment. It now honours
both keys (the legacy one stays, so nobody already opted out is quietly
re-enabled) and mixes in canaryEventProperties(), making "every
telemetry event carries the assignment" actually true.
P2 — partial salvage could drop a tripped breaker.
salvageInstallState() discarded the whole record when markerAt and
bucketSeed were both unusable, taking deParallelRouterTrialFired with it
and re-enrolling a machine whose router already failed. All three fields
are now independently salvageable.
Docs: canary-rollouts.mdx said "disabling telemetry disables the
reporting, not the enrolment" — exactly backwards since the opt-out gate
landed. Corrected; checked for other copies, none.
Tests: 13 new (4 opt-out precedence, 4 legacy-path opt-out and canary
props, 3 route-level host guard, 2 breaker salvage). Fault injection:
each of the four fixes reverted independently fails its own tests
(2 CLI + 1 Studio + 2 Studio).
Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
This commit is contained in:
co-authored by
Claude Opus 5
parent
54534d53c2
commit
f81ab0162e
@@ -80,7 +80,9 @@ tooling, so whoever operates the telemetry backend can split any metric by
|
|||||||
cohort with **nothing configured server-side** — while the decision itself
|
cohort with **nothing configured server-side** — while the decision itself
|
||||||
still happens locally and offline, which the render path requires.
|
still happens locally and offline, which the render path requires.
|
||||||
(Assignments ride the same anonymous, opt-out telemetry pipeline as every
|
(Assignments ride the same anonymous, opt-out telemetry pipeline as every
|
||||||
other event; disabling telemetry disables the reporting, not the enrolment.)
|
other event — and disabling telemetry disables the **enrolment**, not just the
|
||||||
|
reporting: an opted-out install is never bucketed at all. See the note at the
|
||||||
|
top of this page.)
|
||||||
|
|
||||||
Two details worth knowing:
|
Two details worth knowing:
|
||||||
|
|
||||||
|
|||||||
@@ -57,3 +57,57 @@ describe("createStudioServer autoProxy plumbing", () => {
|
|||||||
expect(server.adapter.autoProxy).toBe(true);
|
expect(server.adapter.autoProxy).toBe(true);
|
||||||
});
|
});
|
||||||
});
|
});
|
||||||
|
|
||||||
|
describe("host guarding on identity-bearing responses", () => {
|
||||||
|
const dirs: string[] = [];
|
||||||
|
let server: StudioServer | undefined;
|
||||||
|
|
||||||
|
function tmpProject(): string {
|
||||||
|
const dir = mkdtempSync(join(tmpdir(), "hf-studio-host-test-"));
|
||||||
|
dirs.push(dir);
|
||||||
|
return dir;
|
||||||
|
}
|
||||||
|
|
||||||
|
afterEach(() => {
|
||||||
|
server?.watcher.close();
|
||||||
|
server = undefined;
|
||||||
|
for (const dir of dirs.splice(0)) rmSync(dir, { recursive: true, force: true });
|
||||||
|
});
|
||||||
|
|
||||||
|
// A rebound origin can point its own hostname at 127.0.0.1 and read
|
||||||
|
// responses as same-origin. Guarding only /api/telemetry-identity left the
|
||||||
|
// SPA route as an open side door: fetching `/` returned the same distinct
|
||||||
|
// id and bucket seed inline in the HTML.
|
||||||
|
it("omits identity injection from the SPA response for a hostile Host", async () => {
|
||||||
|
server = createStudioServer({ projectDir: tmpProject() });
|
||||||
|
const res = await server.app.request("/", { headers: { host: "evil.example.com" } });
|
||||||
|
const html = await res.text();
|
||||||
|
expect(html).not.toContain("__HF_CLI_DISTINCT_ID");
|
||||||
|
expect(html).not.toContain("__HF_CLI_BUCKET_SEED");
|
||||||
|
expect(html).not.toContain("__HF_CLI_CANARY_DECISIONS");
|
||||||
|
// Studio still loads — only the identity block is withheld. (The env
|
||||||
|
// script is empty here: it only emits with VITE_STUDIO_* vars set.)
|
||||||
|
expect(res.status).toBe(200);
|
||||||
|
expect(html).toContain("<head>");
|
||||||
|
});
|
||||||
|
|
||||||
|
it("refuses the identity endpoint for a hostile Host", async () => {
|
||||||
|
server = createStudioServer({ projectDir: tmpProject() });
|
||||||
|
const res = await server.app.request("/api/telemetry-identity", {
|
||||||
|
headers: { host: "evil.example.com" },
|
||||||
|
});
|
||||||
|
expect(res.status).toBe(403);
|
||||||
|
expect(await res.text()).not.toContain('distinctId":"');
|
||||||
|
});
|
||||||
|
|
||||||
|
it("serves the identity endpoint on a loopback Host", async () => {
|
||||||
|
server = createStudioServer({ projectDir: tmpProject() });
|
||||||
|
const res = await server.app.request("/api/telemetry-identity", {
|
||||||
|
headers: { host: "127.0.0.1:5173" },
|
||||||
|
});
|
||||||
|
expect(res.status).toBe(200);
|
||||||
|
// The seed is no longer served here at all — Studio gets decisions
|
||||||
|
// injected instead, so nothing needs it over HTTP.
|
||||||
|
expect(Object.keys((await res.json()) as object)).toEqual(["distinctId"]);
|
||||||
|
});
|
||||||
|
});
|
||||||
|
|||||||
@@ -814,7 +814,17 @@ export function createStudioServer(options: StudioServerOptions): StudioServer {
|
|||||||
// Inject before the studio bundle runs. Identity script first (see
|
// Inject before the studio bundle runs. Identity script first (see
|
||||||
// buildStudioHeadScripts) so the CLI distinct id is on `window` by the time
|
// buildStudioHeadScripts) so the CLI distinct id is on `window` by the time
|
||||||
// telemetry init reads it.
|
// telemetry init reads it.
|
||||||
const headScript = buildStudioHeadScripts(buildRuntimeEnvScript());
|
//
|
||||||
|
// Host-guarded for the same reason /api/telemetry-identity is, and it has
|
||||||
|
// to be checked HERE too: guarding only the endpoint leaves this route as
|
||||||
|
// an open side door, since a rebound origin can simply fetch `/` and read
|
||||||
|
// the same distinct id and seed out of the returned HTML. Untrusted Host
|
||||||
|
// still gets a working Studio — it just gets the env script alone, with no
|
||||||
|
// identity, no seed, and no canary decisions.
|
||||||
|
const trustedHost = isLoopbackHost(c.req.header("host"));
|
||||||
|
const headScript = trustedHost
|
||||||
|
? buildStudioHeadScripts(buildRuntimeEnvScript())
|
||||||
|
: buildRuntimeEnvScript();
|
||||||
if (headScript) {
|
if (headScript) {
|
||||||
html = html.replace("<head>", `<head>${headScript}`);
|
html = html.replace("<head>", `<head>${headScript}`);
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -8,7 +8,7 @@ const shouldTrack = vi.fn();
|
|||||||
const readConfig = vi.fn();
|
const readConfig = vi.fn();
|
||||||
// Pinned rather than using the real registry, so these string assertions
|
// Pinned rather than using the real registry, so these string assertions
|
||||||
// don't move every time a canary is added, ramped, or retired.
|
// don't move every time a canary is added, ramped, or retired.
|
||||||
const canaryDecisions = vi.fn<() => Record<string, boolean>>();
|
const canaryDecisions = vi.fn<() => Record<string, { enabled: boolean; forced: boolean }>>();
|
||||||
|
|
||||||
vi.mock("../telemetry/client.js", () => ({
|
vi.mock("../telemetry/client.js", () => ({
|
||||||
shouldTrack: (...args: unknown[]) => shouldTrack(...args),
|
shouldTrack: (...args: unknown[]) => shouldTrack(...args),
|
||||||
@@ -99,10 +99,11 @@ describe("buildCliIdentityScript", () => {
|
|||||||
// stops Studio evaluating independently and enrolling anyway.
|
// stops Studio evaluating independently and enrolling anyway.
|
||||||
it("still publishes canary decisions when telemetry is off, but no identity", () => {
|
it("still publishes canary decisions when telemetry is off, but no identity", () => {
|
||||||
shouldTrack.mockReturnValue(false);
|
shouldTrack.mockReturnValue(false);
|
||||||
canaryDecisions.mockReturnValue({ "de-parallel-router": false });
|
canaryDecisions.mockReturnValue({ "de-parallel-router": { enabled: false, forced: false } });
|
||||||
const script = buildCliIdentityScript();
|
const script = buildCliIdentityScript();
|
||||||
expect(script).toBe(
|
expect(script).toBe(
|
||||||
'<script>window.__HF_CLI_CANARY_DECISIONS={"de-parallel-router":false};</script>',
|
"<script>window.__HF_CLI_CANARY_DECISIONS=" +
|
||||||
|
'{"de-parallel-router":{"enabled":false,"forced":false}};</script>',
|
||||||
);
|
);
|
||||||
expect(script).not.toContain("__HF_CLI_DISTINCT_ID");
|
expect(script).not.toContain("__HF_CLI_DISTINCT_ID");
|
||||||
expect(script).not.toContain("__HF_CLI_BUCKET_SEED");
|
expect(script).not.toContain("__HF_CLI_BUCKET_SEED");
|
||||||
@@ -111,17 +112,20 @@ describe("buildCliIdentityScript", () => {
|
|||||||
it("publishes decisions alongside the identity when telemetry is on", () => {
|
it("publishes decisions alongside the identity when telemetry is on", () => {
|
||||||
shouldTrack.mockReturnValue(true);
|
shouldTrack.mockReturnValue(true);
|
||||||
readConfig.mockReturnValue({ anonymousId: "machine-uuid", bucketSeed: "seed-uuid" });
|
readConfig.mockReturnValue({ anonymousId: "machine-uuid", bucketSeed: "seed-uuid" });
|
||||||
canaryDecisions.mockReturnValue({ "de-parallel-router": true });
|
canaryDecisions.mockReturnValue({ "de-parallel-router": { enabled: true, forced: true } });
|
||||||
expect(buildCliIdentityScript()).toBe(
|
expect(buildCliIdentityScript()).toBe(
|
||||||
'<script>window.__HF_CLI_DISTINCT_ID="machine-uuid";' +
|
'<script>window.__HF_CLI_DISTINCT_ID="machine-uuid";' +
|
||||||
'window.__HF_CLI_BUCKET_SEED="seed-uuid";' +
|
'window.__HF_CLI_BUCKET_SEED="seed-uuid";' +
|
||||||
'window.__HF_CLI_CANARY_DECISIONS={"de-parallel-router":true};</script>',
|
"window.__HF_CLI_CANARY_DECISIONS=" +
|
||||||
|
'{"de-parallel-router":{"enabled":true,"forced":true}};</script>',
|
||||||
);
|
);
|
||||||
});
|
});
|
||||||
|
|
||||||
it("escapes a canary name that tries to close the script tag", () => {
|
it("escapes a canary name that tries to close the script tag", () => {
|
||||||
shouldTrack.mockReturnValue(false);
|
shouldTrack.mockReturnValue(false);
|
||||||
canaryDecisions.mockReturnValue({ "</script><script>alert(1)": true });
|
canaryDecisions.mockReturnValue({
|
||||||
|
"</script><script>alert(1)": { enabled: true, forced: false },
|
||||||
|
});
|
||||||
const script = buildCliIdentityScript();
|
const script = buildCliIdentityScript();
|
||||||
expect(script).not.toContain("</script><script>alert(1)");
|
expect(script).not.toContain("</script><script>alert(1)");
|
||||||
expect(script).toContain("__HF_CLI_CANARY_DECISIONS");
|
expect(script).toContain("__HF_CLI_CANARY_DECISIONS");
|
||||||
|
|||||||
@@ -18,7 +18,7 @@
|
|||||||
|
|
||||||
import { readConfig } from "../telemetry/config.js";
|
import { readConfig } from "../telemetry/config.js";
|
||||||
import { shouldTrack as telemetryShouldTrack } from "../telemetry/client.js";
|
import { shouldTrack as telemetryShouldTrack } from "../telemetry/client.js";
|
||||||
import { canaryDecisionsForStudio } from "../telemetry/canary.js";
|
import { canaryDecisionsForStudio, type CliCanaryDecision } from "../telemetry/canary.js";
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* The CLI's anonymous distinct id to hand to Studio, or null when CLI telemetry
|
* The CLI's anonymous distinct id to hand to Studio, or null when CLI telemetry
|
||||||
@@ -102,7 +102,7 @@ function encodeInlineScriptJson(value: unknown): string {
|
|||||||
* The CLI's resolved canary decisions for a launched Studio, or null when
|
* The CLI's resolved canary decisions for a launched Studio, or null when
|
||||||
* there are none to publish. Fail-silent, like everything else here.
|
* there are none to publish. Fail-silent, like everything else here.
|
||||||
*/
|
*/
|
||||||
function resolveCliCanaryDecisions(): Record<string, boolean> | null {
|
function resolveCliCanaryDecisions(): Record<string, CliCanaryDecision> | null {
|
||||||
try {
|
try {
|
||||||
const decisions = canaryDecisionsForStudio();
|
const decisions = canaryDecisionsForStudio();
|
||||||
return Object.keys(decisions).length > 0 ? decisions : null;
|
return Object.keys(decisions).length > 0 ? decisions : null;
|
||||||
|
|||||||
@@ -118,6 +118,15 @@ export function isCanaryEnabled(name: string): boolean {
|
|||||||
return resolveCanary(name).enabled;
|
return resolveCanary(name).enabled;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* One canary's outcome as handed to Studio: the answer, plus whether it came
|
||||||
|
* from an explicit override rather than a percentage roll.
|
||||||
|
*/
|
||||||
|
export interface CliCanaryDecision {
|
||||||
|
enabled: boolean;
|
||||||
|
forced: boolean;
|
||||||
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Every registered canary's resolved on/off for this process, for handing to
|
* Every registered canary's resolved on/off for this process, for handing to
|
||||||
* a CLI-launched Studio.
|
* a CLI-launched Studio.
|
||||||
@@ -138,9 +147,20 @@ export function isCanaryEnabled(name: string): boolean {
|
|||||||
* impossible: one evaluation, two surfaces. It is also strictly less to
|
* impossible: one evaluation, two surfaces. It is also strictly less to
|
||||||
* expose — booleans about features, not the seed the buckets derive from.
|
* expose — booleans about features, not the seed the buckets derive from.
|
||||||
*/
|
*/
|
||||||
export function canaryDecisionsForStudio(): Record<string, boolean> {
|
export function canaryDecisionsForStudio(): Record<string, CliCanaryDecision> {
|
||||||
const out: Record<string, boolean> = {};
|
const out: Record<string, CliCanaryDecision> = {};
|
||||||
for (const canary of CANARIES) out[canary.name] = resolveCanary(canary.name).enabled;
|
for (const canary of CANARIES) {
|
||||||
|
const { enabled, reason } = resolveCanary(canary.name);
|
||||||
|
out[canary.name] = {
|
||||||
|
enabled,
|
||||||
|
// Provenance, not decoration. Studio has its own telemetry opt-out that
|
||||||
|
// the CLI cannot see, and it must be able to refuse ordinary cohort
|
||||||
|
// enrolment while still honouring a deliberate `HF_CANARY_*` override.
|
||||||
|
// A bare boolean cannot express that difference, so Studio would have
|
||||||
|
// had to either ignore its own opt-out or drop the override.
|
||||||
|
forced: reason === "forced_on" || reason === "forced_off",
|
||||||
|
};
|
||||||
|
}
|
||||||
return out;
|
return out;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -465,3 +465,32 @@ describe("seed backfill write failure is surfaced", () => {
|
|||||||
});
|
});
|
||||||
});
|
});
|
||||||
});
|
});
|
||||||
|
|
||||||
|
describe("a tripped breaker survives even total marker+seed corruption", () => {
|
||||||
|
let readConfig: typeof import("./config.js").readConfig;
|
||||||
|
let STATE_PATH: typeof import("./config.js").STATE_PATH;
|
||||||
|
|
||||||
|
beforeEach(async () => {
|
||||||
|
fsState.files.clear();
|
||||||
|
vi.resetModules();
|
||||||
|
({ readConfig, STATE_PATH } = await import("./config.js"));
|
||||||
|
});
|
||||||
|
|
||||||
|
// Dropping the latch re-enrols a machine into an experimental path that
|
||||||
|
// already FAILED on it — the one outcome install-state exists to prevent.
|
||||||
|
// So the tripped bit is salvageable independently of the other two fields.
|
||||||
|
it("keeps deParallelRouterTrialFired when markerAt and bucketSeed are both unusable", () => {
|
||||||
|
fsState.files.set(
|
||||||
|
STATE_PATH,
|
||||||
|
JSON.stringify({ markerAt: 42, bucketSeed: "", deParallelRouterTrialFired: true }),
|
||||||
|
);
|
||||||
|
const config = readConfig();
|
||||||
|
expect(config.deParallelRouterTrialFired).toBe(true);
|
||||||
|
expect(config.predecessorFound).toBe(true);
|
||||||
|
});
|
||||||
|
|
||||||
|
it("still reports corrupt when none of the three fields survive", () => {
|
||||||
|
fsState.files.set(STATE_PATH, JSON.stringify({ markerAt: 42, bucketSeed: "" }));
|
||||||
|
expect(readConfig().stateFileCorrupt).toBe(true);
|
||||||
|
});
|
||||||
|
});
|
||||||
|
|||||||
@@ -89,13 +89,15 @@ type InstallStateMiss = "absent" | "corrupt";
|
|||||||
function salvageInstallState(parsed: Partial<InstallState>): InstallState | InstallStateMiss {
|
function salvageInstallState(parsed: Partial<InstallState>): InstallState | InstallStateMiss {
|
||||||
const markerAt = typeof parsed.markerAt === "string" ? parsed.markerAt : undefined;
|
const markerAt = typeof parsed.markerAt === "string" ? parsed.markerAt : undefined;
|
||||||
const bucketSeed = parseNonEmptyString(parsed.bucketSeed);
|
const bucketSeed = parseNonEmptyString(parsed.bucketSeed);
|
||||||
// Nothing salvageable in the record at all — treat as corrupt rather than
|
const fired = parsed.deParallelRouterTrialFired === true ? true : undefined;
|
||||||
// inventing a marker, so the miss is still counted as "we knew this machine"
|
// Each of the three is independently salvageable, and the tripped breaker
|
||||||
// instead of masquerading as a fresh install.
|
// most of all: dropping it re-enrols a machine into an experimental path
|
||||||
if (markerAt === undefined && bucketSeed === undefined) return "corrupt";
|
// that already FAILED there, which is the one outcome this file exists to
|
||||||
|
// prevent. `markerAt` is only a timestamp and can be restamped.
|
||||||
|
if (markerAt === undefined && bucketSeed === undefined && fired === undefined) return "corrupt";
|
||||||
return {
|
return {
|
||||||
markerAt: markerAt ?? new Date().toISOString(),
|
markerAt: markerAt ?? new Date().toISOString(),
|
||||||
deParallelRouterTrialFired: parsed.deParallelRouterTrialFired === true ? true : undefined,
|
deParallelRouterTrialFired: fired,
|
||||||
bucketSeed,
|
bucketSeed,
|
||||||
};
|
};
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -228,6 +228,8 @@ describe("telemetry opt-out is canary opt-out", () => {
|
|||||||
|
|
||||||
describe("CLI-launched Studio adopts the CLI's decisions", () => {
|
describe("CLI-launched Studio adopts the CLI's decisions", () => {
|
||||||
const OPT_OUT_KEY = "hyperframes-studio:telemetryDisabled";
|
const OPT_OUT_KEY = "hyperframes-studio:telemetryDisabled";
|
||||||
|
const cohort = (enabled: boolean) => ({ enabled, forced: false });
|
||||||
|
const forced = (enabled: boolean) => ({ enabled, forced: true });
|
||||||
|
|
||||||
afterEach(() => {
|
afterEach(() => {
|
||||||
delete window.__HF_CLI_CANARY_DECISIONS;
|
delete window.__HF_CLI_CANARY_DECISIONS;
|
||||||
@@ -238,40 +240,72 @@ describe("CLI-launched Studio adopts the CLI's decisions", () => {
|
|||||||
// flag it cannot see — left to itself it would evaluate and could enrol.
|
// flag it cannot see — left to itself it would evaluate and could enrol.
|
||||||
it("stays off when the CLI opted out, even though Studio's own flag is unset", () => {
|
it("stays off when the CLI opted out, even though Studio's own flag is unset", () => {
|
||||||
expect(localStorage.getItem(OPT_OUT_KEY)).toBeNull();
|
expect(localStorage.getItem(OPT_OUT_KEY)).toBeNull();
|
||||||
window.__HF_CLI_CANARY_DECISIONS = { "on-everywhere": false };
|
window.__HF_CLI_CANARY_DECISIONS = { "on-everywhere": cohort(false) };
|
||||||
expect(resolveCanary("on-everywhere").enabled).toBe(false);
|
expect(resolveCanary("on-everywhere").enabled).toBe(false);
|
||||||
});
|
});
|
||||||
|
|
||||||
// HF_CANARY_* never crosses into the browser, so before this the CLI was
|
// HF_CANARY_* never crosses into the browser, so before this the CLI was
|
||||||
// forced on and Studio silently guessed from the percentage.
|
// forced on and Studio silently guessed from the percentage.
|
||||||
it("turns on when the CLI forced it on, with no URL param present", () => {
|
it("turns on when the CLI forced it on, with no URL param present", () => {
|
||||||
window.__HF_CLI_CANARY_DECISIONS = { "off-everywhere": true };
|
window.__HF_CLI_CANARY_DECISIONS = { "off-everywhere": forced(true) };
|
||||||
expect(resolveCanary("off-everywhere").enabled).toBe(true);
|
expect(resolveCanary("off-everywhere").enabled).toBe(true);
|
||||||
});
|
});
|
||||||
|
|
||||||
it("beats a contradicting URL override — one render must not run half-enrolled", () => {
|
it("beats a contradicting URL override — one render must not run half-enrolled", () => {
|
||||||
window.__HF_CLI_CANARY_DECISIONS = { "on-everywhere": false };
|
window.__HF_CLI_CANARY_DECISIONS = { "on-everywhere": forced(false) };
|
||||||
setSearch("?hf_canary_on_everywhere=on");
|
setSearch("?hf_canary_on_everywhere=on");
|
||||||
expect(resolveCanary("on-everywhere").enabled).toBe(false);
|
expect(resolveCanary("on-everywhere").enabled).toBe(false);
|
||||||
});
|
});
|
||||||
|
|
||||||
it("beats the seed-derived bucket", () => {
|
it("beats the seed-derived bucket", () => {
|
||||||
window.__HF_CLI_BUCKET_SEED = "5f1c9d2e-0000-4000-8000-aaaaaaaaaaaa";
|
window.__HF_CLI_BUCKET_SEED = "5f1c9d2e-0000-4000-8000-aaaaaaaaaaaa";
|
||||||
window.__HF_CLI_CANARY_DECISIONS = { "on-everywhere": false };
|
window.__HF_CLI_CANARY_DECISIONS = { "on-everywhere": cohort(false) };
|
||||||
expect(resolveCanary("on-everywhere").enabled).toBe(false);
|
expect(resolveCanary("on-everywhere").enabled).toBe(false);
|
||||||
});
|
});
|
||||||
|
|
||||||
it("falls back to local evaluation for a canary the CLI did not publish", () => {
|
it("falls back to local evaluation for a canary the CLI did not publish", () => {
|
||||||
window.__HF_CLI_CANARY_DECISIONS = { "off-everywhere": true };
|
window.__HF_CLI_CANARY_DECISIONS = { "off-everywhere": cohort(true) };
|
||||||
expect(resolveCanary("on-everywhere").enabled).toBe(true);
|
expect(resolveCanary("on-everywhere").enabled).toBe(true);
|
||||||
});
|
});
|
||||||
|
|
||||||
it("ignores a non-boolean value rather than trusting it", () => {
|
it("ignores a malformed entry rather than trusting it", () => {
|
||||||
window.__HF_CLI_CANARY_DECISIONS = { "on-everywhere": "false" } as unknown as Record<
|
window.__HF_CLI_CANARY_DECISIONS = {
|
||||||
string,
|
"on-everywhere": { enabled: "false" },
|
||||||
boolean
|
} as unknown as Record<string, { enabled?: boolean; forced?: boolean }>;
|
||||||
>;
|
|
||||||
// Falls through to local evaluation: on-everywhere is at 100%.
|
// Falls through to local evaluation: on-everywhere is at 100%.
|
||||||
expect(resolveCanary("on-everywhere").enabled).toBe(true);
|
expect(resolveCanary("on-everywhere").enabled).toBe(true);
|
||||||
});
|
});
|
||||||
|
|
||||||
|
// Miguel's P1: a percentage roll from the CLI must NOT be able to enrol a
|
||||||
|
// browser profile that opted out. The two surfaces have independent
|
||||||
|
// opt-outs, and CLI telemetry being on says nothing about this profile.
|
||||||
|
describe("precedence against Studio's own opt-out", () => {
|
||||||
|
beforeEach(() => {
|
||||||
|
localStorage.setItem(OPT_OUT_KEY, "1");
|
||||||
|
});
|
||||||
|
|
||||||
|
it("refuses a CLI COHORT enrolment when this profile opted out", () => {
|
||||||
|
window.__HF_CLI_CANARY_DECISIONS = { "off-everywhere": cohort(true) };
|
||||||
|
expect(resolveCanary("off-everywhere")).toEqual({
|
||||||
|
enabled: false,
|
||||||
|
reason: "telemetry_opt_out",
|
||||||
|
});
|
||||||
|
});
|
||||||
|
|
||||||
|
it("honours a CLI FORCED enrolment even when this profile opted out", () => {
|
||||||
|
// An explicit HF_CANARY_* override is a deliberate operator choice —
|
||||||
|
// the documented escalation channel, same as a local URL override.
|
||||||
|
window.__HF_CLI_CANARY_DECISIONS = { "off-everywhere": forced(true) };
|
||||||
|
expect(resolveCanary("off-everywhere")).toEqual({ enabled: true, reason: "forced_on" });
|
||||||
|
});
|
||||||
|
|
||||||
|
it("honours a CLI forced-OFF when this profile opted out", () => {
|
||||||
|
window.__HF_CLI_CANARY_DECISIONS = { "on-everywhere": forced(false) };
|
||||||
|
expect(resolveCanary("on-everywhere")).toEqual({ enabled: false, reason: "forced_off" });
|
||||||
|
});
|
||||||
|
|
||||||
|
it("still refuses cohort enrolment with no CLI decision at all", () => {
|
||||||
|
expect(resolveCanary("on-everywhere").reason).toBe("telemetry_opt_out");
|
||||||
|
});
|
||||||
|
});
|
||||||
});
|
});
|
||||||
|
|||||||
@@ -53,8 +53,12 @@ export function canaryParamName(name: string): string {
|
|||||||
declare global {
|
declare global {
|
||||||
interface Window {
|
interface Window {
|
||||||
__HF_CLI_BUCKET_SEED?: string;
|
__HF_CLI_BUCKET_SEED?: string;
|
||||||
/** Resolved on/off per canary from the launching CLI. Authoritative. */
|
/**
|
||||||
__HF_CLI_CANARY_DECISIONS?: Record<string, boolean>;
|
* Resolved per-canary outcome from the launching CLI. `forced` marks an
|
||||||
|
* explicit `HF_CANARY_*` override as opposed to a percentage roll — the
|
||||||
|
* two get different precedence against Studio's own opt-out.
|
||||||
|
*/
|
||||||
|
__HF_CLI_CANARY_DECISIONS?: Record<string, { enabled?: boolean; forced?: boolean }>;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -74,11 +78,12 @@ declare global {
|
|||||||
* In all three the CLI has already decided, and one render spanning both
|
* In all three the CLI has already decided, and one render spanning both
|
||||||
* surfaces must not run half-enrolled.
|
* surfaces must not run half-enrolled.
|
||||||
*/
|
*/
|
||||||
function cliDecision(name: string): boolean | undefined {
|
function cliDecision(name: string): { enabled: boolean; forced: boolean } | undefined {
|
||||||
try {
|
try {
|
||||||
if (typeof window === "undefined") return undefined;
|
if (typeof window === "undefined") return undefined;
|
||||||
const decision = window.__HF_CLI_CANARY_DECISIONS?.[name];
|
const decision = window.__HF_CLI_CANARY_DECISIONS?.[name];
|
||||||
return typeof decision === "boolean" ? decision : undefined;
|
if (typeof decision?.enabled !== "boolean") return undefined;
|
||||||
|
return { enabled: decision.enabled, forced: decision.forced === true };
|
||||||
} catch {
|
} catch {
|
||||||
return undefined;
|
return undefined;
|
||||||
}
|
}
|
||||||
@@ -171,27 +176,44 @@ export function __resetStudioCanaryCacheForTests(): void {
|
|||||||
}
|
}
|
||||||
|
|
||||||
/** The uncached decision. Split out so `resolveCanary` is purely the memo. */
|
/** The uncached decision. Split out so `resolveCanary` is purely the memo. */
|
||||||
|
const forcedOutcome = (enabled: boolean): CanaryDecision => ({
|
||||||
|
enabled,
|
||||||
|
reason: enabled ? "forced_on" : "forced_off",
|
||||||
|
});
|
||||||
|
|
||||||
|
const cohortOutcome = (enabled: boolean): CanaryDecision => ({
|
||||||
|
enabled,
|
||||||
|
reason: enabled ? "in_cohort" : "out_of_cohort",
|
||||||
|
});
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Precedence, highest first:
|
||||||
|
*
|
||||||
|
* 1. An explicit `HF_CANARY_*` from the launching CLI. A deliberate operator
|
||||||
|
* choice — the documented escalation channel (dogfooding, bisect,
|
||||||
|
* panic-off) — so it wins outright, including over this profile's
|
||||||
|
* opt-out, exactly as a local URL override does.
|
||||||
|
* 2. A local URL / sessionStorage override, same reasoning.
|
||||||
|
* 3. This profile's telemetry opt-out. Checked BEFORE any percentage-derived
|
||||||
|
* CLI decision: the two surfaces have independent opt-outs, and CLI
|
||||||
|
* telemetry being on says nothing about whether THIS browser profile
|
||||||
|
* agreed to be measured. A cohort roll must never enrol an opted-out
|
||||||
|
* profile.
|
||||||
|
* 4. The CLI's percentage decision — authoritative, since it already applied
|
||||||
|
* the shared seed and re-deriving is what let the surfaces disagree.
|
||||||
|
* 5. Local evaluation (standalone Studio, or a canary the CLI didn't publish).
|
||||||
|
*/
|
||||||
function decideStudioCanary(name: string): CanaryDecision {
|
function decideStudioCanary(name: string): CanaryDecision {
|
||||||
const definition = findCanary(name);
|
const definition = findCanary(name);
|
||||||
if (!definition) return { enabled: false, reason: "out_of_cohort" };
|
if (!definition) return { enabled: false, reason: "out_of_cohort" };
|
||||||
|
|
||||||
// The launching CLI's decision, when there is one, is the whole answer:
|
|
||||||
// it already applied telemetry opt-out, HF_CANARY_* overrides and the
|
|
||||||
// percentage against the shared seed. Re-deriving here is what let the two
|
|
||||||
// surfaces disagree on the same render.
|
|
||||||
const fromCli = cliDecision(definition.name);
|
const fromCli = cliDecision(definition.name);
|
||||||
if (fromCli !== undefined) {
|
if (fromCli?.forced) return forcedOutcome(fromCli.enabled);
|
||||||
return { enabled: fromCli, reason: fromCli ? "forced_on" : "forced_off" };
|
|
||||||
}
|
|
||||||
|
|
||||||
const override = readOverride(definition.name);
|
const override = readOverride(definition.name);
|
||||||
// Standalone Studio. Opting out of telemetry opts you out of canaries —
|
if (override === undefined) {
|
||||||
// same rule as the CLI (see packages/cli/src/telemetry/canary.ts). A
|
if (isOptedOut()) return { enabled: false, reason: "telemetry_opt_out" };
|
||||||
// profile that sends nothing can't be compared against anyone, so
|
if (fromCli !== undefined) return cohortOutcome(fromCli.enabled);
|
||||||
// enrolling it changes that user's code path for no signal. An explicit
|
|
||||||
// override still wins: a deliberate local choice, not silent enrolment.
|
|
||||||
if (override === undefined && isOptedOut()) {
|
|
||||||
return { enabled: false, reason: "telemetry_opt_out" };
|
|
||||||
}
|
}
|
||||||
|
|
||||||
return evaluateCanary({
|
return evaluateCanary({
|
||||||
|
|||||||
@@ -0,0 +1,74 @@
|
|||||||
|
// @vitest-environment happy-dom
|
||||||
|
import { afterEach, beforeEach, describe, expect, it, vi } from "vitest";
|
||||||
|
|
||||||
|
// The `studio:*` path predates telemetry/config.ts and shipped its own
|
||||||
|
// opt-out key and its own send loop, so it sat outside both contracts the
|
||||||
|
// canary work established: the documented opt-out did not silence it, and its
|
||||||
|
// events carried no cohort assignment. These pin both.
|
||||||
|
|
||||||
|
const DOCUMENTED_OPT_OUT = "hyperframes-studio:telemetryDisabled";
|
||||||
|
const LEGACY_OPT_OUT = "hf-studio-telemetry-opt-out";
|
||||||
|
|
||||||
|
vi.mock("../telemetry/canary", () => ({
|
||||||
|
canaryEventProperties: () => ({ "$feature/canary-test-one": "true" }),
|
||||||
|
}));
|
||||||
|
|
||||||
|
describe("studioTelemetry — shared opt-out and canary properties", () => {
|
||||||
|
let trackStudioEvent: typeof import("./studioTelemetry").trackStudioEvent;
|
||||||
|
let fetchMock: ReturnType<typeof vi.fn>;
|
||||||
|
|
||||||
|
beforeEach(async () => {
|
||||||
|
localStorage.clear();
|
||||||
|
vi.resetModules();
|
||||||
|
vi.useFakeTimers();
|
||||||
|
fetchMock = vi.fn(() => Promise.resolve({ ok: true } as Response));
|
||||||
|
vi.stubGlobal("fetch", fetchMock);
|
||||||
|
({ trackStudioEvent } = await import("./studioTelemetry"));
|
||||||
|
});
|
||||||
|
|
||||||
|
afterEach(() => {
|
||||||
|
vi.useRealTimers();
|
||||||
|
vi.unstubAllGlobals();
|
||||||
|
});
|
||||||
|
|
||||||
|
/** Drain the queue and return the events the batch would have sent. */
|
||||||
|
async function sentEvents(): Promise<Array<Record<string, unknown>>> {
|
||||||
|
await vi.runOnlyPendingTimersAsync();
|
||||||
|
if (fetchMock.mock.calls.length === 0) return [];
|
||||||
|
const body = fetchMock.mock.calls[0]?.[1] as { body?: string } | undefined;
|
||||||
|
const parsed = JSON.parse(body?.body ?? "{}") as { batch?: Array<Record<string, unknown>> };
|
||||||
|
return parsed.batch ?? [];
|
||||||
|
}
|
||||||
|
|
||||||
|
it("honours the documented opt-out key", async () => {
|
||||||
|
// Previously only the legacy key was checked, so a user who opted out the
|
||||||
|
// documented way kept emitting every `studio:*` event.
|
||||||
|
localStorage.setItem(DOCUMENTED_OPT_OUT, "1");
|
||||||
|
trackStudioEvent("thing_happened");
|
||||||
|
expect(await sentEvents()).toHaveLength(0);
|
||||||
|
});
|
||||||
|
|
||||||
|
it("still honours the legacy opt-out key", async () => {
|
||||||
|
// Anyone already opted out this way must not be quietly re-enabled.
|
||||||
|
localStorage.setItem(LEGACY_OPT_OUT, "1");
|
||||||
|
trackStudioEvent("thing_happened");
|
||||||
|
expect(await sentEvents()).toHaveLength(0);
|
||||||
|
});
|
||||||
|
|
||||||
|
it("attaches canary assignments to every event", async () => {
|
||||||
|
trackStudioEvent("thing_happened");
|
||||||
|
const events = await sentEvents();
|
||||||
|
expect(events).toHaveLength(1);
|
||||||
|
expect(events[0]?.["properties"]).toMatchObject({
|
||||||
|
"$feature/canary-test-one": "true",
|
||||||
|
});
|
||||||
|
});
|
||||||
|
|
||||||
|
it("lets an explicit property win over the canary mixin", async () => {
|
||||||
|
trackStudioEvent("thing_happened", { "$feature/canary-test-one": "false" });
|
||||||
|
const events = await sentEvents();
|
||||||
|
expect(events[0]?.["properties"]).toMatchObject({
|
||||||
|
"$feature/canary-test-one": "false",
|
||||||
|
});
|
||||||
|
});
|
||||||
|
});
|
||||||
@@ -1,4 +1,6 @@
|
|||||||
import { resolveStudioDistinctId } from "../telemetry/distinctId";
|
import { resolveStudioDistinctId } from "../telemetry/distinctId";
|
||||||
|
import { isOptedOut } from "../telemetry/config";
|
||||||
|
import { canaryEventProperties } from "../telemetry/canary";
|
||||||
|
|
||||||
// PostHog public ingest key — write-only, safe to ship in the client bundle
|
// PostHog public ingest key — write-only, safe to ship in the client bundle
|
||||||
const POSTHOG_API_KEY = "phc_zjjbX0PnWxERXrMHhkEJWj9A9BhGVLRReICgsfTMmpx";
|
const POSTHOG_API_KEY = "phc_zjjbX0PnWxERXrMHhkEJWj9A9BhGVLRReICgsfTMmpx";
|
||||||
@@ -26,7 +28,17 @@ function getDistinctId(): string {
|
|||||||
return resolveStudioDistinctId();
|
return resolveStudioDistinctId();
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Honours BOTH opt-out keys.
|
||||||
|
*
|
||||||
|
* This path predates telemetry/config.ts and shipped its own key, so the
|
||||||
|
* documented `hyperframes-studio:telemetryDisabled` was silently ignored here
|
||||||
|
* — `studio:*` events kept flowing for anyone who opted out the documented
|
||||||
|
* way. The legacy key stays honoured so nobody who already opted out gets
|
||||||
|
* quietly re-enabled by this fix.
|
||||||
|
*/
|
||||||
function isEnabled(): boolean {
|
function isEnabled(): boolean {
|
||||||
|
if (isOptedOut()) return false;
|
||||||
try {
|
try {
|
||||||
return localStorage.getItem("hf-studio-telemetry-opt-out") !== "1";
|
return localStorage.getItem("hf-studio-telemetry-opt-out") !== "1";
|
||||||
} catch {
|
} catch {
|
||||||
@@ -56,7 +68,10 @@ export function trackStudioEvent(event: string, properties: EventProperties = {}
|
|||||||
|
|
||||||
queue.push({
|
queue.push({
|
||||||
event: `studio:${event}`,
|
event: `studio:${event}`,
|
||||||
properties: { ...getSessionProperties(), ...properties },
|
// Canary assignments on every event, matching the CLI and the newer
|
||||||
|
// studio client — "every telemetry event carries the assignment" has to
|
||||||
|
// include this path or a cohort breakdown silently omits `studio:*`.
|
||||||
|
properties: { ...getSessionProperties(), ...canaryEventProperties(), ...properties },
|
||||||
timestamp: new Date().toISOString(),
|
timestamp: new Date().toISOString(),
|
||||||
});
|
});
|
||||||
|
|
||||||
|
|||||||
Reference in New Issue
Block a user