fix(studio): analytics send path and selector facts, not the paths and selectors (#8109)

* fix(studio): analytics send path and selector facts, not the paths and selectors

Both Studio transports now shape every event's values in one place before they leave the machine: a file path becomes its extension, kind, depth and a per-install salted hash; a selector becomes its kind and the same hash; any other value that reads as a path or selector is dropped and named in dropped_fields. Save failures send the error class, a fixed error key and the server's fixed refusal code instead of the error's words. Session fields can no longer be overwritten by a caller.

* fix(studio): save failures keep their dedupe and no selector leaves in shadow mismatches

The editor's failure dedupe keys on the error's own text again, locally. The resolver shadow's animation_not_found no longer repeats the animation id inside mismatches. Error keys and path kinds read from tables; the server's full refusal list is known.

* fix(studio): a paused save queue reports codes, not the save error's words

save_queue_paused spreads the save failure's fixed codes again; its handler lives outside the persistence hook.
This commit is contained in:
Miguel Ángel
2026-11-17 21:58:39 +00:00
committed by GitHub
parent 53a493edc8
commit d872853874
13 changed files with 385 additions and 39 deletions
@@ -27,10 +27,7 @@ export function CompositionBreadcrumb({ stack, onNavigate }: CompositionBreadcru
<button
type="button"
onClick={() => {
trackStudioEvent("navigation", {
action: "back",
target: stack[stack.length - 2]?.label,
});
trackStudioEvent("navigation", { action: "back", level: stack.length - 2 });
onNavigate(stack.length - 2);
}}
className="flex items-center gap-1 px-1.5 py-0.5 rounded-sm text-xs text-text-3 hover:text-text-0 hover:bg-hover active:scale-[0.98] transition-colors"
@@ -52,7 +49,7 @@ export function CompositionBreadcrumb({ stack, onNavigate }: CompositionBreadcru
<button
type="button"
onClick={() => {
trackStudioEvent("navigation", { action: "breadcrumb", target: level.label });
trackStudioEvent("navigation", { action: "breadcrumb", level: i });
onNavigate(i);
}}
className="text-xs text-text-3 hover:text-text-1 transition-colors"
@@ -455,7 +455,7 @@ describe("useDomEditCommits z-index reorder persistence", () => {
expect.objectContaining({
mutation_type: "z-reorder-unmatched",
file_path: "index.html",
error_message: expect.stringContaining("b"),
error_key: "unknown",
}),
);
} finally {
+2 -1
View File
@@ -3,6 +3,7 @@ import { saveProjectFilesWithHistory } from "../utils/studioFileHistory";
import {
StudioFileConflictError,
buildStudioSaveFailureProperties,
getStudioSaveErrorMessage,
heldFileWords,
trackStudioSaveFailure,
type StudioSaveDrainResult,
@@ -76,7 +77,7 @@ export function useEditorSave({
const fingerprint = JSON.stringify([
path,
errorName,
properties.error_message,
getStudioSaveErrorMessage(error),
properties.status_code,
]);
const previous = lastFailureReportRef.current;
@@ -10,7 +10,11 @@ import {
} from "../components/editor/manualEditsDom";
import { STUDIO_MOTION_PATH } from "../components/editor/studioMotion";
import { markScenesStale } from "../player/sceneSwap";
import { createDomEditSaveQueue, type DomEditSaveDrainResult } from "../utils/domEditSaveQueue";
import {
createDomEditSaveQueue,
type DomEditSaveDrainResult,
type DomEditSaveQueueOpenEvent,
} from "../utils/domEditSaveQueue";
import {
flushStudioPendingEdits,
hasStudioPendingEdits,
@@ -53,6 +57,18 @@ export interface UsePreviewPersistenceParams {
reloadPreview: () => void;
}
function saveQueuePaused(event: DomEditSaveQueueOpenEvent): string {
trackStudioEvent("save_queue_paused", {
source: "dom_edit",
...event.errorCodes,
status_code: event.statusCode,
consecutive_failures: event.consecutiveFailures,
});
return event.statusCode === 409
? "Save paused: this file changed elsewhere. Reload and review the latest version before reapplying your edit."
: "Auto-save is paused. Check your connection.";
}
function readIframeDocument(iframe: HTMLIFrameElement): Document | null {
try {
return iframe.contentDocument;
@@ -131,18 +147,9 @@ export function usePreviewPersistence({
if (!domEditSaveQueueRef.current) {
domEditSaveQueueRef.current = createDomEditSaveQueue({
onOpen: (event) => {
const message =
event.statusCode === 409
? "Save paused: this file changed elsewhere. Reload and review the latest version before reapplying your edit."
: "Auto-save is paused. Check your connection.";
const message = saveQueuePaused(event);
setDomEditSaveQueuePaused(message);
showToastRef.current(message, "error");
trackStudioEvent("save_queue_paused", {
source: "dom_edit",
error_message: event.errorMessage,
status_code: event.statusCode,
consecutive_failures: event.consecutiveFailures,
});
},
onReset: () => {
setDomEditSaveQueuePaused(null);
+4 -2
View File
@@ -9,6 +9,7 @@ import { browserTelemetryAllowed } from "./policy";
import { getBrowserSystemMeta } from "./system";
import { canaryEventProperties } from "./canary";
import { recordBreadcrumb } from "./breadcrumbs";
import { shapeEventProperties } from "./eventShape";
import { studioApiFetch } from "../utils/studioApiFetch";
// Write-only PostHog project key, safe to embed in client code.
@@ -41,7 +42,8 @@ export function trackEvent(event: string, properties: EventProperties = {}): voi
// Every studio event passes through here, so this is the one place that can
// build a repro trail without asking each call site to opt in.
recordBreadcrumb(event, properties);
const shaped = shapeEventProperties(event, properties);
recordBreadcrumb(event, shaped);
const sys = getBrowserSystemMeta();
eventQueue.push({
@@ -49,7 +51,7 @@ export function trackEvent(event: string, properties: EventProperties = {}): voi
// Canary assignments as `$feature/canary-<name>`, mirroring the CLI so a
// rollout spanning both surfaces reads as one flag in PostHog. Resolved
// after the shouldTrack guard, so opted-out users never pay for it.
properties: { ...properties, ...sys, ...canaryEventProperties() },
properties: { ...shaped, ...sys, ...canaryEventProperties() },
timestamp: new Date().toISOString(),
});
@@ -0,0 +1,110 @@
// @vitest-environment happy-dom
import { afterEach, beforeEach, describe, expect, it, vi } from "vitest";
vi.mock("./policy", () => ({ browserTelemetryAllowed: () => true }));
vi.mock("./canary", () => ({ canaryEventProperties: () => ({}) }));
const CONTENT = [
"/Users/ana/客户/秘密 video.mp4",
"compositions/intro.html",
"C:\\Users\\ana\\proj\\index.html",
"#hero-title",
'[data-composition-id="intro"]',
".cta",
"forbidden path: compositions/x.html",
];
const PROPERTIES = {
file_path: CONTENT[0],
target_source_file: CONTENT[1],
source_file: CONTENT[2],
target_selector: CONTENT[3],
selector: CONTENT[4],
target_id: "hero-title",
detail: CONTENT[5],
error_message: CONTENT[6],
status_code: 403,
};
type Sent = Array<{ event: string; properties: Record<string, unknown> }>;
describe("Studio analytics leave the machine without paths or selectors", () => {
let fetchMock: ReturnType<typeof vi.fn>;
beforeEach(() => {
localStorage.clear();
sessionStorage.clear();
vi.resetModules();
vi.useFakeTimers();
fetchMock = vi.fn(() => Promise.resolve({ ok: true } as Response));
vi.stubGlobal("fetch", fetchMock);
});
afterEach(() => {
vi.useRealTimers();
vi.unstubAllGlobals();
});
async function sent(): Promise<{ body: string; batch: Sent }> {
await vi.runOnlyPendingTimersAsync();
const bodies = fetchMock.mock.calls.map((call) =>
String((call[1] as { body?: string })?.body ?? ""),
);
const batch = bodies.flatMap((body) => (JSON.parse(body) as { batch: Sent }).batch);
return { body: bodies.join("\n"), batch };
}
async function send(transport: "studio:*" | "studio_*", event: string, properties: object) {
if (transport === "studio:*")
(await import("../utils/studioTelemetry")).trackStudioEvent(event, properties as never);
else (await import("./client")).trackEvent(event, properties as never);
}
it.each(["studio:*", "studio_*"] as const)(
"the %s transport sends none of the content, only facts that still group it",
async (transport) => {
await send(transport, "save_failure", PROPERTIES);
await send(transport, "save_failure", PROPERTIES);
const { body, batch } = await sent();
expect(batch).toHaveLength(2);
for (const text of [...CONTENT, "hero-title", "intro.html", "秘密"]) {
expect(body).not.toContain(JSON.stringify(text).slice(1, -1));
}
const [first, second] = batch.map((e) => e.properties);
expect(first).toMatchObject({
file_path_ext: "mp4",
file_path_kind: "asset",
target_source_file_kind: "composition",
target_source_file_depth: 1,
source_file_ext: "html",
target_selector_kind: "id",
selector_kind: "attribute",
target_id_kind: "id",
status_code: 403,
dropped_fields: "detail,error_message",
});
expect(first?.file_path_hash).toMatch(/^[0-9a-f]{8}$/);
// The same path from the same install groups; the salt that keys it never leaves.
expect(second?.file_path_hash).toBe(first?.file_path_hash);
expect(body).not.toContain(
localStorage.getItem("hyperframes-studio:telemetrySalt") ?? "missing-salt",
);
},
);
it("keeps the session's own fields when a caller sends the same names", async () => {
await send("studio:*", "navigation", { url_hash: "custom", tab_id: "custom" });
const { batch } = await sent();
expect(batch[0]?.properties.url_hash).not.toBe("custom");
expect(batch[0]?.properties.tab_id).not.toBe("custom");
});
it("leaves feedback text the person typed to send", async () => {
await send("studio_*", "studio_feedback", {
comment: "export of compositions/intro.html froze",
});
const { batch } = await sent();
expect(batch[0]?.properties.comment).toBe("export of compositions/intro.html froze");
});
});
+133
View File
@@ -0,0 +1,133 @@
// The one place Studio's event values are shaped before they leave the machine, for both
// transports. A path or a selector becomes facts that still group and debug (extension, kind,
// depth, a per-install hash); any other value that reads as one is dropped and named.
import { redactTelemetryString } from "@hyperframes/core/telemetry-redaction";
import { hashContent } from "../hooks/sdkSelfWriteRegistry";
type Value = string | number | boolean | null | undefined;
type Facts = Record<string, string | number>;
const PATH_KEYS = new Set([
"file_path",
"target_source_file",
"source_file",
"path",
"composition",
"render_composition",
"filename",
]);
const SELECTOR_KEYS = new Set(["target_selector", "selector", "target_id", "animationId"]);
// Text the person typed to send to us on purpose.
const KEPT_AS_TYPED: Record<string, ReadonlySet<string>> = {
studio_feedback: new Set(["comment"]),
};
const SALT_KEY = "hyperframes-studio:telemetrySalt";
let sessionSalt: string | undefined;
function newSalt(): string {
const bytes = new Uint8Array(16);
crypto.getRandomValues(bytes);
return Array.from(bytes, (b) => b.toString(16).padStart(2, "0")).join("");
}
/** Kept on this install and never sent: a hash groups repeats here but names nothing elsewhere. */
function installSalt(): string {
try {
const saved = localStorage.getItem(SALT_KEY);
if (saved && /^[0-9a-f]{32}$/.test(saved)) return saved;
const salt = newSalt();
localStorage.setItem(SALT_KEY, salt);
return salt;
} catch {
sessionSalt ??= newSalt();
return sessionSalt;
}
}
// ponytail: 32-bit FNV-1a groups one install's repeats; a collision only merges two groups.
const hashOf = (value: string): string =>
hashContent(`${installSalt()}\0${value}`).padStart(8, "0");
const KINDS: Record<string, string> = {
html: "composition",
htm: "composition",
js: "script",
mjs: "script",
cjs: "script",
ts: "script",
tsx: "script",
jsx: "script",
css: "style",
scss: "style",
less: "style",
};
function pathFacts(key: string, value: string): Facts {
const paths = value.split(",").filter((p) => p.trim().length > 0);
const first = paths[0] ?? value;
const segments = first.split(/[\\/]/).filter(Boolean);
const name = segments.at(-1) ?? "";
const dot = name.lastIndexOf(".");
const raw = dot > 0 ? name.slice(dot + 1).toLowerCase() : "";
const ext = /^[a-z0-9]{1,8}$/.test(raw) ? raw : "none";
const kind = ext === "none" ? "other" : (KINDS[ext] ?? "asset");
return {
[`${key}_ext`]: ext,
[`${key}_kind`]: kind,
[`${key}_depth`]: Math.max(segments.length - 1, 0),
[`${key}_hash`]: hashOf(first),
...(paths.length > 1 ? { [`${key}_count`]: paths.length } : {}),
};
}
function selectorFacts(key: string, value: string): Facts {
const s = value.trim();
const kind =
key === "target_id"
? "id"
: s.startsWith("#")
? "id"
: s.startsWith(".")
? "class"
: s.startsWith("[")
? "attribute"
: "other";
return { [`${key}_kind`]: kind, [`${key}_hash`]: hashOf(s) };
}
const ATTRIBUTE_SELECTOR = /\[[\w:-]+(?:[~|^$*]?=[^\]]*)?\]/;
const ID_OR_CLASS = /^\s*[#.][A-Za-z_-][\w-]*/;
const HEX_COLOUR = /^#[0-9a-f]{3,8}$/i;
function readsAsPathOrSelector(value: string): boolean {
if (redactTelemetryString(value, Number.MAX_SAFE_INTEGER) !== value) return true;
return (
ATTRIBUTE_SELECTOR.test(value) || (ID_OR_CLASS.test(value) && !HEX_COLOUR.test(value.trim()))
);
}
/** A caller's properties as they may leave the machine; runs before the transport's own fields. */
export function shapeEventProperties<V extends Value>(
event: string,
properties: Record<string, V>,
): Record<string, V | string | number> {
const kept = KEPT_AS_TYPED[event];
const out: Record<string, V | string | number> = {};
const dropped: string[] = [];
for (const [key, value] of Object.entries(properties)) {
if (typeof value !== "string" || value === "" || kept?.has(key)) {
out[key] = value;
} else if (PATH_KEYS.has(key)) {
Object.assign(out, pathFacts(key, value));
} else if (SELECTOR_KEYS.has(key)) {
Object.assign(out, selectorFacts(key, value));
} else if (readsAsPathOrSelector(value)) {
dropped.push(key);
} else {
out[key] = value;
}
}
if (dropped.length > 0) out.dropped_fields = dropped.join(",");
return out;
}
@@ -29,7 +29,12 @@ describe("dom edit save queue", () => {
expect(onOpen).toHaveBeenCalledWith({
consecutiveFailures: 2,
errorMessage: "Still down",
errorCodes: {
error_class: "StudioSaveHttpError",
error_key: "http",
server_why: "other",
block_reason: undefined,
},
statusCode: 503,
});
@@ -146,7 +151,12 @@ describe("dom edit save queue", () => {
expect(onOpen).toHaveBeenCalledWith({
consecutiveFailures: 1,
errorMessage: "File changed elsewhere",
errorCodes: {
error_class: "StudioSaveHttpError",
error_key: "http",
server_why: "other",
block_reason: undefined,
},
statusCode: 409,
});
await expect(queue.enqueue(async () => {})).rejects.toThrow("Auto-save is paused");
@@ -1,13 +1,13 @@
import {
getStudioSaveErrorMessage,
getStudioSaveStatusCode,
studioSaveErrorCodes,
StudioFileConflictError,
type StudioSaveDrainResult,
} from "./studioSaveDiagnostics";
interface DomEditSaveQueueOpenEvent {
export interface DomEditSaveQueueOpenEvent {
consecutiveFailures: number;
errorMessage: string;
errorCodes: ReturnType<typeof studioSaveErrorCodes>;
statusCode: number | null;
}
@@ -58,7 +58,7 @@ export function createDomEditSaveQueue(options: DomEditSaveQueueOptions = {}): D
breakerOpen = true;
options.onOpen?.({
consecutiveFailures,
errorMessage: getStudioSaveErrorMessage(error),
errorCodes: studioSaveErrorCodes(error),
statusCode: getStudioSaveStatusCode(error) ?? null,
});
};
@@ -566,9 +566,7 @@ export async function recordAnimationResolverParity(
...(diskChecked ? { diskChecked: true } : {}),
...(sourceReadFailed ? { sourceReadFailed: true } : {}),
mismatchCount: 1,
mismatches: JSON.stringify([
{ kind: "animation_not_found", animationId } satisfies SdkResolverMismatch,
]),
mismatches: JSON.stringify([{ kind: "animation_not_found" } satisfies SdkResolverMismatch]),
});
} catch {
// never propagate from the shadow path
@@ -75,7 +75,7 @@ describe("studio save diagnostics", () => {
});
});
it("builds save_failure properties with stable diagnostics", () => {
it("builds save_failure properties as fixed codes, never the error's words", () => {
const error = new StudioSaveHttpError("Failed to save index.html (503)", 503);
expect(
@@ -88,7 +88,9 @@ describe("studio save diagnostics", () => {
}),
).toEqual({
source: "code_editor",
error_message: "Failed to save index.html (503)",
error_class: "StudioSaveHttpError",
error_key: "http",
server_why: "other",
status_code: 503,
file_path: "index.html",
mutation_type: "put",
@@ -100,6 +102,21 @@ describe("studio save diagnostics", () => {
});
});
it("names the server's fixed refusal and keeps its words out", async () => {
const response = new Response(
JSON.stringify({ error: "forbidden path: compositions/x.html", why: "protected_path" }),
{ status: 403 },
);
const error = await createStudioSaveHttpError(response, "Failed to save compositions/x.html");
const properties = buildStudioSaveFailureProperties({ source: "dom_edit", error });
expect(properties).toMatchObject({
error_key: "http",
server_why: "protected_path",
status_code: 403,
});
expect(JSON.stringify(properties)).not.toContain("compositions/x.html");
});
it("emits expected direct-edit refusals on edit_blocked", () => {
const error = new Error("This animation is computed at runtime");
@@ -112,7 +129,8 @@ describe("studio save diagnostics", () => {
expect(trackStudioEvent).toHaveBeenCalledWith("edit_blocked", {
source: "gsap_commit",
error_message: error.message,
error_class: "Error",
error_key: "unknown",
status_code: null,
file_path: "index.html",
mutation_type: "drag",
@@ -16,21 +16,37 @@ export interface StudioSaveFailureInput {
targetSourceFile?: string | null;
}
const SERVER_WHY = new Set([
"nul",
"dangling_symlink",
"outside_project",
"protected_path",
"busy",
"permission",
"not_a_file",
"too_large",
"binary",
"link",
"project_dir_missing",
]);
export class StudioSaveHttpError extends Error {
readonly statusCode: number;
readonly alreadyToasted: boolean;
readonly heldFile: boolean;
readonly why: string | null;
constructor(
message: string,
statusCode: number,
options: { alreadyToasted?: boolean; heldFile?: boolean } = {},
options: { alreadyToasted?: boolean; heldFile?: boolean; why?: string | null } = {},
) {
super(message);
this.name = "StudioSaveHttpError";
this.statusCode = statusCode;
this.alreadyToasted = options.alreadyToasted ?? false;
this.heldFile = options.heldFile ?? false;
this.why = options.why ?? null;
}
}
@@ -162,7 +178,7 @@ export function buildStudioSaveFailureProperties(
const attempt = input.attempt ?? getStudioSaveAttempt(input.error) ?? undefined;
return {
source: input.source,
error_message: getStudioSaveErrorMessage(input.error),
...studioSaveErrorCodes(input.error),
status_code: statusCode,
file_path: input.filePath ?? input.targetSourceFile ?? undefined,
mutation_type: input.mutationType ?? undefined,
@@ -175,10 +191,45 @@ export function buildStudioSaveFailureProperties(
};
}
const isNamed = (name: string) => (error: unknown) => error instanceof Error && error.name === name;
const ERROR_KEYS: ReadonlyArray<readonly [string, (error: unknown) => boolean]> = [
["conflict", (error) => findCause(error, StudioFileConflictError) !== null],
["held", (error) => findCause(error, StudioSaveHttpError)?.heldFile === true],
["http", (error) => findCause(error, StudioSaveHttpError) !== null],
["network", (error) => findCause(error, StudioSaveNetworkError) !== null],
["aborted", isNamed("AbortError")],
["blocked", isNamed("GsapEditBlockedError")],
];
export function studioSaveErrorCodes(error: unknown): Record<string, StudioTelemetryValue> {
const failure = error instanceof Error ? error : null;
const http = findCause(error, StudioSaveHttpError);
const errorKey = ERROR_KEYS.find(([, matches]) => matches(error))?.[0] ?? "unknown";
return {
error_class: failure?.name ?? typeof error,
error_key: errorKey,
server_why: http ? (http.why ?? "other") : undefined,
block_reason: blockReason(error),
};
}
function findCause<T extends Error>(error: unknown, type: new (...args: never[]) => T): T | null {
for (let at = error, depth = 0; at && typeof at === "object" && depth < 5; depth++) {
if (at instanceof type) return at;
at = (at as { cause?: unknown }).cause;
}
return null;
}
function blockReason(error: unknown): string | undefined {
if (!(error instanceof Error) || error.name !== "GsapEditBlockedError") return undefined;
const reason = (error as { reason?: unknown }).reason;
return typeof reason === "string" ? reason : undefined;
}
/**
* The specific cause behind a blocked GSAP edit. `error_message` only carries
* the coarse copy, and one of those strings covers six different situations,
* so the message alone cannot say which one a user hit.
* The specific cause behind a blocked GSAP edit: one block reason covers six
* different situations, so the reason alone cannot say which one a user hit.
*/
function gsapBlockDetail(error: unknown): string | undefined {
if (!(error instanceof Error) || error.name !== "GsapEditBlockedError") return undefined;
@@ -234,12 +285,26 @@ export async function createStudioSaveHttpError(
}
const detail = body.trim().slice(0, 300);
const held = heldFileMessage(body, options.path);
const why = serverWhy(body);
const message =
held ??
(detail
? `${fallbackMessage} (${response.status}): ${detail}`
: `${fallbackMessage} (${response.status})`);
return new StudioSaveHttpError(message, response.status, { ...options, heldFile: held !== null });
return new StudioSaveHttpError(message, response.status, {
...options,
heldFile: held !== null,
why,
});
}
function serverWhy(body: string): string | null {
try {
const why: unknown = (JSON.parse(body) as { why?: unknown }).why;
return typeof why === "string" && SERVER_WHY.has(why) ? why : null;
} catch {
return null;
}
}
export const heldFileWords = (error: unknown): string | null =>
+6 -1
View File
@@ -5,6 +5,7 @@ import { canaryEventProperties } from "../telemetry/canary";
import { agentRuntimeProperty } from "../telemetry/agentRuntime";
import { tabIdProperty } from "../telemetry/tabId";
import { studioApiFetch } from "./studioApiFetch";
import { shapeEventProperties } from "../telemetry/eventShape";
// PostHog public ingest key — write-only, safe to ship in the client bundle
const POSTHOG_API_KEY = "phc_zjjbX0PnWxERXrMHhkEJWj9A9BhGVLRReICgsfTMmpx";
@@ -120,7 +121,11 @@ export function trackStudioEvent(event: string, properties: EventProperties = {}
// Canary assignments on every event, matching the CLI and the newer
// studio client — "every telemetry event carries the assignment" has to
// include this path or a cohort breakdown silently omits `studio:*`.
properties: { ...getSessionProperties(), ...canaryEventProperties(), ...properties },
properties: {
...canaryEventProperties(),
...shapeEventProperties(`studio:${event}`, properties),
...getSessionProperties(),
},
timestamp: new Date().toISOString(),
});