From 990f5c314513a379ccceef050da31a633101c9d1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Miguel=20=C3=81ngel?= Date: Tue, 14 Jul 2026 15:46:47 -0400 Subject: [PATCH] =?UTF-8?q?feat(feedback):=20adopt=200=E2=80=9310=20recomm?= =?UTF-8?q?endation=20scale=20(#2438)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * feat(feedback): adopt 10-point recommendation scale * docs(feedback): keep OSS scale contract self-contained --- docs/guides/feedback.mdx | 28 ++++++++++++------- docs/packages/cli.mdx | 12 ++++---- packages/cli/src/commands/feedback.ts | 18 +++++------- packages/cli/src/telemetry/events.test.ts | 1 + packages/cli/src/telemetry/events.ts | 2 ++ packages/cli/src/telemetry/feedback.ts | 9 +++--- packages/cli/src/utils/feedbackIssue.test.ts | 4 +-- packages/cli/src/utils/feedbackIssue.ts | 6 ++-- packages/cli/src/utils/feedbackRating.test.ts | 17 +++++++++++ packages/cli/src/utils/feedbackRating.ts | 8 ++++++ packages/cli/src/utils/submitFeedback.test.ts | 12 ++++++++ packages/cli/src/utils/submitFeedback.ts | 2 ++ .../src/components/StudioFeedbackBar.tsx | 4 +-- packages/studio/src/telemetry/events.test.ts | 10 +++++++ packages/studio/src/telemetry/events.ts | 1 + skills-manifest.json | 2 +- skills/hyperframes-cli/SKILL.md | 4 +-- .../references/preview-render.md | 6 ++-- 18 files changed, 103 insertions(+), 43 deletions(-) create mode 100644 packages/cli/src/utils/feedbackRating.test.ts create mode 100644 packages/cli/src/utils/feedbackRating.ts diff --git a/docs/guides/feedback.mdx b/docs/guides/feedback.mdx index 00b360463..60e73550d 100644 --- a/docs/guides/feedback.mdx +++ b/docs/guides/feedback.mdx @@ -3,11 +3,11 @@ title: Feedback Collection description: "How HyperFrames collects feedback, what data is collected, and how to opt out." --- -HyperFrames occasionally asks how a render went or how a Studio session felt. This page explains why we do it, when prompts appear, what data is collected, and how to disable them. +HyperFrames occasionally asks how likely you are to recommend the render experience or Studio. This page explains why we do it, when prompts appear, what data is collected, and how to disable them. ## Why We Ask -We use anonymous satisfaction scores to understand whether the tool is actually working well — not just whether it runs without errors. A render that takes 10 minutes and produces a broken file counts as a success in logs but a failure in practice. The feedback prompt is the only signal we have for that gap. +We use anonymous recommendation scores to understand whether the tool is actually working well — not just whether it runs without errors. A render that takes 10 minutes and produces a broken file counts as a success in logs but a failure in practice. The feedback prompt is the only signal we have for that gap. No account, email, or identity is tied to responses. Each installation generates a random UUID at setup; that is the only identifier. @@ -18,7 +18,7 @@ No account, email, or identity is tied to responses. Each installation generates After a successful `hyperframes render`, a short prompt may appear: ``` - How was this render? [1=poor 5=great, enter to skip] + How likely are you to recommend HyperFrames? [0=not likely 10=extremely likely, enter to skip] Any details? (enter to skip) ``` @@ -64,15 +64,15 @@ You can submit feedback manually at any time: ```bash # Quick rating -hyperframes feedback --rating 5 +hyperframes feedback --rating 10 # Rating with details -hyperframes feedback --rating 3 --comment "render succeeded but GSAP timeline didn't animate text overlay" +hyperframes feedback --rating 7 --comment "render succeeded but GSAP timeline didn't animate text overlay" ``` | Flag | Description | |------|-------------| -| `--rating` | Satisfaction score, 1–5 (required) | +| `--rating` | Recommendation score, 0–10 (required) | | `--comment` | Optional free-text details | | `--file-issue` | Also open a pre-filled GitHub issue with a published minimal repro (opt-in) | | `--dir` | Project directory to publish as the repro (default: current directory) | @@ -85,7 +85,7 @@ This command collects a doctor summary automatically, flushes telemetry, and exi When a render misbehaves, add `--file-issue` so maintainers can reproduce it: ```bash -hyperframes feedback --rating 2 --comment "GSAP timeline froze on seek" --file-issue +hyperframes feedback --rating 3 --comment "GSAP timeline froze on seek" --file-issue ``` This is **opt-in** and **consented**. With `--file-issue` set, after the usual feedback is sent the CLI: @@ -101,7 +101,7 @@ The issue is **not auto-submitted**: you review and file it under your own GitHu When an AI agent is detected, HyperFrames **skips the interactive readline prompt** and prints a structured hint instead: ``` - [hyperframes] Agent feedback: hyperframes feedback --rating <1-5> --comment "..." + [hyperframes] Agent feedback: hyperframes feedback --rating <0-10> --comment "..." ``` Agents can then submit feedback using the `hyperframes feedback` command above. @@ -130,7 +130,8 @@ Only the existence (or in some cases the value) of these variables is checked | Field | Value | |-------|-------| | `$survey_id` | `render_satisfaction` | -| `$survey_response` | Rating (1–5) | +| `$survey_response` | Raw rating (0–10) | +| `rating_scale` | `10` for the current recommendation scale | | `$survey_response_2` | Free-text comment (only when provided) | | `render_duration_ms` | Time the render took in milliseconds | | `doctor_summary` | System context (see below) | @@ -148,7 +149,8 @@ It may also include `wsl` or sandbox runtime flags when those environments are d | Field | Value | |-------|-------| | `$survey_id` | `studio_experience` | -| `$survey_response` | Rating (1–5) | +| `$survey_response` | Raw rating (0–10) | +| `rating_scale` | `10` for the current recommendation scale | | `$survey_response_2` | Free-text comment (only when provided) | | `source` | `studio` | | `doctor_summary` | Browser context (platform, screen, CPU cores, device memory, network type) | @@ -163,6 +165,12 @@ It may also include `wsl` or sandbox runtime flags when those environments are d Feedback is anonymous. Each installation has a random UUID (`anonymousId`) — there is no account, login, or email association. +## Rating Scale Compatibility + +New feedback stores the raw integer together with `rating_scale: 10`. + +Feedback collected before this change remains on the historical 1–5 satisfaction scale. Historical events have no `rating_scale`, and historical Slack or GitHub records retain their `/5` denominator. Consumers must treat a missing scale as legacy 1–5 data and must not reinterpret those scores as NPS. + ## Config File The CLI persists feedback state in `~/.hyperframes/config.json`: diff --git a/docs/packages/cli.mdx b/docs/packages/cli.mdx index 9204e1a0c..d6efc5c76 100644 --- a/docs/packages/cli.mdx +++ b/docs/packages/cli.mdx @@ -900,22 +900,22 @@ Word-level transcripts (whisper output) are grouped into readable caption cues o ### `feedback` - Submit anonymous satisfaction feedback about your experience: + Submit anonymous recommendation feedback about your experience: ```bash - # Quick rating (1 = poor, 5 = great) - npx hyperframes feedback --rating 5 + # Quick rating (0 = not likely, 10 = extremely likely) + npx hyperframes feedback --rating 10 # Rating with optional details - npx hyperframes feedback --rating 3 --comment "render succeeded but GSAP timeline didn't animate" + npx hyperframes feedback --rating 7 --comment "render succeeded but GSAP timeline didn't animate" # Also file a pre-filled GitHub issue with a published minimal repro (opt-in, consented) - npx hyperframes feedback --rating 2 --comment "GSAP timeline froze on seek" --file-issue + npx hyperframes feedback --rating 3 --comment "GSAP timeline froze on seek" --file-issue ``` | Flag | Description | |------|-------------| - | `--rating` | Satisfaction score, 1–5 (required) | + | `--rating` | Recommendation score, 0–10 (required) | | `--comment` | Optional free-text details | | `--file-issue` | Also open a pre-filled GitHub issue with a published minimal repro (opt-in) | | `--dir` | Project directory to publish as the repro (default: current directory) | diff --git a/packages/cli/src/commands/feedback.ts b/packages/cli/src/commands/feedback.ts index 14e97c058..d4eb23fba 100644 --- a/packages/cli/src/commands/feedback.ts +++ b/packages/cli/src/commands/feedback.ts @@ -11,21 +11,17 @@ import { submitFeedback } from "../utils/submitFeedback.js"; import { buildIssueUrl, HYPERFRAMES_REPO_URL } from "../utils/feedbackIssue.js"; import { VERSION } from "../version.js"; import { c } from "../ui/colors.js"; +import { parseFeedbackRating } from "../utils/feedbackRating.js"; export const examples: Example[] = [ - ["Submit render feedback", 'hyperframes feedback --rating 4 --comment "fast but font missing"'], - ["Quick rating only", "hyperframes feedback --rating 5"], + ["Submit render feedback", 'hyperframes feedback --rating 8 --comment "fast but font missing"'], + ["Quick rating only", "hyperframes feedback --rating 10"], [ "Also file a GitHub issue with a published repro", - 'hyperframes feedback --rating 2 --comment "GSAP timeline froze" --file-issue', + 'hyperframes feedback --rating 3 --comment "GSAP timeline froze" --file-issue', ], ]; -function parseRating(raw: string): number | null { - const n = parseInt(raw, 10); - return n >= 1 && n <= 5 && Number.isFinite(n) ? n : null; -} - function normalizeComment(raw?: string): string | undefined { return raw || undefined; } @@ -122,7 +118,7 @@ export default defineCommand({ args: { rating: { type: "string", - description: "Satisfaction rating (1=poor, 5=great)", + description: "Likelihood to recommend (0=not likely, 10=extremely likely)", required: true, }, comment: { @@ -146,9 +142,9 @@ export default defineCommand({ }, }, async run({ args }) { - const rating = parseRating(args.rating); + const rating = parseFeedbackRating(args.rating); if (rating === null) { - console.error(c.error("Rating must be between 1 and 5")); + console.error(c.error("Rating must be an integer between 0 and 10")); process.exit(1); } diff --git a/packages/cli/src/telemetry/events.test.ts b/packages/cli/src/telemetry/events.test.ts index 490c50f71..17cf69a73 100644 --- a/packages/cli/src/telemetry/events.test.ts +++ b/packages/cli/src/telemetry/events.test.ts @@ -400,6 +400,7 @@ describe("trackRenderFeedback", () => { const [, props] = trackEvent.mock.calls[0] as [string, Record]; expect(props).not.toHaveProperty("render_duration_ms"); expect(props.$survey_response).toBe(4); + expect(props.rating_scale).toBe(10); }); it("includes render_duration_ms when a real duration is supplied", () => { diff --git a/packages/cli/src/telemetry/events.ts b/packages/cli/src/telemetry/events.ts index 38128caa1..cb01725f1 100644 --- a/packages/cli/src/telemetry/events.ts +++ b/packages/cli/src/telemetry/events.ts @@ -1,5 +1,6 @@ import { redactTelemetryString, type OutputResolutionIssueKind } from "@hyperframes/core"; import type { SubTimelineWaitOutcome } from "@hyperframes/engine"; +import { FEEDBACK_RATING_SCALE } from "../utils/feedbackRating.js"; import { flush, trackEvent } from "./client.js"; import { readConfig } from "./config.js"; @@ -600,6 +601,7 @@ export function trackRenderFeedback(props: { trackEvent("survey sent", { $survey_id: "render_satisfaction", $survey_response: props.rating, + rating_scale: FEEDBACK_RATING_SCALE, ...(props.comment ? { $survey_response_2: props.comment } : {}), ...(props.renderDurationMs !== undefined ? { render_duration_ms: props.renderDurationMs } : {}), ...(props.doctorSummary ? { doctor_summary: props.doctorSummary } : {}), diff --git a/packages/cli/src/telemetry/feedback.ts b/packages/cli/src/telemetry/feedback.ts index 4da1c23c3..620c00022 100644 --- a/packages/cli/src/telemetry/feedback.ts +++ b/packages/cli/src/telemetry/feedback.ts @@ -4,6 +4,7 @@ import { shouldTrack } from "./client.js"; import { trackRenderFeedback } from "./events.js"; import { detectAgentRuntime } from "./agent_runtime.js"; import { c } from "../ui/colors.js"; +import { parseFeedbackRating } from "../utils/feedbackRating.js"; const DEFAULT_FEEDBACK_INTERVAL = 15; @@ -50,7 +51,7 @@ export async function maybePromptRenderFeedback(opts: { console.log( c.dim(" [hyperframes] ") + c.dim("Agent feedback: ") + - c.accent('hyperframes feedback --rating <1-5> --comment "..."'), + c.accent('hyperframes feedback --rating <0-10> --comment "..."'), ); return; } @@ -66,11 +67,11 @@ export async function maybePromptRenderFeedback(opts: { writeConfig(config); const answer = await askQuestion( - ` ${c.dim("How was this render?")} ${c.accent("[1=poor 5=great, enter to skip]")} `, + ` ${c.dim("How likely are you to recommend HyperFrames?")} ${c.accent("[0=not likely 10=extremely likely, enter to skip]")} `, ); - const rating = parseInt(answer.trim(), 10); - if (rating >= 1 && rating <= 5) { + const rating = parseFeedbackRating(answer); + if (rating !== null) { // Ask for optional text feedback const details = await askQuestion(` ${c.dim("Any details?")} ${c.accent("(enter to skip)")} `); const trimmedDetails = details.trim(); diff --git a/packages/cli/src/utils/feedbackIssue.test.ts b/packages/cli/src/utils/feedbackIssue.test.ts index 20e2023de..25c7e8cad 100644 --- a/packages/cli/src/utils/feedbackIssue.test.ts +++ b/packages/cli/src/utils/feedbackIssue.test.ts @@ -27,7 +27,7 @@ describe("buildIssueUrl", () => { const url = buildIssueUrl(base); expect(decoded(url, "title")).toBe("[feedback] GSAP timeline froze on seek"); const body = decoded(url, "body"); - expect(body).toContain("2/5"); + expect(body).toContain("2/10"); expect(body).toContain("https://hyperframes.dev/p/abc123"); expect(body).toContain("os=darwin/arm64"); expect(body).toContain("cli=1.2.3"); @@ -35,7 +35,7 @@ describe("buildIssueUrl", () => { it("falls back to a generic title when there is no comment", () => { const url = buildIssueUrl({ ...base, comment: undefined }); - expect(decoded(url, "title")).toBe("Render feedback (rating 2/5)"); + expect(decoded(url, "title")).toBe("Render feedback (rating 2/10)"); }); it("truncates an overlong comment in the body", () => { diff --git a/packages/cli/src/utils/feedbackIssue.ts b/packages/cli/src/utils/feedbackIssue.ts index 3632c6bc2..aae0f0983 100644 --- a/packages/cli/src/utils/feedbackIssue.ts +++ b/packages/cli/src/utils/feedbackIssue.ts @@ -1,3 +1,5 @@ +import { FEEDBACK_RATING_SCALE } from "./feedbackRating.js"; + // Reading package.json at runtime from the single-file bundled CLI is awkward, // so we keep the canonical repo as a constant. It must match the `repository.url` // in packages/cli/package.json. @@ -33,7 +35,7 @@ function truncate(value: string, max: number): string { function buildIssueTitle(rating: number, comment?: string): string { const firstLine = comment?.split("\n")[0]?.trim(); - if (!firstLine) return `Render feedback (rating ${rating}/5)`; + if (!firstLine) return `Render feedback (rating ${rating}/${FEEDBACK_RATING_SCALE})`; return `[feedback] ${truncate(firstLine, TITLE_MAX)}`; } @@ -44,7 +46,7 @@ function buildIssueBody(input: IssueInput): string { : "_Publishing the repro failed, no public link available._"; return [ - `**Rating:** ${input.rating}/5`, + `**Rating:** ${input.rating}/${FEEDBACK_RATING_SCALE}`, "", "## Comment", comment ? truncate(comment, COMMENT_MAX) : "_No comment provided._", diff --git a/packages/cli/src/utils/feedbackRating.test.ts b/packages/cli/src/utils/feedbackRating.test.ts new file mode 100644 index 000000000..603c09263 --- /dev/null +++ b/packages/cli/src/utils/feedbackRating.test.ts @@ -0,0 +1,17 @@ +import { describe, expect, it } from "vitest"; + +import { FEEDBACK_RATING_SCALE, parseFeedbackRating } from "./feedbackRating.js"; + +describe("parseFeedbackRating", () => { + it.each(["0", "10"])("accepts the NPS boundary %s", (raw) => { + expect(parseFeedbackRating(raw)).toBe(Number(raw)); + }); + + it.each(["-1", "11", "4.5", "10x", ""])("rejects invalid rating %j", (raw) => { + expect(parseFeedbackRating(raw)).toBeNull(); + }); + + it("declares the serialized rating scale", () => { + expect(FEEDBACK_RATING_SCALE).toBe(10); + }); +}); diff --git a/packages/cli/src/utils/feedbackRating.ts b/packages/cli/src/utils/feedbackRating.ts new file mode 100644 index 000000000..ff411b35a --- /dev/null +++ b/packages/cli/src/utils/feedbackRating.ts @@ -0,0 +1,8 @@ +export const FEEDBACK_RATING_SCALE = 10; + +export function parseFeedbackRating(raw: string): number | null { + const trimmed = raw.trim(); + if (!/^\d+$/.test(trimmed)) return null; + const rating = Number(trimmed); + return Number.isInteger(rating) && rating >= 0 && rating <= FEEDBACK_RATING_SCALE ? rating : null; +} diff --git a/packages/cli/src/utils/submitFeedback.test.ts b/packages/cli/src/utils/submitFeedback.test.ts index c40b3af47..b69568f2b 100644 --- a/packages/cli/src/utils/submitFeedback.test.ts +++ b/packages/cli/src/utils/submitFeedback.test.ts @@ -38,6 +38,7 @@ describe("submitFeedback", () => { headers: { "content-type": "application/json", heygen_route: "canary" }, body: JSON.stringify({ rating: 4, + rating_scale: 10, comment: "fast but font missing", cli_version: "1.2.3", env: "os=darwin/arm64 node=v22.11.0", @@ -47,6 +48,17 @@ describe("submitFeedback", () => { ); }); + it.each([0, 10])("serializes the NPS boundary %i without changing it", async (rating) => { + const fetchMock = vi.fn(async () => new Response(null, { status: 202 })); + vi.stubGlobal("fetch", fetchMock); + + await submitFeedback({ rating, cliVersion: "1.2.3" }); + + const requestInit = fetchMock.mock.calls[0]?.[1]; + const body = JSON.parse(requestInit?.body as string); + expect(body).toMatchObject({ rating, rating_scale: 10 }); + }); + it("truncates over-long fields to the backend caps", async () => { const fetchMock = vi.fn(async () => new Response(null, { status: 202 })); vi.stubGlobal("fetch", fetchMock); diff --git a/packages/cli/src/utils/submitFeedback.ts b/packages/cli/src/utils/submitFeedback.ts index 9c8087871..3c239ad52 100644 --- a/packages/cli/src/utils/submitFeedback.ts +++ b/packages/cli/src/utils/submitFeedback.ts @@ -1,4 +1,5 @@ import { getPublishApiBaseUrl } from "./publishProject.js"; +import { FEEDBACK_RATING_SCALE } from "./feedbackRating.js"; // Match the backend DTO caps (HyperframesFeedbackRequest). Truncate here so an // over-long field (e.g. a pasted stack trace) is still forwarded truncated, @@ -24,6 +25,7 @@ export async function submitFeedback(input: { method: "POST", body: JSON.stringify({ rating: input.rating, + rating_scale: FEEDBACK_RATING_SCALE, comment: cap(input.comment, MAX_COMMENT), cli_version: cap(input.cliVersion, MAX_CLI_VERSION), env: cap(input.env, MAX_ENV), diff --git a/packages/studio/src/components/StudioFeedbackBar.tsx b/packages/studio/src/components/StudioFeedbackBar.tsx index 90428aa82..7084c2db9 100644 --- a/packages/studio/src/components/StudioFeedbackBar.tsx +++ b/packages/studio/src/components/StudioFeedbackBar.tsx @@ -180,9 +180,9 @@ export const StudioFeedbackBar = memo(function StudioFeedbackBar() { ) : ( <> - How's the Studio experience? + Recommend HyperFrames?
- {[1, 2, 3, 4, 5].map((n) => ( + {Array.from({ length: 11 }, (_, n) => n).map((n) => (