diff --git a/packages/cli/src/commands/check.test.ts b/packages/cli/src/commands/check.test.ts index b5a86f73f..6ea45395d 100644 --- a/packages/cli/src/commands/check.test.ts +++ b/packages/cli/src/commands/check.test.ts @@ -58,6 +58,7 @@ function cleanLint(): ProjectLintResult { results: [ { file: "index.html", + contentHash: "test", result: { ok: true, errorCount: 0, @@ -82,6 +83,7 @@ function lintWith( results: [ { file: "index.html", + contentHash: "test", result: { ok: severity !== "error", errorCount: severity === "error" ? 1 : 0, diff --git a/packages/cli/src/commands/lint.ts b/packages/cli/src/commands/lint.ts index b592fd1e5..149ea9668 100644 --- a/packages/cli/src/commands/lint.ts +++ b/packages/cli/src/commands/lint.ts @@ -12,6 +12,8 @@ export const examples: Example[] = [ import { formatLintFindings } from "../utils/lintFormat.js"; import { lintProject } from "../utils/lintProject.js"; import { resolveProject } from "../utils/project.js"; +import { trackLintRun } from "../telemetry/lintRun.js"; +import { getRunId } from "../telemetry/runId.js"; import { withMeta } from "../utils/updateCheck.js"; export default defineCommand({ @@ -45,7 +47,13 @@ export default defineCommand({ // (publish/transcribe/upgrade/play/present) already use. try { const project = resolveProject(args.dir); + const startedAt = Date.now(); const lintResult = await lintProject(project.dir); + trackLintRun(project.dir, lintResult, { + command: "lint", + durationMs: Date.now() - startedAt, + ...(getRunId() !== undefined ? { runId: getRunId() } : {}), + }); if (args.json) { const allFindings = lintResult.results.flatMap((r) => r.result.findings); diff --git a/packages/cli/src/commands/validate.test.ts b/packages/cli/src/commands/validate.test.ts index f706181ef..2d78f6b39 100644 --- a/packages/cli/src/commands/validate.test.ts +++ b/packages/cli/src/commands/validate.test.ts @@ -232,6 +232,7 @@ describe("extractCompositionErrorsFromLint", () => { results: [ { file: "index.html", + contentHash: "test", result: { ok: findings.length === 0, errorCount: 0, @@ -282,6 +283,7 @@ describe("extractCompositionErrorsFromLint", () => { results: [ { file: "index.html", + contentHash: "test", result: { ok: false, errorCount: 1, @@ -298,6 +300,7 @@ describe("extractCompositionErrorsFromLint", () => { }, { file: "compositions/nested.html", + contentHash: "test", result: { ok: false, errorCount: 1, diff --git a/packages/cli/src/telemetry/events.ts b/packages/cli/src/telemetry/events.ts index 8e9b6420f..02b01701f 100644 --- a/packages/cli/src/telemetry/events.ts +++ b/packages/cli/src/telemetry/events.ts @@ -819,3 +819,76 @@ export function trackCheckReport(props: { ...runIdField(props.runId), }); } + +/** + * One lint pass over a project. `code_counts` is what makes "which rules + * actually fire" answerable; `rule_group_ms` and `slowest_rule` are what make + * "which rules are expensive" answerable. Only lint rule codes and timings are + * sent — never file paths, project names, or composition source. + */ +export function trackLintReport(props: { + /** The command that ran the lint: "lint" or "check". */ + command: string; + durationMs: number; + filesScanned: number; + errorCount: number; + warningCount: number; + infoCount: number; + /** Finding count keyed by lint rule code. */ + codeCounts: Record; + /** Milliseconds spent per rule-source module, summed across files. */ + ruleGroupMs: Record; + /** Slowest single rule as `#`, across every file in the run. */ + slowestRule: string; + slowestRuleMs: number; + /** How many rules this build ran, so a ruleset change is visible in the data. */ + ruleCount: number; + /** + * Rule count per group. `slowest_rule` is positional, so a group that changed + * size between two builds has indices that no longer mean the same thing. + */ + ruleGroupCounts: Record; + runId?: string; +}): void { + trackEvent("lint_report", { + command: props.command, + duration_ms: Math.round(props.durationMs), + files_scanned: props.filesScanned, + error_count: props.errorCount, + warning_count: props.warningCount, + info_count: props.infoCount, + codes: Object.keys(props.codeCounts).sort(), + code_counts: props.codeCounts, + rule_group_ms: props.ruleGroupMs, + slowest_rule: props.slowestRule, + slowest_rule_ms: Math.round(props.slowestRuleMs), + rule_count: props.ruleCount, + rule_group_counts: props.ruleGroupCounts, + ...runIdField(props.runId), + }); +} + +/** + * A finding that survived one or more edits to the file it was reported on. + * + * `cleared: false` with a high `edits` is the signal that matters most: a rule + * an agent kept trying and failing to satisfy. `cleared: true` gives the + * distribution to compare it against — how many edits a normal finding costs. + */ +export function trackLintRuleStreak(props: { + code: string; + severity: string; + edits: number; + cleared: boolean; + command: string; + runId?: string; +}): void { + trackEvent("lint_rule_streak", { + code: props.code, + severity: props.severity, + edits: props.edits, + cleared: props.cleared, + command: props.command, + ...runIdField(props.runId), + }); +} diff --git a/packages/cli/src/telemetry/lintRun.e2e.test.ts b/packages/cli/src/telemetry/lintRun.e2e.test.ts new file mode 100644 index 000000000..93d715fc4 --- /dev/null +++ b/packages/cli/src/telemetry/lintRun.e2e.test.ts @@ -0,0 +1,134 @@ +// End-to-end: a real project on disk -> lintProject -> the exact PostHog +// payloads. Unit tests cover the streak arithmetic; this proves the wiring +// and pins the event shape a dashboard will be built against. + +import { describe, it, expect, vi, beforeEach } from "vitest"; +import { mkdtempSync, mkdirSync, writeFileSync, rmSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; + +const HOME = mkdtempSync(join(tmpdir(), "hf-lintrun-")); +vi.mock("node:os", async (importOriginal) => { + const actual = await importOriginal(); + return { ...actual, homedir: () => HOME }; +}); + +// Capture at the transport boundary so everything client.ts adds +// (cli_version, invocation_id, ...) is visible in the assertions. +const enqueued: Array<{ event: string; properties: Record }> = []; +vi.mock("./transport.js", () => ({ + enqueue: (event: string, properties: Record) => + enqueued.push({ event, properties }), + flush: () => Promise.resolve(), +})); +// shouldTrack() consults these two. A dev build disables telemetry by default, +// which would make this test assert on an empty queue and pass for the wrong +// reason. +vi.mock("./policy.js", async (importOriginal) => { + const actual = await importOriginal(); + return { ...actual, telemetryRuntimeOverride: () => null }; +}); +vi.mock("./config.js", async (importOriginal) => { + const actual = await importOriginal(); + return { ...actual, readConfig: () => ({ ...actual.readConfig(), telemetryEnabled: true }) }; +}); + +const { trackLintRun } = await import("./lintRun.js"); +const { lintProject } = await import("@hyperframes/lint"); + +const COMPOSITION = ` +
+ +
+ + +`; + +function makeProject(html: string): string { + const dir = mkdtempSync(join(tmpdir(), "hf-proj-")); + mkdirSync(join(dir, "compositions"), { recursive: true }); + writeFileSync(join(dir, "index.html"), html, "utf-8"); + return dir; +} + +beforeEach(() => { + enqueued.length = 0; + rmSync(join(HOME, ".hyperframes"), { recursive: true, force: true }); +}); + +describe("trackLintRun end to end", () => { + it("emits one lint_report carrying codes, timings, and the ruleset fingerprint", async () => { + const dir = makeProject(COMPOSITION); + const result = await lintProject(dir); + trackLintRun(dir, result, { command: "lint", durationMs: 12 }); + + const reports = enqueued.filter((e) => e.event === "lint_report"); + expect(reports).toHaveLength(1); + const props = reports[0]!.properties; + + expect(props["command"]).toBe("lint"); + expect(props["files_scanned"]).toBe(1); + expect(props["duration_ms"]).toBe(12); + // The real linter found real problems in this composition. + expect((props["codes"] as string[]).length).toBeGreaterThan(0); + expect(props["error_count"]).toBeGreaterThan(0); + // Timings are attributed per rule group and a slowest rule is identified. + expect(Object.keys(props["rule_group_ms"] as object)).toContain("gsap"); + expect(props["slowest_rule"]).toMatch(/^[a-z]+#\d+$/); + // Version and ruleset fingerprint ride along. + expect(props["cli_version"]).toBeTruthy(); + expect(props["rule_count"]).toBeGreaterThan(0); + // Per-group sizes make the positional `slowest_rule` index comparable + // across builds: a group that changed size renumbered its rules. + const groupCounts = props["rule_group_counts"] as Record; + expect(groupCounts["gsap"]).toBeGreaterThan(0); + expect(Object.values(groupCounts).reduce((a, b) => a + b, 0)).toBe(props["rule_count"]); + // code_counts sums to the number of findings. + const counts = Object.values(props["code_counts"] as Record); + expect(counts.reduce((a, b) => a + b, 0)).toBe( + result.results.flatMap((r) => r.result.findings).length, + ); + + rmSync(dir, { recursive: true, force: true }); + }); + + it("emits lint_rule_streak with cleared:true once an edit removes the finding", async () => { + const dir = makeProject(COMPOSITION); + + const first = await lintProject(dir); + trackLintRun(dir, first, { command: "lint", durationMs: 1 }); + const codes = first.results[0]!.result.findings.map((f) => f.code); + expect(codes).toContain("media_missing_data_start"); + + // Fix exactly that finding and re-lint. + writeFileSync( + join(dir, "index.html"), + COMPOSITION.replace('