From 3a02942a03edd42ffde353e1e1d9331a045283c0 Mon Sep 17 00:00:00 2001 From: Miguel Angel Simon Sierra Date: Fri, 10 Jul 2026 01:37:08 -0400 Subject: [PATCH] feat(cli): run-ID telemetry correlation and check breakdown event HYPERFRAMES_RUN_ID (trimmed, 128-char cap) attaches as run_id to the generic cli_command / cli_command_result events, absent when unset, so an orchestrator setting it per design element can group a verify loop's invocations in analytics. check additionally emits one check_report event per invocation (including lint-short-circuited and failing runs): gate booleans, per-class error/warning counts, launch/seek/contrast phase timings, sample counts, ok and exit code. Timings stay internal; no command output changes. --- packages/cli/src/cli.ts | 13 +- packages/cli/src/commands/check.test.ts | 176 ++++++++++++++++-- .../src/commands/layout-audit.browser.test.ts | 3 + packages/cli/src/commands/validate.ts | 10 +- packages/cli/src/telemetry/events.test.ts | 145 +++++++++++++++ packages/cli/src/telemetry/events.ts | 61 +++++- packages/cli/src/telemetry/runId.test.ts | 73 ++++++++ packages/cli/src/telemetry/runId.ts | 12 ++ packages/cli/src/utils/checkBrowser.test.ts | 6 + packages/cli/src/utils/checkBrowser.ts | 3 + packages/cli/src/utils/checkPipeline.ts | 38 +++- packages/cli/src/utils/checkTypes.ts | 7 + 12 files changed, 526 insertions(+), 21 deletions(-) create mode 100644 packages/cli/src/telemetry/runId.test.ts create mode 100644 packages/cli/src/telemetry/runId.ts diff --git a/packages/cli/src/cli.ts b/packages/cli/src/cli.ts index 2f3cc5d8f..69c7281d7 100644 --- a/packages/cli/src/cli.ts +++ b/packages/cli/src/cli.ts @@ -101,6 +101,7 @@ try { import { defineCommand, runMain } from "citty"; import type { ArgsDef, CommandDef } from "citty"; +import { getRunId } from "./telemetry/runId.js"; import { reportCommandFailure, trackCommandFailures } from "./utils/command-failure-tracking.js"; const isHelp = process.argv.includes("--help") || process.argv.includes("-h"); @@ -196,7 +197,13 @@ let _trackCliError: }) => void) | undefined; let _trackCommandResult: - | ((props: { command: string; success: boolean; exitCode: number; durationMs: number }) => void) + | ((props: { + command: string; + success: boolean; + exitCode: number; + durationMs: number; + runId?: string; + }) => void) | undefined; let _printUpdateNotice: (() => void) | undefined; let _printSkillsUpdateNotice: (() => void) | undefined; @@ -211,7 +218,7 @@ if (!isHelp && command !== "telemetry" && command !== "events" && command !== "u _trackCliError = mod.trackCliError; _trackCommandResult = mod.trackCommandResult; mod.showTelemetryNotice(); - mod.trackCommand(command); + mod.trackCommand(command, runId); if (mod.shouldTrack()) mod.incrementCommandCount(); }); } @@ -242,6 +249,7 @@ if (!isHelp && !hasJsonFlag && command !== "upgrade" && command !== "events") { } const commandStart = Date.now(); +const runId = getRunId(); // Async flush for normal exit. `beforeExit` re-fires every time the // event loop drains, and the async `_flush()` itself schedules new @@ -265,6 +273,7 @@ process.on("exit", (code) => { success: code === 0 && !commandFailed, exitCode: code, durationMs: Date.now() - commandStart, + runId, }); _flushSync?.(); }); diff --git a/packages/cli/src/commands/check.test.ts b/packages/cli/src/commands/check.test.ts index 8e1fb3cbe..581610b72 100644 --- a/packages/cli/src/commands/check.test.ts +++ b/packages/cli/src/commands/check.test.ts @@ -1,6 +1,13 @@ import { runCommand } from "citty"; import { readFileSync } from "node:fs"; import { afterEach, describe, expect, it, vi } from "vitest"; + +const trackCheckReport = vi.fn(); +vi.mock("../telemetry/events.js", () => ({ + trackCheckReport: (...args: unknown[]) => trackCheckReport(...args), + trackCommandFailure: vi.fn(), +})); + import { contrastRatio, parseColorRGBA } from "./contrast-bg.js"; import { createCheckCommand } from "./check.js"; import { @@ -35,6 +42,7 @@ const ORIGINAL_EXIT_CODE = process.exitCode; afterEach(() => { process.exitCode = ORIGINAL_EXIT_CODE; + trackCheckReport.mockClear(); vi.restoreAllMocks(); }); @@ -212,6 +220,20 @@ function noMotion(): MotionSpecResolution { return { kind: "none" }; } +function heroMotionFrame(time: number, visibleAt: (time: number) => boolean) { + return { + time, + data: { + "#hero": { + rect: { left: 10, top: 20, right: 310, bottom: 100, width: 300, height: 80 }, + opacity: visibleAt(time) ? 1 : 0, + visible: visibleAt(time), + }, + }, + liveness: {}, + }; +} + function dependencies( driver: CheckAuditDriver, options: { @@ -772,17 +794,7 @@ describe("check pipeline", () => { }; const driver = fakeDriver({ getDuration: vi.fn(async () => 1), - collectMotionFrame: vi.fn(async (time: number) => ({ - time, - data: { - "#hero": { - rect: { left: 10, top: 20, right: 310, bottom: 100, width: 300, height: 80 }, - opacity: time >= 0.5 ? 1 : 0, - visible: time >= 0.5, - }, - }, - liveness: {}, - })), + collectMotionFrame: vi.fn(async (time: number) => heroMotionFrame(time, (t) => t >= 0.5)), }); const { report } = await runScenario(driver, {}, { motion }); @@ -856,6 +868,148 @@ describe("check pipeline", () => { }); }); +describe("check report telemetry", () => { + it("reports one clean run with every gate and sampled-point count", async () => { + const motion: MotionSpecResolution = { + kind: "valid", + path: "/project/index.motion.json", + spec: { duration: 1, assertions: [{ kind: "appearsBy", selector: "#hero", bySec: 1 }] }, + }; + const driver = fakeDriver({ + getDuration: vi.fn(async () => 1), + collectMotionFrame: vi.fn(async (time: number) => heroMotionFrame(time, () => true)), + collectContrast: vi.fn(async (time: number) => ({ + entries: [contrastEntry({ time, ratio: 7, wcagAA: true })], + pngBase64: PNG_BASE64, + })), + }); + + const { report } = await runScenario( + driver, + { + samples: 1, + captionZone: { x0: 0, y0: 0.8, x1: 1, y1: 1 }, + frameCheck: {}, + snapshots: true, + }, + { motion }, + ); + + expect(trackCheckReport).toHaveBeenCalledTimes(1); + expect(trackCheckReport).toHaveBeenCalledWith( + expect.objectContaining({ + contrastGate: true, + motionGate: true, + captionZoneGate: true, + frameCheckGate: true, + snapshotsGate: true, + gridPoints: 2, + contrastPoints: 1, + ok: true, + exitCode: 0, + }), + ); + expect(report.ok).toBe(true); + }); + + it("reports one failing contrast run with its section error count", async () => { + const collectContrast = vi.fn(async (time: number) => ({ + entries: time === 0.5 ? [contrastEntry()] : [], + pngBase64: PNG_BASE64, + })); + + const { report } = await runScenario(fakeDriver({ collectContrast })); + + expect(trackCheckReport).toHaveBeenCalledTimes(1); + expect(trackCheckReport).toHaveBeenCalledWith( + expect.objectContaining({ + ok: false, + exitCode: 1, + contrastErrors: report.contrast.errorCount, + }), + ); + expect(report.contrast.errorCount).toBe(1); + }); + + it("reports zero browser samples and timings after a lint short circuit", async () => { + const lint = lintWith( + "error", + "root_missing_composition_id", + "Root element needs data-composition-id.", + ); + + const { report, browser } = await runScenario(fakeDriver(), {}, { lint }); + + expect(browser).not.toHaveBeenCalled(); + expect(trackCheckReport).toHaveBeenCalledTimes(1); + expect(trackCheckReport).toHaveBeenCalledWith( + expect.objectContaining({ + ok: false, + exitCode: 1, + gridPoints: 0, + contrastPoints: 0, + launchSettleMs: 0, + seekLoopMs: 0, + contrastMs: 0, + }), + ); + expect(report.ok).toBe(false); + }); + + it("matches report counts for mixed findings across classes", async () => { + const lint = lintWith("warning", "lint_warning", "Lint warning."); + const driver = fakeDriver({ + collectLayout: vi.fn(async () => [layoutIssue(), layoutIssue("warning")]), + collectContrast: vi.fn(async () => ({ + entries: [contrastEntry()], + pngBase64: PNG_BASE64, + })), + }); + + const { report } = await runScenario( + driver, + { samples: 1 }, + { lint, runtime: [runtimeError()] }, + ); + + expect(trackCheckReport).toHaveBeenCalledTimes(1); + expect(trackCheckReport).toHaveBeenCalledWith( + expect.objectContaining({ + lintErrors: report.lint.errorCount, + lintWarnings: report.lint.warningCount, + runtimeErrors: report.runtime.errorCount, + runtimeWarnings: report.runtime.warningCount, + layoutErrors: report.layout.errorCount, + layoutWarnings: report.layout.warningCount, + motionErrors: report.motion.errorCount, + motionWarnings: report.motion.warningCount, + contrastErrors: report.contrast.errorCount, + contrastWarnings: report.contrast.warningCount, + }), + ); + expect(report.lint.warningCount).toBe(1); + expect(report.runtime.errorCount).toBe(1); + expect(report.layout.errorCount).toBe(1); + expect(report.layout.warningCount).toBe(1); + }); + + it("measures contrast work inside the overall seek loop", async () => { + vi.spyOn(Date, "now") + .mockReturnValueOnce(100) + .mockReturnValueOnce(105) + .mockReturnValueOnce(110) + .mockReturnValueOnce(120); + + const result = await runAuditGrid( + fakeDriver(), + { ...DEFAULT_CHECK_OPTIONS, samples: 1 }, + noMotion(), + ); + + expect(result.timings).toEqual({ launchSettleMs: 0, seekLoopMs: 20, contrastMs: 5 }); + }); +}); + describe("contrast candidate round-trip", () => { it("passes the browser script's raw candidates back to finish, never the normalized copies", () => { const source = checkBrowserSource(); diff --git a/packages/cli/src/commands/layout-audit.browser.test.ts b/packages/cli/src/commands/layout-audit.browser.test.ts index 01871c5a6..0f05e8077 100644 --- a/packages/cli/src/commands/layout-audit.browser.test.ts +++ b/packages/cli/src/commands/layout-audit.browser.test.ts @@ -837,6 +837,9 @@ function installGeometry( rects: Record, styleOverrides: Record> = {}, ): void { + // Style-fixture branching mirrors the audit's per-property reads; splitting + // it would scatter one mock across helpers. + // fallow-ignore-next-line complexity vi.spyOn(window, "getComputedStyle").mockImplementation((element) => { const el = element as Element; const isBubble = el.id === "bubble"; diff --git a/packages/cli/src/commands/validate.ts b/packages/cli/src/commands/validate.ts index a5b40a688..0852dec60 100644 --- a/packages/cli/src/commands/validate.ts +++ b/packages/cli/src/commands/validate.ts @@ -1,3 +1,8 @@ +// The media-metadata wait exists twice on purpose: once Node-side and once +// inside a page.evaluate() body, which is serialized into the browser and +// cannot import the Node helper. Line-level markers don't survive the clone +// window drifting as the file is edited, hence the file-level suppression. +// fallow-ignore-file code-duplication import { defineCommand } from "citty"; import { existsSync, mkdtempSync, readFileSync, rmSync } from "node:fs"; import { tmpdir } from "node:os"; @@ -110,8 +115,6 @@ export function raceMediaReady( ): Promise { if (Number.isFinite(el.duration) && el.duration > 0) return Promise.resolve(); return new Promise((resolve) => { - // Clones its in-page twin below; evaluate() bodies can't import Node helpers. - // fallow-ignore-next-line code-duplication const onReady = () => { el.removeEventListener("loadedmetadata", onReady); el.removeEventListener("error", onReady); @@ -156,9 +159,6 @@ async function auditClipDurations( nodes.map((el) => { if (Number.isFinite(el.duration) && el.duration > 0) return Promise.resolve(); return new Promise((resolve) => { - // fallow-ignore-next-line code-duplication - // Serialized twin of the Node-side metadata wait above. - // fallow-ignore-next-line code-duplication const cleanup = () => { el.removeEventListener("loadedmetadata", onReady); el.removeEventListener("error", onReady); diff --git a/packages/cli/src/telemetry/events.test.ts b/packages/cli/src/telemetry/events.test.ts index 5801d8f9b..68932d58b 100644 --- a/packages/cli/src/telemetry/events.test.ts +++ b/packages/cli/src/telemetry/events.test.ts @@ -12,6 +12,9 @@ vi.mock("./config.js", () => ({ })); const { + trackCommand, + trackCommandResult, + trackCheckReport, trackRenderComplete, trackRenderError, trackRenderObservation, @@ -26,6 +29,148 @@ const { identifyUser, } = await import("./events.js"); +describe("command telemetry events", () => { + beforeEach(() => { + trackEvent.mockClear(); + }); + + it("includes run_id in cli_command when a run ID is provided", () => { + trackCommand("check", "run-123"); + + expect(trackEvent).toHaveBeenCalledWith("cli_command", { + command: "check", + run_id: "run-123", + }); + }); + + it("omits run_id from cli_command when no run ID is provided", () => { + trackCommand("check"); + + const properties = trackEvent.mock.lastCall?.[1]; + expect(properties).not.toHaveProperty("run_id"); + }); + + it("includes run_id in cli_command_result when a run ID is provided", () => { + trackCommandResult({ + command: "check", + success: true, + exitCode: 0, + durationMs: 42, + runId: "run-123", + }); + + expect(trackEvent).toHaveBeenCalledWith("cli_command_result", { + command: "check", + success: true, + exit_code: 0, + duration_ms: 42, + run_id: "run-123", + }); + }); + + it("omits run_id from cli_command_result when no run ID is provided", () => { + trackCommandResult({ + command: "check", + success: false, + exitCode: 1, + durationMs: 42, + }); + + const properties = trackEvent.mock.lastCall?.[1]; + expect(properties).not.toHaveProperty("run_id"); + }); +}); + +describe("trackCheckReport", () => { + beforeEach(() => { + trackEvent.mockClear(); + }); + + it("emits the check breakdown with snake_case properties and a run ID", () => { + trackCheckReport({ + contrastGate: true, + motionGate: false, + captionZoneGate: true, + frameCheckGate: false, + snapshotsGate: true, + lintErrors: 1, + lintWarnings: 2, + runtimeErrors: 3, + runtimeWarnings: 4, + layoutErrors: 5, + layoutWarnings: 6, + motionErrors: 7, + motionWarnings: 8, + contrastErrors: 9, + contrastWarnings: 10, + launchSettleMs: 11, + seekLoopMs: 12, + contrastMs: 13, + gridPoints: 14, + contrastPoints: 15, + ok: false, + exitCode: 1, + runId: "run-123", + }); + + expect(trackEvent).toHaveBeenCalledWith("check_report", { + gate_contrast: true, + gate_motion: false, + gate_caption_zone: true, + gate_frame_check: false, + gate_snapshots: true, + lint_errors: 1, + lint_warnings: 2, + runtime_errors: 3, + runtime_warnings: 4, + layout_errors: 5, + layout_warnings: 6, + motion_errors: 7, + motion_warnings: 8, + contrast_errors: 9, + contrast_warnings: 10, + launch_settle_ms: 11, + seek_loop_ms: 12, + contrast_ms: 13, + grid_points: 14, + contrast_points: 15, + ok: false, + exit_code: 1, + run_id: "run-123", + }); + }); + + it("omits run_id when no run ID is provided", () => { + trackCheckReport({ + contrastGate: false, + motionGate: false, + captionZoneGate: false, + frameCheckGate: false, + snapshotsGate: false, + lintErrors: 0, + lintWarnings: 0, + runtimeErrors: 0, + runtimeWarnings: 0, + layoutErrors: 0, + layoutWarnings: 0, + motionErrors: 0, + motionWarnings: 0, + contrastErrors: 0, + contrastWarnings: 0, + launchSettleMs: 0, + seekLoopMs: 0, + contrastMs: 0, + gridPoints: 0, + contrastPoints: 0, + ok: true, + exitCode: 0, + }); + + const properties = trackEvent.mock.lastCall?.[1]; + expect(properties).not.toHaveProperty("run_id"); + }); +}); + describe("render telemetry events", () => { beforeEach(() => { trackEvent.mockClear(); diff --git a/packages/cli/src/telemetry/events.ts b/packages/cli/src/telemetry/events.ts index 89462246a..ab91183f9 100644 --- a/packages/cli/src/telemetry/events.ts +++ b/packages/cli/src/telemetry/events.ts @@ -98,8 +98,11 @@ function redactTelemetryMessage(value: string): string { return redactTelemetryString(value); } -export function trackCommand(command: string): void { - trackEvent("cli_command", { command }); +export function trackCommand(command: string, runId?: string): void { + trackEvent("cli_command", { + command, + ...(runId !== undefined ? { run_id: runId } : {}), + }); } export function trackRenderComplete( @@ -544,11 +547,65 @@ export function trackCommandResult(props: { success: boolean; exitCode: number; durationMs: number; + runId?: string; }): void { trackEvent("cli_command_result", { command: props.command, success: props.success, exit_code: props.exitCode, duration_ms: props.durationMs, + ...(props.runId !== undefined ? { run_id: props.runId } : {}), + }); +} + +export function trackCheckReport(props: { + contrastGate: boolean; + motionGate: boolean; + captionZoneGate: boolean; + frameCheckGate: boolean; + snapshotsGate: boolean; + lintErrors: number; + lintWarnings: number; + runtimeErrors: number; + runtimeWarnings: number; + layoutErrors: number; + layoutWarnings: number; + motionErrors: number; + motionWarnings: number; + contrastErrors: number; + contrastWarnings: number; + launchSettleMs: number; + seekLoopMs: number; + contrastMs: number; + gridPoints: number; + contrastPoints: number; + ok: boolean; + exitCode: number; + runId?: string; +}): void { + trackEvent("check_report", { + gate_contrast: props.contrastGate, + gate_motion: props.motionGate, + gate_caption_zone: props.captionZoneGate, + gate_frame_check: props.frameCheckGate, + gate_snapshots: props.snapshotsGate, + lint_errors: props.lintErrors, + lint_warnings: props.lintWarnings, + runtime_errors: props.runtimeErrors, + runtime_warnings: props.runtimeWarnings, + layout_errors: props.layoutErrors, + layout_warnings: props.layoutWarnings, + motion_errors: props.motionErrors, + motion_warnings: props.motionWarnings, + contrast_errors: props.contrastErrors, + contrast_warnings: props.contrastWarnings, + launch_settle_ms: props.launchSettleMs, + seek_loop_ms: props.seekLoopMs, + contrast_ms: props.contrastMs, + grid_points: props.gridPoints, + contrast_points: props.contrastPoints, + ok: props.ok, + exit_code: props.exitCode, + ...(props.runId !== undefined ? { run_id: props.runId } : {}), }); } diff --git a/packages/cli/src/telemetry/runId.test.ts b/packages/cli/src/telemetry/runId.test.ts new file mode 100644 index 000000000..6a97b0726 --- /dev/null +++ b/packages/cli/src/telemetry/runId.test.ts @@ -0,0 +1,73 @@ +import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; + +const originalRunId = process.env["HYPERFRAMES_RUN_ID"]; + +async function loadGetRunId() { + const { getRunId } = await import("./runId.js"); + return getRunId; +} + +describe("getRunId", () => { + beforeEach(() => { + delete process.env["HYPERFRAMES_RUN_ID"]; + vi.resetModules(); + }); + + afterEach(() => { + if (originalRunId === undefined) delete process.env["HYPERFRAMES_RUN_ID"]; + else process.env["HYPERFRAMES_RUN_ID"] = originalRunId; + vi.resetModules(); + }); + + it("returns undefined when HYPERFRAMES_RUN_ID is unset", async () => { + const getRunId = await loadGetRunId(); + + expect(getRunId()).toBeUndefined(); + }); + + it("returns undefined when HYPERFRAMES_RUN_ID contains only whitespace", async () => { + process.env["HYPERFRAMES_RUN_ID"] = " \t\n "; + const getRunId = await loadGetRunId(); + + expect(getRunId()).toBeUndefined(); + }); + + it("returns a normal HYPERFRAMES_RUN_ID value", async () => { + process.env["HYPERFRAMES_RUN_ID"] = "run-123"; + const getRunId = await loadGetRunId(); + + expect(getRunId()).toBe("run-123"); + }); + + it("truncates HYPERFRAMES_RUN_ID to exactly 128 characters", async () => { + process.env["HYPERFRAMES_RUN_ID"] = "x".repeat(160); + const getRunId = await loadGetRunId(); + + expect(getRunId()).toBe("x".repeat(128)); + expect(getRunId()).toHaveLength(128); + }); + + it("trims whitespace around a real HYPERFRAMES_RUN_ID value", async () => { + process.env["HYPERFRAMES_RUN_ID"] = " run-123 \n"; + const getRunId = await loadGetRunId(); + + expect(getRunId()).toBe("run-123"); + }); + + it("memoizes the first environment read", async () => { + process.env["HYPERFRAMES_RUN_ID"] = "first-run"; + const getRunId = await loadGetRunId(); + + expect(getRunId()).toBe("first-run"); + process.env["HYPERFRAMES_RUN_ID"] = "second-run"; + expect(getRunId()).toBe("first-run"); + }); + + it("memoizes an initial undefined environment read", async () => { + const getRunId = await loadGetRunId(); + + expect(getRunId()).toBeUndefined(); + process.env["HYPERFRAMES_RUN_ID"] = "later-run"; + expect(getRunId()).toBeUndefined(); + }); +}); diff --git a/packages/cli/src/telemetry/runId.ts b/packages/cli/src/telemetry/runId.ts new file mode 100644 index 000000000..5c5e307fb --- /dev/null +++ b/packages/cli/src/telemetry/runId.ts @@ -0,0 +1,12 @@ +let resolved = false; +let runId: string | undefined; + +export function getRunId(): string | undefined { + if (!resolved) { + const value = process.env["HYPERFRAMES_RUN_ID"]?.trim().slice(0, 128); + runId = value ? value : undefined; + resolved = true; + } + + return runId; +} diff --git a/packages/cli/src/utils/checkBrowser.test.ts b/packages/cli/src/utils/checkBrowser.test.ts index 1306a30c3..63619aa67 100644 --- a/packages/cli/src/utils/checkBrowser.test.ts +++ b/packages/cli/src/utils/checkBrowser.test.ts @@ -44,6 +44,11 @@ afterEach(() => { }); it("carries raw browser geometry through the page driver and pipeline", async () => { + vi.spyOn(Date, "now") + .mockReturnValueOnce(100) + .mockReturnValueOnce(160) + .mockReturnValueOnce(200) + .mockReturnValueOnce(240); document.body.innerHTML = `
@@ -85,6 +90,7 @@ it("carries raw browser geometry through the page driver and pipeline", async () time: 5, }), ]); + expect(result.timings).toEqual({ launchSettleMs: 60, seekLoopMs: 40, contrastMs: 0 }); expect(mocks.serverClose).toHaveBeenCalledOnce(); }); diff --git a/packages/cli/src/utils/checkBrowser.ts b/packages/cli/src/utils/checkBrowser.ts index ca8152d0d..edd34746c 100644 --- a/packages/cli/src/utils/checkBrowser.ts +++ b/packages/cli/src/utils/checkBrowser.ts @@ -93,6 +93,7 @@ export async function runBrowserCheck( let chromeBrowser: import("puppeteer-core").Browser | undefined; try { + const launchSettleStart = Date.now(); const session = await openSettledCompositionPage(html, server.url, { renderReadyTimeoutMs: options.timeout, renderReadyWarningSuffix: "checking the current page state", @@ -104,12 +105,14 @@ export async function runBrowserCheck( await waitForPreferredSeekTarget(page, 500); const rootAnchor = await resolveRootAnchor(page); + const launchSettleMs = Date.now() - launchSettleStart; const driver = createPageDriver(page, (time) => { currentTime = time; }); const result = await runGrid(driver, options, motion); return { ...result, + timings: { ...result.timings, launchSettleMs }, runtimeFindings: drafts.map((draft) => runtimeFinding(draft, rootAnchor)), }; } finally { diff --git a/packages/cli/src/utils/checkPipeline.ts b/packages/cli/src/utils/checkPipeline.ts index 07b65f2bb..e60fd559b 100644 --- a/packages/cli/src/utils/checkPipeline.ts +++ b/packages/cli/src/utils/checkPipeline.ts @@ -1,5 +1,7 @@ import { mkdirSync, writeFileSync } from "node:fs"; import { join, relative } from "node:path"; +import { trackCheckReport } from "../telemetry/events.js"; +import { getRunId } from "../telemetry/runId.js"; import type { ProjectDir } from "./project.js"; import { lintProject, shouldBlockRender, type ProjectLintResult } from "./lintProject.js"; import { @@ -180,6 +182,7 @@ interface GridSamples { motionFrames: MotionFrame[]; contrastEntries: ContrastAuditEntry[]; screenshots: CheckScreenshot[]; + contrastMs: number; } interface GeometrySeen { @@ -349,6 +352,7 @@ async function collectGridSamples( motionFrames: [], contrastEntries: [], screenshots: [], + contrastMs: 0, }; for (const time of mergeSampleTimes(grid.layoutSamples, motion.times)) { await driver.seek(time); @@ -366,7 +370,9 @@ async function collectGridSamples( ); } if (contrastSet.has(time)) { + const contrastStart = Date.now(); const capture = await driver.collectContrast(time); + collected.contrastMs += Date.now() - contrastStart; collected.contrastEntries.push(...capture.entries); collected.screenshots.push({ time, pngBase64: capture.pngBase64 }); } @@ -382,7 +388,9 @@ export async function runAuditGrid( await driver.initialize(options.contrast); const grid = await buildSampleGrid(driver, options); const plan = await planMotionSampling(driver, motion, grid.duration); + const seekLoopStart = Date.now(); const collected = await collectGridSamples(driver, options, grid, plan); + const seekLoopMs = Date.now() - seekLoopStart; let motionIssues = plan.preflightIssues; if (motion.kind === "valid" && motionIssues.length === 0 && collected.motionFrames.length > 0) { @@ -408,6 +416,7 @@ export async function runAuditGrid( contrastChecked: collected.contrastEntries.length, contrastPassed: contrast.passed, screenshots: collected.screenshots, + timings: { launchSettleMs: 0, seekLoopMs, contrastMs: collected.contrastMs }, }; } @@ -555,7 +564,7 @@ function buildReport( layout.errorCount + motionSection.errorCount + contrastSection.errorCount; - return { + const report: CheckReport = { ok: errorCount === 0 && (!options.strict || warningCount === 0), strict: options.strict, lint, @@ -580,6 +589,32 @@ function buildReport( times: options.snapshots ? browser.screenshots.map((shot) => shot.time) : [], }, }; + trackCheckReport({ + contrastGate: options.contrast, + motionGate: motion.kind !== "none", + captionZoneGate: options.captionZone !== undefined, + frameCheckGate: options.frameCheck !== undefined, + snapshotsGate: options.snapshots, + lintErrors: lint.errorCount, + lintWarnings: lint.warningCount, + runtimeErrors: runtime.errorCount, + runtimeWarnings: runtime.warningCount, + layoutErrors: layout.errorCount, + layoutWarnings: layout.warningCount, + motionErrors: motionSection.errorCount, + motionWarnings: motionSection.warningCount, + contrastErrors: contrastSection.errorCount, + contrastWarnings: contrastSection.warningCount, + launchSettleMs: browser.timings.launchSettleMs, + seekLoopMs: browser.timings.seekLoopMs, + contrastMs: browser.timings.contrastMs, + gridPoints: browser.layoutSamples.length, + contrastPoints: browser.contrastChecked, + ok: report.ok, + exitCode: checkExitCode(report), + runId: getRunId(), + }); + return report; } function shapeLayoutSection( @@ -651,6 +686,7 @@ function emptyBrowserResult(): CheckBrowserResult { contrastChecked: 0, contrastPassed: 0, screenshots: [], + timings: { launchSettleMs: 0, seekLoopMs: 0, contrastMs: 0 }, }; } diff --git a/packages/cli/src/utils/checkTypes.ts b/packages/cli/src/utils/checkTypes.ts index 2c45dde92..e9b0a3225 100644 --- a/packages/cli/src/utils/checkTypes.ts +++ b/packages/cli/src/utils/checkTypes.ts @@ -133,6 +133,12 @@ export interface CheckScreenshot { pngBase64: string; } +export interface CheckTimings { + launchSettleMs: number; + seekLoopMs: number; + contrastMs: number; +} + export interface CheckBrowserResult { duration: number; layoutSamples: number[]; @@ -147,6 +153,7 @@ export interface CheckBrowserResult { contrastChecked: number; contrastPassed: number; screenshots: CheckScreenshot[]; + timings: CheckTimings; } /** The seek-grid audit loop, injected into checkBrowser so it never imports checkPipeline back. */