feat(cli): run-ID telemetry correlation and check breakdown event

HYPERFRAMES_RUN_ID (trimmed, 128-char cap) attaches as run_id to the
generic cli_command / cli_command_result events, absent when unset, so
an orchestrator setting it per design element can group a verify loop's
invocations in analytics. check additionally emits one check_report
event per invocation (including lint-short-circuited and failing runs):
gate booleans, per-class error/warning counts, launch/seek/contrast
phase timings, sample counts, ok and exit code. Timings stay internal;
no command output changes.
This commit is contained in:
Miguel Angel Simon Sierra
2026-07-10 13:27:52 -04:00
parent 7ab6c2b7a2
commit 3a02942a03
12 changed files with 526 additions and 21 deletions
+165 -11
View File
@@ -1,6 +1,13 @@
import { runCommand } from "citty";
import { readFileSync } from "node:fs";
import { afterEach, describe, expect, it, vi } from "vitest";
const trackCheckReport = vi.fn();
vi.mock("../telemetry/events.js", () => ({
trackCheckReport: (...args: unknown[]) => trackCheckReport(...args),
trackCommandFailure: vi.fn(),
}));
import { contrastRatio, parseColorRGBA } from "./contrast-bg.js";
import { createCheckCommand } from "./check.js";
import {
@@ -35,6 +42,7 @@ const ORIGINAL_EXIT_CODE = process.exitCode;
afterEach(() => {
process.exitCode = ORIGINAL_EXIT_CODE;
trackCheckReport.mockClear();
vi.restoreAllMocks();
});
@@ -212,6 +220,20 @@ function noMotion(): MotionSpecResolution {
return { kind: "none" };
}
function heroMotionFrame(time: number, visibleAt: (time: number) => boolean) {
return {
time,
data: {
"#hero": {
rect: { left: 10, top: 20, right: 310, bottom: 100, width: 300, height: 80 },
opacity: visibleAt(time) ? 1 : 0,
visible: visibleAt(time),
},
},
liveness: {},
};
}
function dependencies(
driver: CheckAuditDriver,
options: {
@@ -772,17 +794,7 @@ describe("check pipeline", () => {
};
const driver = fakeDriver({
getDuration: vi.fn(async () => 1),
collectMotionFrame: vi.fn(async (time: number) => ({
time,
data: {
"#hero": {
rect: { left: 10, top: 20, right: 310, bottom: 100, width: 300, height: 80 },
opacity: time >= 0.5 ? 1 : 0,
visible: time >= 0.5,
},
},
liveness: {},
})),
collectMotionFrame: vi.fn(async (time: number) => heroMotionFrame(time, (t) => t >= 0.5)),
});
const { report } = await runScenario(driver, {}, { motion });
@@ -856,6 +868,148 @@ describe("check pipeline", () => {
});
});
describe("check report telemetry", () => {
it("reports one clean run with every gate and sampled-point count", async () => {
const motion: MotionSpecResolution = {
kind: "valid",
path: "/project/index.motion.json",
spec: { duration: 1, assertions: [{ kind: "appearsBy", selector: "#hero", bySec: 1 }] },
};
const driver = fakeDriver({
getDuration: vi.fn(async () => 1),
collectMotionFrame: vi.fn(async (time: number) => heroMotionFrame(time, () => true)),
collectContrast: vi.fn(async (time: number) => ({
entries: [contrastEntry({ time, ratio: 7, wcagAA: true })],
pngBase64: PNG_BASE64,
})),
});
const { report } = await runScenario(
driver,
{
samples: 1,
captionZone: { x0: 0, y0: 0.8, x1: 1, y1: 1 },
frameCheck: {},
snapshots: true,
},
{ motion },
);
expect(trackCheckReport).toHaveBeenCalledTimes(1);
expect(trackCheckReport).toHaveBeenCalledWith(
expect.objectContaining({
contrastGate: true,
motionGate: true,
captionZoneGate: true,
frameCheckGate: true,
snapshotsGate: true,
gridPoints: 2,
contrastPoints: 1,
ok: true,
exitCode: 0,
}),
);
expect(report.ok).toBe(true);
});
it("reports one failing contrast run with its section error count", async () => {
const collectContrast = vi.fn(async (time: number) => ({
entries: time === 0.5 ? [contrastEntry()] : [],
pngBase64: PNG_BASE64,
}));
const { report } = await runScenario(fakeDriver({ collectContrast }));
expect(trackCheckReport).toHaveBeenCalledTimes(1);
expect(trackCheckReport).toHaveBeenCalledWith(
expect.objectContaining({
ok: false,
exitCode: 1,
contrastErrors: report.contrast.errorCount,
}),
);
expect(report.contrast.errorCount).toBe(1);
});
it("reports zero browser samples and timings after a lint short circuit", async () => {
const lint = lintWith(
"error",
"root_missing_composition_id",
"Root element needs data-composition-id.",
);
const { report, browser } = await runScenario(fakeDriver(), {}, { lint });
expect(browser).not.toHaveBeenCalled();
expect(trackCheckReport).toHaveBeenCalledTimes(1);
expect(trackCheckReport).toHaveBeenCalledWith(
expect.objectContaining({
ok: false,
exitCode: 1,
gridPoints: 0,
contrastPoints: 0,
launchSettleMs: 0,
seekLoopMs: 0,
contrastMs: 0,
}),
);
expect(report.ok).toBe(false);
});
it("matches report counts for mixed findings across classes", async () => {
const lint = lintWith("warning", "lint_warning", "Lint warning.");
const driver = fakeDriver({
collectLayout: vi.fn(async () => [layoutIssue(), layoutIssue("warning")]),
collectContrast: vi.fn(async () => ({
entries: [contrastEntry()],
pngBase64: PNG_BASE64,
})),
});
const { report } = await runScenario(
driver,
{ samples: 1 },
{ lint, runtime: [runtimeError()] },
);
expect(trackCheckReport).toHaveBeenCalledTimes(1);
expect(trackCheckReport).toHaveBeenCalledWith(
expect.objectContaining({
lintErrors: report.lint.errorCount,
lintWarnings: report.lint.warningCount,
runtimeErrors: report.runtime.errorCount,
runtimeWarnings: report.runtime.warningCount,
layoutErrors: report.layout.errorCount,
layoutWarnings: report.layout.warningCount,
motionErrors: report.motion.errorCount,
motionWarnings: report.motion.warningCount,
contrastErrors: report.contrast.errorCount,
contrastWarnings: report.contrast.warningCount,
}),
);
expect(report.lint.warningCount).toBe(1);
expect(report.runtime.errorCount).toBe(1);
expect(report.layout.errorCount).toBe(1);
expect(report.layout.warningCount).toBe(1);
});
it("measures contrast work inside the overall seek loop", async () => {
vi.spyOn(Date, "now")
.mockReturnValueOnce(100)
.mockReturnValueOnce(105)
.mockReturnValueOnce(110)
.mockReturnValueOnce(120);
const result = await runAuditGrid(
fakeDriver(),
{ ...DEFAULT_CHECK_OPTIONS, samples: 1 },
noMotion(),
);
expect(result.timings).toEqual({ launchSettleMs: 0, seekLoopMs: 20, contrastMs: 5 });
});
});
describe("contrast candidate round-trip", () => {
it("passes the browser script's raw candidates back to finish, never the normalized copies", () => {
const source = checkBrowserSource();