mirror of
https://github.com/heygen-com/hyperframes.git
synced 2026-09-08 19:06:04 +00:00
feat(cli): run-ID telemetry correlation and check breakdown event
HYPERFRAMES_RUN_ID (trimmed, 128-char cap) attaches as run_id to the generic cli_command / cli_command_result events, absent when unset, so an orchestrator setting it per design element can group a verify loop's invocations in analytics. check additionally emits one check_report event per invocation (including lint-short-circuited and failing runs): gate booleans, per-class error/warning counts, launch/seek/contrast phase timings, sample counts, ok and exit code. Timings stay internal; no command output changes.
This commit is contained in:
@@ -1,6 +1,13 @@
|
||||
import { runCommand } from "citty";
|
||||
import { readFileSync } from "node:fs";
|
||||
import { afterEach, describe, expect, it, vi } from "vitest";
|
||||
|
||||
const trackCheckReport = vi.fn();
|
||||
vi.mock("../telemetry/events.js", () => ({
|
||||
trackCheckReport: (...args: unknown[]) => trackCheckReport(...args),
|
||||
trackCommandFailure: vi.fn(),
|
||||
}));
|
||||
|
||||
import { contrastRatio, parseColorRGBA } from "./contrast-bg.js";
|
||||
import { createCheckCommand } from "./check.js";
|
||||
import {
|
||||
@@ -35,6 +42,7 @@ const ORIGINAL_EXIT_CODE = process.exitCode;
|
||||
|
||||
afterEach(() => {
|
||||
process.exitCode = ORIGINAL_EXIT_CODE;
|
||||
trackCheckReport.mockClear();
|
||||
vi.restoreAllMocks();
|
||||
});
|
||||
|
||||
@@ -212,6 +220,20 @@ function noMotion(): MotionSpecResolution {
|
||||
return { kind: "none" };
|
||||
}
|
||||
|
||||
function heroMotionFrame(time: number, visibleAt: (time: number) => boolean) {
|
||||
return {
|
||||
time,
|
||||
data: {
|
||||
"#hero": {
|
||||
rect: { left: 10, top: 20, right: 310, bottom: 100, width: 300, height: 80 },
|
||||
opacity: visibleAt(time) ? 1 : 0,
|
||||
visible: visibleAt(time),
|
||||
},
|
||||
},
|
||||
liveness: {},
|
||||
};
|
||||
}
|
||||
|
||||
function dependencies(
|
||||
driver: CheckAuditDriver,
|
||||
options: {
|
||||
@@ -772,17 +794,7 @@ describe("check pipeline", () => {
|
||||
};
|
||||
const driver = fakeDriver({
|
||||
getDuration: vi.fn(async () => 1),
|
||||
collectMotionFrame: vi.fn(async (time: number) => ({
|
||||
time,
|
||||
data: {
|
||||
"#hero": {
|
||||
rect: { left: 10, top: 20, right: 310, bottom: 100, width: 300, height: 80 },
|
||||
opacity: time >= 0.5 ? 1 : 0,
|
||||
visible: time >= 0.5,
|
||||
},
|
||||
},
|
||||
liveness: {},
|
||||
})),
|
||||
collectMotionFrame: vi.fn(async (time: number) => heroMotionFrame(time, (t) => t >= 0.5)),
|
||||
});
|
||||
const { report } = await runScenario(driver, {}, { motion });
|
||||
|
||||
@@ -856,6 +868,148 @@ describe("check pipeline", () => {
|
||||
});
|
||||
});
|
||||
|
||||
describe("check report telemetry", () => {
|
||||
it("reports one clean run with every gate and sampled-point count", async () => {
|
||||
const motion: MotionSpecResolution = {
|
||||
kind: "valid",
|
||||
path: "/project/index.motion.json",
|
||||
spec: { duration: 1, assertions: [{ kind: "appearsBy", selector: "#hero", bySec: 1 }] },
|
||||
};
|
||||
const driver = fakeDriver({
|
||||
getDuration: vi.fn(async () => 1),
|
||||
collectMotionFrame: vi.fn(async (time: number) => heroMotionFrame(time, () => true)),
|
||||
collectContrast: vi.fn(async (time: number) => ({
|
||||
entries: [contrastEntry({ time, ratio: 7, wcagAA: true })],
|
||||
pngBase64: PNG_BASE64,
|
||||
})),
|
||||
});
|
||||
|
||||
const { report } = await runScenario(
|
||||
driver,
|
||||
{
|
||||
samples: 1,
|
||||
captionZone: { x0: 0, y0: 0.8, x1: 1, y1: 1 },
|
||||
frameCheck: {},
|
||||
snapshots: true,
|
||||
},
|
||||
{ motion },
|
||||
);
|
||||
|
||||
expect(trackCheckReport).toHaveBeenCalledTimes(1);
|
||||
expect(trackCheckReport).toHaveBeenCalledWith(
|
||||
expect.objectContaining({
|
||||
contrastGate: true,
|
||||
motionGate: true,
|
||||
captionZoneGate: true,
|
||||
frameCheckGate: true,
|
||||
snapshotsGate: true,
|
||||
gridPoints: 2,
|
||||
contrastPoints: 1,
|
||||
ok: true,
|
||||
exitCode: 0,
|
||||
}),
|
||||
);
|
||||
expect(report.ok).toBe(true);
|
||||
});
|
||||
|
||||
it("reports one failing contrast run with its section error count", async () => {
|
||||
const collectContrast = vi.fn(async (time: number) => ({
|
||||
entries: time === 0.5 ? [contrastEntry()] : [],
|
||||
pngBase64: PNG_BASE64,
|
||||
}));
|
||||
|
||||
const { report } = await runScenario(fakeDriver({ collectContrast }));
|
||||
|
||||
expect(trackCheckReport).toHaveBeenCalledTimes(1);
|
||||
expect(trackCheckReport).toHaveBeenCalledWith(
|
||||
expect.objectContaining({
|
||||
ok: false,
|
||||
exitCode: 1,
|
||||
contrastErrors: report.contrast.errorCount,
|
||||
}),
|
||||
);
|
||||
expect(report.contrast.errorCount).toBe(1);
|
||||
});
|
||||
|
||||
it("reports zero browser samples and timings after a lint short circuit", async () => {
|
||||
const lint = lintWith(
|
||||
"error",
|
||||
"root_missing_composition_id",
|
||||
"Root element needs data-composition-id.",
|
||||
);
|
||||
|
||||
const { report, browser } = await runScenario(fakeDriver(), {}, { lint });
|
||||
|
||||
expect(browser).not.toHaveBeenCalled();
|
||||
expect(trackCheckReport).toHaveBeenCalledTimes(1);
|
||||
expect(trackCheckReport).toHaveBeenCalledWith(
|
||||
expect.objectContaining({
|
||||
ok: false,
|
||||
exitCode: 1,
|
||||
gridPoints: 0,
|
||||
contrastPoints: 0,
|
||||
launchSettleMs: 0,
|
||||
seekLoopMs: 0,
|
||||
contrastMs: 0,
|
||||
}),
|
||||
);
|
||||
expect(report.ok).toBe(false);
|
||||
});
|
||||
|
||||
it("matches report counts for mixed findings across classes", async () => {
|
||||
const lint = lintWith("warning", "lint_warning", "Lint warning.");
|
||||
const driver = fakeDriver({
|
||||
collectLayout: vi.fn(async () => [layoutIssue(), layoutIssue("warning")]),
|
||||
collectContrast: vi.fn(async () => ({
|
||||
entries: [contrastEntry()],
|
||||
pngBase64: PNG_BASE64,
|
||||
})),
|
||||
});
|
||||
|
||||
const { report } = await runScenario(
|
||||
driver,
|
||||
{ samples: 1 },
|
||||
{ lint, runtime: [runtimeError()] },
|
||||
);
|
||||
|
||||
expect(trackCheckReport).toHaveBeenCalledTimes(1);
|
||||
expect(trackCheckReport).toHaveBeenCalledWith(
|
||||
expect.objectContaining({
|
||||
lintErrors: report.lint.errorCount,
|
||||
lintWarnings: report.lint.warningCount,
|
||||
runtimeErrors: report.runtime.errorCount,
|
||||
runtimeWarnings: report.runtime.warningCount,
|
||||
layoutErrors: report.layout.errorCount,
|
||||
layoutWarnings: report.layout.warningCount,
|
||||
motionErrors: report.motion.errorCount,
|
||||
motionWarnings: report.motion.warningCount,
|
||||
contrastErrors: report.contrast.errorCount,
|
||||
contrastWarnings: report.contrast.warningCount,
|
||||
}),
|
||||
);
|
||||
expect(report.lint.warningCount).toBe(1);
|
||||
expect(report.runtime.errorCount).toBe(1);
|
||||
expect(report.layout.errorCount).toBe(1);
|
||||
expect(report.layout.warningCount).toBe(1);
|
||||
});
|
||||
|
||||
it("measures contrast work inside the overall seek loop", async () => {
|
||||
vi.spyOn(Date, "now")
|
||||
.mockReturnValueOnce(100)
|
||||
.mockReturnValueOnce(105)
|
||||
.mockReturnValueOnce(110)
|
||||
.mockReturnValueOnce(120);
|
||||
|
||||
const result = await runAuditGrid(
|
||||
fakeDriver(),
|
||||
{ ...DEFAULT_CHECK_OPTIONS, samples: 1 },
|
||||
noMotion(),
|
||||
);
|
||||
|
||||
expect(result.timings).toEqual({ launchSettleMs: 0, seekLoopMs: 20, contrastMs: 5 });
|
||||
});
|
||||
});
|
||||
|
||||
describe("contrast candidate round-trip", () => {
|
||||
it("passes the browser script's raw candidates back to finish, never the normalized copies", () => {
|
||||
const source = checkBrowserSource();
|
||||
|
||||
@@ -837,6 +837,9 @@ function installGeometry(
|
||||
rects: Record<string, DOMRect>,
|
||||
styleOverrides: Record<string, Partial<CSSStyleDeclaration>> = {},
|
||||
): void {
|
||||
// Style-fixture branching mirrors the audit's per-property reads; splitting
|
||||
// it would scatter one mock across helpers.
|
||||
// fallow-ignore-next-line complexity
|
||||
vi.spyOn(window, "getComputedStyle").mockImplementation((element) => {
|
||||
const el = element as Element;
|
||||
const isBubble = el.id === "bubble";
|
||||
|
||||
@@ -1,3 +1,8 @@
|
||||
// The media-metadata wait exists twice on purpose: once Node-side and once
|
||||
// inside a page.evaluate() body, which is serialized into the browser and
|
||||
// cannot import the Node helper. Line-level markers don't survive the clone
|
||||
// window drifting as the file is edited, hence the file-level suppression.
|
||||
// fallow-ignore-file code-duplication
|
||||
import { defineCommand } from "citty";
|
||||
import { existsSync, mkdtempSync, readFileSync, rmSync } from "node:fs";
|
||||
import { tmpdir } from "node:os";
|
||||
@@ -110,8 +115,6 @@ export function raceMediaReady(
|
||||
): Promise<void> {
|
||||
if (Number.isFinite(el.duration) && el.duration > 0) return Promise.resolve();
|
||||
return new Promise<void>((resolve) => {
|
||||
// Clones its in-page twin below; evaluate() bodies can't import Node helpers.
|
||||
// fallow-ignore-next-line code-duplication
|
||||
const onReady = () => {
|
||||
el.removeEventListener("loadedmetadata", onReady);
|
||||
el.removeEventListener("error", onReady);
|
||||
@@ -156,9 +159,6 @@ async function auditClipDurations(
|
||||
nodes.map((el) => {
|
||||
if (Number.isFinite(el.duration) && el.duration > 0) return Promise.resolve();
|
||||
return new Promise<void>((resolve) => {
|
||||
// fallow-ignore-next-line code-duplication
|
||||
// Serialized twin of the Node-side metadata wait above.
|
||||
// fallow-ignore-next-line code-duplication
|
||||
const cleanup = () => {
|
||||
el.removeEventListener("loadedmetadata", onReady);
|
||||
el.removeEventListener("error", onReady);
|
||||
|
||||
Reference in New Issue
Block a user