From 070b964d7b8cd844a893f0abbe8e0b7f1ac6d879 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Miguel=20=C3=81ngel?= Date: Sun, 8 Nov 2026 14:01:58 +0000 Subject: [PATCH] feat(cli): check replays an unchanged project's last pass and takes --no-craft (#7398) * feat(cli): check returns an unchanged project's last pass in about a second * fix(cli): the check cache hashes files without a stat race and its tests run before the signal test * test(cli): the warm check test compares two real runs, not a run and its cached pass * feat(cli): check --no-craft skips the craft pass for a render preflight * fix(cli): the check cache keys on timeout, env and large files and never replays a moved tree * style(cli): format the check cache * fix(cli): the large-file cache test waits for the clock tick it depends on * fix(cli): the check cache keys on the browser and on scripts under .hyperframes * fix(cli): the check cache ignores the video copies the check itself touches --- docs/packages/cli.mdx | 2 + packages/cli/src/commands/check.test.ts | 209 ++++++++++++++++++++- packages/cli/src/commands/check.ts | 52 ++++- packages/cli/src/utils/checkCache.ts | 139 ++++++++++++++ packages/cli/src/utils/checkPipeline.ts | 4 +- packages/cli/src/utils/checkTypes.ts | 3 + packages/cli/src/warmCheck.process.test.ts | 2 +- 7 files changed, 405 insertions(+), 6 deletions(-) create mode 100644 packages/cli/src/utils/checkCache.ts diff --git a/docs/packages/cli.mdx b/docs/packages/cli.mdx index ae0ffd2cc7..ce2d845bea 100644 --- a/docs/packages/cli.mdx +++ b/docs/packages/cli.mdx @@ -1271,7 +1271,9 @@ cover every finding from the full check. | `--tolerance` | Allowed overflow in px before reporting (default 2) | | `--timeout` | Render-ready budget in ms; also raises page navigation above its 10s floor (default 45000, as snapshot and render) | | `--budget ` | Stop the whole check this long after it starts (at least 30s) and report what it found, with `check_budget_reached` naming the phase it was in. For hosts with their own time limit. | +| `--no-cache` | Run the check even when the project files, flags, CLI and browser are unchanged since the last pass. Without it, an unchanged project returns that pass in about a second, marked as cached; remote scripts and assets are not fetched again for that. A failure is never cached, and a project with `node_modules` or a linked directory is never replayed. | | `--no-contrast` | Skip the WCAG pass while iterating | +| `--no-craft` | Skip the craft pass (dead stretches, unread lines, silent film), the longest part of a full check. The report marks it skipped. Use it for a render preflight; keep it on for the full check before a release. | | `--strict` | Exit non-zero on warnings too (default: errors only) | | `--caption-zone ""` | Opt-in band gate. Flags a text element's DOM box that overlaps the fractional band. Optional `severity` and `seek`. | | `--frame-check` | Opt-in out-of-frame detection for `img`, `svg`, `video`, and `canvas` | diff --git a/packages/cli/src/commands/check.test.ts b/packages/cli/src/commands/check.test.ts index 668ab26950..01c2c0b26a 100644 --- a/packages/cli/src/commands/check.test.ts +++ b/packages/cli/src/commands/check.test.ts @@ -1,6 +1,15 @@ import type { CheckProgress } from "../utils/checkProgress.js"; import { runCommand } from "citty"; -import { mkdirSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from "node:fs"; +import { + mkdirSync, + mkdtempSync, + readFileSync, + rmSync, + statSync, + symlinkSync, + utimesSync, + writeFileSync, +} from "node:fs"; import { tmpdir } from "node:os"; import { join } from "node:path"; import { afterEach, describe, expect, it, vi } from "vitest"; @@ -544,6 +553,24 @@ it("--budget sets when the whole check stops, its report time kept back, counted expect(runPipeline.mock.calls[1]?.[1].budget).toBeUndefined(); }); +it("--no-craft reaches the pipeline as craft: false, and the craft pass is on without it", async () => { + const { report } = await runScenario(fakeDriver()); + const runPipeline = vi.fn(async (_project: ProjectDir, _options: CheckOptions) => report); + vi.spyOn(console, "log").mockImplementation(() => undefined); + const command = createCheckCommand({ + resolveProject: () => PROJECT, + runPipeline, + prepareMedia: vi.fn(async () => []), + withMeta: (value) => value, + }); + + await runCommand(command, { rawArgs: ["--json", "--no-craft"] }); + await runCommand(command, { rawArgs: ["--json"] }); + + expect(runPipeline.mock.calls[0]?.[1].craft).toBe(false); + expect(runPipeline.mock.calls[1]?.[1].craft).toBe(true); +}); + it("parses the caption-zone grammar and enables the frame gate", async () => { const { report } = await runScenario(fakeDriver()); const runPipeline = vi.fn(async (_project: ProjectDir, _options: CheckOptions) => report); @@ -3046,6 +3073,15 @@ describe("check pipeline", () => { expect(checkExitCode(strict.report)).toBe(0); }); + it("--no-craft never samples the craft frames and the report says the pass was skipped", async () => { + const driver = craftDriver(); + const { report } = await runScenario(driver, { craft: false }); + + expect(driver.collectCraftFrame).not.toHaveBeenCalled(); + expect(report.craft.enabled).toBe(false); + expect(report.craft.findings).toEqual([]); + }); + it("seeks each craft frame with WebGL draws on, so a redrawn canvas reads as moving", async () => { const driver = craftDriver(); await runScenario(driver); @@ -3991,6 +4027,177 @@ it("applies transition sample limits within the requested range", async () => { expect(report.layout.transitionSamplesDropped).toBe(3); }); +describe("check cache", () => { + const dirs: string[] = []; + afterEach(() => { + vi.unstubAllEnvs(); + for (const dir of dirs.splice(0)) rmSync(dir, { recursive: true, force: true }); + }); + + function setup(report: CheckReport) { + const browser: { identity: string | undefined } = { identity: "chrome:1" }; + const dir = mkdtempSync(join(tmpdir(), "hf-check-cache-")); + dirs.push(dir); + writeFileSync(join(dir, "index.html"), "one"); + const runPipeline = vi.fn(async (_project: ProjectDir, _options: CheckOptions) => report); + const command = createCheckCommand({ + resolveProject: () => ({ dir, name: "p", indexPath: join(dir, "index.html") }), + runPipeline, + prepareMedia: vi.fn(async () => []), + withMeta: (value) => value, + browserIdentity: async () => browser.identity, + }); + vi.spyOn(console, "log").mockImplementation(() => undefined); + return { + dir, + browser, + runPipeline, + run: (...flags: string[]) => runCommand(command, { rawArgs: ["--json", ...flags] }), + }; + } + + it("answers an unchanged tree from the last pass without running the pipeline", async () => { + const { runPipeline, run } = setup(reportWithFindings()); + await run(); + await run(); + expect(runPipeline).toHaveBeenCalledTimes(1); + }); + + it("re-runs after one byte of the composition changes", async () => { + const { dir, runPipeline, run } = setup(reportWithFindings()); + await run(); + writeFileSync(join(dir, "index.html"), "onf"); + await run(); + expect(runPipeline).toHaveBeenCalledTimes(2); + }); + + it("never stores a failure", async () => { + const { runPipeline, run } = setup(reportWithFindings({ ok: false })); + await run(); + await run(); + expect(runPipeline).toHaveBeenCalledTimes(2); + }); + + it("never stores a pass whose browser did not run", async () => { + const { runPipeline, run } = setup(reportWithFindings({ browserSkipped: true })); + await run(); + await run(); + expect(runPipeline).toHaveBeenCalledTimes(2); + }); + + it("does not store a pass for a tree that changed while it was being checked", async () => { + const { dir, runPipeline, run } = setup(reportWithFindings()); + runPipeline.mockImplementationOnce(async () => { + writeFileSync(join(dir, "index.html"), "written by someone else"); + return reportWithFindings(); + }); + await run(); + writeFileSync(join(dir, "index.html"), "one"); + await run(); + expect(runPipeline).toHaveBeenCalledTimes(2); + }); + + it("re-runs a large file changed to the same size with its mtime put back", async () => { + const { dir, runPipeline, run } = setup(reportWithFindings()); + const big = join(dir, "clip.bin"); + const mtime = 1_700_000_000; + writeFileSync(big, Buffer.alloc(3 * 1024 * 1024, 1)); + utimesSync(big, mtime, mtime); + await run(); + const writtenAt = statSync(big).ctimeMs; + await vi.waitFor( + () => { + writeFileSync(big, Buffer.alloc(3 * 1024 * 1024, 2)); + utimesSync(big, mtime, mtime); + if (statSync(big).ctimeMs === writtenAt) throw new Error("the clock has not ticked"); + }, + { interval: 1 }, + ); + await run(); + expect(runPipeline).toHaveBeenCalledTimes(2); + }); + + it("never caches a project with node_modules or a linked directory", async () => { + const { dir, runPipeline, run } = setup(reportWithFindings()); + mkdirSync(join(dir, "node_modules")); + await run(); + await run(); + rmSync(join(dir, "node_modules"), { recursive: true }); + mkdirSync(join(dir, "real")); + symlinkSync(join(dir, "real"), join(dir, "linked"), "dir"); + await run(); + await run(); + expect(runPipeline).toHaveBeenCalledTimes(4); + }); + + it("re-runs when --timeout or the runtime URL changes", async () => { + const { runPipeline, run } = setup(reportWithFindings()); + vi.stubEnv("HYPERFRAME_RUNTIME_URL", "http://localhost:1/runtime.js"); + await run(); + await run("--timeout", "20s"); + vi.stubEnv("HYPERFRAME_RUNTIME_URL", "http://localhost:2/runtime.js"); + await run(); + vi.unstubAllEnvs(); + expect(runPipeline).toHaveBeenCalledTimes(3); + }); + + it("re-runs when the browser binary changes, and never caches without one", async () => { + const { browser, runPipeline, run } = setup(reportWithFindings()); + await run(); + browser.identity = "chrome:2"; + await run(); + browser.identity = undefined; + await run(); + await run(); + expect(runPipeline).toHaveBeenCalledTimes(4); + }); + + it("re-runs when a script under .hyperframes changes, but not when the CLI's own files do", async () => { + const { dir, runPipeline, run } = setup(reportWithFindings()); + mkdirSync(join(dir, ".hyperframes", "artifacts"), { recursive: true }); + writeFileSync(join(dir, ".hyperframes", "probe.js"), "console.log(1)"); + await run(); + writeFileSync(join(dir, ".hyperframes", "artifacts", "report.json"), "{}"); + await run(); + expect(runPipeline).toHaveBeenCalledTimes(1); + writeFileSync(join(dir, ".hyperframes", "probe.js"), "console.error(1)"); + await run(); + expect(runPipeline).toHaveBeenCalledTimes(2); + }); + + it("still hits when the check touches its video copies under .transcode-cache", async () => { + const { dir, runPipeline, run } = setup(reportWithFindings()); + const copy = join(dir, ".transcode-cache", "clip.mp4"); + mkdirSync(join(dir, ".transcode-cache")); + writeFileSync(copy, Buffer.alloc(3 * 1024 * 1024, 1)); + runPipeline.mockImplementationOnce(async () => { + utimesSync(copy, new Date(), new Date()); + return reportWithFindings(); + }); + await run(); + await run(); + expect(runPipeline).toHaveBeenCalledTimes(1); + }); + + it("never caches a project whose .hyperframes links to another directory", async () => { + const { dir, runPipeline, run } = setup(reportWithFindings()); + mkdirSync(join(dir, "real")); + mkdirSync(join(dir, ".hyperframes")); + symlinkSync(join(dir, "real"), join(dir, ".hyperframes", "linked"), "dir"); + await run(); + await run(); + expect(runPipeline).toHaveBeenCalledTimes(2); + }); + + it("re-runs when the flags differ, or with --no-cache, or under --budget", async () => { + const { runPipeline, run } = setup(reportWithFindings()); + await run(); + await run("--strict"); + await run("--no-cache"); + await run("--budget", "8m"); + expect(runPipeline).toHaveBeenCalledTimes(4); + }); +}); describe("check stopped by a signal", () => { it("SIGTERM prints the report so far, then exits 143", async () => { const { deps, runBrowserCheck } = dependencies(fakeDriver()); diff --git a/packages/cli/src/commands/check.ts b/packages/cli/src/commands/check.ts index a747d47453..3f4cc6eb23 100644 --- a/packages/cli/src/commands/check.ts +++ b/packages/cli/src/commands/check.ts @@ -44,6 +44,14 @@ import { parseTimeoutMs } from "../utils/timeoutArg.js"; import { failureMessage, jsonFailure } from "../utils/jsonError.js"; import { recordLastCheck } from "../telemetry/config.js"; import { failureCodes } from "../utils/checkFailureCodes.js"; +import { + browserIdentity, + checkCacheKey, + checkCacheable, + readCachedPass, + storePass, +} from "../utils/checkCache.js"; +import { VERSION } from "../version.js"; export const examples: Example[] = [ ["Run the full verification gate", "hyperframes check"], @@ -61,6 +69,7 @@ export interface CheckCommandDependencies { runPipeline(project: ProjectDir, options: CheckOptions): Promise; prepareMedia(project: ProjectDir, options: CheckOptions): Promise; withMeta(value: object): object; + browserIdentity?(): Promise; } const DEFAULT_COMMAND_DEPENDENCIES: CheckCommandDependencies = { @@ -227,6 +236,12 @@ const CHECK_COMMAND_ARGS = { description: "Stop the whole check this long after it starts (ms, or 90s, 8m; at least 30s) and report what it found so far", }, + cache: { + type: "boolean", + description: + "Return the stored result when the project, options and CLI are unchanged since the last pass (pass --no-cache to always re-run)", + default: true, + }, timeout: { type: "string", description: @@ -238,6 +253,12 @@ const CHECK_COMMAND_ARGS = { description: "Run the WCAG AA contrast pass (enabled by default)", default: true, }, + craft: { + type: "boolean", + description: + "Run the craft pass (dead stretches, unread lines, silent films; enabled by default). --no-craft skips it, e.g. for a render preflight", + default: true, + }, strict: { type: "boolean", description: "Exit non-zero on warnings too", @@ -313,9 +334,7 @@ export function createCheckCommand( return; } recordLastCheck({ at: new Date().toISOString(), codes: {} }); - const report = await withProjectLock(project.dir, "check", () => - dependencies.runPipeline(project, options), - ); + const report = await checkOrReplay(dependencies, project, options); recordLastCheck({ at: new Date().toISOString(), exitCode: checkExitCode(report), @@ -349,6 +368,27 @@ export function createCheckCommand( }); } +async function checkOrReplay( + dependencies: CheckCommandDependencies, + project: ProjectDir, + options: CheckOptions, +): Promise { + const identify = dependencies.browserIdentity ?? browserIdentity; + return withProjectLock(project.dir, "check", async () => { + const keyNow = async () => + checkCacheable(options) + ? checkCacheKey(project.dir, options, VERSION, await identify()) + : undefined; + const key = await keyNow(); + const cached = key ? readCachedPass(project.dir, key) : undefined; + if (cached) return cached; + const report = await dependencies.runPipeline(project, options); + const treeUnchangedByTheRun = key !== undefined && (await keyNow()) === key; + if (treeUnchangedByTheRun) storePass(project.dir, key, report); + return report; + }); +} + function reportCheckFailure( error: unknown, asJson: boolean, @@ -450,7 +490,9 @@ function parseCheckOptions(args: Record): CheckOptions { tolerance: nonNegativeNumber(args.tolerance, DEFAULT_CHECK_OPTIONS.tolerance), timeout: parseTimeoutMs(args.timeout, DEFAULT_CHECK_OPTIONS.timeout, 500), budget: parseBudget(args.budget), + cache: args.cache !== false, contrast: args.contrast !== false, + craft: args.craft !== false, ...(checkReview()?.skipContrastInMotion ? { skipContrastInMotion: true } : {}), strict: args.strict === true, snapshots: args.snapshots === true, @@ -693,6 +735,10 @@ function browserSkippedLine(report: CheckReport): string { } function printHumanReport(report: CheckReport): void { + if (report.cached) + console.log( + `${c.dim("◇")} Nothing changed since the pass at ${report.cached.at}; showing that result. Remote scripts and assets were not fetched again. Run with --no-cache to check again.`, + ); if (report.scope) { console.log( `Scope: ${report.scope.range ? report.scope.range.join("–") + "s" : "full timeline"}; ${report.scope.hiddenFindings} findings hidden by component filter. Counts/status include hidden findings.`, diff --git a/packages/cli/src/utils/checkCache.ts b/packages/cli/src/utils/checkCache.ts new file mode 100644 index 0000000000..e63375c58c --- /dev/null +++ b/packages/cli/src/utils/checkCache.ts @@ -0,0 +1,139 @@ +import { createHash } from "node:crypto"; +import { + closeSync, + fstatSync, + mkdirSync, + openSync, + readFileSync, + readdirSync, + writeFileSync, +} from "node:fs"; +import { join } from "node:path"; +import { CACHE_DIR_NAME as PROXY_CACHE_DIR_NAME } from "@hyperframes/studio-server/proxy-transcoder"; +import type { CheckOptions, CheckReport } from "./checkTypes.js"; + +const PROJECT_STATE_DIR = ".hyperframes"; +// What the CLI itself writes under .hyperframes; anything else there (a script the page loads) is project input. +const TOOL_OWNED = new Set([ + "artifacts", + "cache", + "history-id", + "video-state.json", + "video-state-lock", + "project.lock", +]); +// Bigger files (video, audio) are not read: size, mtime and ctime (which a tool cannot restore) stand for them. +const HASH_CONTENT_UP_TO = 2 * 1024 * 1024; +// Environment that changes what a check loads or how long it waits, so it is part of the key. +const VERDICT_ENV = /^(HYPERFRAMES?|PRODUCER|HF)_/; + +class Uncacheable extends Error {} + +function cachePath(projectDir: string): string { + return join(projectDir, ".hyperframes", "cache", "check.json"); +} + +/** The browser binary a check would launch, by path and file identity; undefined when none is installed. */ +export async function browserIdentity(): Promise { + const { findBrowser } = await import("../browser/manager.js"); + const found = await findBrowser(); + if (!found) return undefined; + try { + const fd = openSync(found.executablePath, "r"); + try { + const { size, mtimeMs, ctimeMs, ino } = fstatSync(fd); + return `${found.executablePath}:${size}:${mtimeMs}:${ctimeMs}:${ino}`; + } finally { + closeSync(fd); + } + } catch { + return undefined; + } +} + +function fileFingerprint(path: string): string { + const fd = openSync(path, "r"); + try { + const stat = fstatSync(fd); + if (stat.isDirectory()) throw new Uncacheable(`${path} is a linked directory`); + return stat.size <= HASH_CONTENT_UP_TO + ? createHash("sha256").update(readFileSync(fd)).digest("hex") + : `${stat.size}:${stat.mtimeMs}:${stat.ctimeMs}:${stat.ino}`; + } finally { + closeSync(fd); + } +} + +function hashTree(dir: string, prefix: string, hash: ReturnType): void { + const entries = readdirSync(dir, { withFileTypes: true }).sort((a, b) => + a.name < b.name ? -1 : 1, + ); + for (const entry of entries) { + if (entry.name === ".git" || entry.name === PROXY_CACHE_DIR_NAME) continue; + if (prefix === `${PROJECT_STATE_DIR}/` && TOOL_OWNED.has(entry.name)) continue; + if (entry.name === "node_modules") + throw new Uncacheable("node_modules can hold code the page runs"); + const path = join(dir, entry.name); + const name = `${prefix}${entry.name}`; + if (entry.isDirectory()) hashTree(path, `${name}/`, hash); + else hash.update(`${name}\0${fileFingerprint(path)}\0`); + } +} + +/** Everything that can change a check verdict: the project files, the CLI build and the options. */ +export function checkCacheKey( + projectDir: string, + options: CheckOptions, + version: string, + browser: string | undefined, +): string | undefined { + if (browser === undefined) return undefined; // no installed browser: the check would download one + const hash = createHash("sha256").update(`${version}\0${browser}\0`); + const { cache: _cache, ...verdictOptions } = options; + const env = Object.entries(process.env) + .filter(([name]) => VERDICT_ENV.test(name)) + .sort(); + hash.update(`${JSON.stringify([verdictOptions, env])}\0`); + try { + hashTree(projectDir, "", hash); + } catch { + return undefined; // an unreadable or linked tree is not cached; the check itself says what is wrong + } + return hash.digest("hex"); +} + +/** A budgeted run is partial, and these options need artifacts a replayed report cannot recreate. */ +export function checkCacheable(options: CheckOptions): boolean { + return ( + options.cache !== false && + !options.budget && + !options.snapshots && + !options.findingFrames && + !options.inspect + ); +} + +export function readCachedPass(projectDir: string, key: string): CheckReport | undefined { + try { + const stored = JSON.parse(readFileSync(cachePath(projectDir), "utf8")); + if (stored.key !== key || !stored.report?.ok) return undefined; + return { ...stored.report, cached: { at: stored.at } }; + } catch { + return undefined; + } +} + +/** Only a clean pass that ran the browser is stored; a failure is always re-run. */ +export function storePass(projectDir: string, key: string, report: CheckReport): void { + if (!report.ok || report.browserSkipped) return; + try { + mkdirSync(join(projectDir, ".hyperframes", "cache"), { recursive: true }); + const { saved: _saved, ...clean } = report as CheckReport & { saved?: unknown }; + writeFileSync( + cachePath(projectDir), + JSON.stringify({ key, at: new Date().toISOString(), report: clean }), + ); + } catch { + // A cache that cannot be written only costs the next run its speed-up. + } +} diff --git a/packages/cli/src/utils/checkPipeline.ts b/packages/cli/src/utils/checkPipeline.ts index ffc1aba874..e77aec7e80 100644 --- a/packages/cli/src/utils/checkPipeline.ts +++ b/packages/cli/src/utils/checkPipeline.ts @@ -621,7 +621,9 @@ async function collectGridSamples( } function craftEnabled(driver: CheckAuditDriver, options: CheckOptions): boolean { - return !options.range && typeof driver.collectCraftFrame === "function"; + return ( + options.craft !== false && !options.range && typeof driver.collectCraftFrame === "function" + ); } async function collectContrastAt( diff --git a/packages/cli/src/utils/checkTypes.ts b/packages/cli/src/utils/checkTypes.ts index 2c09d2675a..8a4580fcd1 100644 --- a/packages/cli/src/utils/checkTypes.ts +++ b/packages/cli/src/utils/checkTypes.ts @@ -38,6 +38,8 @@ export interface CheckOptions { autoProxy?: boolean; browserGpuMode?: BrowserGpuMode; includeUnmounted?: boolean; + cache?: boolean; + craft?: boolean; } export interface CaptionZoneOptions { @@ -396,6 +398,7 @@ export interface CheckReport { notes: string[]; }; ok: boolean; + cached?: { at: string }; strict: boolean; browserSkipped: boolean; lint: CheckSection & { filesScanned: number }; diff --git a/packages/cli/src/warmCheck.process.test.ts b/packages/cli/src/warmCheck.process.test.ts index 43b3aae345..b8738101d6 100644 --- a/packages/cli/src/warmCheck.process.test.ts +++ b/packages/cli/src/warmCheck.process.test.ts @@ -15,7 +15,7 @@ import { import { Frame, encodeFrame, frameReader, warmCheckEndpoint } from "./warmCheckProtocol.js"; const here = dirname(fileURLToPath(import.meta.url)); -const CHECK_ARGS = ["--json", "--no-browser-gpu", "--samples=1"]; +const CHECK_ARGS = ["--json", "--no-browser-gpu", "--samples=1", "--no-cache"]; const CHROME = /chrom/i; function temporary(prefix: string, base = tmpdir()): string {