feat(cli): check replays an unchanged project's last pass and takes --no-craft (#7398)

* feat(cli): check returns an unchanged project's last pass in about a second

* fix(cli): the check cache hashes files without a stat race and its tests run before the signal test

* test(cli): the warm check test compares two real runs, not a run and its cached pass

* feat(cli): check --no-craft skips the craft pass for a render preflight

* fix(cli): the check cache keys on timeout, env and large files and never replays a moved tree

* style(cli): format the check cache

* fix(cli): the large-file cache test waits for the clock tick it depends on

* fix(cli): the check cache keys on the browser and on scripts under .hyperframes

* fix(cli): the check cache ignores the video copies the check itself touches
This commit is contained in:
Miguel Ángel
2026-11-08 14:01:58 +00:00
committed by GitHub
parent 12906e0bfb
commit 070b964d7b
7 changed files with 405 additions and 6 deletions
+2
View File
@@ -1271,7 +1271,9 @@ cover every finding from the full check.
| `--tolerance` | Allowed overflow in px before reporting (default 2) |
| `--timeout` | Render-ready budget in ms; also raises page navigation above its 10s floor (default 45000, as snapshot and render) |
| `--budget <ms\|90s\|8m>` | Stop the whole check this long after it starts (at least 30s) and report what it found, with `check_budget_reached` naming the phase it was in. For hosts with their own time limit. |
| `--no-cache` | Run the check even when the project files, flags, CLI and browser are unchanged since the last pass. Without it, an unchanged project returns that pass in about a second, marked as cached; remote scripts and assets are not fetched again for that. A failure is never cached, and a project with `node_modules` or a linked directory is never replayed. |
| `--no-contrast` | Skip the WCAG pass while iterating |
| `--no-craft` | Skip the craft pass (dead stretches, unread lines, silent film), the longest part of a full check. The report marks it skipped. Use it for a render preflight; keep it on for the full check before a release. |
| `--strict` | Exit non-zero on warnings too (default: errors only) |
| `--caption-zone "<x0=..;y0=..;x1=..;y1=..>"` | Opt-in band gate. Flags a text element's DOM box that overlaps the fractional band. Optional `severity` and `seek`. |
| `--frame-check` | Opt-in out-of-frame detection for `img`, `svg`, `video`, and `canvas` |
+208 -1
View File
@@ -1,6 +1,15 @@
import type { CheckProgress } from "../utils/checkProgress.js";
import { runCommand } from "citty";
import { mkdirSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from "node:fs";
import {
mkdirSync,
mkdtempSync,
readFileSync,
rmSync,
statSync,
symlinkSync,
utimesSync,
writeFileSync,
} from "node:fs";
import { tmpdir } from "node:os";
import { join } from "node:path";
import { afterEach, describe, expect, it, vi } from "vitest";
@@ -544,6 +553,24 @@ it("--budget sets when the whole check stops, its report time kept back, counted
expect(runPipeline.mock.calls[1]?.[1].budget).toBeUndefined();
});
it("--no-craft reaches the pipeline as craft: false, and the craft pass is on without it", async () => {
const { report } = await runScenario(fakeDriver());
const runPipeline = vi.fn(async (_project: ProjectDir, _options: CheckOptions) => report);
vi.spyOn(console, "log").mockImplementation(() => undefined);
const command = createCheckCommand({
resolveProject: () => PROJECT,
runPipeline,
prepareMedia: vi.fn(async () => []),
withMeta: (value) => value,
});
await runCommand(command, { rawArgs: ["--json", "--no-craft"] });
await runCommand(command, { rawArgs: ["--json"] });
expect(runPipeline.mock.calls[0]?.[1].craft).toBe(false);
expect(runPipeline.mock.calls[1]?.[1].craft).toBe(true);
});
it("parses the caption-zone grammar and enables the frame gate", async () => {
const { report } = await runScenario(fakeDriver());
const runPipeline = vi.fn(async (_project: ProjectDir, _options: CheckOptions) => report);
@@ -3046,6 +3073,15 @@ describe("check pipeline", () => {
expect(checkExitCode(strict.report)).toBe(0);
});
it("--no-craft never samples the craft frames and the report says the pass was skipped", async () => {
const driver = craftDriver();
const { report } = await runScenario(driver, { craft: false });
expect(driver.collectCraftFrame).not.toHaveBeenCalled();
expect(report.craft.enabled).toBe(false);
expect(report.craft.findings).toEqual([]);
});
it("seeks each craft frame with WebGL draws on, so a redrawn canvas reads as moving", async () => {
const driver = craftDriver();
await runScenario(driver);
@@ -3991,6 +4027,177 @@ it("applies transition sample limits within the requested range", async () => {
expect(report.layout.transitionSamplesDropped).toBe(3);
});
describe("check cache", () => {
const dirs: string[] = [];
afterEach(() => {
vi.unstubAllEnvs();
for (const dir of dirs.splice(0)) rmSync(dir, { recursive: true, force: true });
});
function setup(report: CheckReport) {
const browser: { identity: string | undefined } = { identity: "chrome:1" };
const dir = mkdtempSync(join(tmpdir(), "hf-check-cache-"));
dirs.push(dir);
writeFileSync(join(dir, "index.html"), "<html>one</html>");
const runPipeline = vi.fn(async (_project: ProjectDir, _options: CheckOptions) => report);
const command = createCheckCommand({
resolveProject: () => ({ dir, name: "p", indexPath: join(dir, "index.html") }),
runPipeline,
prepareMedia: vi.fn(async () => []),
withMeta: (value) => value,
browserIdentity: async () => browser.identity,
});
vi.spyOn(console, "log").mockImplementation(() => undefined);
return {
dir,
browser,
runPipeline,
run: (...flags: string[]) => runCommand(command, { rawArgs: ["--json", ...flags] }),
};
}
it("answers an unchanged tree from the last pass without running the pipeline", async () => {
const { runPipeline, run } = setup(reportWithFindings());
await run();
await run();
expect(runPipeline).toHaveBeenCalledTimes(1);
});
it("re-runs after one byte of the composition changes", async () => {
const { dir, runPipeline, run } = setup(reportWithFindings());
await run();
writeFileSync(join(dir, "index.html"), "<html>onf</html>");
await run();
expect(runPipeline).toHaveBeenCalledTimes(2);
});
it("never stores a failure", async () => {
const { runPipeline, run } = setup(reportWithFindings({ ok: false }));
await run();
await run();
expect(runPipeline).toHaveBeenCalledTimes(2);
});
it("never stores a pass whose browser did not run", async () => {
const { runPipeline, run } = setup(reportWithFindings({ browserSkipped: true }));
await run();
await run();
expect(runPipeline).toHaveBeenCalledTimes(2);
});
it("does not store a pass for a tree that changed while it was being checked", async () => {
const { dir, runPipeline, run } = setup(reportWithFindings());
runPipeline.mockImplementationOnce(async () => {
writeFileSync(join(dir, "index.html"), "<html>written by someone else</html>");
return reportWithFindings();
});
await run();
writeFileSync(join(dir, "index.html"), "<html>one</html>");
await run();
expect(runPipeline).toHaveBeenCalledTimes(2);
});
it("re-runs a large file changed to the same size with its mtime put back", async () => {
const { dir, runPipeline, run } = setup(reportWithFindings());
const big = join(dir, "clip.bin");
const mtime = 1_700_000_000;
writeFileSync(big, Buffer.alloc(3 * 1024 * 1024, 1));
utimesSync(big, mtime, mtime);
await run();
const writtenAt = statSync(big).ctimeMs;
await vi.waitFor(
() => {
writeFileSync(big, Buffer.alloc(3 * 1024 * 1024, 2));
utimesSync(big, mtime, mtime);
if (statSync(big).ctimeMs === writtenAt) throw new Error("the clock has not ticked");
},
{ interval: 1 },
);
await run();
expect(runPipeline).toHaveBeenCalledTimes(2);
});
it("never caches a project with node_modules or a linked directory", async () => {
const { dir, runPipeline, run } = setup(reportWithFindings());
mkdirSync(join(dir, "node_modules"));
await run();
await run();
rmSync(join(dir, "node_modules"), { recursive: true });
mkdirSync(join(dir, "real"));
symlinkSync(join(dir, "real"), join(dir, "linked"), "dir");
await run();
await run();
expect(runPipeline).toHaveBeenCalledTimes(4);
});
it("re-runs when --timeout or the runtime URL changes", async () => {
const { runPipeline, run } = setup(reportWithFindings());
vi.stubEnv("HYPERFRAME_RUNTIME_URL", "http://localhost:1/runtime.js");
await run();
await run("--timeout", "20s");
vi.stubEnv("HYPERFRAME_RUNTIME_URL", "http://localhost:2/runtime.js");
await run();
vi.unstubAllEnvs();
expect(runPipeline).toHaveBeenCalledTimes(3);
});
it("re-runs when the browser binary changes, and never caches without one", async () => {
const { browser, runPipeline, run } = setup(reportWithFindings());
await run();
browser.identity = "chrome:2";
await run();
browser.identity = undefined;
await run();
await run();
expect(runPipeline).toHaveBeenCalledTimes(4);
});
it("re-runs when a script under .hyperframes changes, but not when the CLI's own files do", async () => {
const { dir, runPipeline, run } = setup(reportWithFindings());
mkdirSync(join(dir, ".hyperframes", "artifacts"), { recursive: true });
writeFileSync(join(dir, ".hyperframes", "probe.js"), "console.log(1)");
await run();
writeFileSync(join(dir, ".hyperframes", "artifacts", "report.json"), "{}");
await run();
expect(runPipeline).toHaveBeenCalledTimes(1);
writeFileSync(join(dir, ".hyperframes", "probe.js"), "console.error(1)");
await run();
expect(runPipeline).toHaveBeenCalledTimes(2);
});
it("still hits when the check touches its video copies under .transcode-cache", async () => {
const { dir, runPipeline, run } = setup(reportWithFindings());
const copy = join(dir, ".transcode-cache", "clip.mp4");
mkdirSync(join(dir, ".transcode-cache"));
writeFileSync(copy, Buffer.alloc(3 * 1024 * 1024, 1));
runPipeline.mockImplementationOnce(async () => {
utimesSync(copy, new Date(), new Date());
return reportWithFindings();
});
await run();
await run();
expect(runPipeline).toHaveBeenCalledTimes(1);
});
it("never caches a project whose .hyperframes links to another directory", async () => {
const { dir, runPipeline, run } = setup(reportWithFindings());
mkdirSync(join(dir, "real"));
mkdirSync(join(dir, ".hyperframes"));
symlinkSync(join(dir, "real"), join(dir, ".hyperframes", "linked"), "dir");
await run();
await run();
expect(runPipeline).toHaveBeenCalledTimes(2);
});
it("re-runs when the flags differ, or with --no-cache, or under --budget", async () => {
const { runPipeline, run } = setup(reportWithFindings());
await run();
await run("--strict");
await run("--no-cache");
await run("--budget", "8m");
expect(runPipeline).toHaveBeenCalledTimes(4);
});
});
describe("check stopped by a signal", () => {
it("SIGTERM prints the report so far, then exits 143", async () => {
const { deps, runBrowserCheck } = dependencies(fakeDriver());
+49 -3
View File
@@ -44,6 +44,14 @@ import { parseTimeoutMs } from "../utils/timeoutArg.js";
import { failureMessage, jsonFailure } from "../utils/jsonError.js";
import { recordLastCheck } from "../telemetry/config.js";
import { failureCodes } from "../utils/checkFailureCodes.js";
import {
browserIdentity,
checkCacheKey,
checkCacheable,
readCachedPass,
storePass,
} from "../utils/checkCache.js";
import { VERSION } from "../version.js";
export const examples: Example[] = [
["Run the full verification gate", "hyperframes check"],
@@ -61,6 +69,7 @@ export interface CheckCommandDependencies {
runPipeline(project: ProjectDir, options: CheckOptions): Promise<SavedCheckReport>;
prepareMedia(project: ProjectDir, options: CheckOptions): Promise<PreparedMediaFile[]>;
withMeta(value: object): object;
browserIdentity?(): Promise<string | undefined>;
}
const DEFAULT_COMMAND_DEPENDENCIES: CheckCommandDependencies = {
@@ -227,6 +236,12 @@ const CHECK_COMMAND_ARGS = {
description:
"Stop the whole check this long after it starts (ms, or 90s, 8m; at least 30s) and report what it found so far",
},
cache: {
type: "boolean",
description:
"Return the stored result when the project, options and CLI are unchanged since the last pass (pass --no-cache to always re-run)",
default: true,
},
timeout: {
type: "string",
description:
@@ -238,6 +253,12 @@ const CHECK_COMMAND_ARGS = {
description: "Run the WCAG AA contrast pass (enabled by default)",
default: true,
},
craft: {
type: "boolean",
description:
"Run the craft pass (dead stretches, unread lines, silent films; enabled by default). --no-craft skips it, e.g. for a render preflight",
default: true,
},
strict: {
type: "boolean",
description: "Exit non-zero on warnings too",
@@ -313,9 +334,7 @@ export function createCheckCommand(
return;
}
recordLastCheck({ at: new Date().toISOString(), codes: {} });
const report = await withProjectLock(project.dir, "check", () =>
dependencies.runPipeline(project, options),
);
const report = await checkOrReplay(dependencies, project, options);
recordLastCheck({
at: new Date().toISOString(),
exitCode: checkExitCode(report),
@@ -349,6 +368,27 @@ export function createCheckCommand(
});
}
async function checkOrReplay(
dependencies: CheckCommandDependencies,
project: ProjectDir,
options: CheckOptions,
): Promise<SavedCheckReport> {
const identify = dependencies.browserIdentity ?? browserIdentity;
return withProjectLock(project.dir, "check", async () => {
const keyNow = async () =>
checkCacheable(options)
? checkCacheKey(project.dir, options, VERSION, await identify())
: undefined;
const key = await keyNow();
const cached = key ? readCachedPass(project.dir, key) : undefined;
if (cached) return cached;
const report = await dependencies.runPipeline(project, options);
const treeUnchangedByTheRun = key !== undefined && (await keyNow()) === key;
if (treeUnchangedByTheRun) storePass(project.dir, key, report);
return report;
});
}
function reportCheckFailure(
error: unknown,
asJson: boolean,
@@ -450,7 +490,9 @@ function parseCheckOptions(args: Record<string, unknown>): CheckOptions {
tolerance: nonNegativeNumber(args.tolerance, DEFAULT_CHECK_OPTIONS.tolerance),
timeout: parseTimeoutMs(args.timeout, DEFAULT_CHECK_OPTIONS.timeout, 500),
budget: parseBudget(args.budget),
cache: args.cache !== false,
contrast: args.contrast !== false,
craft: args.craft !== false,
...(checkReview()?.skipContrastInMotion ? { skipContrastInMotion: true } : {}),
strict: args.strict === true,
snapshots: args.snapshots === true,
@@ -693,6 +735,10 @@ function browserSkippedLine(report: CheckReport): string {
}
function printHumanReport(report: CheckReport): void {
if (report.cached)
console.log(
`${c.dim("◇")} Nothing changed since the pass at ${report.cached.at}; showing that result. Remote scripts and assets were not fetched again. Run with --no-cache to check again.`,
);
if (report.scope) {
console.log(
`Scope: ${report.scope.range ? report.scope.range.join("–") + "s" : "full timeline"}; ${report.scope.hiddenFindings} findings hidden by component filter. Counts/status include hidden findings.`,
+139
View File
@@ -0,0 +1,139 @@
import { createHash } from "node:crypto";
import {
closeSync,
fstatSync,
mkdirSync,
openSync,
readFileSync,
readdirSync,
writeFileSync,
} from "node:fs";
import { join } from "node:path";
import { CACHE_DIR_NAME as PROXY_CACHE_DIR_NAME } from "@hyperframes/studio-server/proxy-transcoder";
import type { CheckOptions, CheckReport } from "./checkTypes.js";
const PROJECT_STATE_DIR = ".hyperframes";
// What the CLI itself writes under .hyperframes; anything else there (a script the page loads) is project input.
const TOOL_OWNED = new Set([
"artifacts",
"cache",
"history-id",
"video-state.json",
"video-state-lock",
"project.lock",
]);
// Bigger files (video, audio) are not read: size, mtime and ctime (which a tool cannot restore) stand for them.
const HASH_CONTENT_UP_TO = 2 * 1024 * 1024;
// Environment that changes what a check loads or how long it waits, so it is part of the key.
const VERDICT_ENV = /^(HYPERFRAMES?|PRODUCER|HF)_/;
class Uncacheable extends Error {}
function cachePath(projectDir: string): string {
return join(projectDir, ".hyperframes", "cache", "check.json");
}
/** The browser binary a check would launch, by path and file identity; undefined when none is installed. */
export async function browserIdentity(): Promise<string | undefined> {
const { findBrowser } = await import("../browser/manager.js");
const found = await findBrowser();
if (!found) return undefined;
try {
const fd = openSync(found.executablePath, "r");
try {
const { size, mtimeMs, ctimeMs, ino } = fstatSync(fd);
return `${found.executablePath}:${size}:${mtimeMs}:${ctimeMs}:${ino}`;
} finally {
closeSync(fd);
}
} catch {
return undefined;
}
}
function fileFingerprint(path: string): string {
const fd = openSync(path, "r");
try {
const stat = fstatSync(fd);
if (stat.isDirectory()) throw new Uncacheable(`${path} is a linked directory`);
return stat.size <= HASH_CONTENT_UP_TO
? createHash("sha256").update(readFileSync(fd)).digest("hex")
: `${stat.size}:${stat.mtimeMs}:${stat.ctimeMs}:${stat.ino}`;
} finally {
closeSync(fd);
}
}
function hashTree(dir: string, prefix: string, hash: ReturnType<typeof createHash>): void {
const entries = readdirSync(dir, { withFileTypes: true }).sort((a, b) =>
a.name < b.name ? -1 : 1,
);
for (const entry of entries) {
if (entry.name === ".git" || entry.name === PROXY_CACHE_DIR_NAME) continue;
if (prefix === `${PROJECT_STATE_DIR}/` && TOOL_OWNED.has(entry.name)) continue;
if (entry.name === "node_modules")
throw new Uncacheable("node_modules can hold code the page runs");
const path = join(dir, entry.name);
const name = `${prefix}${entry.name}`;
if (entry.isDirectory()) hashTree(path, `${name}/`, hash);
else hash.update(`${name}\0${fileFingerprint(path)}\0`);
}
}
/** Everything that can change a check verdict: the project files, the CLI build and the options. */
export function checkCacheKey(
projectDir: string,
options: CheckOptions,
version: string,
browser: string | undefined,
): string | undefined {
if (browser === undefined) return undefined; // no installed browser: the check would download one
const hash = createHash("sha256").update(`${version}\0${browser}\0`);
const { cache: _cache, ...verdictOptions } = options;
const env = Object.entries(process.env)
.filter(([name]) => VERDICT_ENV.test(name))
.sort();
hash.update(`${JSON.stringify([verdictOptions, env])}\0`);
try {
hashTree(projectDir, "", hash);
} catch {
return undefined; // an unreadable or linked tree is not cached; the check itself says what is wrong
}
return hash.digest("hex");
}
/** A budgeted run is partial, and these options need artifacts a replayed report cannot recreate. */
export function checkCacheable(options: CheckOptions): boolean {
return (
options.cache !== false &&
!options.budget &&
!options.snapshots &&
!options.findingFrames &&
!options.inspect
);
}
export function readCachedPass(projectDir: string, key: string): CheckReport | undefined {
try {
const stored = JSON.parse(readFileSync(cachePath(projectDir), "utf8"));
if (stored.key !== key || !stored.report?.ok) return undefined;
return { ...stored.report, cached: { at: stored.at } };
} catch {
return undefined;
}
}
/** Only a clean pass that ran the browser is stored; a failure is always re-run. */
export function storePass(projectDir: string, key: string, report: CheckReport): void {
if (!report.ok || report.browserSkipped) return;
try {
mkdirSync(join(projectDir, ".hyperframes", "cache"), { recursive: true });
const { saved: _saved, ...clean } = report as CheckReport & { saved?: unknown };
writeFileSync(
cachePath(projectDir),
JSON.stringify({ key, at: new Date().toISOString(), report: clean }),
);
} catch {
// A cache that cannot be written only costs the next run its speed-up.
}
}
+3 -1
View File
@@ -621,7 +621,9 @@ async function collectGridSamples(
}
function craftEnabled(driver: CheckAuditDriver, options: CheckOptions): boolean {
return !options.range && typeof driver.collectCraftFrame === "function";
return (
options.craft !== false && !options.range && typeof driver.collectCraftFrame === "function"
);
}
async function collectContrastAt(
+3
View File
@@ -38,6 +38,8 @@ export interface CheckOptions {
autoProxy?: boolean;
browserGpuMode?: BrowserGpuMode;
includeUnmounted?: boolean;
cache?: boolean;
craft?: boolean;
}
export interface CaptionZoneOptions {
@@ -396,6 +398,7 @@ export interface CheckReport {
notes: string[];
};
ok: boolean;
cached?: { at: string };
strict: boolean;
browserSkipped: boolean;
lint: CheckSection & { filesScanned: number };
+1 -1
View File
@@ -15,7 +15,7 @@ import {
import { Frame, encodeFrame, frameReader, warmCheckEndpoint } from "./warmCheckProtocol.js";
const here = dirname(fileURLToPath(import.meta.url));
const CHECK_ARGS = ["--json", "--no-browser-gpu", "--samples=1"];
const CHECK_ARGS = ["--json", "--no-browser-gpu", "--samples=1", "--no-cache"];
const CHROME = /chrom/i;
function temporary(prefix: string, base = tmpdir()): string {