feat(cli): check --budget stops the whole check in time and reports what it found (#7245)

* feat(cli): check --budget stops the whole check in time and reports what it found

* fix(cli): keep check's comment share where it was

* fix(cli): a --budget that runs out before any check step is waiting is not an unhandled rejection
This commit is contained in:
Miguel Ángel
2026-11-05 14:45:30 +00:00
committed by GitHub
parent c1977fab3e
commit b7d72c9478
9 changed files with 183 additions and 10 deletions
+1
View File
@@ -1249,6 +1249,7 @@ cover every finding from the full check.
| `--collapse-static` / `--no-collapse-static` | Fold a repeated static finding into one row (default: on) |
| `--tolerance` | Allowed overflow in px before reporting (default 2) |
| `--timeout` | Render-ready budget in ms; also raises page navigation above its 10s floor (default 45000, as snapshot and render) |
| `--budget <ms\|90s\|8m>` | Stop the whole check this long after it starts (at least 30s) and report what it found, with `check_budget_reached` naming the phase it was in. For hosts with their own time limit. |
| `--no-contrast` | Skip the WCAG pass while iterating |
| `--strict` | Exit non-zero on warnings too (default: errors only) |
| `--caption-zone "<x0=..;y0=..;x1=..;y1=..>"` | Opt-in band gate. Flags a text element's DOM box that overlaps the fractional band. Optional `severity` and `seek`. |
+74
View File
@@ -518,6 +518,28 @@ it("waits for the composition as long as snapshot and render do unless --timeout
expect(runPipeline.mock.calls[1]?.[1].timeout).toBe(20_000);
});
it("--budget sets when the whole check stops, its report time kept back, counted from the command's start", async () => {
const { report } = await runScenario(fakeDriver());
const runPipeline = vi.fn(async (_project: ProjectDir, _options: CheckOptions) => report);
vi.spyOn(console, "log").mockImplementation(() => undefined);
const command = createCheckCommand({
resolveProject: () => PROJECT,
runPipeline,
prepareMedia: vi.fn(async () => []),
withMeta: (value) => value,
});
const startedAt = Date.now();
await runCommand(command, { rawArgs: ["--json", "--budget", "8m"] });
await runCommand(command, { rawArgs: ["--json"] });
const budget = runPipeline.mock.calls[0]?.[1].budget;
expect(budget?.ms).toBe(480_000);
expect(budget?.stopAt).toBeGreaterThanOrEqual(startedAt + 470_000);
expect(budget?.stopAt).toBeLessThanOrEqual(Date.now() + 470_000);
expect(runPipeline.mock.calls[1]?.[1].budget).toBeUndefined();
});
it("parses the caption-zone grammar and enables the frame gate", async () => {
const { report } = await runScenario(fakeDriver());
const runPipeline = vi.fn(async (_project: ProjectDir, _options: CheckOptions) => report);
@@ -2497,6 +2519,58 @@ describe("check pipeline", () => {
}
});
it("--budget ends a check that is still within its idle window and keeps what its finished samples found", async () => {
vi.useFakeTimers();
try {
let aborted!: AbortSignal;
const driver = fakeDriver({
collectLayout: vi.fn(async (time: number) => [
layoutIssue("warning", { time, code: "container_overflow" }),
]),
collectOverlapGrid: vi.fn(
() =>
new Promise<never>((_, reject) =>
aborted.addEventListener("abort", () => reject(aborted.reason)),
),
),
});
const { deps } = dependencies(driver);
deps.runBrowserCheck = async (_project, options, motion, deadline) => {
aborted = deadline!.signal;
deadline!.start();
return runAuditGrid(
driver,
options,
motion,
undefined,
deadline!.enter,
undefined,
deadline!.keepPartial,
);
};
const budget = { ms: 70_000, stopAt: Date.now() + 60_000 };
const pending = runCheckPipeline(
PROJECT,
{ ...DEFAULT_CHECK_OPTIONS, timeout: 400_000, budget },
deps,
);
await vi.advanceTimersByTimeAsync(60_000);
const report = await pending;
const reached = report.runtime.findings.find(
(finding) => finding.code === "check_budget_reached",
);
expect(reached?.message).toMatch(
/^Check reached its --budget of 70 s while running the overlap pass .*Seconds per phase: /,
);
expect(report.ok).toBe(false);
expect(report.layout.findings.some((finding) => finding.code === "container_overflow")).toBe(
true,
);
} finally {
vi.useRealTimers();
}
});
it("--strict flips a warnings-only result from exit 0 to exit 1", async () => {
const warningDriver = () =>
fakeDriver({
+13
View File
@@ -215,6 +215,11 @@ const CHECK_COMMAND_ARGS = {
description: "Allowed pixel overflow before reporting an issue (default: 2)",
default: "2",
},
budget: {
type: "string",
description:
"Stop the whole check this long after it starts (ms, or 90s, 8m; at least 30s) and report what it found so far",
},
timeout: {
type: "string",
description:
@@ -348,6 +353,13 @@ function reportCheckFailure(
}
const REPORT_GRACE_MS = 5_000;
const MIN_BUDGET_MS = 30_000;
const BUDGET_REPORT_MS = 10_000;
function parseBudget(raw: unknown): CheckOptions["budget"] {
const ms = parseTimeoutMs(raw, 0, MIN_BUDGET_MS, console.error, "--budget");
return ms > 0 ? { ms, stopAt: Date.now() + ms - BUDGET_REPORT_MS } : undefined;
}
function untilReported(): { done: () => void } {
let done = (): void => undefined;
@@ -423,6 +435,7 @@ function parseCheckOptions(args: Record<string, unknown>): CheckOptions {
collapseStatic: args["collapse-static"] !== false,
tolerance: nonNegativeNumber(args.tolerance, DEFAULT_CHECK_OPTIONS.tolerance),
timeout: parseTimeoutMs(args.timeout, DEFAULT_CHECK_OPTIONS.timeout, 500),
budget: parseBudget(args.budget),
contrast: args.contrast !== false,
...(checkReview()?.skipContrastInMotion ? { skipContrastInMotion: true } : {}),
strict: args.strict === true,
@@ -1637,6 +1637,43 @@ describe("preResolveHostileMediaProxies", () => {
}
});
it("gives the copies only what is left of the check's --budget when that is under the copy budget", async () => {
vi.useFakeTimers();
try {
const projectDir = mkProjectDir();
mocks.scanProjectMediaCodecMap.mockResolvedValue({
"/clip.mov": {
codecName: "hevc",
browserHostile: true,
representativeMime: null,
hasAlpha: false,
},
});
mocks.resolveProxy.mockImplementation(
() =>
new Promise<string>((_, reject) =>
mocks.proxySignal?.addEventListener("abort", () => reject(new Error("aborted"))),
),
);
vi.spyOn(console, "error").mockImplementation(() => undefined);
const deadline = new CheckDeadline(undefined, { ms: 40_000, stopAt: Date.now() + 30_000 });
const pending = runBrowserCheck(
{ ...PROJECT, dir: projectDir, indexPath: `${projectDir}/index.html` },
DEFAULT_CHECK_OPTIONS,
{ kind: "none" },
runAuditGrid,
deadline,
);
const outcome = expect(pending).rejects.toThrow(/took longer than 30 s: clip\.mov/);
await vi.advanceTimersByTimeAsync(30_000);
await outcome;
await deadline.close();
} finally {
vi.useRealTimers();
}
});
it("stops reporting a copy once it is done, so a finished one is not read as stuck", async () => {
vi.useFakeTimers();
try {
+6 -1
View File
@@ -443,7 +443,12 @@ export async function runBrowserCheck(
findings: drafts,
failedPaths,
limits,
} = await preResolveHostileMediaProxies(project.dir, html, options.autoProxy, COPY_BUDGET_MS);
} = await preResolveHostileMediaProxies(
project.dir,
html,
options.autoProxy,
Math.min(COPY_BUDGET_MS, deadline?.remainingMs() ?? COPY_BUDGET_MS),
);
deadline?.enter("launch");
const server = await serveStaticProjectHtml(
project.dir,
+35 -2
View File
@@ -49,6 +49,14 @@ export class CheckCopiesUnfinishedError extends Error {
}
}
export class CheckBudgetError extends Error {
constructor(budgetMs: number, where: string) {
super(
`Check reached its --budget of ${budgetMs / 1000} s${where || "."} The report holds what it found before then.`,
);
}
}
export class CheckStoppedError extends Error {
constructor(signal: string) {
super(`Check stopped before finishing (${signal}).`);
@@ -71,10 +79,28 @@ export class CheckDeadline {
private readonly spent = new Map<CheckPhase, number>();
private readonly partials: Array<() => Partial<CheckBrowserResult>> = [];
constructor(private readonly idleMs = CHECK_IDLE_MS) {
private readonly budgetTimer: NodeJS.Timeout | undefined;
private readonly stopAt: number | undefined;
constructor(
private readonly idleMs = CHECK_IDLE_MS,
budget?: { ms: number; stopAt: number },
) {
this.expiry = new Promise((_, reject) => {
this.rejectExpiry = reject;
});
this.expiry.catch(() => undefined);
this.budgetTimer =
budget &&
setTimeout(
() => this.expire(new CheckBudgetError(budget.ms, this.where())),
Math.max(0, budget.stopAt - Date.now()),
);
this.stopAt = budget?.stopAt;
}
remainingMs(): number | undefined {
return this.stopAt === undefined ? undefined : Math.max(0, this.stopAt - Date.now());
}
keepPartial = (read: () => Partial<CheckBrowserResult>): void => {
@@ -99,13 +125,18 @@ export class CheckDeadline {
}
private describe(): string {
const where = this.where();
return where && ` (no progress for ${this.idleMs / 1000} s after Chrome was ready),${where}`;
}
private where(): string {
if (!this.work) return "";
const open = Date.now() - this.work.since;
const parts = [...this.spent].map(([phase, ms]) => {
const seconds = ((phase === this.work?.phase ? ms + open : ms) / 1000).toFixed(1);
return `${PHASE_LABELS[phase]} ${seconds}${BEFORE_DEADLINE.has(phase) ? " (before the deadline)" : ""}`;
});
return ` (no progress for ${this.idleMs / 1000} s after Chrome was ready), while ${PHASE_LABELS[this.work.phase]}${this.work.detail ? ` (${this.work.detail})` : ""}. Seconds per phase: ${parts.join(", ")}.`;
return ` while ${PHASE_LABELS[this.work.phase]}${this.work.detail ? ` (${this.work.detail})` : ""}. Seconds per phase: ${parts.join(", ")}.`;
}
start = (): void => {
@@ -125,6 +156,7 @@ export class CheckDeadline {
private expire(error: Error): void {
clearTimeout(this.timer);
clearTimeout(this.budgetTimer);
this.expired = error;
this.rejectExpiry(error);
this.abort.abort(error);
@@ -173,6 +205,7 @@ export class CheckDeadline {
async close(): Promise<void> {
this.phase = "closed";
clearTimeout(this.timer);
clearTimeout(this.budgetTimer);
const closes = await Promise.allSettled([...this.browsers].map(closeManagedBrowser));
const errors = closes
.filter((entry) => entry.status === "rejected")
+13 -5
View File
@@ -2,6 +2,7 @@ import { DEFAULT_CONFIG } from "@hyperframes/engine";
import { BrowserCleanupError, onStopSignal } from "../browser/launch.js";
import type { CheckProgress } from "./checkProgress.js";
import {
CheckBudgetError,
CheckDeadline,
CheckCopiesUnfinishedError,
CheckDeadlineError,
@@ -1734,7 +1735,7 @@ export async function runCheckPipeline(
]
: [];
const deadline = new CheckDeadline(idleWindow(options.timeout));
const deadline = new CheckDeadline(idleWindow(options.timeout), options.budget);
const forgetStop = onStopSignal(deadline.stop);
let outcome: { kind: "report"; report: CheckReport } | { kind: "failed"; error: unknown };
try {
@@ -1839,7 +1840,7 @@ async function withFindingCrops(
deadline: CheckDeadline,
): Promise<CheckReport> {
const cropRequests = selectFindingCropRequests(report);
if (cropRequests.length === 0) return report;
if (cropRequests.length === 0 || deadline.signal.aborted) return report;
try {
deadline.enter("crops");
const findingFiles = await deadline.run(() =>
@@ -1847,7 +1848,7 @@ async function withFindingCrops(
);
return { ...report, snapshots: { ...report.snapshots, findingFiles } };
} catch (error) {
if (error instanceof CheckDeadlineError) {
if (isCheckExpiry(error)) {
const finding = runtimeFailure(error);
return { ...report, ok: false, runtime: section([...report.runtime.findings, finding]) };
}
@@ -1881,7 +1882,7 @@ async function withOptionalFindingFrames(
const captured = await deadline.run(() => capture(project, options, requests, deadline));
return withFindingFrameFields(report, plan, captured);
} catch (error) {
if (error instanceof CheckDeadlineError || error instanceof CheckStoppedError) {
if (isCheckExpiry(error)) {
const finding = runtimeFailure(error);
return { ...report, ok: false, runtime: section([...report.runtime.findings, finding]) };
}
@@ -1891,6 +1892,11 @@ async function withOptionalFindingFrames(
}
}
const isCheckExpiry = (error: unknown): boolean =>
error instanceof CheckDeadlineError ||
error instanceof CheckStoppedError ||
error instanceof CheckBudgetError;
const MAX_FINDING_CROPS = 12;
/** Which error findings get a `finding-NN-<code>.png` crop for `check --snapshots`:
@@ -2263,7 +2269,9 @@ function runtimeFailure(error: unknown, code?: string): CheckFinding {
code ??
(error instanceof CheckCopiesUnfinishedError
? "media_copies_unfinished"
: "check_runtime_failure"),
: error instanceof CheckBudgetError
? "check_budget_reached"
: "check_runtime_failure"),
"error",
normalizeErrorMessage(error),
error instanceof CheckDeadlineError ? (error.sourceFile ?? "index.html") : "index.html",
+1
View File
@@ -23,6 +23,7 @@ export interface CheckOptions {
collapseStatic: boolean;
tolerance: number;
timeout: number;
budget?: { ms: number; stopAt: number };
contrast: boolean;
skipContrastInMotion?: boolean;
strict: boolean;
+3 -2
View File
@@ -7,12 +7,13 @@ export function parseTimeoutMs(
fallback: number,
floorMs = 0,
warn: (message: string) => void = console.error,
flag = "--timeout",
): number {
if (raw === undefined || raw === null || raw === "") return fallback;
const text = String(raw).trim();
const match = /^(\d+(?:\.\d+)?)\s*(ms|s|m)?$/i.exec(text);
if (!match) {
warn(`[hyperframes] --timeout "${text}" is not a duration; using ${fallback} ms.`);
warn(`[hyperframes] ${flag} "${text}" is not a duration; using ${fallback} ms.`);
return fallback;
}
const unit = match[2]?.toLowerCase() as keyof typeof UNIT_MS | undefined;
@@ -21,7 +22,7 @@ export function parseTimeoutMs(
const used = Math.max(ms, floorMs);
if (!unit && ms < 1_000) {
warn(
`[hyperframes] --timeout ${text} is ${ms} ms, so ${used} ms is used. Write ${text}s for seconds.`,
`[hyperframes] ${flag} ${text} is ${ms} ms, so ${used} ms is used. Write ${text}s for seconds.`,
);
}
return used;