diff --git a/docs/packages/cli.mdx b/docs/packages/cli.mdx index 18af9f3ea6..1e4646c17a 100644 --- a/docs/packages/cli.mdx +++ b/docs/packages/cli.mdx @@ -1249,6 +1249,7 @@ cover every finding from the full check. | `--collapse-static` / `--no-collapse-static` | Fold a repeated static finding into one row (default: on) | | `--tolerance` | Allowed overflow in px before reporting (default 2) | | `--timeout` | Render-ready budget in ms; also raises page navigation above its 10s floor (default 45000, as snapshot and render) | +| `--budget ` | Stop the whole check this long after it starts (at least 30s) and report what it found, with `check_budget_reached` naming the phase it was in. For hosts with their own time limit. | | `--no-contrast` | Skip the WCAG pass while iterating | | `--strict` | Exit non-zero on warnings too (default: errors only) | | `--caption-zone ""` | Opt-in band gate. Flags a text element's DOM box that overlaps the fractional band. Optional `severity` and `seek`. | diff --git a/packages/cli/src/commands/check.test.ts b/packages/cli/src/commands/check.test.ts index 93073696a4..ddc146961a 100644 --- a/packages/cli/src/commands/check.test.ts +++ b/packages/cli/src/commands/check.test.ts @@ -518,6 +518,28 @@ it("waits for the composition as long as snapshot and render do unless --timeout expect(runPipeline.mock.calls[1]?.[1].timeout).toBe(20_000); }); +it("--budget sets when the whole check stops, its report time kept back, counted from the command's start", async () => { + const { report } = await runScenario(fakeDriver()); + const runPipeline = vi.fn(async (_project: ProjectDir, _options: CheckOptions) => report); + vi.spyOn(console, "log").mockImplementation(() => undefined); + const command = createCheckCommand({ + resolveProject: () => PROJECT, + runPipeline, + prepareMedia: vi.fn(async () => []), + withMeta: (value) => value, + }); + + const startedAt = Date.now(); + await runCommand(command, { rawArgs: ["--json", "--budget", "8m"] }); + await runCommand(command, { rawArgs: ["--json"] }); + + const budget = runPipeline.mock.calls[0]?.[1].budget; + expect(budget?.ms).toBe(480_000); + expect(budget?.stopAt).toBeGreaterThanOrEqual(startedAt + 470_000); + expect(budget?.stopAt).toBeLessThanOrEqual(Date.now() + 470_000); + expect(runPipeline.mock.calls[1]?.[1].budget).toBeUndefined(); +}); + it("parses the caption-zone grammar and enables the frame gate", async () => { const { report } = await runScenario(fakeDriver()); const runPipeline = vi.fn(async (_project: ProjectDir, _options: CheckOptions) => report); @@ -2497,6 +2519,58 @@ describe("check pipeline", () => { } }); + it("--budget ends a check that is still within its idle window and keeps what its finished samples found", async () => { + vi.useFakeTimers(); + try { + let aborted!: AbortSignal; + const driver = fakeDriver({ + collectLayout: vi.fn(async (time: number) => [ + layoutIssue("warning", { time, code: "container_overflow" }), + ]), + collectOverlapGrid: vi.fn( + () => + new Promise((_, reject) => + aborted.addEventListener("abort", () => reject(aborted.reason)), + ), + ), + }); + const { deps } = dependencies(driver); + deps.runBrowserCheck = async (_project, options, motion, deadline) => { + aborted = deadline!.signal; + deadline!.start(); + return runAuditGrid( + driver, + options, + motion, + undefined, + deadline!.enter, + undefined, + deadline!.keepPartial, + ); + }; + const budget = { ms: 70_000, stopAt: Date.now() + 60_000 }; + const pending = runCheckPipeline( + PROJECT, + { ...DEFAULT_CHECK_OPTIONS, timeout: 400_000, budget }, + deps, + ); + await vi.advanceTimersByTimeAsync(60_000); + const report = await pending; + const reached = report.runtime.findings.find( + (finding) => finding.code === "check_budget_reached", + ); + expect(reached?.message).toMatch( + /^Check reached its --budget of 70 s while running the overlap pass .*Seconds per phase: /, + ); + expect(report.ok).toBe(false); + expect(report.layout.findings.some((finding) => finding.code === "container_overflow")).toBe( + true, + ); + } finally { + vi.useRealTimers(); + } + }); + it("--strict flips a warnings-only result from exit 0 to exit 1", async () => { const warningDriver = () => fakeDriver({ diff --git a/packages/cli/src/commands/check.ts b/packages/cli/src/commands/check.ts index 272d0630e8..0601d01629 100644 --- a/packages/cli/src/commands/check.ts +++ b/packages/cli/src/commands/check.ts @@ -215,6 +215,11 @@ const CHECK_COMMAND_ARGS = { description: "Allowed pixel overflow before reporting an issue (default: 2)", default: "2", }, + budget: { + type: "string", + description: + "Stop the whole check this long after it starts (ms, or 90s, 8m; at least 30s) and report what it found so far", + }, timeout: { type: "string", description: @@ -348,6 +353,13 @@ function reportCheckFailure( } const REPORT_GRACE_MS = 5_000; +const MIN_BUDGET_MS = 30_000; +const BUDGET_REPORT_MS = 10_000; + +function parseBudget(raw: unknown): CheckOptions["budget"] { + const ms = parseTimeoutMs(raw, 0, MIN_BUDGET_MS, console.error, "--budget"); + return ms > 0 ? { ms, stopAt: Date.now() + ms - BUDGET_REPORT_MS } : undefined; +} function untilReported(): { done: () => void } { let done = (): void => undefined; @@ -423,6 +435,7 @@ function parseCheckOptions(args: Record): CheckOptions { collapseStatic: args["collapse-static"] !== false, tolerance: nonNegativeNumber(args.tolerance, DEFAULT_CHECK_OPTIONS.tolerance), timeout: parseTimeoutMs(args.timeout, DEFAULT_CHECK_OPTIONS.timeout, 500), + budget: parseBudget(args.budget), contrast: args.contrast !== false, ...(checkReview()?.skipContrastInMotion ? { skipContrastInMotion: true } : {}), strict: args.strict === true, diff --git a/packages/cli/src/utils/checkBrowser.test.ts b/packages/cli/src/utils/checkBrowser.test.ts index fd113534c4..670f91c793 100644 --- a/packages/cli/src/utils/checkBrowser.test.ts +++ b/packages/cli/src/utils/checkBrowser.test.ts @@ -1637,6 +1637,43 @@ describe("preResolveHostileMediaProxies", () => { } }); + it("gives the copies only what is left of the check's --budget when that is under the copy budget", async () => { + vi.useFakeTimers(); + try { + const projectDir = mkProjectDir(); + mocks.scanProjectMediaCodecMap.mockResolvedValue({ + "/clip.mov": { + codecName: "hevc", + browserHostile: true, + representativeMime: null, + hasAlpha: false, + }, + }); + mocks.resolveProxy.mockImplementation( + () => + new Promise((_, reject) => + mocks.proxySignal?.addEventListener("abort", () => reject(new Error("aborted"))), + ), + ); + vi.spyOn(console, "error").mockImplementation(() => undefined); + const deadline = new CheckDeadline(undefined, { ms: 40_000, stopAt: Date.now() + 30_000 }); + + const pending = runBrowserCheck( + { ...PROJECT, dir: projectDir, indexPath: `${projectDir}/index.html` }, + DEFAULT_CHECK_OPTIONS, + { kind: "none" }, + runAuditGrid, + deadline, + ); + const outcome = expect(pending).rejects.toThrow(/took longer than 30 s: clip\.mov/); + await vi.advanceTimersByTimeAsync(30_000); + await outcome; + await deadline.close(); + } finally { + vi.useRealTimers(); + } + }); + it("stops reporting a copy once it is done, so a finished one is not read as stuck", async () => { vi.useFakeTimers(); try { diff --git a/packages/cli/src/utils/checkBrowser.ts b/packages/cli/src/utils/checkBrowser.ts index 7a46166cee..42566682f2 100644 --- a/packages/cli/src/utils/checkBrowser.ts +++ b/packages/cli/src/utils/checkBrowser.ts @@ -443,7 +443,12 @@ export async function runBrowserCheck( findings: drafts, failedPaths, limits, - } = await preResolveHostileMediaProxies(project.dir, html, options.autoProxy, COPY_BUDGET_MS); + } = await preResolveHostileMediaProxies( + project.dir, + html, + options.autoProxy, + Math.min(COPY_BUDGET_MS, deadline?.remainingMs() ?? COPY_BUDGET_MS), + ); deadline?.enter("launch"); const server = await serveStaticProjectHtml( project.dir, diff --git a/packages/cli/src/utils/checkDeadline.ts b/packages/cli/src/utils/checkDeadline.ts index 5793ff9d13..b41f8c5a32 100644 --- a/packages/cli/src/utils/checkDeadline.ts +++ b/packages/cli/src/utils/checkDeadline.ts @@ -49,6 +49,14 @@ export class CheckCopiesUnfinishedError extends Error { } } +export class CheckBudgetError extends Error { + constructor(budgetMs: number, where: string) { + super( + `Check reached its --budget of ${budgetMs / 1000} s${where || "."} The report holds what it found before then.`, + ); + } +} + export class CheckStoppedError extends Error { constructor(signal: string) { super(`Check stopped before finishing (${signal}).`); @@ -71,10 +79,28 @@ export class CheckDeadline { private readonly spent = new Map(); private readonly partials: Array<() => Partial> = []; - constructor(private readonly idleMs = CHECK_IDLE_MS) { + private readonly budgetTimer: NodeJS.Timeout | undefined; + private readonly stopAt: number | undefined; + + constructor( + private readonly idleMs = CHECK_IDLE_MS, + budget?: { ms: number; stopAt: number }, + ) { this.expiry = new Promise((_, reject) => { this.rejectExpiry = reject; }); + this.expiry.catch(() => undefined); + this.budgetTimer = + budget && + setTimeout( + () => this.expire(new CheckBudgetError(budget.ms, this.where())), + Math.max(0, budget.stopAt - Date.now()), + ); + this.stopAt = budget?.stopAt; + } + + remainingMs(): number | undefined { + return this.stopAt === undefined ? undefined : Math.max(0, this.stopAt - Date.now()); } keepPartial = (read: () => Partial): void => { @@ -99,13 +125,18 @@ export class CheckDeadline { } private describe(): string { + const where = this.where(); + return where && ` (no progress for ${this.idleMs / 1000} s after Chrome was ready),${where}`; + } + + private where(): string { if (!this.work) return ""; const open = Date.now() - this.work.since; const parts = [...this.spent].map(([phase, ms]) => { const seconds = ((phase === this.work?.phase ? ms + open : ms) / 1000).toFixed(1); return `${PHASE_LABELS[phase]} ${seconds}${BEFORE_DEADLINE.has(phase) ? " (before the deadline)" : ""}`; }); - return ` (no progress for ${this.idleMs / 1000} s after Chrome was ready), while ${PHASE_LABELS[this.work.phase]}${this.work.detail ? ` (${this.work.detail})` : ""}. Seconds per phase: ${parts.join(", ")}.`; + return ` while ${PHASE_LABELS[this.work.phase]}${this.work.detail ? ` (${this.work.detail})` : ""}. Seconds per phase: ${parts.join(", ")}.`; } start = (): void => { @@ -125,6 +156,7 @@ export class CheckDeadline { private expire(error: Error): void { clearTimeout(this.timer); + clearTimeout(this.budgetTimer); this.expired = error; this.rejectExpiry(error); this.abort.abort(error); @@ -173,6 +205,7 @@ export class CheckDeadline { async close(): Promise { this.phase = "closed"; clearTimeout(this.timer); + clearTimeout(this.budgetTimer); const closes = await Promise.allSettled([...this.browsers].map(closeManagedBrowser)); const errors = closes .filter((entry) => entry.status === "rejected") diff --git a/packages/cli/src/utils/checkPipeline.ts b/packages/cli/src/utils/checkPipeline.ts index 7e21afce02..97817aacc2 100644 --- a/packages/cli/src/utils/checkPipeline.ts +++ b/packages/cli/src/utils/checkPipeline.ts @@ -2,6 +2,7 @@ import { DEFAULT_CONFIG } from "@hyperframes/engine"; import { BrowserCleanupError, onStopSignal } from "../browser/launch.js"; import type { CheckProgress } from "./checkProgress.js"; import { + CheckBudgetError, CheckDeadline, CheckCopiesUnfinishedError, CheckDeadlineError, @@ -1734,7 +1735,7 @@ export async function runCheckPipeline( ] : []; - const deadline = new CheckDeadline(idleWindow(options.timeout)); + const deadline = new CheckDeadline(idleWindow(options.timeout), options.budget); const forgetStop = onStopSignal(deadline.stop); let outcome: { kind: "report"; report: CheckReport } | { kind: "failed"; error: unknown }; try { @@ -1839,7 +1840,7 @@ async function withFindingCrops( deadline: CheckDeadline, ): Promise { const cropRequests = selectFindingCropRequests(report); - if (cropRequests.length === 0) return report; + if (cropRequests.length === 0 || deadline.signal.aborted) return report; try { deadline.enter("crops"); const findingFiles = await deadline.run(() => @@ -1847,7 +1848,7 @@ async function withFindingCrops( ); return { ...report, snapshots: { ...report.snapshots, findingFiles } }; } catch (error) { - if (error instanceof CheckDeadlineError) { + if (isCheckExpiry(error)) { const finding = runtimeFailure(error); return { ...report, ok: false, runtime: section([...report.runtime.findings, finding]) }; } @@ -1881,7 +1882,7 @@ async function withOptionalFindingFrames( const captured = await deadline.run(() => capture(project, options, requests, deadline)); return withFindingFrameFields(report, plan, captured); } catch (error) { - if (error instanceof CheckDeadlineError || error instanceof CheckStoppedError) { + if (isCheckExpiry(error)) { const finding = runtimeFailure(error); return { ...report, ok: false, runtime: section([...report.runtime.findings, finding]) }; } @@ -1891,6 +1892,11 @@ async function withOptionalFindingFrames( } } +const isCheckExpiry = (error: unknown): boolean => + error instanceof CheckDeadlineError || + error instanceof CheckStoppedError || + error instanceof CheckBudgetError; + const MAX_FINDING_CROPS = 12; /** Which error findings get a `finding-NN-.png` crop for `check --snapshots`: @@ -2263,7 +2269,9 @@ function runtimeFailure(error: unknown, code?: string): CheckFinding { code ?? (error instanceof CheckCopiesUnfinishedError ? "media_copies_unfinished" - : "check_runtime_failure"), + : error instanceof CheckBudgetError + ? "check_budget_reached" + : "check_runtime_failure"), "error", normalizeErrorMessage(error), error instanceof CheckDeadlineError ? (error.sourceFile ?? "index.html") : "index.html", diff --git a/packages/cli/src/utils/checkTypes.ts b/packages/cli/src/utils/checkTypes.ts index ecac9a423d..d2ca6bd3b8 100644 --- a/packages/cli/src/utils/checkTypes.ts +++ b/packages/cli/src/utils/checkTypes.ts @@ -23,6 +23,7 @@ export interface CheckOptions { collapseStatic: boolean; tolerance: number; timeout: number; + budget?: { ms: number; stopAt: number }; contrast: boolean; skipContrastInMotion?: boolean; strict: boolean; diff --git a/packages/cli/src/utils/timeoutArg.ts b/packages/cli/src/utils/timeoutArg.ts index f7b8fd16f5..c20e036c71 100644 --- a/packages/cli/src/utils/timeoutArg.ts +++ b/packages/cli/src/utils/timeoutArg.ts @@ -7,12 +7,13 @@ export function parseTimeoutMs( fallback: number, floorMs = 0, warn: (message: string) => void = console.error, + flag = "--timeout", ): number { if (raw === undefined || raw === null || raw === "") return fallback; const text = String(raw).trim(); const match = /^(\d+(?:\.\d+)?)\s*(ms|s|m)?$/i.exec(text); if (!match) { - warn(`[hyperframes] --timeout "${text}" is not a duration; using ${fallback} ms.`); + warn(`[hyperframes] ${flag} "${text}" is not a duration; using ${fallback} ms.`); return fallback; } const unit = match[2]?.toLowerCase() as keyof typeof UNIT_MS | undefined; @@ -21,7 +22,7 @@ export function parseTimeoutMs( const used = Math.max(ms, floorMs); if (!unit && ms < 1_000) { warn( - `[hyperframes] --timeout ${text} is ${ms} ms, so ${used} ms is used. Write ${text}s for seconds.`, + `[hyperframes] ${flag} ${text} is ${ms} ms, so ${used} ms is used. Write ${text}s for seconds.`, ); } return used;