mirror of
https://github.com/heygen-com/hyperframes.git
synced 2026-11-18 07:50:13 +00:00
feat(cli): check --budget stops the whole check in time and reports what it found (#7245)
* feat(cli): check --budget stops the whole check in time and reports what it found * fix(cli): keep check's comment share where it was * fix(cli): a --budget that runs out before any check step is waiting is not an unhandled rejection
This commit is contained in:
@@ -1249,6 +1249,7 @@ cover every finding from the full check.
|
||||
| `--collapse-static` / `--no-collapse-static` | Fold a repeated static finding into one row (default: on) |
|
||||
| `--tolerance` | Allowed overflow in px before reporting (default 2) |
|
||||
| `--timeout` | Render-ready budget in ms; also raises page navigation above its 10s floor (default 45000, as snapshot and render) |
|
||||
| `--budget <ms\|90s\|8m>` | Stop the whole check this long after it starts (at least 30s) and report what it found, with `check_budget_reached` naming the phase it was in. For hosts with their own time limit. |
|
||||
| `--no-contrast` | Skip the WCAG pass while iterating |
|
||||
| `--strict` | Exit non-zero on warnings too (default: errors only) |
|
||||
| `--caption-zone "<x0=..;y0=..;x1=..;y1=..>"` | Opt-in band gate. Flags a text element's DOM box that overlaps the fractional band. Optional `severity` and `seek`. |
|
||||
|
||||
@@ -518,6 +518,28 @@ it("waits for the composition as long as snapshot and render do unless --timeout
|
||||
expect(runPipeline.mock.calls[1]?.[1].timeout).toBe(20_000);
|
||||
});
|
||||
|
||||
it("--budget sets when the whole check stops, its report time kept back, counted from the command's start", async () => {
|
||||
const { report } = await runScenario(fakeDriver());
|
||||
const runPipeline = vi.fn(async (_project: ProjectDir, _options: CheckOptions) => report);
|
||||
vi.spyOn(console, "log").mockImplementation(() => undefined);
|
||||
const command = createCheckCommand({
|
||||
resolveProject: () => PROJECT,
|
||||
runPipeline,
|
||||
prepareMedia: vi.fn(async () => []),
|
||||
withMeta: (value) => value,
|
||||
});
|
||||
|
||||
const startedAt = Date.now();
|
||||
await runCommand(command, { rawArgs: ["--json", "--budget", "8m"] });
|
||||
await runCommand(command, { rawArgs: ["--json"] });
|
||||
|
||||
const budget = runPipeline.mock.calls[0]?.[1].budget;
|
||||
expect(budget?.ms).toBe(480_000);
|
||||
expect(budget?.stopAt).toBeGreaterThanOrEqual(startedAt + 470_000);
|
||||
expect(budget?.stopAt).toBeLessThanOrEqual(Date.now() + 470_000);
|
||||
expect(runPipeline.mock.calls[1]?.[1].budget).toBeUndefined();
|
||||
});
|
||||
|
||||
it("parses the caption-zone grammar and enables the frame gate", async () => {
|
||||
const { report } = await runScenario(fakeDriver());
|
||||
const runPipeline = vi.fn(async (_project: ProjectDir, _options: CheckOptions) => report);
|
||||
@@ -2497,6 +2519,58 @@ describe("check pipeline", () => {
|
||||
}
|
||||
});
|
||||
|
||||
it("--budget ends a check that is still within its idle window and keeps what its finished samples found", async () => {
|
||||
vi.useFakeTimers();
|
||||
try {
|
||||
let aborted!: AbortSignal;
|
||||
const driver = fakeDriver({
|
||||
collectLayout: vi.fn(async (time: number) => [
|
||||
layoutIssue("warning", { time, code: "container_overflow" }),
|
||||
]),
|
||||
collectOverlapGrid: vi.fn(
|
||||
() =>
|
||||
new Promise<never>((_, reject) =>
|
||||
aborted.addEventListener("abort", () => reject(aborted.reason)),
|
||||
),
|
||||
),
|
||||
});
|
||||
const { deps } = dependencies(driver);
|
||||
deps.runBrowserCheck = async (_project, options, motion, deadline) => {
|
||||
aborted = deadline!.signal;
|
||||
deadline!.start();
|
||||
return runAuditGrid(
|
||||
driver,
|
||||
options,
|
||||
motion,
|
||||
undefined,
|
||||
deadline!.enter,
|
||||
undefined,
|
||||
deadline!.keepPartial,
|
||||
);
|
||||
};
|
||||
const budget = { ms: 70_000, stopAt: Date.now() + 60_000 };
|
||||
const pending = runCheckPipeline(
|
||||
PROJECT,
|
||||
{ ...DEFAULT_CHECK_OPTIONS, timeout: 400_000, budget },
|
||||
deps,
|
||||
);
|
||||
await vi.advanceTimersByTimeAsync(60_000);
|
||||
const report = await pending;
|
||||
const reached = report.runtime.findings.find(
|
||||
(finding) => finding.code === "check_budget_reached",
|
||||
);
|
||||
expect(reached?.message).toMatch(
|
||||
/^Check reached its --budget of 70 s while running the overlap pass .*Seconds per phase: /,
|
||||
);
|
||||
expect(report.ok).toBe(false);
|
||||
expect(report.layout.findings.some((finding) => finding.code === "container_overflow")).toBe(
|
||||
true,
|
||||
);
|
||||
} finally {
|
||||
vi.useRealTimers();
|
||||
}
|
||||
});
|
||||
|
||||
it("--strict flips a warnings-only result from exit 0 to exit 1", async () => {
|
||||
const warningDriver = () =>
|
||||
fakeDriver({
|
||||
|
||||
@@ -215,6 +215,11 @@ const CHECK_COMMAND_ARGS = {
|
||||
description: "Allowed pixel overflow before reporting an issue (default: 2)",
|
||||
default: "2",
|
||||
},
|
||||
budget: {
|
||||
type: "string",
|
||||
description:
|
||||
"Stop the whole check this long after it starts (ms, or 90s, 8m; at least 30s) and report what it found so far",
|
||||
},
|
||||
timeout: {
|
||||
type: "string",
|
||||
description:
|
||||
@@ -348,6 +353,13 @@ function reportCheckFailure(
|
||||
}
|
||||
|
||||
const REPORT_GRACE_MS = 5_000;
|
||||
const MIN_BUDGET_MS = 30_000;
|
||||
const BUDGET_REPORT_MS = 10_000;
|
||||
|
||||
function parseBudget(raw: unknown): CheckOptions["budget"] {
|
||||
const ms = parseTimeoutMs(raw, 0, MIN_BUDGET_MS, console.error, "--budget");
|
||||
return ms > 0 ? { ms, stopAt: Date.now() + ms - BUDGET_REPORT_MS } : undefined;
|
||||
}
|
||||
|
||||
function untilReported(): { done: () => void } {
|
||||
let done = (): void => undefined;
|
||||
@@ -423,6 +435,7 @@ function parseCheckOptions(args: Record<string, unknown>): CheckOptions {
|
||||
collapseStatic: args["collapse-static"] !== false,
|
||||
tolerance: nonNegativeNumber(args.tolerance, DEFAULT_CHECK_OPTIONS.tolerance),
|
||||
timeout: parseTimeoutMs(args.timeout, DEFAULT_CHECK_OPTIONS.timeout, 500),
|
||||
budget: parseBudget(args.budget),
|
||||
contrast: args.contrast !== false,
|
||||
...(checkReview()?.skipContrastInMotion ? { skipContrastInMotion: true } : {}),
|
||||
strict: args.strict === true,
|
||||
|
||||
@@ -1637,6 +1637,43 @@ describe("preResolveHostileMediaProxies", () => {
|
||||
}
|
||||
});
|
||||
|
||||
it("gives the copies only what is left of the check's --budget when that is under the copy budget", async () => {
|
||||
vi.useFakeTimers();
|
||||
try {
|
||||
const projectDir = mkProjectDir();
|
||||
mocks.scanProjectMediaCodecMap.mockResolvedValue({
|
||||
"/clip.mov": {
|
||||
codecName: "hevc",
|
||||
browserHostile: true,
|
||||
representativeMime: null,
|
||||
hasAlpha: false,
|
||||
},
|
||||
});
|
||||
mocks.resolveProxy.mockImplementation(
|
||||
() =>
|
||||
new Promise<string>((_, reject) =>
|
||||
mocks.proxySignal?.addEventListener("abort", () => reject(new Error("aborted"))),
|
||||
),
|
||||
);
|
||||
vi.spyOn(console, "error").mockImplementation(() => undefined);
|
||||
const deadline = new CheckDeadline(undefined, { ms: 40_000, stopAt: Date.now() + 30_000 });
|
||||
|
||||
const pending = runBrowserCheck(
|
||||
{ ...PROJECT, dir: projectDir, indexPath: `${projectDir}/index.html` },
|
||||
DEFAULT_CHECK_OPTIONS,
|
||||
{ kind: "none" },
|
||||
runAuditGrid,
|
||||
deadline,
|
||||
);
|
||||
const outcome = expect(pending).rejects.toThrow(/took longer than 30 s: clip\.mov/);
|
||||
await vi.advanceTimersByTimeAsync(30_000);
|
||||
await outcome;
|
||||
await deadline.close();
|
||||
} finally {
|
||||
vi.useRealTimers();
|
||||
}
|
||||
});
|
||||
|
||||
it("stops reporting a copy once it is done, so a finished one is not read as stuck", async () => {
|
||||
vi.useFakeTimers();
|
||||
try {
|
||||
|
||||
@@ -443,7 +443,12 @@ export async function runBrowserCheck(
|
||||
findings: drafts,
|
||||
failedPaths,
|
||||
limits,
|
||||
} = await preResolveHostileMediaProxies(project.dir, html, options.autoProxy, COPY_BUDGET_MS);
|
||||
} = await preResolveHostileMediaProxies(
|
||||
project.dir,
|
||||
html,
|
||||
options.autoProxy,
|
||||
Math.min(COPY_BUDGET_MS, deadline?.remainingMs() ?? COPY_BUDGET_MS),
|
||||
);
|
||||
deadline?.enter("launch");
|
||||
const server = await serveStaticProjectHtml(
|
||||
project.dir,
|
||||
|
||||
@@ -49,6 +49,14 @@ export class CheckCopiesUnfinishedError extends Error {
|
||||
}
|
||||
}
|
||||
|
||||
export class CheckBudgetError extends Error {
|
||||
constructor(budgetMs: number, where: string) {
|
||||
super(
|
||||
`Check reached its --budget of ${budgetMs / 1000} s${where || "."} The report holds what it found before then.`,
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
export class CheckStoppedError extends Error {
|
||||
constructor(signal: string) {
|
||||
super(`Check stopped before finishing (${signal}).`);
|
||||
@@ -71,10 +79,28 @@ export class CheckDeadline {
|
||||
private readonly spent = new Map<CheckPhase, number>();
|
||||
private readonly partials: Array<() => Partial<CheckBrowserResult>> = [];
|
||||
|
||||
constructor(private readonly idleMs = CHECK_IDLE_MS) {
|
||||
private readonly budgetTimer: NodeJS.Timeout | undefined;
|
||||
private readonly stopAt: number | undefined;
|
||||
|
||||
constructor(
|
||||
private readonly idleMs = CHECK_IDLE_MS,
|
||||
budget?: { ms: number; stopAt: number },
|
||||
) {
|
||||
this.expiry = new Promise((_, reject) => {
|
||||
this.rejectExpiry = reject;
|
||||
});
|
||||
this.expiry.catch(() => undefined);
|
||||
this.budgetTimer =
|
||||
budget &&
|
||||
setTimeout(
|
||||
() => this.expire(new CheckBudgetError(budget.ms, this.where())),
|
||||
Math.max(0, budget.stopAt - Date.now()),
|
||||
);
|
||||
this.stopAt = budget?.stopAt;
|
||||
}
|
||||
|
||||
remainingMs(): number | undefined {
|
||||
return this.stopAt === undefined ? undefined : Math.max(0, this.stopAt - Date.now());
|
||||
}
|
||||
|
||||
keepPartial = (read: () => Partial<CheckBrowserResult>): void => {
|
||||
@@ -99,13 +125,18 @@ export class CheckDeadline {
|
||||
}
|
||||
|
||||
private describe(): string {
|
||||
const where = this.where();
|
||||
return where && ` (no progress for ${this.idleMs / 1000} s after Chrome was ready),${where}`;
|
||||
}
|
||||
|
||||
private where(): string {
|
||||
if (!this.work) return "";
|
||||
const open = Date.now() - this.work.since;
|
||||
const parts = [...this.spent].map(([phase, ms]) => {
|
||||
const seconds = ((phase === this.work?.phase ? ms + open : ms) / 1000).toFixed(1);
|
||||
return `${PHASE_LABELS[phase]} ${seconds}${BEFORE_DEADLINE.has(phase) ? " (before the deadline)" : ""}`;
|
||||
});
|
||||
return ` (no progress for ${this.idleMs / 1000} s after Chrome was ready), while ${PHASE_LABELS[this.work.phase]}${this.work.detail ? ` (${this.work.detail})` : ""}. Seconds per phase: ${parts.join(", ")}.`;
|
||||
return ` while ${PHASE_LABELS[this.work.phase]}${this.work.detail ? ` (${this.work.detail})` : ""}. Seconds per phase: ${parts.join(", ")}.`;
|
||||
}
|
||||
|
||||
start = (): void => {
|
||||
@@ -125,6 +156,7 @@ export class CheckDeadline {
|
||||
|
||||
private expire(error: Error): void {
|
||||
clearTimeout(this.timer);
|
||||
clearTimeout(this.budgetTimer);
|
||||
this.expired = error;
|
||||
this.rejectExpiry(error);
|
||||
this.abort.abort(error);
|
||||
@@ -173,6 +205,7 @@ export class CheckDeadline {
|
||||
async close(): Promise<void> {
|
||||
this.phase = "closed";
|
||||
clearTimeout(this.timer);
|
||||
clearTimeout(this.budgetTimer);
|
||||
const closes = await Promise.allSettled([...this.browsers].map(closeManagedBrowser));
|
||||
const errors = closes
|
||||
.filter((entry) => entry.status === "rejected")
|
||||
|
||||
@@ -2,6 +2,7 @@ import { DEFAULT_CONFIG } from "@hyperframes/engine";
|
||||
import { BrowserCleanupError, onStopSignal } from "../browser/launch.js";
|
||||
import type { CheckProgress } from "./checkProgress.js";
|
||||
import {
|
||||
CheckBudgetError,
|
||||
CheckDeadline,
|
||||
CheckCopiesUnfinishedError,
|
||||
CheckDeadlineError,
|
||||
@@ -1734,7 +1735,7 @@ export async function runCheckPipeline(
|
||||
]
|
||||
: [];
|
||||
|
||||
const deadline = new CheckDeadline(idleWindow(options.timeout));
|
||||
const deadline = new CheckDeadline(idleWindow(options.timeout), options.budget);
|
||||
const forgetStop = onStopSignal(deadline.stop);
|
||||
let outcome: { kind: "report"; report: CheckReport } | { kind: "failed"; error: unknown };
|
||||
try {
|
||||
@@ -1839,7 +1840,7 @@ async function withFindingCrops(
|
||||
deadline: CheckDeadline,
|
||||
): Promise<CheckReport> {
|
||||
const cropRequests = selectFindingCropRequests(report);
|
||||
if (cropRequests.length === 0) return report;
|
||||
if (cropRequests.length === 0 || deadline.signal.aborted) return report;
|
||||
try {
|
||||
deadline.enter("crops");
|
||||
const findingFiles = await deadline.run(() =>
|
||||
@@ -1847,7 +1848,7 @@ async function withFindingCrops(
|
||||
);
|
||||
return { ...report, snapshots: { ...report.snapshots, findingFiles } };
|
||||
} catch (error) {
|
||||
if (error instanceof CheckDeadlineError) {
|
||||
if (isCheckExpiry(error)) {
|
||||
const finding = runtimeFailure(error);
|
||||
return { ...report, ok: false, runtime: section([...report.runtime.findings, finding]) };
|
||||
}
|
||||
@@ -1881,7 +1882,7 @@ async function withOptionalFindingFrames(
|
||||
const captured = await deadline.run(() => capture(project, options, requests, deadline));
|
||||
return withFindingFrameFields(report, plan, captured);
|
||||
} catch (error) {
|
||||
if (error instanceof CheckDeadlineError || error instanceof CheckStoppedError) {
|
||||
if (isCheckExpiry(error)) {
|
||||
const finding = runtimeFailure(error);
|
||||
return { ...report, ok: false, runtime: section([...report.runtime.findings, finding]) };
|
||||
}
|
||||
@@ -1891,6 +1892,11 @@ async function withOptionalFindingFrames(
|
||||
}
|
||||
}
|
||||
|
||||
const isCheckExpiry = (error: unknown): boolean =>
|
||||
error instanceof CheckDeadlineError ||
|
||||
error instanceof CheckStoppedError ||
|
||||
error instanceof CheckBudgetError;
|
||||
|
||||
const MAX_FINDING_CROPS = 12;
|
||||
|
||||
/** Which error findings get a `finding-NN-<code>.png` crop for `check --snapshots`:
|
||||
@@ -2263,7 +2269,9 @@ function runtimeFailure(error: unknown, code?: string): CheckFinding {
|
||||
code ??
|
||||
(error instanceof CheckCopiesUnfinishedError
|
||||
? "media_copies_unfinished"
|
||||
: "check_runtime_failure"),
|
||||
: error instanceof CheckBudgetError
|
||||
? "check_budget_reached"
|
||||
: "check_runtime_failure"),
|
||||
"error",
|
||||
normalizeErrorMessage(error),
|
||||
error instanceof CheckDeadlineError ? (error.sourceFile ?? "index.html") : "index.html",
|
||||
|
||||
@@ -23,6 +23,7 @@ export interface CheckOptions {
|
||||
collapseStatic: boolean;
|
||||
tolerance: number;
|
||||
timeout: number;
|
||||
budget?: { ms: number; stopAt: number };
|
||||
contrast: boolean;
|
||||
skipContrastInMotion?: boolean;
|
||||
strict: boolean;
|
||||
|
||||
@@ -7,12 +7,13 @@ export function parseTimeoutMs(
|
||||
fallback: number,
|
||||
floorMs = 0,
|
||||
warn: (message: string) => void = console.error,
|
||||
flag = "--timeout",
|
||||
): number {
|
||||
if (raw === undefined || raw === null || raw === "") return fallback;
|
||||
const text = String(raw).trim();
|
||||
const match = /^(\d+(?:\.\d+)?)\s*(ms|s|m)?$/i.exec(text);
|
||||
if (!match) {
|
||||
warn(`[hyperframes] --timeout "${text}" is not a duration; using ${fallback} ms.`);
|
||||
warn(`[hyperframes] ${flag} "${text}" is not a duration; using ${fallback} ms.`);
|
||||
return fallback;
|
||||
}
|
||||
const unit = match[2]?.toLowerCase() as keyof typeof UNIT_MS | undefined;
|
||||
@@ -21,7 +22,7 @@ export function parseTimeoutMs(
|
||||
const used = Math.max(ms, floorMs);
|
||||
if (!unit && ms < 1_000) {
|
||||
warn(
|
||||
`[hyperframes] --timeout ${text} is ${ms} ms, so ${used} ms is used. Write ${text}s for seconds.`,
|
||||
`[hyperframes] ${flag} ${text} is ${ms} ms, so ${used} ms is used. Write ${text}s for seconds.`,
|
||||
);
|
||||
}
|
||||
return used;
|
||||
|
||||
Reference in New Issue
Block a user