Files
hyperframes/packages/cli/src/utils/checkDeadline.ts
T
Miguel Ángel e827de256f fix(check): a Chrome that never becomes ready ends the check with a named error (#7360)
* fix(check): a Chrome that never becomes ready ends the check with a named error

* fix(check): name the Chrome start window in code instead of comments
2026-11-07 11:25:11 +00:00

271 lines
9.2 KiB
TypeScript

import type { Browser, Page } from "puppeteer-core";
import { normalizeErrorMessage } from "./errorMessage.js";
import { withRemainingBudget } from "../capture/captureTimeout.js";
import { BrowserCleanupError, closeManagedBrowser } from "../browser/launch.js";
import type { CheckBrowserResult } from "./checkTypes.js";
const CHECK_IDLE_MS = 120_000;
export const idleWindow = (timeoutMs: number): number =>
Math.max(CHECK_IDLE_MS, timeoutMs + 10_000);
const SCRIPT_START = "[hyperframes:script:start] ";
const SCRIPT_FINISH = "[hyperframes:script:finish] ";
const PHASE_LABELS = {
copies: "making video copies",
launch: "starting Chrome",
load: "loading the page",
settle: "waiting for the page to be ready",
samples: "checking samples",
analysis: "analyzing the samples",
overlap: "running the overlap pass",
crops: "capturing finding crops",
frames: "capturing finding frames",
} as const;
export type CheckPhase = keyof typeof PHASE_LABELS;
const BEFORE_DEADLINE: ReadonlySet<CheckPhase> = new Set(["copies"]);
const CHROME_START_WINDOW_MS = 180_000;
export class CheckDeadlineError extends Error {
constructor(
readonly sourceFile?: string,
phases = "",
) {
super(
(sourceFile === undefined
? "Check did not finish within its deadline"
: `${sourceFile}: its script did not finish within the check deadline`) + (phases || "."),
);
}
}
export const COPY_BUDGET_MS = 120_000;
const HEARTBEAT_MS = 30_000;
export class CheckCopiesUnfinishedError extends Error {
constructor(budgetMs: number, unfinished: readonly string[]) {
super(
`Making browser-playable video copies took longer than ${budgetMs / 1000} s: ${unfinished.join(", ")}. ` +
"Finished pieces are kept. Run `hyperframes check --prepare-media` (no time limit) to finish them, then run check again.",
);
}
}
export class CheckBudgetError extends Error {
constructor(budgetMs: number, where: string) {
super(
`Check reached its --budget of ${budgetMs / 1000} s${where || "."} The report holds what it found before then.`,
);
}
}
export class CheckStoppedError extends Error {
constructor(signal: string) {
super(`Check stopped before finishing (${signal}).`);
}
}
export class CheckChromeStartError extends Error {
constructor(waitedMs: number, where: string) {
super(
`Chrome did not start within ${waitedMs / 1000} s${where || "."} ` +
"On a first run it may still be downloading: run `hyperframes browser ensure`, then run check again.",
);
}
}
/** Ends a check that makes no progress for its idle window; each phase entered or operation settled restarts it. */
export class CheckDeadline {
private readonly abort = new AbortController();
private readonly operations = new Set<Promise<unknown>>();
readonly signal = this.abort.signal;
private readonly browsers = new Set<Browser>();
private readonly scripts: string[] = [];
private expired: Error | undefined;
private timer: NodeJS.Timeout | undefined;
private readonly expiry: Promise<never>;
private rejectExpiry!: (error: Error) => void;
private phase: "waiting" | "started" | "closed" = "waiting";
private work: { phase: CheckPhase; since: number; detail?: string } | undefined;
private readonly spent = new Map<CheckPhase, number>();
private readonly partials: Array<() => Partial<CheckBrowserResult>> = [];
private heartbeat: NodeJS.Timeout | undefined;
private readonly budgetTimer: NodeJS.Timeout | undefined;
private readonly stopAt: number | undefined;
constructor(
private readonly idleMs = CHECK_IDLE_MS,
budget?: { ms: number; stopAt: number },
) {
this.expiry = new Promise((_, reject) => {
this.rejectExpiry = reject;
});
this.expiry.catch(() => undefined);
this.budgetTimer =
budget &&
setTimeout(
() => this.expire(new CheckBudgetError(budget.ms, this.where())),
Math.max(0, budget.stopAt - Date.now()),
);
this.stopAt = budget?.stopAt;
}
remainingMs(): number | undefined {
return this.stopAt === undefined ? undefined : Math.max(0, this.stopAt - Date.now());
}
keepPartial = (read: () => Partial<CheckBrowserResult>): void => {
this.partials.push(read);
};
partialResult(): Partial<CheckBrowserResult> {
return Object.assign({}, ...this.partials.map((read) => read()));
}
enter = (phase: CheckPhase, detail?: string): void => {
if (this.expired) return;
const now = Date.now();
if (this.work?.phase !== phase) this.announce(phase, PHASE_LABELS[phase]);
if (this.work) this.spend(this.work.phase, now - this.work.since);
this.work = { phase, since: now, detail };
this.spend(phase, 0);
this.beat();
this.rearm();
};
private announce(phase: CheckPhase, text: string): void {
if (phase !== "copies") console.error(`[hyperframes] check: ${text}`);
}
private spend(phase: CheckPhase, ms: number): void {
this.spent.set(phase, (this.spent.get(phase) ?? 0) + ms);
}
private describe(): string {
const where = this.where();
return where && ` (no progress for ${this.idleMs / 1000} s after Chrome was ready),${where}`;
}
private where(): string {
if (!this.work) return "";
const open = Date.now() - this.work.since;
const parts = [...this.spent].map(([phase, ms]) => {
const seconds = ((phase === this.work?.phase ? ms + open : ms) / 1000).toFixed(1);
return `${PHASE_LABELS[phase]} ${seconds}${BEFORE_DEADLINE.has(phase) ? " (before the deadline)" : ""}`;
});
return ` while ${PHASE_LABELS[this.work.phase]}${this.work.detail ? ` (${this.work.detail})` : ""}. Seconds per phase: ${parts.join(", ")}.`;
}
start = (): void => {
if (this.phase !== "waiting") return;
this.phase = "started";
this.enter("load");
};
private beat(): void {
if (this.heartbeat || this.phase === "closed" || this.expired) return;
this.heartbeat = setInterval(() => {
if (this.work) {
const seconds = Math.round((Date.now() - this.work.since) / 1000);
const { phase } = this.work;
this.announce(phase, `still ${PHASE_LABELS[phase]} (${seconds} s in this step)`);
}
}, HEARTBEAT_MS);
this.heartbeat.unref();
}
private rearm(): void {
if (this.phase === "closed" || this.expired) return;
const launching = this.work?.phase === "launch" && this.phase === "waiting";
if (this.phase !== "started" && !launching) return;
clearTimeout(this.timer);
const window = launching ? Math.max(CHROME_START_WINDOW_MS, this.idleMs) : this.idleMs;
this.timer = setTimeout(
() =>
this.expire(
launching
? new CheckChromeStartError(window, this.where())
: new CheckDeadlineError(this.scripts.at(-1), this.describe()),
),
window,
);
}
private expire(error: Error): void {
clearInterval(this.heartbeat);
clearTimeout(this.timer);
clearTimeout(this.budgetTimer);
this.expired = error;
this.rejectExpiry(error);
this.abort.abort(error);
}
stop = (signal: string): void => {
if (this.expired || this.phase === "closed") return;
this.expire(new CheckStoppedError(signal));
};
ownBrowser = (browser: Browser): void => {
if (this.expired) throw this.expired;
if (this.phase === "closed") throw new Error("Check has already finished.");
this.browsers.add(browser);
};
async preparePage(page: Page): Promise<void> {
page.on("console", (message) => {
const text = message.text();
if (text.startsWith(SCRIPT_START)) this.scripts.push(text.slice(SCRIPT_START.length));
if (text.startsWith(SCRIPT_FINISH)) {
const index = this.scripts.lastIndexOf(text.slice(SCRIPT_FINISH.length));
if (index !== -1) this.scripts.splice(index, 1);
}
});
await page.evaluateOnNewDocument(
(start, finish) => {
Reflect.set(window, "__hfScriptState", (phase: "start" | "finish", source: string) => {
console.debug((phase === "start" ? start : finish) + source);
});
},
SCRIPT_START,
SCRIPT_FINISH,
);
}
run<T>(operation: () => Promise<T>): Promise<T> {
if (this.expired) return Promise.reject(this.expired);
const pending = operation();
this.operations.add(pending);
const settled = (): void => this.rearm();
pending.then(settled, settled);
return Promise.race([pending, this.expiry]);
}
async close(): Promise<void> {
this.phase = "closed";
clearInterval(this.heartbeat);
clearTimeout(this.timer);
clearTimeout(this.budgetTimer);
const closes = await Promise.allSettled([...this.browsers].map(closeManagedBrowser));
const errors = closes
.filter((entry) => entry.status === "rejected")
.map((entry) => entry.reason);
try {
const drained = await withRemainingBudget(
Promise.allSettled(this.operations),
1_000,
"draining check cleanup",
);
for (const entry of drained) {
if (entry.status === "rejected" && entry.reason instanceof BrowserCleanupError)
errors.push(entry.reason);
}
} catch (error) {
errors.push(error);
}
if (errors.length > 0)
throw new BrowserCleanupError(errors, errors.map(normalizeErrorMessage).join("; "));
}
}