From 778a005641db50f2f37ef8d40cf813c58a30cee4 Mon Sep 17 00:00:00 2001 From: Vance Ingalls Date: Mon, 16 Nov 2026 12:10:39 -0800 Subject: [PATCH] fix(lint): run Studio and producer lint off the host's own thread (#7974) * fix(lint): run Studio and producer lint off the host's own thread A composition that sends the linter's parser into a runaway froze or crashed the long-lived hosts that lint it in their own process. CLI Studio's single-page lint now runs in a child process with a heap cap and a 15 s limit. Producer /lint (runHyperframeLint) runs on a worker thread with the same limits. A page that times out, runs out of memory, or whose child or thread ends without an answer gets one lint_aborted error finding and is never linted in the host process. Studio's project lint keeps its timeout and gains a heap cap. Co-Authored-By: Claude Sonnet 5.5 * fix(producer): cap concurrent /lint workers at two Each lint worker can hold up to 512 MB, so parallel /lint requests multiplied that without bound. At most two run at once; the rest wait in order, and a request still waiting when its own 15 s runs out gets the lint_aborted result. Co-Authored-By: Claude Sonnet 5.5 --------- Co-authored-by: Claude Sonnet 5.5 --- .fallowrc.jsonc | 4 + packages/cli/src/renderSetupWorker.ts | 9 ++ .../server/studioServer.lintProcess.test.ts | 16 ++- packages/cli/src/server/studioServer.ts | 14 +- packages/cli/src/utils/cancellableProcess.ts | 10 +- packages/cli/src/utils/lintHtmlWorker.test.ts | 50 +++++++ packages/cli/src/utils/lintHtmlWorker.ts | 68 ++++++++++ packages/producer/buildEntries.mjs | 5 +- .../producer/scripts/integration-tests.json | 2 + packages/producer/src/buildEntries.test.ts | 3 +- .../producer/src/server.lintWorker.test.ts | 52 ++++++++ .../services/__fixtures__/lintWorkerStuck.mjs | 17 +++ .../producer/src/services/hyperframeLint.ts | 123 +++++++++++++++++- .../services/hyperframeLint.worker.test.ts | 72 ++++++++++ .../src/services/hyperframeLintWorker.ts | 12 ++ 15 files changed, 439 insertions(+), 18 deletions(-) create mode 100644 packages/cli/src/utils/lintHtmlWorker.test.ts create mode 100644 packages/cli/src/utils/lintHtmlWorker.ts create mode 100644 packages/producer/src/server.lintWorker.test.ts create mode 100644 packages/producer/src/services/__fixtures__/lintWorkerStuck.mjs create mode 100644 packages/producer/src/services/hyperframeLint.worker.test.ts create mode 100644 packages/producer/src/services/hyperframeLintWorker.ts diff --git a/.fallowrc.jsonc b/.fallowrc.jsonc index bdb3909ab1..9ae69f725b 100644 --- a/.fallowrc.jsonc +++ b/.fallowrc.jsonc @@ -55,6 +55,8 @@ "packages/producer/src/services/shaderTransitionWorker.ts", // Off-main-thread /health endpoint, spawned by path from healthWorker.ts. "packages/producer/src/services/healthWorkerThread.ts", + // Worker thread of runHyperframeLint, spawned by path from hyperframeLint.ts. + "packages/producer/src/services/hyperframeLintWorker.ts", // Render-setup worker, spawned by path from cancellableProcess.ts. "packages/cli/src/renderSetupWorker.ts", // Sub-composition build worker, started by path from subCompositionBuild.ts (.dev.mjs from source under Node). @@ -73,6 +75,8 @@ // Test fixture worker, spawned by path via the pools' workerEntryPath // option from the crash-recovery tests; has no import-graph referrer. "packages/producer/src/services/__fixtures__/crashOnMessageWorker.mjs", + // Test fixture worker for runHyperframeLint, passed by path as its workerEntry option. + "packages/producer/src/services/__fixtures__/lintWorkerStuck.mjs", "scripts/**/*.{ts,mjs,js}", "scripts/*/run.mjs", // Keyframe UI components — wired dynamically via EaseCurveSection/MotionPanel. diff --git a/packages/cli/src/renderSetupWorker.ts b/packages/cli/src/renderSetupWorker.ts index 7e5b1ac44b..308efc232e 100644 --- a/packages/cli/src/renderSetupWorker.ts +++ b/packages/cli/src/renderSetupWorker.ts @@ -1,6 +1,8 @@ import "./utils/hideChildConsoles.js"; import "./utils/windowsExeSearch.js"; +import { readFile, rm } from "node:fs/promises"; import { constants, setPriority } from "node:os"; +import { lintHyperframeHtml } from "@hyperframes/lint"; import { runBrowserInstall } from "./browser/installRun.js"; import { ensureBrowser, releaseOwnedBrowserInstallLock } from "./browser/manager.js"; import { lintProject } from "./utils/lintProject.js"; @@ -48,6 +50,13 @@ async function runMode(): Promise { input.host ? { host: input.host } : undefined, ); } + if (mode === "lint-html") { + try { + return await lintHyperframeHtml(await readFile(input.htmlPath, "utf-8"), input.opts); + } finally { + await rm(input.htmlPath, { force: true }); + } + } if (mode === "orphan-cleanup") { const killed = killOrphanedProcesses(); if (killed > 0) await new Promise((resolve) => setTimeout(resolve, 600)); diff --git a/packages/cli/src/server/studioServer.lintProcess.test.ts b/packages/cli/src/server/studioServer.lintProcess.test.ts index 169918be5c..cf4cc3e0d5 100644 --- a/packages/cli/src/server/studioServer.lintProcess.test.ts +++ b/packages/cli/src/server/studioServer.lintProcess.test.ts @@ -3,12 +3,15 @@ import * as fs from "node:fs"; import * as path from "node:path"; import { tmpdir } from "node:os"; -// Studio must lint outside its own process; any in-process project lint fails the request. +// Studio must lint outside its own process; any in-process project or page lint fails the request. vi.mock("@hyperframes/lint", async (importOriginal) => ({ ...(await importOriginal()), lintProject: () => { throw new Error("project lint ran on the Studio server's event loop"); }, + lintHyperframeHtml: () => { + throw new Error("page lint ran on the Studio server's event loop"); + }, })); const { createStudioServer } = await import("./studioServer.js"); @@ -24,6 +27,9 @@ beforeEach(() => {
`, ); + // Not linked from the entry, so the project lint skips it and the route lints it as a page. + fs.mkdirSync(path.join(root, "loose")); + fs.writeFileSync(path.join(root, "loose", "page.html"), `
no composition
`); server = createStudioServer({ projectDir: root, projectName: "film" }); }); @@ -40,4 +46,12 @@ describe("studio lint", () => { expect(response.status).toBe(200); expect(body.findings?.some((f) => f.message.includes("window.__timelines"))).toBe(true); }, 60_000); + + it("lints a page the project lint skips in a separate process too", async () => { + const response = await server.app.request("/api/projects/film/lint"); + const body = (await response.json()) as { findings?: Array<{ file?: string }> }; + + expect(response.status).toBe(200); + expect(body.findings?.some((f) => f.file === "loose/page.html")).toBe(true); + }, 60_000); }); diff --git a/packages/cli/src/server/studioServer.ts b/packages/cli/src/server/studioServer.ts index 47eb6d375f..210f3c76d9 100644 --- a/packages/cli/src/server/studioServer.ts +++ b/packages/cli/src/server/studioServer.ts @@ -49,6 +49,7 @@ import { recordFolder } from "../utils/projectRecords.js"; import { outsideAssetsAllowed } from "../utils/receivedRender.js"; import { insideHostApp } from "../utils/desktopApp.js"; import { runRenderSetupWorker } from "../utils/cancellableProcess.js"; +import { lintHtmlOutOfProcess } from "../utils/lintHtmlWorker.js"; import { acquireRenderLock, describeRenderLockHolder } from "../utils/renderLock.js"; import type { ProjectLintResult } from "@hyperframes/lint"; import { resolveRenderBrowser } from "../browser/preflight.js"; @@ -110,6 +111,7 @@ const STUDIO_MANUAL_EDITS_PATH = ".hyperframes/studio-manual-edits.json"; const RENDER_SHUTDOWN_WAIT_MS = 2_000; const LINT_WORKER_TIMEOUT_MS = 120_000; +const LINT_WORKER_HEAP_MB = 1024; // Vite emits only content-hashed files under dist/assets; hand-authored // public/ files land at the dist root. The route is the signal because the @@ -618,10 +620,7 @@ export function createStudioServer(options: StudioServerOptions): StudioServer { if (resolve(dir) === resolve(projectDir)) cachedProjectSignature = null; }, - async lint(html, opts) { - const { lintHyperframeHtml } = await import("@hyperframes/lint"); - return await lintHyperframeHtml(html, { ...opts, host: "studio" }); - }, + lint: (html, opts) => lintHtmlOutOfProcess(html, { ...opts, host: "studio" }), // Out of process: linting a large composition is seconds of synchronous parsing, and on // this event loop it stalls every other Studio request, the preview included. @@ -629,7 +628,12 @@ export function createStudioServer(options: StudioServerOptions): StudioServer { runRenderSetupWorker( "lint", { projectDir: dir, host: "studio", background: true }, - { maxBufferBytes: 8 * 1024 * 1024, signal, timeoutMs: LINT_WORKER_TIMEOUT_MS }, + { + execArgv: [`--max-old-space-size=${LINT_WORKER_HEAP_MB}`], + maxBufferBytes: 8 * 1024 * 1024, + signal, + timeoutMs: LINT_WORKER_TIMEOUT_MS, + }, ), runtimeUrl: "/api/runtime.js", diff --git a/packages/cli/src/utils/cancellableProcess.ts b/packages/cli/src/utils/cancellableProcess.ts index 05a783ae24..bfeedcc398 100644 --- a/packages/cli/src/utils/cancellableProcess.ts +++ b/packages/cli/src/utils/cancellableProcess.ts @@ -210,17 +210,19 @@ function tsxLoaderFromCliInstall(): string { } export async function runRenderSetupWorker( - mode: "browser" | "browser-install" | "lint" | "orphan-cleanup", + mode: "browser" | "browser-install" | "lint" | "lint-html" | "orphan-cleanup", input: unknown, - options: CancellableProcessOptions, + { execArgv: nodeFlags = [], ...options }: CancellableProcessOptions & { execArgv?: string[] }, ): Promise { const sourceMode = import.meta.url.endsWith(".ts"); const workerUrl = new URL( sourceMode ? "../renderSetupWorker.ts" : "./renderSetupWorker.js", import.meta.url, ); - const execArgv = - sourceMode && !process.versions.bun ? ["--import", tsxLoaderFromCliInstall()] : []; + const execArgv = [ + ...nodeFlags, + ...(sourceMode && !process.versions.bun ? ["--import", tsxLoaderFromCliInstall()] : []), + ]; let result: CancellableProcessResult; try { result = await runCancellableProcess( diff --git a/packages/cli/src/utils/lintHtmlWorker.test.ts b/packages/cli/src/utils/lintHtmlWorker.test.ts new file mode 100644 index 0000000000..4058efaabe --- /dev/null +++ b/packages/cli/src/utils/lintHtmlWorker.test.ts @@ -0,0 +1,50 @@ +import { existsSync } from "node:fs"; +import { dirname } from "node:path"; +import { describe, expect, it } from "vitest"; +import { lintHyperframeHtml } from "@hyperframes/lint"; +import { runCancellableProcess, runRenderSetupWorker } from "./cancellableProcess.js"; +import { lintHtmlOutOfProcess } from "./lintHtmlWorker.js"; + +const PAGE = `
`; +const OPTS = { filePath: "index.html", host: "studio" as const }; +const withoutTimings = ({ timings: _timings, ...rest }: Record) => rest; + +describe("lintHtmlOutOfProcess", () => { + // The second page is past the ~128 KB an env var can carry, so it only lints if the page travels in a file. + it.each([PAGE, PAGE.replace("", `\n`)])( + "lints a %#-th page in a child to what the linter returns in process", + async (html) => { + let pageDir = ""; + const run = ((mode, input, options) => { + pageDir = dirname((input as { htmlPath: string }).htmlPath); + return runRenderSetupWorker(mode, input, options); + }) as typeof runRenderSetupWorker; + const result = await lintHtmlOutOfProcess(html, OPTS, { run }); + expect(withoutTimings({ ...result })).toEqual( + withoutTimings({ ...(await lintHyperframeHtml(html, OPTS)) }), + ); + expect(existsSync(pageDir)).toBe(false); + }, + 30_000, + ); + + it("answers a child that never answers with the lint_aborted error, and does not lint here", async () => { + // A real child that never exits, stopped by the runner's own time limit. + const hang = ((_mode, _input, options) => + runCancellableProcess(process.execPath, ["--eval", "setInterval(() => {}, 1000)"], { + timeoutMs: options.timeoutMs, + })) as typeof runRenderSetupWorker; + const began = performance.now(); + const result = await lintHtmlOutOfProcess(PAGE, OPTS, { run: hang, timeoutMs: 300 }); + expect(performance.now() - began).toBeLessThan(10_000); + expect([result.ok, result.errorCount, result.warningCount, result.infoCount]).toEqual([ + false, + 1, + 0, + 0, + ]); + expect(result.findings.map(({ code, severity, file }) => [code, severity, file])).toEqual([ + ["lint_aborted", "error", "index.html"], + ]); + }); +}); diff --git a/packages/cli/src/utils/lintHtmlWorker.ts b/packages/cli/src/utils/lintHtmlWorker.ts new file mode 100644 index 0000000000..34819058ae --- /dev/null +++ b/packages/cli/src/utils/lintHtmlWorker.ts @@ -0,0 +1,68 @@ +import { mkdtemp, rm, writeFile } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import type { HyperframeLinterOptions, HyperframeLintResult } from "@hyperframes/lint"; +import { runRenderSetupWorker } from "./cancellableProcess.js"; + +/** A real composition lints in about a second at a few tens of MB of heap; this is where a runaway is stopped. */ +const LINT_HTML_HEAP_MB = 512; +const LINT_HTML_TIMEOUT_MS = 15_000; + +type LintHtmlTestHooks = { run?: typeof runRenderSetupWorker; timeoutMs?: number }; + +/** The linter's own result shape for a page it could not check: one error, never linted in this process. */ +function abortedLintResult( + opts: HyperframeLinterOptions | undefined, + afterMs: number, +): HyperframeLintResult { + return { + ok: false, + errorCount: 1, + warningCount: 0, + infoCount: 0, + findings: [ + { + code: "lint_aborted", + severity: "error", + message: + "This composition's script could not be checked: linting it ran too long or out of memory, so it was stopped. A very large or deeply nested script is the usual cause.", + ...(opts?.filePath && { file: opts.filePath }), + }, + ], + timings: { totalMs: afterMs, groupMs: {}, slowestRule: "", slowestRuleMs: 0 }, + }; +} + +/** + * Lints one page in a child process with a heap cap and a time limit, so a long-lived host (Studio) + * survives a page that sends the parser into a runaway: that page's lint is the `lint_aborted` + * error. The child timing out, dying or exiting without a result never falls back to linting here. + */ +export async function lintHtmlOutOfProcess( + html: string, + opts: HyperframeLinterOptions | undefined, + { run = runRenderSetupWorker, timeoutMs = LINT_HTML_TIMEOUT_MS }: LintHtmlTestHooks = {}, +): Promise { + // The runner hands its input over in an env var, which a large page would overflow: the page goes in a file. + const dir = await mkdtemp(join(tmpdir(), "hf-lint-")); + const htmlPath = join(dir, "page.html"); + try { + await writeFile(htmlPath, html); + const began = performance.now(); + try { + return await run( + "lint-html", + { htmlPath, opts }, + { + execArgv: [`--max-old-space-size=${LINT_HTML_HEAP_MB}`], + maxBufferBytes: 8 * 1024 * 1024, + timeoutMs, + }, + ); + } catch { + return abortedLintResult(opts, Math.round(performance.now() - began)); + } + } finally { + await rm(dir, { recursive: true, force: true }); + } +} diff --git a/packages/producer/buildEntries.mjs b/packages/producer/buildEntries.mjs index b6dc34c708..ff6c205592 100644 --- a/packages/producer/buildEntries.mjs +++ b/packages/producer/buildEntries.mjs @@ -1,5 +1,5 @@ -// Workers sit beside the bundles because healthWorker.ts and shaderTransitionWorkerPool.ts -// look for them in their own module's directory. +// Workers sit beside the bundles because healthWorker.ts, hyperframeLint.ts and +// shaderTransitionWorkerPool.ts look for them in their own module's directory. export const BUNDLES = [ { entry: "src/index.ts", outfile: "dist/index.js" }, { entry: "src/server.ts", outfile: "dist/public-server.js" }, @@ -8,4 +8,5 @@ export const BUNDLES = [ export const WORKERS = [ { entry: "src/services/shaderTransitionWorker.ts", outfile: "dist/shaderTransitionWorker.js" }, { entry: "src/services/healthWorkerThread.ts", outfile: "dist/healthWorkerThread.js" }, + { entry: "src/services/hyperframeLintWorker.ts", outfile: "dist/hyperframeLintWorker.js" }, ]; diff --git a/packages/producer/scripts/integration-tests.json b/packages/producer/scripts/integration-tests.json index 59e6a5bb8a..37d8eed4f0 100644 --- a/packages/producer/scripts/integration-tests.json +++ b/packages/producer/scripts/integration-tests.json @@ -30,6 +30,8 @@ "src/services/fileServer.listenError.test.ts", "src/services/outsideProjectAssets.test.ts", "src/services/healthWorker.test.ts", + "src/services/hyperframeLint.worker.test.ts", + "src/server.lintWorker.test.ts", "src/services/scriptFailureAttribution.test.ts", "src/services/strictVariablesSession.test.ts", "src/services/hyperframeLint.file-race.test.ts", diff --git a/packages/producer/src/buildEntries.test.ts b/packages/producer/src/buildEntries.test.ts index 8b8bcb2141..e4b79454cf 100644 --- a/packages/producer/src/buildEntries.test.ts +++ b/packages/producer/src/buildEntries.test.ts @@ -6,9 +6,10 @@ describe("published build layout", () => { it("emits each worker beside the bundles whose worker lookup probes their own directory", () => { const bundleDirs = new Set(BUNDLES.map(({ outfile }) => dirname(outfile))); expect([...bundleDirs]).toEqual(["dist"]); - expect(WORKERS.map(({ outfile }) => dirname(outfile))).toEqual(["dist", "dist"]); + expect(WORKERS.map(({ outfile }) => dirname(outfile))).toEqual(["dist", "dist", "dist"]); expect(WORKERS.map(({ outfile }) => basename(outfile)).sort()).toEqual([ "healthWorkerThread.js", + "hyperframeLintWorker.js", "shaderTransitionWorker.js", ]); }); diff --git a/packages/producer/src/server.lintWorker.test.ts b/packages/producer/src/server.lintWorker.test.ts new file mode 100644 index 0000000000..755576b6af --- /dev/null +++ b/packages/producer/src/server.lintWorker.test.ts @@ -0,0 +1,52 @@ +import { mkdtempSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { dirname, join, resolve } from "node:path"; +import { fileURLToPath } from "node:url"; +import { describe, expect, it, vi } from "vitest"; + +// Every /lint page runs on a worker that never answers, stopped by a short limit. +vi.mock("./services/hyperframeLint.js", async (importOriginal) => { + const actual = await importOriginal(); + const stuck = resolve( + dirname(fileURLToPath(import.meta.url)), + "services/__fixtures__/lintWorkerStuck.mjs", + ); + return { + ...actual, + runHyperframeLint: (prepared: Parameters[0]) => + actual.runHyperframeLint(prepared, { workerEntry: stuck, timeoutMs: 500 }), + }; +}); + +import { createProducerApp } from "./server.js"; + +describe("POST /lint", () => { + it("answers 200 with lint_aborted for a page whose lint never ends, and the server keeps answering", async () => { + const root = mkdtempSync(join(tmpdir(), "hf-lint-route-")); + const app = createProducerApp({ + projectsDir: join(root, "projects"), + rendersDir: join(root, "renders"), + }); + const lint = Promise.resolve( + app.request("http://localhost:9847/lint", { + method: "POST", + headers: { host: "localhost:9847" }, + body: JSON.stringify({ html: "hang" }), + }), + ); + // While that page is stuck, another request is answered. + const health = await app.request("http://localhost:9847/health", { + headers: { host: "localhost:9847" }, + }); + expect(health.status).toBe(200); + + const res = await lint; + expect(res.status).toBe(200); + const body = (await res.json()) as { + success: boolean; + result: { findings: { code: string }[] }; + }; + expect(body.success).toBe(true); + expect(body.result.findings.map((finding) => finding.code)).toEqual(["lint_aborted"]); + }, 30_000); +}); diff --git a/packages/producer/src/services/__fixtures__/lintWorkerStuck.mjs b/packages/producer/src/services/__fixtures__/lintWorkerStuck.mjs new file mode 100644 index 0000000000..d4fd9e2b2a --- /dev/null +++ b/packages/producer/src/services/__fixtures__/lintWorkerStuck.mjs @@ -0,0 +1,17 @@ +// Stands in for hyperframeLintWorker.ts on a page that sends the parser into a runaway: "hang" spins and never +// answers, "alloc" allocates until the thread's heap cap ends it, "heap" answers with the thread's own heap limit. +import { appendFileSync } from "node:fs"; +import { getHeapStatistics } from "node:v8"; +import { parentPort, workerData } from "node:worker_threads"; + +if (process.env.HF_LINT_FIXTURE_LOG) appendFileSync(process.env.HF_LINT_FIXTURE_LOG, "x"); + +if (workerData.html === "heap") { + parentPort.postMessage({ + result: { heapLimitMb: getHeapStatistics().heap_size_limit / 1048576 }, + }); +} else if (workerData.html === "hang") { + for (;;); +} else { + for (const kept = []; ; ) kept.push(new Array(100_000).fill(1)); +} diff --git a/packages/producer/src/services/hyperframeLint.ts b/packages/producer/src/services/hyperframeLint.ts index fa5475649b..29b96138ab 100644 --- a/packages/producer/src/services/hyperframeLint.ts +++ b/packages/producer/src/services/hyperframeLint.ts @@ -7,8 +7,10 @@ import { closeSync, constants, } from "node:fs"; -import { resolve, join, relative, isAbsolute, sep } from "node:path"; -import { lintHyperframeHtml, type HyperframeLintResult } from "@hyperframes/lint"; +import { resolve, join, relative, isAbsolute, sep, dirname } from "node:path"; +import { fileURLToPath } from "node:url"; +import { Worker } from "node:worker_threads"; +import type { HyperframeLintResult } from "@hyperframes/lint"; export interface PreparedHyperframeLintInput { entryFile: string; @@ -162,11 +164,122 @@ export function prepareHyperframeLintBody( return { error: "Missing lint source: provide projectDir, files, or html" }; } +const LINT_HEAP_MB = 512; +const LINT_TIMEOUT_MS = 15_000; +const LINT_MAX_WORKERS = 2; + +const waiting: Array<() => void> = []; +let running = 0; + +function takeLintSlot(limit: number, waitMs: number): Promise { + if (running < limit) { + running++; + return Promise.resolve(true); + } + return new Promise((resolve) => { + const entry = () => { + clearTimeout(timer); + running++; + resolve(true); + }; + const timer = setTimeout( + () => { + waiting.splice(waiting.indexOf(entry), 1); + resolve(false); + }, + Math.max(0, waitMs), + ); + waiting.push(entry); + }); +} + +function freeLintSlot(): void { + running--; + waiting.shift()?.(); +} + +export interface HyperframeLintWorkerOptions { + workerEntry?: string; + timeoutMs?: number; + heapMb?: number; + maxWorkers?: number; +} + +function resolveLintWorkerEntry(): string | undefined { + const here = dirname(fileURLToPath(import.meta.url)); + for (const name of ["hyperframeLintWorker.js", "hyperframeLintWorker.ts"]) { + const candidate = join(here, name); + if (existsSync(candidate)) return candidate; + } + return undefined; +} + +function abortedLintResult(filePath: string, afterMs: number): HyperframeLintResult { + return { + ok: false, + errorCount: 1, + warningCount: 0, + infoCount: 0, + findings: [ + { + code: "lint_aborted", + severity: "error", + message: + "This composition's script could not be checked: linting it ran too long or out of memory, so it was stopped. A very large or deeply nested script is the usual cause.", + file: filePath, + }, + ], + timings: { totalMs: afterMs, groupMs: {}, slowestRule: "", slowestRuleMs: 0 }, + }; +} + export async function runHyperframeLint( prepared: PreparedHyperframeLintInput, + options: HyperframeLintWorkerOptions = {}, ): Promise { - return lintHyperframeHtml(prepared.html, { - filePath: prepared.entryFile, - compSrcPath: prepared.entryFile, + const began = performance.now(); + const aborted = () => + abortedLintResult(prepared.entryFile, Math.round(performance.now() - began)); + const entry = options.workerEntry ?? resolveLintWorkerEntry(); + if (!entry) return aborted(); + + const timeoutMs = options.timeoutMs ?? LINT_TIMEOUT_MS; + const gotSlot = await takeLintSlot(options.maxWorkers ?? LINT_MAX_WORKERS, timeoutMs); + if (!gotSlot) return aborted(); + const remainingMs = Math.max(0, timeoutMs - (performance.now() - began)); + + return new Promise((resolvePromise, reject) => { + let worker: Worker; + try { + worker = new Worker(entry, { + workerData: { + html: prepared.html, + opts: { filePath: prepared.entryFile, compSrcPath: prepared.entryFile }, + }, + resourceLimits: { maxOldGenerationSizeMb: options.heapMb ?? LINT_HEAP_MB }, + stdout: false, + stderr: false, + }); + } catch { + freeLintSlot(); + resolvePromise(aborted()); + return; + } + let settled = false; + const settle = (finish: () => void) => { + if (settled) return; + settled = true; + clearTimeout(timer); + void worker.terminate().finally(freeLintSlot); + finish(); + }; + const timer = setTimeout(() => settle(() => resolvePromise(aborted())), remainingMs); + worker.on("message", (message: { result?: HyperframeLintResult; error?: string }) => + settle(() => + message.result ? resolvePromise(message.result) : reject(new Error(message.error)), + ), + ); + worker.on("error", () => settle(() => resolvePromise(aborted()))); + worker.on("exit", () => settle(() => resolvePromise(aborted()))); }); } diff --git a/packages/producer/src/services/hyperframeLint.worker.test.ts b/packages/producer/src/services/hyperframeLint.worker.test.ts new file mode 100644 index 0000000000..f311978b27 --- /dev/null +++ b/packages/producer/src/services/hyperframeLint.worker.test.ts @@ -0,0 +1,72 @@ +import { mkdtempSync, readFileSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { dirname, join, resolve } from "node:path"; +import { fileURLToPath } from "node:url"; +import { getHeapStatistics } from "node:v8"; +import { describe, expect, it } from "vitest"; +import { lintHyperframeHtml } from "@hyperframes/lint"; +import { runHyperframeLint } from "./hyperframeLint.js"; + +const STUCK = resolve(dirname(fileURLToPath(import.meta.url)), "__fixtures__/lintWorkerStuck.mjs"); +const PAGE = `
`; +const prepared = (html: string) => ({ entryFile: "index.html", html, source: "html" as const }); +const withoutTimings = ({ timings: _timings, ...rest }: Record) => rest; + +describe("runHyperframeLint", () => { + it("lints a page on a worker thread to what the linter returns in process", async () => { + const result = await runHyperframeLint(prepared(PAGE)); + const direct = await lintHyperframeHtml(PAGE, { + filePath: "index.html", + compSrcPath: "index.html", + }); + expect(withoutTimings({ ...result })).toEqual(withoutTimings({ ...direct })); + }, 30_000); + + it("caps the worker's heap", async () => { + const result = await runHyperframeLint(prepared("heap"), { workerEntry: STUCK, heapMb: 64 }); + // Uncapped, a thread has the main thread's limit; the cap (plus its young generation) is far below it. + const { heapLimitMb } = result as unknown as { heapLimitMb: number }; + expect(heapLimitMb).toBeLessThan(getHeapStatistics().heap_size_limit / 1048576 / 2); + }); + + it.each([ + ["runs past its time limit", "hang", { timeoutMs: 300 }], + ["runs out of memory", "alloc", { timeoutMs: 20_000, heapMb: 64 }], + ["has no worker to run on", PAGE, { workerEntry: "/nonexistent/worker.mjs", timeoutMs: 300 }], + ])( + "a page that %s is answered with the lint_aborted error", + async (_name, html, limits) => { + const began = performance.now(); + const result = await runHyperframeLint(prepared(html), { workerEntry: STUCK, ...limits }); + expect(performance.now() - began).toBeLessThan(10_000); + expect([result.ok, result.errorCount, result.warningCount, result.infoCount]).toEqual([ + false, + 1, + 0, + 0, + ]); + expect(result.findings.map(({ code, severity, file }) => [code, severity, file])).toEqual([ + ["lint_aborted", "error", "index.html"], + ]); + }, + 30_000, + ); + + it("with every worker slot taken, a waiting page is answered lint_aborted when its time is up and no extra worker starts", async () => { + const log = join(mkdtempSync(join(tmpdir(), "hf-lint-slots-")), "starts"); + process.env.HF_LINT_FIXTURE_LOG = log; + try { + const limits = { workerEntry: STUCK, maxWorkers: 1 }; + const holder = runHyperframeLint(prepared("hang"), { ...limits, timeoutMs: 1_500 }); + await new Promise((resolve) => setTimeout(resolve, 300)); + const began = performance.now(); + const waiter = await runHyperframeLint(prepared("hang"), { ...limits, timeoutMs: 300 }); + expect(performance.now() - began).toBeLessThan(1_000); + expect(waiter.findings.map(({ code }) => code)).toEqual(["lint_aborted"]); + expect(readFileSync(log, "utf-8")).toBe("x"); + expect((await holder).findings.map(({ code }) => code)).toEqual(["lint_aborted"]); + } finally { + delete process.env.HF_LINT_FIXTURE_LOG; + } + }, 30_000); +}); diff --git a/packages/producer/src/services/hyperframeLintWorker.ts b/packages/producer/src/services/hyperframeLintWorker.ts new file mode 100644 index 0000000000..62ed8b6d58 --- /dev/null +++ b/packages/producer/src/services/hyperframeLintWorker.ts @@ -0,0 +1,12 @@ +// Lints workerData's page on its own heap, so a parser runaway ends this thread, not the server. +// One thread per request: /lint is not hot, and a stuck page cannot hold up another. +import { parentPort, workerData } from "node:worker_threads"; +import { lintHyperframeHtml, type HyperframeLinterOptions } from "@hyperframes/lint"; + +const { html, opts } = workerData as { html: string; opts?: HyperframeLinterOptions }; + +try { + parentPort?.postMessage({ result: await lintHyperframeHtml(html, opts) }); +} catch (error) { + parentPort?.postMessage({ error: error instanceof Error ? error.message : String(error) }); +}