mirror of
https://github.com/heygen-com/hyperframes.git
synced 2026-11-18 07:50:13 +00:00
fix(lint): run Studio and producer lint off the host's own thread (#7974)
* fix(lint): run Studio and producer lint off the host's own thread A composition that sends the linter's parser into a runaway froze or crashed the long-lived hosts that lint it in their own process. CLI Studio's single-page lint now runs in a child process with a heap cap and a 15 s limit. Producer /lint (runHyperframeLint) runs on a worker thread with the same limits. A page that times out, runs out of memory, or whose child or thread ends without an answer gets one lint_aborted error finding and is never linted in the host process. Studio's project lint keeps its timeout and gains a heap cap. Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com> * fix(producer): cap concurrent /lint workers at two Each lint worker can hold up to 512 MB, so parallel /lint requests multiplied that without bound. At most two run at once; the rest wait in order, and a request still waiting when its own 15 s runs out gets the lint_aborted result. Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com> --------- Co-authored-by: Claude Sonnet 5.5 <noreply@anthropic.com>
This commit is contained in:
co-authored by
Claude Sonnet 5.5
parent
95e4e7f880
commit
778a005641
@@ -55,6 +55,8 @@
|
||||
"packages/producer/src/services/shaderTransitionWorker.ts",
|
||||
// Off-main-thread /health endpoint, spawned by path from healthWorker.ts.
|
||||
"packages/producer/src/services/healthWorkerThread.ts",
|
||||
// Worker thread of runHyperframeLint, spawned by path from hyperframeLint.ts.
|
||||
"packages/producer/src/services/hyperframeLintWorker.ts",
|
||||
// Render-setup worker, spawned by path from cancellableProcess.ts.
|
||||
"packages/cli/src/renderSetupWorker.ts",
|
||||
// Sub-composition build worker, started by path from subCompositionBuild.ts (.dev.mjs from source under Node).
|
||||
@@ -73,6 +75,8 @@
|
||||
// Test fixture worker, spawned by path via the pools' workerEntryPath
|
||||
// option from the crash-recovery tests; has no import-graph referrer.
|
||||
"packages/producer/src/services/__fixtures__/crashOnMessageWorker.mjs",
|
||||
// Test fixture worker for runHyperframeLint, passed by path as its workerEntry option.
|
||||
"packages/producer/src/services/__fixtures__/lintWorkerStuck.mjs",
|
||||
"scripts/**/*.{ts,mjs,js}",
|
||||
"scripts/*/run.mjs",
|
||||
// Keyframe UI components — wired dynamically via EaseCurveSection/MotionPanel.
|
||||
|
||||
@@ -1,6 +1,8 @@
|
||||
import "./utils/hideChildConsoles.js";
|
||||
import "./utils/windowsExeSearch.js";
|
||||
import { readFile, rm } from "node:fs/promises";
|
||||
import { constants, setPriority } from "node:os";
|
||||
import { lintHyperframeHtml } from "@hyperframes/lint";
|
||||
import { runBrowserInstall } from "./browser/installRun.js";
|
||||
import { ensureBrowser, releaseOwnedBrowserInstallLock } from "./browser/manager.js";
|
||||
import { lintProject } from "./utils/lintProject.js";
|
||||
@@ -48,6 +50,13 @@ async function runMode(): Promise<unknown> {
|
||||
input.host ? { host: input.host } : undefined,
|
||||
);
|
||||
}
|
||||
if (mode === "lint-html") {
|
||||
try {
|
||||
return await lintHyperframeHtml(await readFile(input.htmlPath, "utf-8"), input.opts);
|
||||
} finally {
|
||||
await rm(input.htmlPath, { force: true });
|
||||
}
|
||||
}
|
||||
if (mode === "orphan-cleanup") {
|
||||
const killed = killOrphanedProcesses();
|
||||
if (killed > 0) await new Promise((resolve) => setTimeout(resolve, 600));
|
||||
|
||||
@@ -3,12 +3,15 @@ import * as fs from "node:fs";
|
||||
import * as path from "node:path";
|
||||
import { tmpdir } from "node:os";
|
||||
|
||||
// Studio must lint outside its own process; any in-process project lint fails the request.
|
||||
// Studio must lint outside its own process; any in-process project or page lint fails the request.
|
||||
vi.mock("@hyperframes/lint", async (importOriginal) => ({
|
||||
...(await importOriginal<typeof import("@hyperframes/lint")>()),
|
||||
lintProject: () => {
|
||||
throw new Error("project lint ran on the Studio server's event loop");
|
||||
},
|
||||
lintHyperframeHtml: () => {
|
||||
throw new Error("page lint ran on the Studio server's event loop");
|
||||
},
|
||||
}));
|
||||
|
||||
const { createStudioServer } = await import("./studioServer.js");
|
||||
@@ -24,6 +27,9 @@ beforeEach(() => {
|
||||
<div id="root" data-composition-id="main" data-width="320" data-height="180"></div>
|
||||
</body></html>`,
|
||||
);
|
||||
// Not linked from the entry, so the project lint skips it and the route lints it as a page.
|
||||
fs.mkdirSync(path.join(root, "loose"));
|
||||
fs.writeFileSync(path.join(root, "loose", "page.html"), `<div>no composition</div>`);
|
||||
server = createStudioServer({ projectDir: root, projectName: "film" });
|
||||
});
|
||||
|
||||
@@ -40,4 +46,12 @@ describe("studio lint", () => {
|
||||
expect(response.status).toBe(200);
|
||||
expect(body.findings?.some((f) => f.message.includes("window.__timelines"))).toBe(true);
|
||||
}, 60_000);
|
||||
|
||||
it("lints a page the project lint skips in a separate process too", async () => {
|
||||
const response = await server.app.request("/api/projects/film/lint");
|
||||
const body = (await response.json()) as { findings?: Array<{ file?: string }> };
|
||||
|
||||
expect(response.status).toBe(200);
|
||||
expect(body.findings?.some((f) => f.file === "loose/page.html")).toBe(true);
|
||||
}, 60_000);
|
||||
});
|
||||
|
||||
@@ -49,6 +49,7 @@ import { recordFolder } from "../utils/projectRecords.js";
|
||||
import { outsideAssetsAllowed } from "../utils/receivedRender.js";
|
||||
import { insideHostApp } from "../utils/desktopApp.js";
|
||||
import { runRenderSetupWorker } from "../utils/cancellableProcess.js";
|
||||
import { lintHtmlOutOfProcess } from "../utils/lintHtmlWorker.js";
|
||||
import { acquireRenderLock, describeRenderLockHolder } from "../utils/renderLock.js";
|
||||
import type { ProjectLintResult } from "@hyperframes/lint";
|
||||
import { resolveRenderBrowser } from "../browser/preflight.js";
|
||||
@@ -110,6 +111,7 @@ const STUDIO_MANUAL_EDITS_PATH = ".hyperframes/studio-manual-edits.json";
|
||||
const RENDER_SHUTDOWN_WAIT_MS = 2_000;
|
||||
|
||||
const LINT_WORKER_TIMEOUT_MS = 120_000;
|
||||
const LINT_WORKER_HEAP_MB = 1024;
|
||||
|
||||
// Vite emits only content-hashed files under dist/assets; hand-authored
|
||||
// public/ files land at the dist root. The route is the signal because the
|
||||
@@ -618,10 +620,7 @@ export function createStudioServer(options: StudioServerOptions): StudioServer {
|
||||
if (resolve(dir) === resolve(projectDir)) cachedProjectSignature = null;
|
||||
},
|
||||
|
||||
async lint(html, opts) {
|
||||
const { lintHyperframeHtml } = await import("@hyperframes/lint");
|
||||
return await lintHyperframeHtml(html, { ...opts, host: "studio" });
|
||||
},
|
||||
lint: (html, opts) => lintHtmlOutOfProcess(html, { ...opts, host: "studio" }),
|
||||
|
||||
// Out of process: linting a large composition is seconds of synchronous parsing, and on
|
||||
// this event loop it stalls every other Studio request, the preview included.
|
||||
@@ -629,7 +628,12 @@ export function createStudioServer(options: StudioServerOptions): StudioServer {
|
||||
runRenderSetupWorker<ProjectLintResult>(
|
||||
"lint",
|
||||
{ projectDir: dir, host: "studio", background: true },
|
||||
{ maxBufferBytes: 8 * 1024 * 1024, signal, timeoutMs: LINT_WORKER_TIMEOUT_MS },
|
||||
{
|
||||
execArgv: [`--max-old-space-size=${LINT_WORKER_HEAP_MB}`],
|
||||
maxBufferBytes: 8 * 1024 * 1024,
|
||||
signal,
|
||||
timeoutMs: LINT_WORKER_TIMEOUT_MS,
|
||||
},
|
||||
),
|
||||
|
||||
runtimeUrl: "/api/runtime.js",
|
||||
|
||||
@@ -210,17 +210,19 @@ function tsxLoaderFromCliInstall(): string {
|
||||
}
|
||||
|
||||
export async function runRenderSetupWorker<T>(
|
||||
mode: "browser" | "browser-install" | "lint" | "orphan-cleanup",
|
||||
mode: "browser" | "browser-install" | "lint" | "lint-html" | "orphan-cleanup",
|
||||
input: unknown,
|
||||
options: CancellableProcessOptions,
|
||||
{ execArgv: nodeFlags = [], ...options }: CancellableProcessOptions & { execArgv?: string[] },
|
||||
): Promise<T> {
|
||||
const sourceMode = import.meta.url.endsWith(".ts");
|
||||
const workerUrl = new URL(
|
||||
sourceMode ? "../renderSetupWorker.ts" : "./renderSetupWorker.js",
|
||||
import.meta.url,
|
||||
);
|
||||
const execArgv =
|
||||
sourceMode && !process.versions.bun ? ["--import", tsxLoaderFromCliInstall()] : [];
|
||||
const execArgv = [
|
||||
...nodeFlags,
|
||||
...(sourceMode && !process.versions.bun ? ["--import", tsxLoaderFromCliInstall()] : []),
|
||||
];
|
||||
let result: CancellableProcessResult;
|
||||
try {
|
||||
result = await runCancellableProcess(
|
||||
|
||||
@@ -0,0 +1,50 @@
|
||||
import { existsSync } from "node:fs";
|
||||
import { dirname } from "node:path";
|
||||
import { describe, expect, it } from "vitest";
|
||||
import { lintHyperframeHtml } from "@hyperframes/lint";
|
||||
import { runCancellableProcess, runRenderSetupWorker } from "./cancellableProcess.js";
|
||||
import { lintHtmlOutOfProcess } from "./lintHtmlWorker.js";
|
||||
|
||||
const PAGE = `<html><body><div id="root" data-composition-id="main" data-width="1920" data-height="1080"></div></body></html>`;
|
||||
const OPTS = { filePath: "index.html", host: "studio" as const };
|
||||
const withoutTimings = ({ timings: _timings, ...rest }: Record<string, unknown>) => rest;
|
||||
|
||||
describe("lintHtmlOutOfProcess", () => {
|
||||
// The second page is past the ~128 KB an env var can carry, so it only lints if the page travels in a file.
|
||||
it.each([PAGE, PAGE.replace("</body>", `<!-- ${"x".repeat(300_000)} -->\n</body>`)])(
|
||||
"lints a %#-th page in a child to what the linter returns in process",
|
||||
async (html) => {
|
||||
let pageDir = "";
|
||||
const run = ((mode, input, options) => {
|
||||
pageDir = dirname((input as { htmlPath: string }).htmlPath);
|
||||
return runRenderSetupWorker(mode, input, options);
|
||||
}) as typeof runRenderSetupWorker;
|
||||
const result = await lintHtmlOutOfProcess(html, OPTS, { run });
|
||||
expect(withoutTimings({ ...result })).toEqual(
|
||||
withoutTimings({ ...(await lintHyperframeHtml(html, OPTS)) }),
|
||||
);
|
||||
expect(existsSync(pageDir)).toBe(false);
|
||||
},
|
||||
30_000,
|
||||
);
|
||||
|
||||
it("answers a child that never answers with the lint_aborted error, and does not lint here", async () => {
|
||||
// A real child that never exits, stopped by the runner's own time limit.
|
||||
const hang = ((_mode, _input, options) =>
|
||||
runCancellableProcess(process.execPath, ["--eval", "setInterval(() => {}, 1000)"], {
|
||||
timeoutMs: options.timeoutMs,
|
||||
})) as typeof runRenderSetupWorker;
|
||||
const began = performance.now();
|
||||
const result = await lintHtmlOutOfProcess(PAGE, OPTS, { run: hang, timeoutMs: 300 });
|
||||
expect(performance.now() - began).toBeLessThan(10_000);
|
||||
expect([result.ok, result.errorCount, result.warningCount, result.infoCount]).toEqual([
|
||||
false,
|
||||
1,
|
||||
0,
|
||||
0,
|
||||
]);
|
||||
expect(result.findings.map(({ code, severity, file }) => [code, severity, file])).toEqual([
|
||||
["lint_aborted", "error", "index.html"],
|
||||
]);
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,68 @@
|
||||
import { mkdtemp, rm, writeFile } from "node:fs/promises";
|
||||
import { tmpdir } from "node:os";
|
||||
import { join } from "node:path";
|
||||
import type { HyperframeLinterOptions, HyperframeLintResult } from "@hyperframes/lint";
|
||||
import { runRenderSetupWorker } from "./cancellableProcess.js";
|
||||
|
||||
/** A real composition lints in about a second at a few tens of MB of heap; this is where a runaway is stopped. */
|
||||
const LINT_HTML_HEAP_MB = 512;
|
||||
const LINT_HTML_TIMEOUT_MS = 15_000;
|
||||
|
||||
type LintHtmlTestHooks = { run?: typeof runRenderSetupWorker; timeoutMs?: number };
|
||||
|
||||
/** The linter's own result shape for a page it could not check: one error, never linted in this process. */
|
||||
function abortedLintResult(
|
||||
opts: HyperframeLinterOptions | undefined,
|
||||
afterMs: number,
|
||||
): HyperframeLintResult {
|
||||
return {
|
||||
ok: false,
|
||||
errorCount: 1,
|
||||
warningCount: 0,
|
||||
infoCount: 0,
|
||||
findings: [
|
||||
{
|
||||
code: "lint_aborted",
|
||||
severity: "error",
|
||||
message:
|
||||
"This composition's script could not be checked: linting it ran too long or out of memory, so it was stopped. A very large or deeply nested script is the usual cause.",
|
||||
...(opts?.filePath && { file: opts.filePath }),
|
||||
},
|
||||
],
|
||||
timings: { totalMs: afterMs, groupMs: {}, slowestRule: "", slowestRuleMs: 0 },
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Lints one page in a child process with a heap cap and a time limit, so a long-lived host (Studio)
|
||||
* survives a page that sends the parser into a runaway: that page's lint is the `lint_aborted`
|
||||
* error. The child timing out, dying or exiting without a result never falls back to linting here.
|
||||
*/
|
||||
export async function lintHtmlOutOfProcess(
|
||||
html: string,
|
||||
opts: HyperframeLinterOptions | undefined,
|
||||
{ run = runRenderSetupWorker, timeoutMs = LINT_HTML_TIMEOUT_MS }: LintHtmlTestHooks = {},
|
||||
): Promise<HyperframeLintResult> {
|
||||
// The runner hands its input over in an env var, which a large page would overflow: the page goes in a file.
|
||||
const dir = await mkdtemp(join(tmpdir(), "hf-lint-"));
|
||||
const htmlPath = join(dir, "page.html");
|
||||
try {
|
||||
await writeFile(htmlPath, html);
|
||||
const began = performance.now();
|
||||
try {
|
||||
return await run<HyperframeLintResult>(
|
||||
"lint-html",
|
||||
{ htmlPath, opts },
|
||||
{
|
||||
execArgv: [`--max-old-space-size=${LINT_HTML_HEAP_MB}`],
|
||||
maxBufferBytes: 8 * 1024 * 1024,
|
||||
timeoutMs,
|
||||
},
|
||||
);
|
||||
} catch {
|
||||
return abortedLintResult(opts, Math.round(performance.now() - began));
|
||||
}
|
||||
} finally {
|
||||
await rm(dir, { recursive: true, force: true });
|
||||
}
|
||||
}
|
||||
@@ -1,5 +1,5 @@
|
||||
// Workers sit beside the bundles because healthWorker.ts and shaderTransitionWorkerPool.ts
|
||||
// look for them in their own module's directory.
|
||||
// Workers sit beside the bundles because healthWorker.ts, hyperframeLint.ts and
|
||||
// shaderTransitionWorkerPool.ts look for them in their own module's directory.
|
||||
export const BUNDLES = [
|
||||
{ entry: "src/index.ts", outfile: "dist/index.js" },
|
||||
{ entry: "src/server.ts", outfile: "dist/public-server.js" },
|
||||
@@ -8,4 +8,5 @@ export const BUNDLES = [
|
||||
export const WORKERS = [
|
||||
{ entry: "src/services/shaderTransitionWorker.ts", outfile: "dist/shaderTransitionWorker.js" },
|
||||
{ entry: "src/services/healthWorkerThread.ts", outfile: "dist/healthWorkerThread.js" },
|
||||
{ entry: "src/services/hyperframeLintWorker.ts", outfile: "dist/hyperframeLintWorker.js" },
|
||||
];
|
||||
|
||||
@@ -30,6 +30,8 @@
|
||||
"src/services/fileServer.listenError.test.ts",
|
||||
"src/services/outsideProjectAssets.test.ts",
|
||||
"src/services/healthWorker.test.ts",
|
||||
"src/services/hyperframeLint.worker.test.ts",
|
||||
"src/server.lintWorker.test.ts",
|
||||
"src/services/scriptFailureAttribution.test.ts",
|
||||
"src/services/strictVariablesSession.test.ts",
|
||||
"src/services/hyperframeLint.file-race.test.ts",
|
||||
|
||||
@@ -6,9 +6,10 @@ describe("published build layout", () => {
|
||||
it("emits each worker beside the bundles whose worker lookup probes their own directory", () => {
|
||||
const bundleDirs = new Set(BUNDLES.map(({ outfile }) => dirname(outfile)));
|
||||
expect([...bundleDirs]).toEqual(["dist"]);
|
||||
expect(WORKERS.map(({ outfile }) => dirname(outfile))).toEqual(["dist", "dist"]);
|
||||
expect(WORKERS.map(({ outfile }) => dirname(outfile))).toEqual(["dist", "dist", "dist"]);
|
||||
expect(WORKERS.map(({ outfile }) => basename(outfile)).sort()).toEqual([
|
||||
"healthWorkerThread.js",
|
||||
"hyperframeLintWorker.js",
|
||||
"shaderTransitionWorker.js",
|
||||
]);
|
||||
});
|
||||
|
||||
@@ -0,0 +1,52 @@
|
||||
import { mkdtempSync } from "node:fs";
|
||||
import { tmpdir } from "node:os";
|
||||
import { dirname, join, resolve } from "node:path";
|
||||
import { fileURLToPath } from "node:url";
|
||||
import { describe, expect, it, vi } from "vitest";
|
||||
|
||||
// Every /lint page runs on a worker that never answers, stopped by a short limit.
|
||||
vi.mock("./services/hyperframeLint.js", async (importOriginal) => {
|
||||
const actual = await importOriginal<typeof import("./services/hyperframeLint.js")>();
|
||||
const stuck = resolve(
|
||||
dirname(fileURLToPath(import.meta.url)),
|
||||
"services/__fixtures__/lintWorkerStuck.mjs",
|
||||
);
|
||||
return {
|
||||
...actual,
|
||||
runHyperframeLint: (prepared: Parameters<typeof actual.runHyperframeLint>[0]) =>
|
||||
actual.runHyperframeLint(prepared, { workerEntry: stuck, timeoutMs: 500 }),
|
||||
};
|
||||
});
|
||||
|
||||
import { createProducerApp } from "./server.js";
|
||||
|
||||
describe("POST /lint", () => {
|
||||
it("answers 200 with lint_aborted for a page whose lint never ends, and the server keeps answering", async () => {
|
||||
const root = mkdtempSync(join(tmpdir(), "hf-lint-route-"));
|
||||
const app = createProducerApp({
|
||||
projectsDir: join(root, "projects"),
|
||||
rendersDir: join(root, "renders"),
|
||||
});
|
||||
const lint = Promise.resolve(
|
||||
app.request("http://localhost:9847/lint", {
|
||||
method: "POST",
|
||||
headers: { host: "localhost:9847" },
|
||||
body: JSON.stringify({ html: "hang" }),
|
||||
}),
|
||||
);
|
||||
// While that page is stuck, another request is answered.
|
||||
const health = await app.request("http://localhost:9847/health", {
|
||||
headers: { host: "localhost:9847" },
|
||||
});
|
||||
expect(health.status).toBe(200);
|
||||
|
||||
const res = await lint;
|
||||
expect(res.status).toBe(200);
|
||||
const body = (await res.json()) as {
|
||||
success: boolean;
|
||||
result: { findings: { code: string }[] };
|
||||
};
|
||||
expect(body.success).toBe(true);
|
||||
expect(body.result.findings.map((finding) => finding.code)).toEqual(["lint_aborted"]);
|
||||
}, 30_000);
|
||||
});
|
||||
@@ -0,0 +1,17 @@
|
||||
// Stands in for hyperframeLintWorker.ts on a page that sends the parser into a runaway: "hang" spins and never
|
||||
// answers, "alloc" allocates until the thread's heap cap ends it, "heap" answers with the thread's own heap limit.
|
||||
import { appendFileSync } from "node:fs";
|
||||
import { getHeapStatistics } from "node:v8";
|
||||
import { parentPort, workerData } from "node:worker_threads";
|
||||
|
||||
if (process.env.HF_LINT_FIXTURE_LOG) appendFileSync(process.env.HF_LINT_FIXTURE_LOG, "x");
|
||||
|
||||
if (workerData.html === "heap") {
|
||||
parentPort.postMessage({
|
||||
result: { heapLimitMb: getHeapStatistics().heap_size_limit / 1048576 },
|
||||
});
|
||||
} else if (workerData.html === "hang") {
|
||||
for (;;);
|
||||
} else {
|
||||
for (const kept = []; ; ) kept.push(new Array(100_000).fill(1));
|
||||
}
|
||||
@@ -7,8 +7,10 @@ import {
|
||||
closeSync,
|
||||
constants,
|
||||
} from "node:fs";
|
||||
import { resolve, join, relative, isAbsolute, sep } from "node:path";
|
||||
import { lintHyperframeHtml, type HyperframeLintResult } from "@hyperframes/lint";
|
||||
import { resolve, join, relative, isAbsolute, sep, dirname } from "node:path";
|
||||
import { fileURLToPath } from "node:url";
|
||||
import { Worker } from "node:worker_threads";
|
||||
import type { HyperframeLintResult } from "@hyperframes/lint";
|
||||
|
||||
export interface PreparedHyperframeLintInput {
|
||||
entryFile: string;
|
||||
@@ -162,11 +164,122 @@ export function prepareHyperframeLintBody(
|
||||
return { error: "Missing lint source: provide projectDir, files, or html" };
|
||||
}
|
||||
|
||||
const LINT_HEAP_MB = 512;
|
||||
const LINT_TIMEOUT_MS = 15_000;
|
||||
const LINT_MAX_WORKERS = 2;
|
||||
|
||||
const waiting: Array<() => void> = [];
|
||||
let running = 0;
|
||||
|
||||
function takeLintSlot(limit: number, waitMs: number): Promise<boolean> {
|
||||
if (running < limit) {
|
||||
running++;
|
||||
return Promise.resolve(true);
|
||||
}
|
||||
return new Promise((resolve) => {
|
||||
const entry = () => {
|
||||
clearTimeout(timer);
|
||||
running++;
|
||||
resolve(true);
|
||||
};
|
||||
const timer = setTimeout(
|
||||
() => {
|
||||
waiting.splice(waiting.indexOf(entry), 1);
|
||||
resolve(false);
|
||||
},
|
||||
Math.max(0, waitMs),
|
||||
);
|
||||
waiting.push(entry);
|
||||
});
|
||||
}
|
||||
|
||||
function freeLintSlot(): void {
|
||||
running--;
|
||||
waiting.shift()?.();
|
||||
}
|
||||
|
||||
export interface HyperframeLintWorkerOptions {
|
||||
workerEntry?: string;
|
||||
timeoutMs?: number;
|
||||
heapMb?: number;
|
||||
maxWorkers?: number;
|
||||
}
|
||||
|
||||
function resolveLintWorkerEntry(): string | undefined {
|
||||
const here = dirname(fileURLToPath(import.meta.url));
|
||||
for (const name of ["hyperframeLintWorker.js", "hyperframeLintWorker.ts"]) {
|
||||
const candidate = join(here, name);
|
||||
if (existsSync(candidate)) return candidate;
|
||||
}
|
||||
return undefined;
|
||||
}
|
||||
|
||||
function abortedLintResult(filePath: string, afterMs: number): HyperframeLintResult {
|
||||
return {
|
||||
ok: false,
|
||||
errorCount: 1,
|
||||
warningCount: 0,
|
||||
infoCount: 0,
|
||||
findings: [
|
||||
{
|
||||
code: "lint_aborted",
|
||||
severity: "error",
|
||||
message:
|
||||
"This composition's script could not be checked: linting it ran too long or out of memory, so it was stopped. A very large or deeply nested script is the usual cause.",
|
||||
file: filePath,
|
||||
},
|
||||
],
|
||||
timings: { totalMs: afterMs, groupMs: {}, slowestRule: "", slowestRuleMs: 0 },
|
||||
};
|
||||
}
|
||||
|
||||
export async function runHyperframeLint(
|
||||
prepared: PreparedHyperframeLintInput,
|
||||
options: HyperframeLintWorkerOptions = {},
|
||||
): Promise<HyperframeLintResult> {
|
||||
return lintHyperframeHtml(prepared.html, {
|
||||
filePath: prepared.entryFile,
|
||||
compSrcPath: prepared.entryFile,
|
||||
const began = performance.now();
|
||||
const aborted = () =>
|
||||
abortedLintResult(prepared.entryFile, Math.round(performance.now() - began));
|
||||
const entry = options.workerEntry ?? resolveLintWorkerEntry();
|
||||
if (!entry) return aborted();
|
||||
|
||||
const timeoutMs = options.timeoutMs ?? LINT_TIMEOUT_MS;
|
||||
const gotSlot = await takeLintSlot(options.maxWorkers ?? LINT_MAX_WORKERS, timeoutMs);
|
||||
if (!gotSlot) return aborted();
|
||||
const remainingMs = Math.max(0, timeoutMs - (performance.now() - began));
|
||||
|
||||
return new Promise<HyperframeLintResult>((resolvePromise, reject) => {
|
||||
let worker: Worker;
|
||||
try {
|
||||
worker = new Worker(entry, {
|
||||
workerData: {
|
||||
html: prepared.html,
|
||||
opts: { filePath: prepared.entryFile, compSrcPath: prepared.entryFile },
|
||||
},
|
||||
resourceLimits: { maxOldGenerationSizeMb: options.heapMb ?? LINT_HEAP_MB },
|
||||
stdout: false,
|
||||
stderr: false,
|
||||
});
|
||||
} catch {
|
||||
freeLintSlot();
|
||||
resolvePromise(aborted());
|
||||
return;
|
||||
}
|
||||
let settled = false;
|
||||
const settle = (finish: () => void) => {
|
||||
if (settled) return;
|
||||
settled = true;
|
||||
clearTimeout(timer);
|
||||
void worker.terminate().finally(freeLintSlot);
|
||||
finish();
|
||||
};
|
||||
const timer = setTimeout(() => settle(() => resolvePromise(aborted())), remainingMs);
|
||||
worker.on("message", (message: { result?: HyperframeLintResult; error?: string }) =>
|
||||
settle(() =>
|
||||
message.result ? resolvePromise(message.result) : reject(new Error(message.error)),
|
||||
),
|
||||
);
|
||||
worker.on("error", () => settle(() => resolvePromise(aborted())));
|
||||
worker.on("exit", () => settle(() => resolvePromise(aborted())));
|
||||
});
|
||||
}
|
||||
|
||||
@@ -0,0 +1,72 @@
|
||||
import { mkdtempSync, readFileSync } from "node:fs";
|
||||
import { tmpdir } from "node:os";
|
||||
import { dirname, join, resolve } from "node:path";
|
||||
import { fileURLToPath } from "node:url";
|
||||
import { getHeapStatistics } from "node:v8";
|
||||
import { describe, expect, it } from "vitest";
|
||||
import { lintHyperframeHtml } from "@hyperframes/lint";
|
||||
import { runHyperframeLint } from "./hyperframeLint.js";
|
||||
|
||||
const STUCK = resolve(dirname(fileURLToPath(import.meta.url)), "__fixtures__/lintWorkerStuck.mjs");
|
||||
const PAGE = `<html><body><div id="root" data-composition-id="main" data-width="1920" data-height="1080"></div></body></html>`;
|
||||
const prepared = (html: string) => ({ entryFile: "index.html", html, source: "html" as const });
|
||||
const withoutTimings = ({ timings: _timings, ...rest }: Record<string, unknown>) => rest;
|
||||
|
||||
describe("runHyperframeLint", () => {
|
||||
it("lints a page on a worker thread to what the linter returns in process", async () => {
|
||||
const result = await runHyperframeLint(prepared(PAGE));
|
||||
const direct = await lintHyperframeHtml(PAGE, {
|
||||
filePath: "index.html",
|
||||
compSrcPath: "index.html",
|
||||
});
|
||||
expect(withoutTimings({ ...result })).toEqual(withoutTimings({ ...direct }));
|
||||
}, 30_000);
|
||||
|
||||
it("caps the worker's heap", async () => {
|
||||
const result = await runHyperframeLint(prepared("heap"), { workerEntry: STUCK, heapMb: 64 });
|
||||
// Uncapped, a thread has the main thread's limit; the cap (plus its young generation) is far below it.
|
||||
const { heapLimitMb } = result as unknown as { heapLimitMb: number };
|
||||
expect(heapLimitMb).toBeLessThan(getHeapStatistics().heap_size_limit / 1048576 / 2);
|
||||
});
|
||||
|
||||
it.each([
|
||||
["runs past its time limit", "hang", { timeoutMs: 300 }],
|
||||
["runs out of memory", "alloc", { timeoutMs: 20_000, heapMb: 64 }],
|
||||
["has no worker to run on", PAGE, { workerEntry: "/nonexistent/worker.mjs", timeoutMs: 300 }],
|
||||
])(
|
||||
"a page that %s is answered with the lint_aborted error",
|
||||
async (_name, html, limits) => {
|
||||
const began = performance.now();
|
||||
const result = await runHyperframeLint(prepared(html), { workerEntry: STUCK, ...limits });
|
||||
expect(performance.now() - began).toBeLessThan(10_000);
|
||||
expect([result.ok, result.errorCount, result.warningCount, result.infoCount]).toEqual([
|
||||
false,
|
||||
1,
|
||||
0,
|
||||
0,
|
||||
]);
|
||||
expect(result.findings.map(({ code, severity, file }) => [code, severity, file])).toEqual([
|
||||
["lint_aborted", "error", "index.html"],
|
||||
]);
|
||||
},
|
||||
30_000,
|
||||
);
|
||||
|
||||
it("with every worker slot taken, a waiting page is answered lint_aborted when its time is up and no extra worker starts", async () => {
|
||||
const log = join(mkdtempSync(join(tmpdir(), "hf-lint-slots-")), "starts");
|
||||
process.env.HF_LINT_FIXTURE_LOG = log;
|
||||
try {
|
||||
const limits = { workerEntry: STUCK, maxWorkers: 1 };
|
||||
const holder = runHyperframeLint(prepared("hang"), { ...limits, timeoutMs: 1_500 });
|
||||
await new Promise((resolve) => setTimeout(resolve, 300));
|
||||
const began = performance.now();
|
||||
const waiter = await runHyperframeLint(prepared("hang"), { ...limits, timeoutMs: 300 });
|
||||
expect(performance.now() - began).toBeLessThan(1_000);
|
||||
expect(waiter.findings.map(({ code }) => code)).toEqual(["lint_aborted"]);
|
||||
expect(readFileSync(log, "utf-8")).toBe("x");
|
||||
expect((await holder).findings.map(({ code }) => code)).toEqual(["lint_aborted"]);
|
||||
} finally {
|
||||
delete process.env.HF_LINT_FIXTURE_LOG;
|
||||
}
|
||||
}, 30_000);
|
||||
});
|
||||
@@ -0,0 +1,12 @@
|
||||
// Lints workerData's page on its own heap, so a parser runaway ends this thread, not the server.
|
||||
// One thread per request: /lint is not hot, and a stuck page cannot hold up another.
|
||||
import { parentPort, workerData } from "node:worker_threads";
|
||||
import { lintHyperframeHtml, type HyperframeLinterOptions } from "@hyperframes/lint";
|
||||
|
||||
const { html, opts } = workerData as { html: string; opts?: HyperframeLinterOptions };
|
||||
|
||||
try {
|
||||
parentPort?.postMessage({ result: await lintHyperframeHtml(html, opts) });
|
||||
} catch (error) {
|
||||
parentPort?.postMessage({ error: error instanceof Error ? error.message : String(error) });
|
||||
}
|
||||
Reference in New Issue
Block a user