diff --git a/packages/cli/src/server/studioServer.ts b/packages/cli/src/server/studioServer.ts index 4be70c217e..ba6a24ac97 100644 --- a/packages/cli/src/server/studioServer.ts +++ b/packages/cli/src/server/studioServer.ts @@ -106,6 +106,8 @@ const STUDIO_MANUAL_EDITS_PATH = ".hyperframes/studio-manual-edits.json"; // before that watchdog can fire and skip this file's browser cleanup. const RENDER_SHUTDOWN_WAIT_MS = 2_000; +const LINT_WORKER_TIMEOUT_MS = 120_000; + // Vite emits only content-hashed files under dist/assets; hand-authored // public/ files land at the dist root. The route is the signal because the // filename is not: rollup's base64url hash may itself contain a hyphen. @@ -605,11 +607,11 @@ export function createStudioServer(options: StudioServerOptions): StudioServer { // Out of process: linting a large composition is seconds of synchronous parsing, and on // this event loop it stalls every other Studio request, the preview included. - lintProject: (dir: string) => + lintProject: (dir: string, signal?: AbortSignal) => runRenderSetupWorker( "lint", { projectDir: dir, host: "studio", background: true }, - { maxBufferBytes: 8 * 1024 * 1024 }, + { maxBufferBytes: 8 * 1024 * 1024, signal, timeoutMs: LINT_WORKER_TIMEOUT_MS }, ), runtimeUrl: "/api/runtime.js", diff --git a/packages/studio-server/src/createStudioApi.originGuard.test.ts b/packages/studio-server/src/createStudioApi.originGuard.test.ts index 1c03ad9f41..2445cda227 100644 --- a/packages/studio-server/src/createStudioApi.originGuard.test.ts +++ b/packages/studio-server/src/createStudioApi.originGuard.test.ts @@ -306,6 +306,45 @@ describe("createStudioApi media analysis from another site", () => { }); }); +describe("createStudioApi work-starting reads from another site", () => { + function appCountingWork() { + const projectDir = mkdtempSync(join(tmpdir(), "hf-origin-work-")); + tempDirs.push(projectDir); + const calls = { lint: 0 }; + const app = createStudioApi({ + ...stubAdapter(projectDir), + lintProject: () => { + calls.lint++; + return { results: [] }; + }, + }); + return { app, calls }; + } + + it.each([ + ["lint", "cross-site"], + ["lint", "same-site"], + ["media/metadata?path=talk.mp4", "cross-site"], + ["image-thumbnail/photo.jpg", "cross-site"], + ])("refuses %s marked %s without starting any work", async (route, site) => { + const { app, calls } = appCountingWork(); + const res = await app.request(`http://localhost/projects/demo/${route}`, { + headers: { Host: "localhost", "Sec-Fetch-Site": site }, + }); + expect(res.status).toBe(403); + expect(calls.lint).toBe(0); + }); + + it("answers Studio's own lint request", async () => { + const { app, calls } = appCountingWork(); + const res = await app.request("http://localhost/projects/demo/lint", { + headers: { Host: "localhost", "Sec-Fetch-Site": "same-origin" }, + }); + expect(res.status).toBe(200); + expect(calls.lint).toBe(1); + }); +}); + describe("previewAllowedHostsFromEnv", () => { it("parses a comma separated list, trimming blanks", () => { expect( diff --git a/packages/studio-server/src/createStudioApi.ts b/packages/studio-server/src/createStudioApi.ts index c6ba77dc02..edb657734a 100644 --- a/packages/studio-server/src/createStudioApi.ts +++ b/packages/studio-server/src/createStudioApi.ts @@ -141,6 +141,9 @@ export function createStudioApi(adapter: StudioApiAdapter, options: StudioApiOpt api.use("/projects/:id/thumbnail/*", refuseForAnotherSite("thumbnail")); api.use("/projects/:id/waveform/*", refuseForAnotherSite("waveform")); api.use("/projects/:id/peaks/*", refuseForAnotherSite("peaks")); + api.use("/projects/:id/image-thumbnail/*", refuseForAnotherSite("image thumbnail")); + api.use("/projects/:id/media/metadata", refuseForAnotherSite("media metadata")); + api.use("/projects/:id/lint", refuseForAnotherSite("lint")); // This API is unauthenticated and simple cross-origin POSTs need no // preflight, so the guard rejects writes the shared predicate refuses. api.use(async function rejectCrossOriginWrites(c, next) { diff --git a/packages/studio-server/src/routes/lint.runs.test.ts b/packages/studio-server/src/routes/lint.runs.test.ts new file mode 100644 index 0000000000..62d0c245b4 --- /dev/null +++ b/packages/studio-server/src/routes/lint.runs.test.ts @@ -0,0 +1,146 @@ +// @vitest-environment node +// Node environment: the abort tests need a request's AbortSignal to reach the route. +import { afterEach, describe, expect, it, vi } from "vitest"; +import { Hono } from "hono"; +import { mkdtempSync, rmSync, writeFileSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { registerLintRoutes } from "./lint"; +import { stubAdapter } from "./stubAdapter.test-helpers"; +import type { ProjectLintResult, StudioApiAdapter } from "../types"; + +const tempDirs: string[] = []; + +afterEach(() => { + for (const dir of tempDirs.splice(0)) rmSync(dir, { recursive: true, force: true }); +}); + +function projectDir(): string { + const dir = mkdtempSync(join(tmpdir(), "hf-lint-runs-")); + tempDirs.push(dir); + writeFileSync(join(dir, "index.html"), "real"); + return dir; +} + +const CLEAN: ProjectLintResult = { results: [{ file: "index.html", result: { findings: [] } }] }; + +/** Stands in for the lint worker: each run stays live until it is aborted or finished. */ +function workerPool() { + let live = 0; + let peak = 0; + let started = 0; + const finishers: Array<() => void> = []; + const lintProject = (_dir: string, signal?: AbortSignal) => + new Promise((resolve, reject) => { + started++; + live++; + peak = Math.max(peak, live); + let exited = false; + const exit = (settle: () => void) => { + if (exited) return; + exited = true; + live--; + settle(); + }; + signal?.addEventListener("abort", () => exit(() => reject(signal.reason)), { once: true }); + finishers.push(() => exit(() => resolve(CLEAN))); + }); + return { + lintProject, + finishAll: () => finishers.splice(0).forEach((finish) => finish()), + stats: () => ({ live, peak, started }), + }; +} + +function appFor(dirsById: Record, lintProject: StudioApiAdapter["lintProject"]) { + const adapter: StudioApiAdapter = { + ...stubAdapter(""), + resolveProject: async (id: string) => { + const dir = dirsById[id]; + return dir ? { id, dir } : null; + }, + lintProject, + }; + const app = new Hono(); + registerLintRoutes(app, adapter); + return app; +} + +const tick = () => new Promise((resolve) => setTimeout(resolve, 0)); + +describe("project lint runs", () => { + it("shares one run between concurrent requests for the same project", async () => { + const pool = workerPool(); + const app = appFor({ demo: projectDir() }, pool.lintProject); + + const requests = Array.from({ length: 5 }, () => + app.request("http://localhost/projects/demo/lint"), + ); + await vi.waitFor(() => expect(pool.stats().started).toBe(1)); + await tick(); + expect(pool.stats().started).toBe(1); + pool.finishAll(); + + for (const res of await Promise.all(requests)) expect(res.status).toBe(200); + expect(pool.stats().started).toBe(1); + }); + + it("starts a fresh run when the project changed after a run began", async () => { + const pool = workerPool(); + const dir = projectDir(); + const app = appFor({ demo: dir }, pool.lintProject); + + const before = app.request("http://localhost/projects/demo/lint"); + await vi.waitFor(() => expect(pool.stats().started).toBe(1)); + writeFileSync(join(dir, "scene.html"), "new scene"); + const after = app.request("http://localhost/projects/demo/lint"); + await vi.waitFor(() => expect(pool.stats().started).toBe(2)); + pool.finishAll(); + for (const res of await Promise.all([before, after])) expect(res.status).toBe(200); + }); + + it("aborts a run once no request waits on it", async () => { + const pool = workerPool(); + const app = appFor({ demo: projectDir() }, pool.lintProject); + const first = new AbortController(); + const second = new AbortController(); + + const requests = [first, second].map((controller) => + app.request("http://localhost/projects/demo/lint", { signal: controller.signal }), + ); + await vi.waitFor(() => expect(pool.stats().live).toBe(1)); + first.abort(); + await tick(); + expect(pool.stats().live).toBe(1); + second.abort(); + await tick(); + expect(pool.stats().live).toBe(0); + await Promise.allSettled(requests); + }); + + it("never keeps more than two runs live while requests start and abort", async () => { + const pool = workerPool(); + const dirs = Object.fromEntries( + Array.from({ length: 6 }, (_, index) => [`p${index}`, projectDir()]), + ); + const app = appFor(dirs, pool.lintProject); + const pending: Promise[] = []; + + for (let round = 0; round < 20; round++) { + const controllers = Object.keys(dirs).map((id) => { + const controller = new AbortController(); + pending.push( + app.request(`http://localhost/projects/${id}/lint`, { signal: controller.signal }), + ); + return controller; + }); + await vi.waitFor(() => expect(pool.stats().live).toBe(2)); + for (const controller of controllers) controller.abort(); + await vi.waitFor(() => expect(pool.stats().live).toBe(0)); + } + + expect(pool.stats().peak).toBe(2); + expect(pool.stats().live).toBe(0); + await Promise.allSettled(pending); + }); +}); diff --git a/packages/studio-server/src/routes/lint.test.ts b/packages/studio-server/src/routes/lint.test.ts index 07953634a1..4cef0769d2 100644 --- a/packages/studio-server/src/routes/lint.test.ts +++ b/packages/studio-server/src/routes/lint.test.ts @@ -93,7 +93,7 @@ describe("registerLintRoutes — dot-directory exclusion (#1384)", () => { }; expect(response.status).toBe(200); - expect(projectLint).toHaveBeenCalledWith(projectDir); + expect(projectLint).toHaveBeenCalledWith(projectDir, expect.any(AbortSignal)); expect(singleFileLint).not.toHaveBeenCalled(); expect(payload.findings).toContainEqual( expect.objectContaining({ diff --git a/packages/studio-server/src/routes/lint.ts b/packages/studio-server/src/routes/lint.ts index ecc34842d5..9d0128ac67 100644 --- a/packages/studio-server/src/routes/lint.ts +++ b/packages/studio-server/src/routes/lint.ts @@ -3,18 +3,21 @@ import { readFileSync } from "node:fs"; import { join } from "node:path"; import type { LintResult, StudioApiAdapter } from "../types.js"; import { isInHiddenOrVendorDir, walkDir } from "../helpers/safePath.js"; +import { resolveProjectSignature } from "../helpers/projectSignature.js"; +import { ThumbnailGenerationCoordinator } from "./thumbnailGenerationCoordinator.js"; type LintFinding = LintResult["findings"][number]; async function collectProjectFindings( adapter: StudioApiAdapter, projectDir: string, + signal: AbortSignal, ): Promise<{ findings: LintFinding[]; coveredFiles: Set }> { const findings: LintFinding[] = []; const coveredFiles = new Set(); if (!adapter.lintProject) return { findings, coveredFiles }; - const result = await adapter.lintProject(projectDir); + const result = await adapter.lintProject(projectDir, signal); for (const entry of result.results) { coveredFiles.add(entry.file); for (const finding of entry.result.findings) { @@ -46,22 +49,38 @@ async function lintUncoveredHtml( return findings; } +async function lintProjectDir( + adapter: StudioApiAdapter, + projectDir: string, + signal: AbortSignal, +): Promise { + const htmlFiles = walkDir(projectDir).filter( + (f) => f.endsWith(".html") && !isInHiddenOrVendorDir(f), + ); + const projectLint = await collectProjectFindings(adapter, projectDir, signal); + const extraFindings = await lintUncoveredHtml( + adapter, + projectDir, + htmlFiles, + projectLint.coveredFiles, + ); + return [...projectLint.findings, ...extraFindings]; +} + +const MAX_CONCURRENT_LINT_RUNS = 2; + export function registerLintRoutes(api: Hono, adapter: StudioApiAdapter): void { + const runs = new ThumbnailGenerationCoordinator(MAX_CONCURRENT_LINT_RUNS); api.get("/projects/:id/lint", async (c) => { const project = await adapter.resolveProject(c.req.param("id")); if (!project) return c.json({ error: "not found" }, 404); try { - const htmlFiles = walkDir(project.dir).filter( - (f) => f.endsWith(".html") && !isInHiddenOrVendorDir(f), + const signature = await resolveProjectSignature(adapter, project.dir); + const key = JSON.stringify([project.dir, signature]); + const findings = await runs.acquire(key, c.req.raw.signal, (signal) => + lintProjectDir(adapter, project.dir, signal), ); - const projectLint = await collectProjectFindings(adapter, project.dir); - const extraFindings = await lintUncoveredHtml( - adapter, - project.dir, - htmlFiles, - projectLint.coveredFiles, - ); - return c.json({ findings: [...projectLint.findings, ...extraFindings] }); + return c.json({ findings }); } catch (err) { const msg = err instanceof Error ? err.message : String(err); return c.json({ error: `Lint failed: ${msg}` }, 500); diff --git a/packages/studio-server/src/routes/media.ts b/packages/studio-server/src/routes/media.ts index 47515434b3..df2f48976c 100644 --- a/packages/studio-server/src/routes/media.ts +++ b/packages/studio-server/src/routes/media.ts @@ -1,6 +1,6 @@ import type { Hono } from "hono"; import { streamSSE } from "hono/streaming"; -import { existsSync } from "node:fs"; +import { existsSync, statSync } from "node:fs"; import { basename, dirname, extname, join, posix } from "node:path"; import type { MediaProcessingJobState, StudioApiAdapter } from "../types.js"; import { requestedProjectPath } from "../helpers/requestSubPath.js"; @@ -10,8 +10,9 @@ import { resolveWithinProject, resolveWritableWithinProject, } from "../helpers/safePath.js"; -import { probeMediaMetadata } from "../helpers/mediaMetadata.js"; +import { probeMediaMetadata, type MediaMetadata } from "../helpers/mediaMetadata.js"; import { forgetJobsFinishedLongerThan } from "../helpers/finishedJobs.js"; +import { ThumbnailGenerationCoordinator } from "./thumbnailGenerationCoordinator.js"; const VIDEO_EXTENSIONS = new Set([ ".mp4", @@ -43,6 +44,8 @@ interface BackgroundRemovalBody { type ProbeMediaMetadata = typeof probeMediaMetadata; +const MAX_CONCURRENT_METADATA_PROBES = 4; + function isVideoPath(path: string): boolean { return VIDEO_EXTENSIONS.has(extname(path).toLowerCase()); } @@ -134,6 +137,7 @@ export function registerMediaRoutes( const mediaJobs = new Map(); const TTL_MS = 300_000; const readMediaMetadata = options.probeMediaMetadata ?? probeMediaMetadata; + const probes = new ThumbnailGenerationCoordinator(MAX_CONCURRENT_METADATA_PROBES); function cleanupFinishedJobs(): void { forgetJobsFinishedLongerThan( @@ -156,9 +160,12 @@ export function registerMediaRoutes( const filePath = resolveWithinProject(project.dir, assetPath); if (!filePath) return c.json({ error: "forbidden" }, 403); - if (!existsSync(filePath)) return c.json({ error: "media not found" }, 404); + const stat = statSync(filePath, { throwIfNoEntry: false }); + if (!stat) return c.json({ error: "media not found" }, 404); - return c.json({ path: assetPath, metadata: await readMediaMetadata(filePath) }); + const key = JSON.stringify([filePath, stat.dev, stat.ino, stat.size, stat.mtimeMs]); + const metadata = await probes.acquire(key, c.req.raw.signal, () => readMediaMetadata(filePath)); + return c.json({ path: assetPath, metadata }); }); api.post( diff --git a/packages/studio-server/src/routes/thumbnailGenerationCoordinator.ts b/packages/studio-server/src/routes/thumbnailGenerationCoordinator.ts index 6804df3b0e..0c89898dd5 100644 --- a/packages/studio-server/src/routes/thumbnailGenerationCoordinator.ts +++ b/packages/studio-server/src/routes/thumbnailGenerationCoordinator.ts @@ -1,22 +1,22 @@ -export type ThumbnailGenerationValue = Buffer | null; -export type ThumbnailGenerationWork = (signal: AbortSignal) => Promise; +type ThumbnailGenerationValue = Buffer | null; +type GenerationWork = (signal: AbortSignal) => Promise; -interface GenerationEntry { +interface GenerationEntry { key: string; controller: AbortController; leases: number; state: "queued" | "active"; - work: ThumbnailGenerationWork; - promise: Promise; - resolve: (value: ThumbnailGenerationValue) => void; + work: GenerationWork; + promise: Promise; + resolve: (value: T) => void; reject: (reason: unknown) => void; } /** Sole server owner for same-key dedupe, concurrency, cancellation, and queue order. */ -export class ThumbnailGenerationCoordinator { - private readonly entries = new Map(); - private readonly queue: GenerationEntry[] = []; - private readonly activeEntries = new Set(); +export class ThumbnailGenerationCoordinator { + private readonly entries = new Map>(); + private readonly queue: GenerationEntry[] = []; + private readonly activeEntries = new Set>(); private active = 0; constructor(private readonly concurrency = 1) { @@ -25,18 +25,14 @@ export class ThumbnailGenerationCoordinator { } } - acquire( - key: string, - signal: AbortSignal, - work: ThumbnailGenerationWork, - ): Promise { + acquire(key: string, signal: AbortSignal, work: GenerationWork): Promise { if (signal.aborted) return Promise.reject(new DOMException("Aborted", "AbortError")); let entry = this.entries.get(key); if (!entry) { - let resolve!: (value: ThumbnailGenerationValue) => void; + let resolve!: (value: T) => void; let reject!: (reason: unknown) => void; - const promise = new Promise((resolvePromise, rejectPromise) => { + const promise = new Promise((resolvePromise, rejectPromise) => { resolve = resolvePromise; reject = rejectPromise; }); @@ -63,7 +59,7 @@ export class ThumbnailGenerationCoordinator { return new Set([...this.entries.keys(), ...[...this.activeEntries].map((entry) => entry.key)]); } - private lease(entry: GenerationEntry, signal: AbortSignal): Promise { + private lease(entry: GenerationEntry, signal: AbortSignal): Promise { return new Promise((resolve, reject) => { let released = false; const release = () => { @@ -112,7 +108,7 @@ export class ThumbnailGenerationCoordinator { } } - private async run(entry: GenerationEntry): Promise { + private async run(entry: GenerationEntry): Promise { try { entry.resolve(await entry.work(entry.controller.signal)); } catch (error) { diff --git a/packages/studio-server/src/types.ts b/packages/studio-server/src/types.ts index e775fef09c..b5c453b7e4 100644 --- a/packages/studio-server/src/types.ts +++ b/packages/studio-server/src/types.ts @@ -138,7 +138,10 @@ export interface StudioApiAdapter { * adapters provide this; the single-file method remains as a compatibility * fallback for third-party adapters compiled against older releases. */ - lintProject?: (projectDir: string) => Promise | ProjectLintResult; + lintProject?: ( + projectDir: string, + signal?: AbortSignal, + ) => Promise | ProjectLintResult; /** URL to the hyperframe runtime JS (injected into preview HTML). */ runtimeUrl: string;