mirror of
https://github.com/heygen-com/hyperframes.git
synced 2026-11-18 07:50:13 +00:00
fix(studio-server): other sites cannot start lint or probe work (#7894)
* wip: paused before review * fix(studio-server): key shared lint runs on project content; test cross-site refusals
This commit is contained in:
@@ -106,6 +106,8 @@ const STUDIO_MANUAL_EDITS_PATH = ".hyperframes/studio-manual-edits.json";
|
||||
// before that watchdog can fire and skip this file's browser cleanup.
|
||||
const RENDER_SHUTDOWN_WAIT_MS = 2_000;
|
||||
|
||||
const LINT_WORKER_TIMEOUT_MS = 120_000;
|
||||
|
||||
// Vite emits only content-hashed files under dist/assets; hand-authored
|
||||
// public/ files land at the dist root. The route is the signal because the
|
||||
// filename is not: rollup's base64url hash may itself contain a hyphen.
|
||||
@@ -605,11 +607,11 @@ export function createStudioServer(options: StudioServerOptions): StudioServer {
|
||||
|
||||
// Out of process: linting a large composition is seconds of synchronous parsing, and on
|
||||
// this event loop it stalls every other Studio request, the preview included.
|
||||
lintProject: (dir: string) =>
|
||||
lintProject: (dir: string, signal?: AbortSignal) =>
|
||||
runRenderSetupWorker<ProjectLintResult>(
|
||||
"lint",
|
||||
{ projectDir: dir, host: "studio", background: true },
|
||||
{ maxBufferBytes: 8 * 1024 * 1024 },
|
||||
{ maxBufferBytes: 8 * 1024 * 1024, signal, timeoutMs: LINT_WORKER_TIMEOUT_MS },
|
||||
),
|
||||
|
||||
runtimeUrl: "/api/runtime.js",
|
||||
|
||||
@@ -306,6 +306,45 @@ describe("createStudioApi media analysis from another site", () => {
|
||||
});
|
||||
});
|
||||
|
||||
describe("createStudioApi work-starting reads from another site", () => {
|
||||
function appCountingWork() {
|
||||
const projectDir = mkdtempSync(join(tmpdir(), "hf-origin-work-"));
|
||||
tempDirs.push(projectDir);
|
||||
const calls = { lint: 0 };
|
||||
const app = createStudioApi({
|
||||
...stubAdapter(projectDir),
|
||||
lintProject: () => {
|
||||
calls.lint++;
|
||||
return { results: [] };
|
||||
},
|
||||
});
|
||||
return { app, calls };
|
||||
}
|
||||
|
||||
it.each([
|
||||
["lint", "cross-site"],
|
||||
["lint", "same-site"],
|
||||
["media/metadata?path=talk.mp4", "cross-site"],
|
||||
["image-thumbnail/photo.jpg", "cross-site"],
|
||||
])("refuses %s marked %s without starting any work", async (route, site) => {
|
||||
const { app, calls } = appCountingWork();
|
||||
const res = await app.request(`http://localhost/projects/demo/${route}`, {
|
||||
headers: { Host: "localhost", "Sec-Fetch-Site": site },
|
||||
});
|
||||
expect(res.status).toBe(403);
|
||||
expect(calls.lint).toBe(0);
|
||||
});
|
||||
|
||||
it("answers Studio's own lint request", async () => {
|
||||
const { app, calls } = appCountingWork();
|
||||
const res = await app.request("http://localhost/projects/demo/lint", {
|
||||
headers: { Host: "localhost", "Sec-Fetch-Site": "same-origin" },
|
||||
});
|
||||
expect(res.status).toBe(200);
|
||||
expect(calls.lint).toBe(1);
|
||||
});
|
||||
});
|
||||
|
||||
describe("previewAllowedHostsFromEnv", () => {
|
||||
it("parses a comma separated list, trimming blanks", () => {
|
||||
expect(
|
||||
|
||||
@@ -141,6 +141,9 @@ export function createStudioApi(adapter: StudioApiAdapter, options: StudioApiOpt
|
||||
api.use("/projects/:id/thumbnail/*", refuseForAnotherSite("thumbnail"));
|
||||
api.use("/projects/:id/waveform/*", refuseForAnotherSite("waveform"));
|
||||
api.use("/projects/:id/peaks/*", refuseForAnotherSite("peaks"));
|
||||
api.use("/projects/:id/image-thumbnail/*", refuseForAnotherSite("image thumbnail"));
|
||||
api.use("/projects/:id/media/metadata", refuseForAnotherSite("media metadata"));
|
||||
api.use("/projects/:id/lint", refuseForAnotherSite("lint"));
|
||||
// This API is unauthenticated and simple cross-origin POSTs need no
|
||||
// preflight, so the guard rejects writes the shared predicate refuses.
|
||||
api.use(async function rejectCrossOriginWrites(c, next) {
|
||||
|
||||
@@ -0,0 +1,146 @@
|
||||
// @vitest-environment node
|
||||
// Node environment: the abort tests need a request's AbortSignal to reach the route.
|
||||
import { afterEach, describe, expect, it, vi } from "vitest";
|
||||
import { Hono } from "hono";
|
||||
import { mkdtempSync, rmSync, writeFileSync } from "node:fs";
|
||||
import { tmpdir } from "node:os";
|
||||
import { join } from "node:path";
|
||||
import { registerLintRoutes } from "./lint";
|
||||
import { stubAdapter } from "./stubAdapter.test-helpers";
|
||||
import type { ProjectLintResult, StudioApiAdapter } from "../types";
|
||||
|
||||
const tempDirs: string[] = [];
|
||||
|
||||
afterEach(() => {
|
||||
for (const dir of tempDirs.splice(0)) rmSync(dir, { recursive: true, force: true });
|
||||
});
|
||||
|
||||
function projectDir(): string {
|
||||
const dir = mkdtempSync(join(tmpdir(), "hf-lint-runs-"));
|
||||
tempDirs.push(dir);
|
||||
writeFileSync(join(dir, "index.html"), "<html><body>real</body></html>");
|
||||
return dir;
|
||||
}
|
||||
|
||||
const CLEAN: ProjectLintResult = { results: [{ file: "index.html", result: { findings: [] } }] };
|
||||
|
||||
/** Stands in for the lint worker: each run stays live until it is aborted or finished. */
|
||||
function workerPool() {
|
||||
let live = 0;
|
||||
let peak = 0;
|
||||
let started = 0;
|
||||
const finishers: Array<() => void> = [];
|
||||
const lintProject = (_dir: string, signal?: AbortSignal) =>
|
||||
new Promise<ProjectLintResult>((resolve, reject) => {
|
||||
started++;
|
||||
live++;
|
||||
peak = Math.max(peak, live);
|
||||
let exited = false;
|
||||
const exit = (settle: () => void) => {
|
||||
if (exited) return;
|
||||
exited = true;
|
||||
live--;
|
||||
settle();
|
||||
};
|
||||
signal?.addEventListener("abort", () => exit(() => reject(signal.reason)), { once: true });
|
||||
finishers.push(() => exit(() => resolve(CLEAN)));
|
||||
});
|
||||
return {
|
||||
lintProject,
|
||||
finishAll: () => finishers.splice(0).forEach((finish) => finish()),
|
||||
stats: () => ({ live, peak, started }),
|
||||
};
|
||||
}
|
||||
|
||||
function appFor(dirsById: Record<string, string>, lintProject: StudioApiAdapter["lintProject"]) {
|
||||
const adapter: StudioApiAdapter = {
|
||||
...stubAdapter(""),
|
||||
resolveProject: async (id: string) => {
|
||||
const dir = dirsById[id];
|
||||
return dir ? { id, dir } : null;
|
||||
},
|
||||
lintProject,
|
||||
};
|
||||
const app = new Hono();
|
||||
registerLintRoutes(app, adapter);
|
||||
return app;
|
||||
}
|
||||
|
||||
const tick = () => new Promise((resolve) => setTimeout(resolve, 0));
|
||||
|
||||
describe("project lint runs", () => {
|
||||
it("shares one run between concurrent requests for the same project", async () => {
|
||||
const pool = workerPool();
|
||||
const app = appFor({ demo: projectDir() }, pool.lintProject);
|
||||
|
||||
const requests = Array.from({ length: 5 }, () =>
|
||||
app.request("http://localhost/projects/demo/lint"),
|
||||
);
|
||||
await vi.waitFor(() => expect(pool.stats().started).toBe(1));
|
||||
await tick();
|
||||
expect(pool.stats().started).toBe(1);
|
||||
pool.finishAll();
|
||||
|
||||
for (const res of await Promise.all(requests)) expect(res.status).toBe(200);
|
||||
expect(pool.stats().started).toBe(1);
|
||||
});
|
||||
|
||||
it("starts a fresh run when the project changed after a run began", async () => {
|
||||
const pool = workerPool();
|
||||
const dir = projectDir();
|
||||
const app = appFor({ demo: dir }, pool.lintProject);
|
||||
|
||||
const before = app.request("http://localhost/projects/demo/lint");
|
||||
await vi.waitFor(() => expect(pool.stats().started).toBe(1));
|
||||
writeFileSync(join(dir, "scene.html"), "<html><body>new scene</body></html>");
|
||||
const after = app.request("http://localhost/projects/demo/lint");
|
||||
await vi.waitFor(() => expect(pool.stats().started).toBe(2));
|
||||
pool.finishAll();
|
||||
for (const res of await Promise.all([before, after])) expect(res.status).toBe(200);
|
||||
});
|
||||
|
||||
it("aborts a run once no request waits on it", async () => {
|
||||
const pool = workerPool();
|
||||
const app = appFor({ demo: projectDir() }, pool.lintProject);
|
||||
const first = new AbortController();
|
||||
const second = new AbortController();
|
||||
|
||||
const requests = [first, second].map((controller) =>
|
||||
app.request("http://localhost/projects/demo/lint", { signal: controller.signal }),
|
||||
);
|
||||
await vi.waitFor(() => expect(pool.stats().live).toBe(1));
|
||||
first.abort();
|
||||
await tick();
|
||||
expect(pool.stats().live).toBe(1);
|
||||
second.abort();
|
||||
await tick();
|
||||
expect(pool.stats().live).toBe(0);
|
||||
await Promise.allSettled(requests);
|
||||
});
|
||||
|
||||
it("never keeps more than two runs live while requests start and abort", async () => {
|
||||
const pool = workerPool();
|
||||
const dirs = Object.fromEntries(
|
||||
Array.from({ length: 6 }, (_, index) => [`p${index}`, projectDir()]),
|
||||
);
|
||||
const app = appFor(dirs, pool.lintProject);
|
||||
const pending: Promise<Response>[] = [];
|
||||
|
||||
for (let round = 0; round < 20; round++) {
|
||||
const controllers = Object.keys(dirs).map((id) => {
|
||||
const controller = new AbortController();
|
||||
pending.push(
|
||||
app.request(`http://localhost/projects/${id}/lint`, { signal: controller.signal }),
|
||||
);
|
||||
return controller;
|
||||
});
|
||||
await vi.waitFor(() => expect(pool.stats().live).toBe(2));
|
||||
for (const controller of controllers) controller.abort();
|
||||
await vi.waitFor(() => expect(pool.stats().live).toBe(0));
|
||||
}
|
||||
|
||||
expect(pool.stats().peak).toBe(2);
|
||||
expect(pool.stats().live).toBe(0);
|
||||
await Promise.allSettled(pending);
|
||||
});
|
||||
});
|
||||
@@ -93,7 +93,7 @@ describe("registerLintRoutes — dot-directory exclusion (#1384)", () => {
|
||||
};
|
||||
|
||||
expect(response.status).toBe(200);
|
||||
expect(projectLint).toHaveBeenCalledWith(projectDir);
|
||||
expect(projectLint).toHaveBeenCalledWith(projectDir, expect.any(AbortSignal));
|
||||
expect(singleFileLint).not.toHaveBeenCalled();
|
||||
expect(payload.findings).toContainEqual(
|
||||
expect.objectContaining({
|
||||
|
||||
@@ -3,18 +3,21 @@ import { readFileSync } from "node:fs";
|
||||
import { join } from "node:path";
|
||||
import type { LintResult, StudioApiAdapter } from "../types.js";
|
||||
import { isInHiddenOrVendorDir, walkDir } from "../helpers/safePath.js";
|
||||
import { resolveProjectSignature } from "../helpers/projectSignature.js";
|
||||
import { ThumbnailGenerationCoordinator } from "./thumbnailGenerationCoordinator.js";
|
||||
|
||||
type LintFinding = LintResult["findings"][number];
|
||||
|
||||
async function collectProjectFindings(
|
||||
adapter: StudioApiAdapter,
|
||||
projectDir: string,
|
||||
signal: AbortSignal,
|
||||
): Promise<{ findings: LintFinding[]; coveredFiles: Set<string> }> {
|
||||
const findings: LintFinding[] = [];
|
||||
const coveredFiles = new Set<string>();
|
||||
if (!adapter.lintProject) return { findings, coveredFiles };
|
||||
|
||||
const result = await adapter.lintProject(projectDir);
|
||||
const result = await adapter.lintProject(projectDir, signal);
|
||||
for (const entry of result.results) {
|
||||
coveredFiles.add(entry.file);
|
||||
for (const finding of entry.result.findings) {
|
||||
@@ -46,22 +49,38 @@ async function lintUncoveredHtml(
|
||||
return findings;
|
||||
}
|
||||
|
||||
async function lintProjectDir(
|
||||
adapter: StudioApiAdapter,
|
||||
projectDir: string,
|
||||
signal: AbortSignal,
|
||||
): Promise<LintFinding[]> {
|
||||
const htmlFiles = walkDir(projectDir).filter(
|
||||
(f) => f.endsWith(".html") && !isInHiddenOrVendorDir(f),
|
||||
);
|
||||
const projectLint = await collectProjectFindings(adapter, projectDir, signal);
|
||||
const extraFindings = await lintUncoveredHtml(
|
||||
adapter,
|
||||
projectDir,
|
||||
htmlFiles,
|
||||
projectLint.coveredFiles,
|
||||
);
|
||||
return [...projectLint.findings, ...extraFindings];
|
||||
}
|
||||
|
||||
const MAX_CONCURRENT_LINT_RUNS = 2;
|
||||
|
||||
export function registerLintRoutes(api: Hono, adapter: StudioApiAdapter): void {
|
||||
const runs = new ThumbnailGenerationCoordinator<LintFinding[]>(MAX_CONCURRENT_LINT_RUNS);
|
||||
api.get("/projects/:id/lint", async (c) => {
|
||||
const project = await adapter.resolveProject(c.req.param("id"));
|
||||
if (!project) return c.json({ error: "not found" }, 404);
|
||||
try {
|
||||
const htmlFiles = walkDir(project.dir).filter(
|
||||
(f) => f.endsWith(".html") && !isInHiddenOrVendorDir(f),
|
||||
const signature = await resolveProjectSignature(adapter, project.dir);
|
||||
const key = JSON.stringify([project.dir, signature]);
|
||||
const findings = await runs.acquire(key, c.req.raw.signal, (signal) =>
|
||||
lintProjectDir(adapter, project.dir, signal),
|
||||
);
|
||||
const projectLint = await collectProjectFindings(adapter, project.dir);
|
||||
const extraFindings = await lintUncoveredHtml(
|
||||
adapter,
|
||||
project.dir,
|
||||
htmlFiles,
|
||||
projectLint.coveredFiles,
|
||||
);
|
||||
return c.json({ findings: [...projectLint.findings, ...extraFindings] });
|
||||
return c.json({ findings });
|
||||
} catch (err) {
|
||||
const msg = err instanceof Error ? err.message : String(err);
|
||||
return c.json({ error: `Lint failed: ${msg}` }, 500);
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
import type { Hono } from "hono";
|
||||
import { streamSSE } from "hono/streaming";
|
||||
import { existsSync } from "node:fs";
|
||||
import { existsSync, statSync } from "node:fs";
|
||||
import { basename, dirname, extname, join, posix } from "node:path";
|
||||
import type { MediaProcessingJobState, StudioApiAdapter } from "../types.js";
|
||||
import { requestedProjectPath } from "../helpers/requestSubPath.js";
|
||||
@@ -10,8 +10,9 @@ import {
|
||||
resolveWithinProject,
|
||||
resolveWritableWithinProject,
|
||||
} from "../helpers/safePath.js";
|
||||
import { probeMediaMetadata } from "../helpers/mediaMetadata.js";
|
||||
import { probeMediaMetadata, type MediaMetadata } from "../helpers/mediaMetadata.js";
|
||||
import { forgetJobsFinishedLongerThan } from "../helpers/finishedJobs.js";
|
||||
import { ThumbnailGenerationCoordinator } from "./thumbnailGenerationCoordinator.js";
|
||||
|
||||
const VIDEO_EXTENSIONS = new Set([
|
||||
".mp4",
|
||||
@@ -43,6 +44,8 @@ interface BackgroundRemovalBody {
|
||||
|
||||
type ProbeMediaMetadata = typeof probeMediaMetadata;
|
||||
|
||||
const MAX_CONCURRENT_METADATA_PROBES = 4;
|
||||
|
||||
function isVideoPath(path: string): boolean {
|
||||
return VIDEO_EXTENSIONS.has(extname(path).toLowerCase());
|
||||
}
|
||||
@@ -134,6 +137,7 @@ export function registerMediaRoutes(
|
||||
const mediaJobs = new Map<string, MediaProcessingJobState>();
|
||||
const TTL_MS = 300_000;
|
||||
const readMediaMetadata = options.probeMediaMetadata ?? probeMediaMetadata;
|
||||
const probes = new ThumbnailGenerationCoordinator<MediaMetadata>(MAX_CONCURRENT_METADATA_PROBES);
|
||||
|
||||
function cleanupFinishedJobs(): void {
|
||||
forgetJobsFinishedLongerThan(
|
||||
@@ -156,9 +160,12 @@ export function registerMediaRoutes(
|
||||
|
||||
const filePath = resolveWithinProject(project.dir, assetPath);
|
||||
if (!filePath) return c.json({ error: "forbidden" }, 403);
|
||||
if (!existsSync(filePath)) return c.json({ error: "media not found" }, 404);
|
||||
const stat = statSync(filePath, { throwIfNoEntry: false });
|
||||
if (!stat) return c.json({ error: "media not found" }, 404);
|
||||
|
||||
return c.json({ path: assetPath, metadata: await readMediaMetadata(filePath) });
|
||||
const key = JSON.stringify([filePath, stat.dev, stat.ino, stat.size, stat.mtimeMs]);
|
||||
const metadata = await probes.acquire(key, c.req.raw.signal, () => readMediaMetadata(filePath));
|
||||
return c.json({ path: assetPath, metadata });
|
||||
});
|
||||
|
||||
api.post(
|
||||
|
||||
@@ -1,22 +1,22 @@
|
||||
export type ThumbnailGenerationValue = Buffer | null;
|
||||
export type ThumbnailGenerationWork = (signal: AbortSignal) => Promise<ThumbnailGenerationValue>;
|
||||
type ThumbnailGenerationValue = Buffer | null;
|
||||
type GenerationWork<T> = (signal: AbortSignal) => Promise<T>;
|
||||
|
||||
interface GenerationEntry {
|
||||
interface GenerationEntry<T> {
|
||||
key: string;
|
||||
controller: AbortController;
|
||||
leases: number;
|
||||
state: "queued" | "active";
|
||||
work: ThumbnailGenerationWork;
|
||||
promise: Promise<ThumbnailGenerationValue>;
|
||||
resolve: (value: ThumbnailGenerationValue) => void;
|
||||
work: GenerationWork<T>;
|
||||
promise: Promise<T>;
|
||||
resolve: (value: T) => void;
|
||||
reject: (reason: unknown) => void;
|
||||
}
|
||||
|
||||
/** Sole server owner for same-key dedupe, concurrency, cancellation, and queue order. */
|
||||
export class ThumbnailGenerationCoordinator {
|
||||
private readonly entries = new Map<string, GenerationEntry>();
|
||||
private readonly queue: GenerationEntry[] = [];
|
||||
private readonly activeEntries = new Set<GenerationEntry>();
|
||||
export class ThumbnailGenerationCoordinator<T = ThumbnailGenerationValue> {
|
||||
private readonly entries = new Map<string, GenerationEntry<T>>();
|
||||
private readonly queue: GenerationEntry<T>[] = [];
|
||||
private readonly activeEntries = new Set<GenerationEntry<T>>();
|
||||
private active = 0;
|
||||
|
||||
constructor(private readonly concurrency = 1) {
|
||||
@@ -25,18 +25,14 @@ export class ThumbnailGenerationCoordinator {
|
||||
}
|
||||
}
|
||||
|
||||
acquire(
|
||||
key: string,
|
||||
signal: AbortSignal,
|
||||
work: ThumbnailGenerationWork,
|
||||
): Promise<ThumbnailGenerationValue> {
|
||||
acquire(key: string, signal: AbortSignal, work: GenerationWork<T>): Promise<T> {
|
||||
if (signal.aborted) return Promise.reject(new DOMException("Aborted", "AbortError"));
|
||||
|
||||
let entry = this.entries.get(key);
|
||||
if (!entry) {
|
||||
let resolve!: (value: ThumbnailGenerationValue) => void;
|
||||
let resolve!: (value: T) => void;
|
||||
let reject!: (reason: unknown) => void;
|
||||
const promise = new Promise<ThumbnailGenerationValue>((resolvePromise, rejectPromise) => {
|
||||
const promise = new Promise<T>((resolvePromise, rejectPromise) => {
|
||||
resolve = resolvePromise;
|
||||
reject = rejectPromise;
|
||||
});
|
||||
@@ -63,7 +59,7 @@ export class ThumbnailGenerationCoordinator {
|
||||
return new Set([...this.entries.keys(), ...[...this.activeEntries].map((entry) => entry.key)]);
|
||||
}
|
||||
|
||||
private lease(entry: GenerationEntry, signal: AbortSignal): Promise<ThumbnailGenerationValue> {
|
||||
private lease(entry: GenerationEntry<T>, signal: AbortSignal): Promise<T> {
|
||||
return new Promise((resolve, reject) => {
|
||||
let released = false;
|
||||
const release = () => {
|
||||
@@ -112,7 +108,7 @@ export class ThumbnailGenerationCoordinator {
|
||||
}
|
||||
}
|
||||
|
||||
private async run(entry: GenerationEntry): Promise<void> {
|
||||
private async run(entry: GenerationEntry<T>): Promise<void> {
|
||||
try {
|
||||
entry.resolve(await entry.work(entry.controller.signal));
|
||||
} catch (error) {
|
||||
|
||||
@@ -138,7 +138,10 @@ export interface StudioApiAdapter {
|
||||
* adapters provide this; the single-file method remains as a compatibility
|
||||
* fallback for third-party adapters compiled against older releases.
|
||||
*/
|
||||
lintProject?: (projectDir: string) => Promise<ProjectLintResult> | ProjectLintResult;
|
||||
lintProject?: (
|
||||
projectDir: string,
|
||||
signal?: AbortSignal,
|
||||
) => Promise<ProjectLintResult> | ProjectLintResult;
|
||||
|
||||
/** URL to the hyperframe runtime JS (injected into preview HTML). */
|
||||
runtimeUrl: string;
|
||||
|
||||
Reference in New Issue
Block a user