mirror of
https://github.com/heygen-com/hyperframes.git
synced 2026-11-18 07:50:13 +00:00
fix(producer): linear probe media check, and renders refuse composition HTML over 16 MB (#7992)
* fix(producer): linear probe media check, and renders refuse composition HTML over 8 MB The probe stage tested every inline script body with /<(?:video|audio)\b[^>]*>/i. A body full of unclosed <video openers made that test take quadratic time on the render server's main thread. hasClosedMediaTag finds the first opener and then looks for one '>' after it, which gives the same answer in linear time. Compile and probe work on a page also runs on the main thread, and its cost grows with the page's size. Entry and sub-composition HTML files over 8 MB are now refused before they are read, with the COMPOSITION_TOO_LARGE error code. * fix(producer): raise the composition HTML limit to 16 MB The largest real render pages are about 7 MB of HTML, so 8 MB left too little room. 16 MB is more than twice that, and still bounds one file's compile to about 12 s on dense markup.
This commit is contained in:
@@ -414,6 +414,8 @@ npx hyperframes check
|
||||
|
||||
Keep the first exact error rather than only the final “render failed” message. See [Troubleshooting](/help) for the next checks.
|
||||
|
||||
A render refuses any composition HTML file, the entry or a sub-composition, that is larger than 16 MB. The error code is `COMPOSITION_TOO_LARGE`. Keep images, fonts and data in separate files and reference them by path instead of inlining them.
|
||||
|
||||
<Tip>
|
||||
Always watch the exported file itself. Preview proves that the project can play; the output file
|
||||
proves that the delivery is correct.
|
||||
|
||||
@@ -0,0 +1,47 @@
|
||||
import { statSync } from "node:fs";
|
||||
|
||||
/**
|
||||
* The largest entry or sub-composition HTML file a render compiles. Compiling and probing a page
|
||||
* runs on the render server's main thread, and their cost grows with the page's size, so one huge
|
||||
* page would hold up every other render in the same process.
|
||||
*/
|
||||
export const MAX_COMPOSITION_HTML_BYTES = 16 * 1024 * 1024;
|
||||
|
||||
const COMPOSITION_TOO_LARGE = "COMPOSITION_TOO_LARGE" as const;
|
||||
|
||||
const megabytes = (bytes: number) => `${(bytes / (1024 * 1024)).toFixed(1)} MB`;
|
||||
|
||||
class CompositionTooLargeError extends Error {
|
||||
readonly code = COMPOSITION_TOO_LARGE;
|
||||
// The producer server reads these off the error for the wire envelope.
|
||||
readonly owner = "user" as const;
|
||||
readonly retryable = false as const;
|
||||
readonly publicMetadata: Readonly<{ bytes: number; limitBytes: number }>;
|
||||
|
||||
constructor(file: string, bytes: number) {
|
||||
super(
|
||||
`Composition file "${file}" is ${megabytes(bytes)}, over the ` +
|
||||
`${megabytes(MAX_COMPOSITION_HTML_BYTES)} limit for one composition HTML file. ` +
|
||||
"Keep large data such as images, fonts and JSON in separate asset files and reference " +
|
||||
"them by path instead of inlining them.",
|
||||
);
|
||||
this.name = "CompositionTooLargeError";
|
||||
this.publicMetadata = { bytes, limitBytes: MAX_COMPOSITION_HTML_BYTES };
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Throws {@link CompositionTooLargeError} when `path` is a file over the limit, before anything
|
||||
* reads it. A missing path or a folder passes, so the caller's own read reports it as before.
|
||||
*/
|
||||
export function assertCompositionFileSize(path: string, file: string): void {
|
||||
let bytes: number;
|
||||
try {
|
||||
const stats = statSync(path);
|
||||
if (!stats.isFile()) return;
|
||||
bytes = stats.size;
|
||||
} catch {
|
||||
return;
|
||||
}
|
||||
if (bytes > MAX_COMPOSITION_HTML_BYTES) throw new CompositionTooLargeError(file, bytes);
|
||||
}
|
||||
@@ -0,0 +1,78 @@
|
||||
import { mkdtempSync, writeFileSync } from "node:fs";
|
||||
import { tmpdir } from "node:os";
|
||||
import { join } from "node:path";
|
||||
import { describe, expect, it } from "vitest";
|
||||
import { assertCompositionFileSize, MAX_COMPOSITION_HTML_BYTES } from "./compositionSizeLimit.js";
|
||||
import { compileForRender } from "./htmlCompiler.js";
|
||||
import { extractSafeRenderErrorCode } from "./safeRenderErrorCode.js";
|
||||
|
||||
const ROOT = `<!doctype html>
|
||||
<html><body>
|
||||
<div data-composition-id="root" data-width="1920" data-height="1080" data-duration="2">
|
||||
<div data-composition-id="scene" data-composition-src="scene.html" data-start="0" data-duration="2"></div>
|
||||
<script>
|
||||
window.__timelines = window.__timelines || {};
|
||||
window.__timelines.root = { duration: function() { return 2; } };
|
||||
</script>
|
||||
</div>
|
||||
</body></html>`;
|
||||
const SCENE = `<template><div data-composition-id="scene"><h1>Scene</h1></div></template>`;
|
||||
|
||||
/** `html` followed by a comment that brings the file to exactly `bytes`. */
|
||||
const paddedTo = (html: string, bytes: number) =>
|
||||
`${html}<!--${"x".repeat(bytes - Buffer.byteLength(html) - 7)}-->`;
|
||||
|
||||
function project(files: Record<string, string>): string {
|
||||
const dir = mkdtempSync(join(tmpdir(), "hf-compiler-size-"));
|
||||
for (const [name, text] of Object.entries(files)) writeFileSync(join(dir, name), text);
|
||||
return dir;
|
||||
}
|
||||
|
||||
async function compileError(dir: string): Promise<unknown> {
|
||||
return compileForRender(dir, join(dir, "index.html"), dir).then(
|
||||
() => undefined,
|
||||
(error: unknown) => error,
|
||||
);
|
||||
}
|
||||
|
||||
describe("composition HTML size limit", () => {
|
||||
it("refuses an entry file over the limit before compiling it", async () => {
|
||||
const dir = project({
|
||||
"index.html": paddedTo(ROOT, MAX_COMPOSITION_HTML_BYTES + 1),
|
||||
"scene.html": SCENE,
|
||||
});
|
||||
const started = performance.now();
|
||||
const error = await compileError(dir);
|
||||
expect(performance.now() - started).toBeLessThan(1_000);
|
||||
expect(extractSafeRenderErrorCode(error)).toBe("COMPOSITION_TOO_LARGE");
|
||||
expect(String(error)).toContain('"index.html"');
|
||||
expect(String(error)).toContain("16.0 MB limit");
|
||||
});
|
||||
|
||||
it("refuses a sub-composition file over the limit", async () => {
|
||||
const dir = project({
|
||||
"index.html": ROOT,
|
||||
"scene.html": paddedTo(SCENE, MAX_COMPOSITION_HTML_BYTES + 1),
|
||||
});
|
||||
const error = await compileError(dir);
|
||||
expect(extractSafeRenderErrorCode(error)).toBe("COMPOSITION_TOO_LARGE");
|
||||
expect(String(error)).toContain('"scene.html"');
|
||||
});
|
||||
|
||||
it("compiles files at the limit", async () => {
|
||||
const dir = project({
|
||||
"index.html": paddedTo(ROOT, MAX_COMPOSITION_HTML_BYTES),
|
||||
"scene.html": SCENE,
|
||||
});
|
||||
const compiled = await compileForRender(dir, join(dir, "index.html"), dir);
|
||||
expect(compiled.html).toContain("<h1>Scene</h1>");
|
||||
});
|
||||
|
||||
it("leaves missing paths and folders to the caller's own read", () => {
|
||||
const dir = project({});
|
||||
expect(() =>
|
||||
assertCompositionFileSize(join(dir, "missing.html"), "missing.html"),
|
||||
).not.toThrow();
|
||||
expect(() => assertCompositionFileSize(dir, "folder")).not.toThrow();
|
||||
});
|
||||
});
|
||||
@@ -101,6 +101,7 @@ import {
|
||||
} from "@hyperframes/parsers/asset-resolution";
|
||||
import { isPrivateFileUnder, isStudioPrivatePath } from "@hyperframes/core/safe-path";
|
||||
import { extractMediaMetadata, extractAudioMetadata } from "../utils/ffprobe.js";
|
||||
import { assertCompositionFileSize } from "./compositionSizeLimit.js";
|
||||
import {
|
||||
escapesAssetRoot,
|
||||
isPathInside,
|
||||
@@ -340,6 +341,7 @@ function assertSubCompositionsUsable(
|
||||
// silence here rather than pretend it surfaces an error somewhere else.
|
||||
if (visited.has(filePath)) continue;
|
||||
|
||||
assertCompositionFileSize(filePath, srcPath);
|
||||
const read = readProjectFile(filePath);
|
||||
if (read.kind === "missing") {
|
||||
problems.push({ srcPath, detail: "the file does not exist" });
|
||||
@@ -880,6 +882,7 @@ async function parseSubCompositions(
|
||||
continue;
|
||||
}
|
||||
|
||||
assertCompositionFileSize(filePath, srcPath);
|
||||
const read = readProjectFile(filePath);
|
||||
if (read.kind !== "file") {
|
||||
continue;
|
||||
@@ -1170,6 +1173,7 @@ function inlineSubCompositions(
|
||||
let compHtml = subCompositions.get(srcPath) || null;
|
||||
if (!compHtml) {
|
||||
const filePath = resolveCompositionFile(projectDir, srcPath, assetRoot);
|
||||
if (filePath) assertCompositionFileSize(filePath, srcPath);
|
||||
const read =
|
||||
filePath && !isPrivateCompositionFile(projectDir, filePath)
|
||||
? readProjectFile(filePath)
|
||||
@@ -2217,6 +2221,7 @@ export async function compileForRender(
|
||||
if (!COMPOSITION_FILE.test(htmlPath)) {
|
||||
throw new Error(`Entry file ${htmlPath} is not an .html file, so it is refused`);
|
||||
}
|
||||
assertCompositionFileSize(htmlPath, basename(htmlPath));
|
||||
const entryHtml = rebaseDirectEntryAssetPaths(readTextFile(htmlPath), projectDir, htmlPath);
|
||||
const rawHtml = bindVariableMediaSources(
|
||||
normalizeCompositionSrcs(rewriteUnresolvableGsapToCdn(entryHtml, projectDir), projectDir),
|
||||
|
||||
@@ -4,6 +4,7 @@ import {
|
||||
countTagsWithAttr,
|
||||
countedTagNames,
|
||||
firstCompositionIdTag,
|
||||
hasClosedMediaTag,
|
||||
inlineScriptBlocks,
|
||||
remoteSrcUrls,
|
||||
removeDelimitedRuns,
|
||||
@@ -291,6 +292,12 @@ describe("htmlTagScans agree with the regexes they replace", { timeout: 60_000 }
|
||||
}
|
||||
});
|
||||
|
||||
it("closed media tag test", () => {
|
||||
for (const text of both(7, FOCUSED.attrs)) {
|
||||
expect(hasClosedMediaTag(text), text).toBe(/<(?:video|audio)\b[^>]*>/i.test(text));
|
||||
}
|
||||
});
|
||||
|
||||
it("remote src urls", () => {
|
||||
for (const html of both(5, FOCUSED.remote)) {
|
||||
for (const tag of ["media", "source", "img"] as const) {
|
||||
@@ -340,6 +347,7 @@ describe("htmlTagScans run in linear time", () => {
|
||||
],
|
||||
["crossorigin", (h) => stripCrossoriginAttr(h, "video"), ["<video ", "<video crossorigin "]],
|
||||
["preload", stripPreloadNone, ["<video ", '<video preload="none" ']],
|
||||
["closed media tag", hasClosedMediaTag, ["<video", "<audio ", "<VIDEO"]],
|
||||
["remote media", (h) => remoteSrcUrls(h, "media"), ["<video ", '<video src="http://a" ']],
|
||||
["remote img", (h) => remoteSrcUrls(h, "img"), ["<img ", "<img src='https://a "]],
|
||||
["script/style removal", removeScriptAndStyleBlocks, ["<script ", "<style>", "<script>"]],
|
||||
|
||||
@@ -222,6 +222,11 @@ export function stripPreloadNone(html: string): string {
|
||||
});
|
||||
}
|
||||
|
||||
export function hasClosedMediaTag(text: string): boolean {
|
||||
const open = /<(?:video|audio)\b/i.exec(text);
|
||||
return open !== null && text.includes(">", open.index + open[0].length);
|
||||
}
|
||||
|
||||
// For <img> only a real `src` counts, never `data-src` or `data-*-src` (lazy-loader placeholders
|
||||
// whose URL is not what Chrome paints). `srcset` never counts, since `\s*=` must follow `src`.
|
||||
const REMOTE_SRC_TAGS = {
|
||||
|
||||
@@ -30,6 +30,7 @@ import {
|
||||
type CompileStageResult,
|
||||
} from "./compileStage.js";
|
||||
import type { RenderJob } from "../../renderOrchestrator.js";
|
||||
import { MAX_COMPOSITION_HTML_BYTES } from "../../compositionSizeLimit.js";
|
||||
|
||||
const noopLog = {
|
||||
error: () => {},
|
||||
@@ -495,6 +496,13 @@ describe("resolveEntryHtmlPath", () => {
|
||||
expect(existsSync(join(fixture.workDir, "standalone-entry.html"))).toBe(false);
|
||||
});
|
||||
|
||||
it("refuses an entry file over the composition size limit before reading it", () => {
|
||||
fixture = setupFixture(PLAIN_HTML);
|
||||
const bigPath = join(fixture.workDir, "project", "big.html");
|
||||
writeFileSync(bigPath, " ".repeat(MAX_COMPOSITION_HTML_BYTES + 1), "utf-8");
|
||||
expect(() => resolve("big.html")).toThrow(/over the 16\.0 MB limit/);
|
||||
});
|
||||
|
||||
it("returns a non-template sub-composition file untouched", () => {
|
||||
fixture = setupFixture(PLAIN_HTML);
|
||||
const subPath = join(fixture.workDir, "project", "full.html");
|
||||
|
||||
@@ -42,6 +42,7 @@ import { privateCacheRoot, type ChromaSubsampling, type EngineConfig } from "@hy
|
||||
import { suggestMatchingPreset, type CanvasResolution } from "@hyperframes/core";
|
||||
import { extractStandaloneEntryFromIndex, isTemplateEntry } from "@hyperframes/core/compiler";
|
||||
import { readTextFile } from "@hyperframes/parsers/asset-resolution";
|
||||
import { assertCompositionFileSize } from "../../compositionSizeLimit.js";
|
||||
import type { CompiledComposition } from "../../htmlCompiler.js";
|
||||
import { compileForRender } from "../../htmlCompiler.js";
|
||||
import type { ProducerLogger } from "../../../logger.js";
|
||||
@@ -185,6 +186,7 @@ export function resolveEntryHtmlPath(input: {
|
||||
`Entry file "${relative(projectDir, path)}" leads outside the project folder`,
|
||||
);
|
||||
}
|
||||
assertCompositionFileSize(path, relative(projectDir, path));
|
||||
return readTextFile(path);
|
||||
};
|
||||
const htmlPath = join(projectDir, entryFile);
|
||||
|
||||
@@ -74,6 +74,7 @@ import {
|
||||
type FileServerHandle,
|
||||
VIRTUAL_TIME_SHIM,
|
||||
} from "../../fileServer.js";
|
||||
import { hasClosedMediaTag } from "../../htmlTagScans.js";
|
||||
import { servedPath } from "../../servedPath.js";
|
||||
import type { ProducerLogger } from "../../../logger.js";
|
||||
import {
|
||||
@@ -303,7 +304,7 @@ function hasRuntimeMediaChanges(html: string): boolean {
|
||||
(script) =>
|
||||
/\bcreateElement\s*\(\s*["'`](?:video|audio)["'`]\s*\)/i.test(script) ||
|
||||
/\bnew\s+(?:Audio|Video)\s*\(/i.test(script) ||
|
||||
/<(?:video|audio)\b[^>]*>/i.test(script),
|
||||
hasClosedMediaTag(script),
|
||||
);
|
||||
if (insertedMedia) return true;
|
||||
|
||||
|
||||
@@ -11,6 +11,7 @@ const SAFE_RENDER_ERROR_CODES = new Set<string>([
|
||||
"RENDER_STAGE_STALLED",
|
||||
"PAGE_NEVER_READY",
|
||||
"VARIABLES_INVALID",
|
||||
"COMPOSITION_TOO_LARGE",
|
||||
]);
|
||||
|
||||
/**
|
||||
|
||||
Reference in New Issue
Block a user