fix(producer): fetch each remote script and stylesheet once, a few at a time, within the size limit (#8153)

* test(producer): remote scripts and stylesheets inlined at compile time stay within the size limit

* fix(producer): fetch each remote script and stylesheet once, a few at a time, within the size limit

Every inlined copy is charged to the render's expansion budget as it arrives;
going over fails the compile with COMPOSITION_TOO_LARGE and stops the
downloads still queued. Remote CSS is spliced in literally, so `$` sequences
in it are no longer expanded.

* fix(producer): drop the budget label comment the comment gate counts

* fix(producer): count a remote script when it arrives and stop the queue on an integrity mismatch

* test(producer): font normalization tests answer Google Fonts from a stub, not the network

* style(producer): wrap the stubbed font response
This commit is contained in:
Miguel Ángel
2026-11-18 12:16:51 +00:00
committed by GitHub
parent 75992d82ad
commit 4bdf9181b4
5 changed files with 236 additions and 55 deletions
+1
View File
@@ -394,6 +394,7 @@ export {
} from "./utils/loopbackOnly.js";
export { readWebGlVendorInfoFromCanvas } from "./utils/readWebGlVendorInfoFromCanvas.js";
export { createConcurrencyLimit } from "./utils/concurrencyLimit.js";
export {
extractMediaMetadata,
@@ -49,6 +49,9 @@ const fileTooLarge = (file: string, bytes: number) =>
const limitText = `One render generates at most ${megabytes(MAX_EXPANDED_COMPOSITION_BYTES)}`;
export const REMOTE_SCRIPTS = "remote scripts";
export const REMOTE_STYLESHEETS = "remote stylesheets";
function generatedTooLarge(file: string, bytes: number): string {
if (file === "composition variables") {
return (
@@ -62,6 +65,12 @@ function generatedTooLarge(file: string, bytes: number): string {
"Use shorter URLs or bind fewer elements."
);
}
if (file === REMOTE_SCRIPTS || file === REMOTE_STYLESHEETS) {
return (
`Inlined ${file} add up to ${megabytes(bytes)}. ${limitText}. ` +
"Bundle scripts and fonts with the project, or link each one once."
);
}
return (
`Scripts, styles and rebased asset paths generated for compositions, ending at "${file}", add up to ` +
`${megabytes(bytes)}. One render generates at most ` +
@@ -2,13 +2,19 @@ import { mkdirSync, mkdtempSync, writeFileSync } from "node:fs";
import { tmpdir } from "node:os";
import { dirname, join } from "node:path";
import { parseHTML } from "linkedom";
import { describe, expect, it, vi } from "vitest";
import { afterEach, describe, expect, it, vi } from "vitest";
import { _setDownloadFetchForTests } from "@hyperframes/engine";
import {
assertCompositionFileSize,
MAX_COMPOSITION_HTML_BYTES,
MAX_EXPANDED_COMPOSITION_BYTES,
} from "./compositionSizeLimit.js";
import { compileForRender, MAX_VARIABLE_MEDIA_SRC_BYTES } from "./htmlCompiler.js";
import {
compileForRender,
inlineExternalScripts,
localizeRemoteFontFaces,
MAX_VARIABLE_MEDIA_SRC_BYTES,
} from "./htmlCompiler.js";
import { extractSafeRenderErrorCode } from "./safeRenderErrorCode.js";
const ROOT = `<!doctype html>
@@ -397,3 +403,88 @@ describe("asset paths rebased from a composition's folder", () => {
expect(extractSafeRenderErrorCode(error)).toBe("COMPOSITION_TOO_LARGE");
});
});
describe("remote scripts and stylesheets inlined at compile time", () => {
afterEach(() => _setDownloadFetchForTests());
const MiB = 1024 * 1024;
/** Serves `body(url)` and records each request and the most requests open at once. */
function serve(body: (url: string) => string) {
const seen = { requests: [] as string[], open: 0, mostOpen: 0 };
_setDownloadFetchForTests(async (url: string) => {
seen.requests.push(url);
seen.mostOpen = Math.max(seen.mostOpen, ++seen.open);
await new Promise((resolve) => setImmediate(resolve));
seen.open -= 1;
return new Response(body(url));
});
return seen;
}
const scriptTags = (urls: string[]) =>
`<html><body>${urls.map((url) => `<script src="${url}"></script>`).join("")}</body></html>`;
it("fetches a repeated script once and refuses copies past the render's limit", async () => {
const seen = serve(() => `/*${"x".repeat(MiB)}*/`);
const url = "https://cdn.example.com/big.js";
const error = await inlineExternalScripts(scriptTags(Array(40).fill(url))).catch((e) => e);
expect(extractSafeRenderErrorCode(error)).toBe("COMPOSITION_TOO_LARGE");
expect(seen.requests).toEqual([url]);
});
it("downloads a few scripts at a time and stops the queued ones once over the limit", async () => {
const seen = serve(() => `/*${"x".repeat(8 * MiB)}*/`);
const urls = Array.from({ length: 20 }, (_, i) => `https://cdn.example.com/s${i}.js`);
const error = await inlineExternalScripts(scriptTags(urls)).catch((e) => e);
expect(extractSafeRenderErrorCode(error)).toBe("COMPOSITION_TOO_LARGE");
expect(seen.mostOpen).toBe(4);
expect(seen.requests.length).toBeLessThan(urls.length);
});
it("stops queued downloads once a script fails its integrity check", async () => {
const seen = serve(() => `/*${"x".repeat(MiB)}*/`);
const urls = Array.from({ length: 40 }, (_, i) => `https://cdn.example.com/i${i}.js`);
const html = scriptTags(urls).replaceAll("<script ", '<script integrity="sha384-YQ==" ');
await expect(inlineExternalScripts(html)).rejects.toThrow("integrity mismatch");
expect(seen.requests.length).toBeLessThan(urls.length);
});
it("still inlines a few ordinary scripts", async () => {
serve((url) => `window.loaded = ${JSON.stringify(url)};`);
const html = await inlineExternalScripts(
scriptTags(["https://cdn.example.com/a.js", "https://cdn.example.com/a.js"]),
);
expect(html.split('window.loaded = "https://cdn.example.com/a.js";').length - 1).toBe(2);
});
const linkTags = (urls: string[]) =>
urls.map((url) => `<link rel="stylesheet" href="${url}">`).join("\n");
it("inserts a remote stylesheet's text literally and fetches a repeated link once", async () => {
const family = "A$&$'$`$$B";
const seen = serve(() => `@font-face { font-family: "${family}"; src: local("${family}"); }`);
const url = "https://cdn.example.com/fonts.css";
const { html } = await localizeRemoteFontFaces(
`<p>before</p>\n${linkTags([url, url])}\n<p>after</p>`,
mkdtempSync(join(tmpdir(), "hf-remote-sheet-")),
);
expect(seen.requests).toEqual([url]);
expect(html.split(`font-family: "${family}"`).length - 1).toBe(2);
expect(html.split("<p>before</p>").length - 1).toBe(1);
expect(html.split("<p>after</p>").length - 1).toBe(1);
});
it.each([
["distinct", (i: number) => `https://cdn.example.com/f${i}.css`],
["repeated", () => "https://cdn.example.com/f.css"],
])("refuses %s remote font stylesheets that add up past the render's limit", async (_, url) => {
const face = `@font-face { font-family: "F"; src: local("${"x".repeat(MiB)}"); }`;
const seen = serve(() => face);
const urls = Array.from({ length: 40 }, (_, i) => url(i));
const error = await localizeRemoteFontFaces(
linkTags(urls),
mkdtempSync(join(tmpdir(), "hf-remote-sheet-")),
).catch((e) => e);
expect(extractSafeRenderErrorCode(error)).toBe("COMPOSITION_TOO_LARGE");
expect(seen.requests.length).toBeLessThan(urls.length);
});
});
@@ -1,5 +1,5 @@
// fallow-ignore-file code-duplication
import { describe, expect, it, mock, beforeAll } from "bun:test";
import { afterEach, beforeAll, beforeEach, describe, expect, it, mock } from "bun:test";
import { createHash } from "node:crypto";
import { mkdtempSync, writeFileSync, mkdirSync, rmSync, readFileSync, readdirSync } from "node:fs";
import { tmpdir } from "node:os";
@@ -8,6 +8,7 @@ import { runInThisContext } from "node:vm";
import { parseHTML } from "linkedom";
import { interpolateVolumeGain } from "@hyperframes/core/media-volume-envelope";
import { redactTelemetryString } from "@hyperframes/core";
import { _clearGoogleFontCssCacheForTests } from "@hyperframes/core/fonts/embed";
import { defaultLogger } from "../logger.js";
import { NotMediaPayloadError, _setDownloadFetchForTests } from "@hyperframes/engine";
import {
@@ -664,8 +665,8 @@ describe("inlineExternalScripts", () => {
<script src="https://cdn.example.com/gsap.min.js"></script>
</body></html>`;
const result = await inlineExternalScripts(html);
// Both identical script tags should be fetched and replaced independently.
expect(fetchCount).toBe(2);
// The URL is fetched once and both tags are replaced.
expect(fetchCount).toBe(1);
expect(
result.match(/\/\* inlined: https:\/\/cdn\.example\.com\/gsap\.min\.js \*\//g)?.length,
).toBe(2);
@@ -1472,6 +1473,32 @@ describe("detectShaderTransitionUsage", () => {
});
describe("system-primary font normalization", () => {
// Google Fonts answers from a stub and a fresh cache, so a cold cache never waits on the network.
const FACE = "https://fonts.gstatic.com/s/stub/face.woff2";
let restore = () => {};
beforeEach(() => {
const previousFetch = globalThis.fetch;
const previousCacheDir = process.env.HYPERFRAMES_FONT_CACHE_DIR;
process.env.HYPERFRAMES_FONT_CACHE_DIR = mkdtempSync(join(tmpdir(), "hf-font-cache-"));
_clearGoogleFontCssCacheForTests();
globalThis.fetch = (async (input: unknown) => {
const url = new URL(String(input));
if (url.href === FACE) return new Response(new Uint8Array(64));
if (url.hostname !== "fonts.googleapis.com") throw new TypeError(`offline: ${url.href}`);
const family = (url.searchParams.get("family") ?? "").split(":")[0];
return new Response(
`@font-face { font-family: '${family}'; src: url(${FACE}) format('woff2'); }`,
);
}) as unknown as typeof fetch;
restore = () => {
_clearGoogleFontCssCacheForTests();
globalThis.fetch = previousFetch;
if (previousCacheDir === undefined) delete process.env.HYPERFRAMES_FONT_CACHE_DIR;
else process.env.HYPERFRAMES_FONT_CACHE_DIR = previousCacheDir;
};
});
afterEach(() => restore());
it("promotes Inter before system/generic primary stacks before distributed plan validation", async () => {
const projectDir = mkdtempSync(join(tmpdir(), "hf-system-primary-font-"));
writeFileSync(
+103 -50
View File
@@ -103,7 +103,12 @@ import {
import { isPrivateFileUnder, isStudioPrivatePath } from "@hyperframes/core/safe-path";
import { linksOutside } from "@hyperframes/studio-server/outside-file";
import { extractMediaMetadata, extractAudioMetadata } from "../utils/ffprobe.js";
import { assertCompositionFileSize, CompositionExpansionBudget } from "./compositionSizeLimit.js";
import {
assertCompositionFileSize,
CompositionExpansionBudget,
REMOTE_SCRIPTS,
REMOTE_STYLESHEETS,
} from "./compositionSizeLimit.js";
import {
escapesAssetRoot,
isPathInside,
@@ -122,6 +127,7 @@ import {
type MediaProbeProfile,
analyzeKeyframeIntervals,
assertMediaPayload,
createConcurrencyLimit,
isPrivateUnderAssetRoots,
NotMediaPayloadError,
parseMasterFxChain,
@@ -1488,11 +1494,18 @@ async function downloadScript(src: string, mirror: string | undefined): Promise<
return fetchPublicHttpsBytes(src, { maxBytes: MAX_SCRIPT_BYTES, timeoutMs: 15_000 });
}
const MAX_CONCURRENT_SCRIPT_FETCHES = 4;
/**
* Download external CDN scripts and inline them into the HTML so rendering
* works without network access (Docker, CI, restricted environments).
* works without network access (Docker, CI, restricted environments). Each URL
* is fetched once, a few at a time, and every inlined copy is charged to the
* render's budget as it arrives, so going over stops the downloads still queued.
*/
export async function inlineExternalScripts(html: string): Promise<string> {
export async function inlineExternalScripts(
html: string,
budget = new CompositionExpansionBudget(),
): Promise<string> {
const fullHtml = ensureFullDocument(html);
const wrappedFragment = fullHtml !== html;
const { document } = parseHTML(fullHtml);
@@ -1509,21 +1522,54 @@ export async function inlineExternalScripts(html: string): Promise<string> {
if (externalScripts.length === 0) return html;
const mirror = operatorGsapMirror();
const limit = createConcurrencyLimit(MAX_CONCURRENT_SCRIPT_FETCHES);
const stop = new AbortController();
const fail = (error: unknown): never => {
stop.abort(error);
throw error;
};
const charge = (bytes: number) => {
try {
budget.chargeGenerated(REMOTE_SCRIPTS, bytes);
} catch (error) {
fail(error);
}
};
// A download is charged as it arrives, so bodies held for later tags are counted too.
const fetches = new Map<string, Promise<Uint8Array>>();
const fetchOnce = (src: string) => {
let bytes = fetches.get(src);
if (!bytes) {
bytes = limit(async () => {
if (stop.signal.aborted) throw stop.signal.reason;
const downloaded = await downloadScript(src, mirror);
charge(downloaded.byteLength);
return downloaded;
}, stop.signal);
fetches.set(src, bytes);
}
return bytes;
};
const placed = new Set<string>();
const downloads = await Promise.allSettled(
externalScripts.map(async ({ el, src }) => {
const integrity = el.getAttribute("integrity") || "";
// The host's own copy needs no network; one that fails the tag's integrity metadata is not used.
const bundled = bundledScript(src);
if (bundled && matchesScriptIntegrity(bundled, integrity)) {
return { src, text: new TextDecoder().decode(bundled) };
}
const bytes = await downloadScript(src, mirror);
if (!matchesScriptIntegrity(bytes, integrity)) {
throw new ScriptIntegrityError(`Subresource integrity mismatch for ${src}`);
let bytes = bundledScript(src);
if (bytes && matchesScriptIntegrity(bytes, integrity)) {
charge(bytes.byteLength);
} else {
bytes = await fetchOnce(src);
if (!matchesScriptIntegrity(bytes, integrity)) {
fail(new ScriptIntegrityError(`Subresource integrity mismatch for ${src}`));
}
if (placed.has(src)) charge(bytes.byteLength);
placed.add(src);
}
return { src, text: new TextDecoder().decode(bytes) };
}),
);
if (stop.signal.aborted) throw stop.signal.reason;
for (let i = 0; i < downloads.length; i++) {
const download = downloads[i]!;
@@ -1904,6 +1950,13 @@ function matchingBraces(css: string): Map<number, number> {
return closes;
}
/** The address of a stylesheet link this compile inlines the font rules of, if it is one. */
function remoteFontStylesheetHref(tag: string): string | undefined {
const href = /\bhref=["']([^"']+)["']/i.exec(tag)?.[1];
if (!href || !/^https?:\/\//i.test(href) || isGoogleFontsUrl(href)) return undefined;
return href;
}
/**
* Find all external `<link rel="stylesheet">` tags pointing to non-Google
* CDNs, fetch their CSS, and replace the `<link>` with an inline `<style>`
@@ -1914,51 +1967,50 @@ function matchingBraces(css: string): Map<number, number> {
* handles those. If a fetch fails or the CSS has no `@font-face` blocks,
* the original `<link>` tag is preserved (graceful degradation).
*/
// fallow-ignore-next-line complexity
async function inlineExternalFontStylesheets(html: string): Promise<string> {
const hrefRe = /\bhref=["']([^"']+)["']/i;
const linkMatches: { fullMatch: string; href: string }[] = [];
async function inlineExternalFontStylesheets(
html: string,
budget = new CompositionExpansionBudget(),
): Promise<string> {
const hrefs = new Set<string>();
for (const { parts } of stylesheetLinkMatches(html)) {
const tag = parts[0];
const hrefMatch = hrefRe.exec(tag);
if (!hrefMatch?.[1]) continue;
const href = hrefMatch[1];
if (!/^https?:\/\//i.test(href)) continue;
if (isGoogleFontsUrl(href)) continue;
linkMatches.push({ fullMatch: tag, href });
const href = remoteFontStylesheetHref(parts[0]);
if (href) hrefs.add(href);
}
if (hrefs.size === 0) return html;
if (linkMatches.length === 0) return html;
// Each sheet is fetched once and only its font rules are kept, charged as they arrive.
const MAX_CONCURRENT_STYLESHEET_FETCHES = 4;
const fetches: { fullMatch: string; href: string; css: string | null }[] = [];
for (let i = 0; i < linkMatches.length; i += MAX_CONCURRENT_STYLESHEET_FETCHES) {
const batch = linkMatches.slice(i, i + MAX_CONCURRENT_STYLESHEET_FETCHES);
const results = await Promise.all(
batch.map(async ({ fullMatch, href }) => {
const css = await fetchExternalStylesheetCss(href);
return { fullMatch, href, css };
}),
);
fetches.push(...results);
}
let result = html;
for (const { fullMatch, href, css } of fetches) {
if (css === null) continue;
const fontFaceBlocks = extractFontFaceBlocks(css);
if (fontFaceBlocks.length === 0) continue;
const identity = safeDownloadUrlIdentity(href);
const inlineStyle = `<style>/* Inlined external font stylesheet */\n${fontFaceBlocks.join("\n")}\n</style>`;
result = result.replace(fullMatch, inlineStyle);
defaultLogger.debug("[Compiler] Inlined external @font-face rule(s)", {
count: fontFaceBlocks.length,
urlFingerprint: identity.urlFingerprint,
host: identity.host,
const inlineByHref = new Map<string, string>();
const unique = [...hrefs];
for (let i = 0; i < unique.length; i += MAX_CONCURRENT_STYLESHEET_FETCHES) {
const batch = unique.slice(i, i + MAX_CONCURRENT_STYLESHEET_FETCHES);
const sheets = await Promise.all(batch.map(fetchExternalStylesheetCss));
batch.forEach((href, index) => {
const css = sheets[index];
const fontFaceBlocks = css ? extractFontFaceBlocks(css) : [];
if (fontFaceBlocks.length === 0) return;
const inlineStyle = `<style>/* Inlined external font stylesheet */\n${fontFaceBlocks.join("\n")}\n</style>`;
budget.chargeGenerated(REMOTE_STYLESHEETS, inlineStyle.length);
inlineByHref.set(href, inlineStyle);
const identity = safeDownloadUrlIdentity(href);
defaultLogger.debug("[Compiler] Inlined external @font-face rule(s)", {
count: fontFaceBlocks.length,
urlFingerprint: identity.urlFingerprint,
host: identity.host,
});
});
}
return result;
const placed = new Set<string>();
return replaceMatches(html, stylesheetLinkMatches(html), (tag) => {
const href = remoteFontStylesheetHref(tag);
const inlineStyle = href ? inlineByHref.get(href) : undefined;
if (!href || !inlineStyle) return tag;
// The first copy was charged when it arrived; each repeat adds the whole sheet again.
if (placed.has(href)) budget.chargeGenerated(REMOTE_STYLESHEETS, inlineStyle.length);
placed.add(href);
return inlineStyle;
});
}
// Families a linked project stylesheet declares with an @font-face file the project ships.
@@ -2450,6 +2502,7 @@ export async function compileForRender(
// Remote font stylesheets join the page first, or the font check below cannot see them.
const fontSheetHtml = await inlineExternalFontStylesheets(
promoteCssImportsToLinkTags(sanitizedHtml),
budget,
);
const normalizedFontHtml = normalizeSystemFontPrimaryFamilies(
injectTextRenderingRule(coalesceHeadStylesAndBodyScripts(fontSheetHtml)),
@@ -2464,7 +2517,7 @@ export async function compileForRender(
});
// CDN scripts are inlined after coalescing so they stay separate from the merged inline runs.
const assembledHtml = await inlineExternalScripts(coalescedHtml);
const assembledHtml = await inlineExternalScripts(coalescedHtml, budget);
// Inject studio position seek re-apply script when positions are baked into HTML.
// GSAP overwrites the `translate` CSS property on every frame seek; this script