diff --git a/packages/engine/src/index.ts b/packages/engine/src/index.ts index a76a4a32a2..4d9a755445 100644 --- a/packages/engine/src/index.ts +++ b/packages/engine/src/index.ts @@ -394,6 +394,7 @@ export { } from "./utils/loopbackOnly.js"; export { readWebGlVendorInfoFromCanvas } from "./utils/readWebGlVendorInfoFromCanvas.js"; +export { createConcurrencyLimit } from "./utils/concurrencyLimit.js"; export { extractMediaMetadata, diff --git a/packages/producer/src/services/compositionSizeLimit.ts b/packages/producer/src/services/compositionSizeLimit.ts index 28ea57ddc2..9e859481f1 100644 --- a/packages/producer/src/services/compositionSizeLimit.ts +++ b/packages/producer/src/services/compositionSizeLimit.ts @@ -49,6 +49,9 @@ const fileTooLarge = (file: string, bytes: number) => const limitText = `One render generates at most ${megabytes(MAX_EXPANDED_COMPOSITION_BYTES)}`; +export const REMOTE_SCRIPTS = "remote scripts"; +export const REMOTE_STYLESHEETS = "remote stylesheets"; + function generatedTooLarge(file: string, bytes: number): string { if (file === "composition variables") { return ( @@ -62,6 +65,12 @@ function generatedTooLarge(file: string, bytes: number): string { "Use shorter URLs or bind fewer elements." ); } + if (file === REMOTE_SCRIPTS || file === REMOTE_STYLESHEETS) { + return ( + `Inlined ${file} add up to ${megabytes(bytes)}. ${limitText}. ` + + "Bundle scripts and fonts with the project, or link each one once." + ); + } return ( `Scripts, styles and rebased asset paths generated for compositions, ending at "${file}", add up to ` + `${megabytes(bytes)}. One render generates at most ` + diff --git a/packages/producer/src/services/htmlCompiler.sizeLimit.test.ts b/packages/producer/src/services/htmlCompiler.sizeLimit.test.ts index 293c7cde53..9fe3082bed 100644 --- a/packages/producer/src/services/htmlCompiler.sizeLimit.test.ts +++ b/packages/producer/src/services/htmlCompiler.sizeLimit.test.ts @@ -2,13 +2,19 @@ import { mkdirSync, mkdtempSync, writeFileSync } from "node:fs"; import { tmpdir } from "node:os"; import { dirname, join } from "node:path"; import { parseHTML } from "linkedom"; -import { describe, expect, it, vi } from "vitest"; +import { afterEach, describe, expect, it, vi } from "vitest"; +import { _setDownloadFetchForTests } from "@hyperframes/engine"; import { assertCompositionFileSize, MAX_COMPOSITION_HTML_BYTES, MAX_EXPANDED_COMPOSITION_BYTES, } from "./compositionSizeLimit.js"; -import { compileForRender, MAX_VARIABLE_MEDIA_SRC_BYTES } from "./htmlCompiler.js"; +import { + compileForRender, + inlineExternalScripts, + localizeRemoteFontFaces, + MAX_VARIABLE_MEDIA_SRC_BYTES, +} from "./htmlCompiler.js"; import { extractSafeRenderErrorCode } from "./safeRenderErrorCode.js"; const ROOT = ` @@ -397,3 +403,88 @@ describe("asset paths rebased from a composition's folder", () => { expect(extractSafeRenderErrorCode(error)).toBe("COMPOSITION_TOO_LARGE"); }); }); + +describe("remote scripts and stylesheets inlined at compile time", () => { + afterEach(() => _setDownloadFetchForTests()); + const MiB = 1024 * 1024; + + /** Serves `body(url)` and records each request and the most requests open at once. */ + function serve(body: (url: string) => string) { + const seen = { requests: [] as string[], open: 0, mostOpen: 0 }; + _setDownloadFetchForTests(async (url: string) => { + seen.requests.push(url); + seen.mostOpen = Math.max(seen.mostOpen, ++seen.open); + await new Promise((resolve) => setImmediate(resolve)); + seen.open -= 1; + return new Response(body(url)); + }); + return seen; + } + const scriptTags = (urls: string[]) => + `${urls.map((url) => ``).join("")}`; + + it("fetches a repeated script once and refuses copies past the render's limit", async () => { + const seen = serve(() => `/*${"x".repeat(MiB)}*/`); + const url = "https://cdn.example.com/big.js"; + const error = await inlineExternalScripts(scriptTags(Array(40).fill(url))).catch((e) => e); + expect(extractSafeRenderErrorCode(error)).toBe("COMPOSITION_TOO_LARGE"); + expect(seen.requests).toEqual([url]); + }); + + it("downloads a few scripts at a time and stops the queued ones once over the limit", async () => { + const seen = serve(() => `/*${"x".repeat(8 * MiB)}*/`); + const urls = Array.from({ length: 20 }, (_, i) => `https://cdn.example.com/s${i}.js`); + const error = await inlineExternalScripts(scriptTags(urls)).catch((e) => e); + expect(extractSafeRenderErrorCode(error)).toBe("COMPOSITION_TOO_LARGE"); + expect(seen.mostOpen).toBe(4); + expect(seen.requests.length).toBeLessThan(urls.length); + }); + + it("stops queued downloads once a script fails its integrity check", async () => { + const seen = serve(() => `/*${"x".repeat(MiB)}*/`); + const urls = Array.from({ length: 40 }, (_, i) => `https://cdn.example.com/i${i}.js`); + const html = scriptTags(urls).replaceAll(" `; const result = await inlineExternalScripts(html); - // Both identical script tags should be fetched and replaced independently. - expect(fetchCount).toBe(2); + // The URL is fetched once and both tags are replaced. + expect(fetchCount).toBe(1); expect( result.match(/\/\* inlined: https:\/\/cdn\.example\.com\/gsap\.min\.js \*\//g)?.length, ).toBe(2); @@ -1472,6 +1473,32 @@ describe("detectShaderTransitionUsage", () => { }); describe("system-primary font normalization", () => { + // Google Fonts answers from a stub and a fresh cache, so a cold cache never waits on the network. + const FACE = "https://fonts.gstatic.com/s/stub/face.woff2"; + let restore = () => {}; + beforeEach(() => { + const previousFetch = globalThis.fetch; + const previousCacheDir = process.env.HYPERFRAMES_FONT_CACHE_DIR; + process.env.HYPERFRAMES_FONT_CACHE_DIR = mkdtempSync(join(tmpdir(), "hf-font-cache-")); + _clearGoogleFontCssCacheForTests(); + globalThis.fetch = (async (input: unknown) => { + const url = new URL(String(input)); + if (url.href === FACE) return new Response(new Uint8Array(64)); + if (url.hostname !== "fonts.googleapis.com") throw new TypeError(`offline: ${url.href}`); + const family = (url.searchParams.get("family") ?? "").split(":")[0]; + return new Response( + `@font-face { font-family: '${family}'; src: url(${FACE}) format('woff2'); }`, + ); + }) as unknown as typeof fetch; + restore = () => { + _clearGoogleFontCssCacheForTests(); + globalThis.fetch = previousFetch; + if (previousCacheDir === undefined) delete process.env.HYPERFRAMES_FONT_CACHE_DIR; + else process.env.HYPERFRAMES_FONT_CACHE_DIR = previousCacheDir; + }; + }); + afterEach(() => restore()); + it("promotes Inter before system/generic primary stacks before distributed plan validation", async () => { const projectDir = mkdtempSync(join(tmpdir(), "hf-system-primary-font-")); writeFileSync( diff --git a/packages/producer/src/services/htmlCompiler.ts b/packages/producer/src/services/htmlCompiler.ts index cb4f92c9ee..a70aec3eee 100644 --- a/packages/producer/src/services/htmlCompiler.ts +++ b/packages/producer/src/services/htmlCompiler.ts @@ -103,7 +103,12 @@ import { import { isPrivateFileUnder, isStudioPrivatePath } from "@hyperframes/core/safe-path"; import { linksOutside } from "@hyperframes/studio-server/outside-file"; import { extractMediaMetadata, extractAudioMetadata } from "../utils/ffprobe.js"; -import { assertCompositionFileSize, CompositionExpansionBudget } from "./compositionSizeLimit.js"; +import { + assertCompositionFileSize, + CompositionExpansionBudget, + REMOTE_SCRIPTS, + REMOTE_STYLESHEETS, +} from "./compositionSizeLimit.js"; import { escapesAssetRoot, isPathInside, @@ -122,6 +127,7 @@ import { type MediaProbeProfile, analyzeKeyframeIntervals, assertMediaPayload, + createConcurrencyLimit, isPrivateUnderAssetRoots, NotMediaPayloadError, parseMasterFxChain, @@ -1488,11 +1494,18 @@ async function downloadScript(src: string, mirror: string | undefined): Promise< return fetchPublicHttpsBytes(src, { maxBytes: MAX_SCRIPT_BYTES, timeoutMs: 15_000 }); } +const MAX_CONCURRENT_SCRIPT_FETCHES = 4; + /** * Download external CDN scripts and inline them into the HTML so rendering - * works without network access (Docker, CI, restricted environments). + * works without network access (Docker, CI, restricted environments). Each URL + * is fetched once, a few at a time, and every inlined copy is charged to the + * render's budget as it arrives, so going over stops the downloads still queued. */ -export async function inlineExternalScripts(html: string): Promise { +export async function inlineExternalScripts( + html: string, + budget = new CompositionExpansionBudget(), +): Promise { const fullHtml = ensureFullDocument(html); const wrappedFragment = fullHtml !== html; const { document } = parseHTML(fullHtml); @@ -1509,21 +1522,54 @@ export async function inlineExternalScripts(html: string): Promise { if (externalScripts.length === 0) return html; const mirror = operatorGsapMirror(); + const limit = createConcurrencyLimit(MAX_CONCURRENT_SCRIPT_FETCHES); + const stop = new AbortController(); + const fail = (error: unknown): never => { + stop.abort(error); + throw error; + }; + const charge = (bytes: number) => { + try { + budget.chargeGenerated(REMOTE_SCRIPTS, bytes); + } catch (error) { + fail(error); + } + }; + // A download is charged as it arrives, so bodies held for later tags are counted too. + const fetches = new Map>(); + const fetchOnce = (src: string) => { + let bytes = fetches.get(src); + if (!bytes) { + bytes = limit(async () => { + if (stop.signal.aborted) throw stop.signal.reason; + const downloaded = await downloadScript(src, mirror); + charge(downloaded.byteLength); + return downloaded; + }, stop.signal); + fetches.set(src, bytes); + } + return bytes; + }; + const placed = new Set(); const downloads = await Promise.allSettled( externalScripts.map(async ({ el, src }) => { const integrity = el.getAttribute("integrity") || ""; // The host's own copy needs no network; one that fails the tag's integrity metadata is not used. - const bundled = bundledScript(src); - if (bundled && matchesScriptIntegrity(bundled, integrity)) { - return { src, text: new TextDecoder().decode(bundled) }; - } - const bytes = await downloadScript(src, mirror); - if (!matchesScriptIntegrity(bytes, integrity)) { - throw new ScriptIntegrityError(`Subresource integrity mismatch for ${src}`); + let bytes = bundledScript(src); + if (bytes && matchesScriptIntegrity(bytes, integrity)) { + charge(bytes.byteLength); + } else { + bytes = await fetchOnce(src); + if (!matchesScriptIntegrity(bytes, integrity)) { + fail(new ScriptIntegrityError(`Subresource integrity mismatch for ${src}`)); + } + if (placed.has(src)) charge(bytes.byteLength); + placed.add(src); } return { src, text: new TextDecoder().decode(bytes) }; }), ); + if (stop.signal.aborted) throw stop.signal.reason; for (let i = 0; i < downloads.length; i++) { const download = downloads[i]!; @@ -1904,6 +1950,13 @@ function matchingBraces(css: string): Map { return closes; } +/** The address of a stylesheet link this compile inlines the font rules of, if it is one. */ +function remoteFontStylesheetHref(tag: string): string | undefined { + const href = /\bhref=["']([^"']+)["']/i.exec(tag)?.[1]; + if (!href || !/^https?:\/\//i.test(href) || isGoogleFontsUrl(href)) return undefined; + return href; +} + /** * Find all external `` tags pointing to non-Google * CDNs, fetch their CSS, and replace the `` with an inline ``; - result = result.replace(fullMatch, inlineStyle); - defaultLogger.debug("[Compiler] Inlined external @font-face rule(s)", { - count: fontFaceBlocks.length, - urlFingerprint: identity.urlFingerprint, - host: identity.host, + const inlineByHref = new Map(); + const unique = [...hrefs]; + for (let i = 0; i < unique.length; i += MAX_CONCURRENT_STYLESHEET_FETCHES) { + const batch = unique.slice(i, i + MAX_CONCURRENT_STYLESHEET_FETCHES); + const sheets = await Promise.all(batch.map(fetchExternalStylesheetCss)); + batch.forEach((href, index) => { + const css = sheets[index]; + const fontFaceBlocks = css ? extractFontFaceBlocks(css) : []; + if (fontFaceBlocks.length === 0) return; + const inlineStyle = ``; + budget.chargeGenerated(REMOTE_STYLESHEETS, inlineStyle.length); + inlineByHref.set(href, inlineStyle); + const identity = safeDownloadUrlIdentity(href); + defaultLogger.debug("[Compiler] Inlined external @font-face rule(s)", { + count: fontFaceBlocks.length, + urlFingerprint: identity.urlFingerprint, + host: identity.host, + }); }); } - return result; + + const placed = new Set(); + return replaceMatches(html, stylesheetLinkMatches(html), (tag) => { + const href = remoteFontStylesheetHref(tag); + const inlineStyle = href ? inlineByHref.get(href) : undefined; + if (!href || !inlineStyle) return tag; + // The first copy was charged when it arrived; each repeat adds the whole sheet again. + if (placed.has(href)) budget.chargeGenerated(REMOTE_STYLESHEETS, inlineStyle.length); + placed.add(href); + return inlineStyle; + }); } // Families a linked project stylesheet declares with an @font-face file the project ships. @@ -2450,6 +2502,7 @@ export async function compileForRender( // Remote font stylesheets join the page first, or the font check below cannot see them. const fontSheetHtml = await inlineExternalFontStylesheets( promoteCssImportsToLinkTags(sanitizedHtml), + budget, ); const normalizedFontHtml = normalizeSystemFontPrimaryFamilies( injectTextRenderingRule(coalesceHeadStylesAndBodyScripts(fontSheetHtml)), @@ -2464,7 +2517,7 @@ export async function compileForRender( }); // CDN scripts are inlined after coalescing so they stay separate from the merged inline runs. - const assembledHtml = await inlineExternalScripts(coalescedHtml); + const assembledHtml = await inlineExternalScripts(coalescedHtml, budget); // Inject studio position seek re-apply script when positions are baked into HTML. // GSAP overwrites the `translate` CSS property on every frame seek; this script