fix(producer): compile and render-telemetry tag scans run in linear time (#7896)

* fix(producer): compile and render-telemetry tag scans run in linear time

The compiler's gsap script rewrite, crossorigin and preload strips, remote
media and image src scans and inline-script scan, and the render telemetry's
tag counts, now use forward scans that return exactly what the regexes they
replace returned. Those regexes rescanned to the end of the page from every
tag opener that never closed, or from every character of a long whitespace
run in one tag. The telemetry's script and style strip stops after 8 passes.

* test(producer): give the scan fuzz tests room on CI and raise the producer size budget
This commit is contained in:
Michael Wang
2026-11-16 02:08:44 +00:00
committed by GitHub
parent 478313ba0e
commit 82fdf406c4
6 changed files with 863 additions and 87 deletions
+28 -66
View File
@@ -29,6 +29,14 @@ import { createReadStream, existsSync, mkdirSync, readFileSync } from "fs";
import { join, dirname, resolve, basename, relative, sep } from "path";
import { parseHTML } from "linkedom";
import { bundledScript } from "./bundledScripts.js";
import {
inlineScriptBlocks,
remoteSrcUrls,
removeDelimitedRuns,
replaceGsapDistScripts,
stripCrossoriginAttr,
stripPreloadNone,
} from "./htmlTagScans.js";
import {
COMPOSITION_SOURCE_URL,
compileTimingAttrs,
@@ -393,22 +401,15 @@ export interface RenderModeHints {
reasons: RenderModeHint[];
}
const INLINE_SCRIPT_PATTERN = /<script\b([^>]*)>([\s\S]*?)<\/script>/gi;
const COMPILER_MOUNT_BLOCK_START = "/* __HF_COMPILER_MOUNT_START__ */";
const COMPILER_MOUNT_BLOCK_END = "/* __HF_COMPILER_MOUNT_END__ */";
function stripJsComments(source: string): string {
return source.replace(/\/\/.*$/gm, "").replace(/\/\*[\s\S]*?\*\//g, "");
return removeDelimitedRuns(source.replace(/\/\/.*$/gm, ""), "/*", "*/");
}
function stripCompilerMountBootstrap(source: string): string {
return source.replace(
new RegExp(
`${COMPILER_MOUNT_BLOCK_START.replace(/[.*+?^${}()|[\]\\]/g, "\\$&")}[\\s\\S]*?${COMPILER_MOUNT_BLOCK_END.replace(/[.*+?^${}()|[\]\\]/g, "\\$&")}`,
"g",
),
"",
);
return removeDelimitedRuns(source, COMPILER_MOUNT_BLOCK_START, COMPILER_MOUNT_BLOCK_END);
}
/** The composition the runtime reads data-motion-blur from: `data-root="true"`, else the first. */
@@ -449,12 +450,9 @@ export function detectRenderModeHints(html: string, pageMotionBlur?: "off"): Ren
});
}
let scriptMatch: RegExpExecArray | null;
const scriptPattern = new RegExp(INLINE_SCRIPT_PATTERN.source, INLINE_SCRIPT_PATTERN.flags);
while ((scriptMatch = scriptPattern.exec(html)) !== null) {
const attrs = scriptMatch[1] || "";
for (const { attrs, body } of inlineScriptBlocks(html)) {
if (/\bsrc\s*=/i.test(attrs)) continue;
const content = stripJsComments(stripCompilerMountBootstrap(scriptMatch[2] || ""));
const content = stripJsComments(stripCompilerMountBootstrap(body));
if (!/requestAnimationFrame\s*\(/.test(content)) continue;
reasons.push({
code: "requestAnimationFrame",
@@ -551,12 +549,9 @@ const SHADER_TRANSITION_USAGE_PATTERN =
/\b(?:(?:window|globalThis)\s*\.\s*)?HyperShader\s*\.\s*init\s*\(|\b__hf\s*\.\s*transitions\s*=/;
export function detectShaderTransitionUsage(html: string): boolean {
let scriptMatch: RegExpExecArray | null;
const scriptPattern = new RegExp(INLINE_SCRIPT_PATTERN.source, INLINE_SCRIPT_PATTERN.flags);
while ((scriptMatch = scriptPattern.exec(html)) !== null) {
const attrs = scriptMatch[1] || "";
for (const { attrs, body } of inlineScriptBlocks(html)) {
if (/\bsrc\s*=/i.test(attrs)) continue;
const content = stripJsComments(stripCompilerMountBootstrap(scriptMatch[2] || ""));
const content = stripJsComments(stripCompilerMountBootstrap(body));
if (SHADER_TRANSITION_USAGE_PATTERN.test(content)) return true;
}
@@ -809,13 +804,13 @@ async function compileHtmlFile(
// injected frame images, so the browser never needs to load the source.
// Without this, videos with crossorigin="anonymous" targeting CORS-restricted
// origins (e.g. S3 without CORS headers) keep readyState=0, blocking page setup.
compiledHtml = compiledHtml.replace(/(<video\b[^>]*)\s+crossorigin(?:=["'][^"']*["'])?/gi, "$1");
compiledHtml = stripCrossoriginAttr(compiledHtml, "video");
// Strip crossorigin from img elements. The renderer captures DOM frames visually —
// no canvas readback — so CORS compliance is unnecessary. External images from
// CORS-restricted origins (e.g. S3) render blank when crossorigin forces a failed
// CORS request against the renderer's localhost file server.
compiledHtml = compiledHtml.replace(/(<img\b[^>]*)\s+crossorigin(?:=["'][^"']*["'])?/gi, "$1");
compiledHtml = stripCrossoriginAttr(compiledHtml, "img");
// Strip crossorigin from audio elements. Audio is processed out-of-band via
// FFmpeg; the browser's CORS policy for audio elements is irrelevant to
@@ -824,7 +819,7 @@ async function compileHtmlFile(
// headers reject — leaving audio elements in a failed network state. The
// FFmpeg audio path reads the src URL directly and is unaffected by browser
// CORS, so stripping the attribute has no side effects.
compiledHtml = compiledHtml.replace(/(<audio\b[^>]*)\s+crossorigin(?:=["'][^"']*["'])?/gi, "$1");
compiledHtml = stripCrossoriginAttr(compiledHtml, "audio");
return { html: compiledHtml, unresolvedCompositions };
}
@@ -1540,22 +1535,6 @@ export function collectExternalAssets(
}
export const REMOTE_MEDIA_SUBDIR = "_remote_media";
// Match opening tags of <video> or <audio> elements that carry an HTTP(S) src.
// Uses [^>]* to span attributes — safe for composition elements that won't
// have `>` inside quoted attribute values (data-title etc.).
const REMOTE_MEDIA_TAG_RE =
/<(?:video|audio)\b[^>]*?\bsrc\s*=\s*["'](https?:\/\/[^"']+)["'][^>]*>/gi;
// <source src> on media elements (picture uses srcset, not src).
const REMOTE_SOURCE_TAG_RE = /<source\b[^>]*?\bsrc\s*=\s*["'](https?:\/\/[^"']+)["'][^>]*>/gi;
// Match <img> tags (including agent-pipeline-emitted variants where `src` is
// not the first attribute). Producer-side localisation is the primary fix for
// the remote-<img> flicker; frameCapture's `pollImagesReady`/`decodeAllImages`
// are the defense-in-depth layer for any remote URL that bypasses this step.
// The `(?<![\w-])` lookbehind pins the match to a real `src` attribute so we
// don't rewrite `data-src` / `data-*-src` (lazy-loader placeholders whose URL
// is not what Chrome actually paints). `srcset` is excluded by the `\s*=`.
const REMOTE_IMG_TAG_RE = /<img\b[^>]*?(?<![\w-])src\s*=\s*["'](https?:\/\/[^"']+)["'][^>]*>/gi;
/**
* Download a set of remote URLs in parallel into `remoteDir`, build the
* `{ relPath → absPath }` asset map, and rewrite every occurrence of each
@@ -1637,14 +1616,7 @@ export async function localizeRemoteMediaSources(
html: string,
downloadDir: string,
): Promise<{ html: string; remoteMediaAssets: Map<string, string> }> {
const urlSet = new Set<string>();
for (const tagRe of [REMOTE_MEDIA_TAG_RE, REMOTE_SOURCE_TAG_RE]) {
const re = new RegExp(tagRe.source, tagRe.flags);
let m: RegExpExecArray | null;
while ((m = re.exec(html)) !== null) {
if (m[1]) urlSet.add(m[1]);
}
}
const urlSet = new Set([...remoteSrcUrls(html, "media"), ...remoteSrcUrls(html, "source")]);
const localized = await downloadAndRewriteUrls(
urlSet,
html,
@@ -1700,12 +1672,8 @@ export async function localizeRemoteImageSources(
html: string,
downloadDir: string,
): Promise<{ html: string; remoteMediaAssets: Map<string, string> }> {
const urlSet = new Set<string>();
const re = new RegExp(REMOTE_IMG_TAG_RE.source, REMOTE_IMG_TAG_RE.flags);
let m: RegExpExecArray | null;
while ((m = re.exec(html)) !== null) {
if (m[1]) urlSet.add(m[1]);
}
// frameCapture's `pollImagesReady`/`decodeAllImages` cover any remote URL this misses.
const urlSet = new Set(remoteSrcUrls(html, "img"));
return downloadAndRewriteUrls(
urlSet,
html,
@@ -2145,16 +2113,13 @@ function rewriteUnresolvableGsapToCdn(
const file = resolve(projectDir, path);
return !namesAnotherNetworkRoot(projectDir, file) && existsSync(file);
};
return html.replace(
/(<script\b[^>]*\bsrc=["'])([^"']*gsap[^"']*\/dist\/([^"']+))(["'][^>]*>)/gi,
(full, prefix, src, file, suffix) => {
if (/^https?:\/\//i.test(src)) return full;
if (exists(compSrcPath ? rewriteAssetPath(compSrcPath, src, exists) : src)) return full;
gsapDist ??= renderGsapDist();
defaultLogger.debug(`[Compiler] Rewriting missing gsap script: ${src} → ${gsapDist}${file}`);
return `${prefix}${gsapDist}${file}${suffix}`;
},
);
return replaceGsapDistScripts(html, (full, prefix, src, file, suffix) => {
if (/^https?:\/\//i.test(src)) return full;
if (exists(compSrcPath ? rewriteAssetPath(compSrcPath, src, exists) : src)) return full;
gsapDist ??= renderGsapDist();
defaultLogger.debug(`[Compiler] Rewriting missing gsap script: ${src} → ${gsapDist}${file}`);
return `${prefix}${gsapDist}${file}${suffix}`;
});
}
/**
@@ -2292,10 +2257,7 @@ export async function compileForRender(
// media upfront for frame capture. Users add this to reduce browser memory in
// preview, but it causes the headless renderer to never load the media, leading
// to 45s timeout failures.
const sanitizedHtml = inlinedHtml.replace(
/(<(?:video|audio)\b[^>]*?)\s+preload\s*=\s*["']none["']/gi,
"$1",
);
const sanitizedHtml = stripPreloadNone(inlinedHtml);
const renderModeHints = detectRenderModeHints(sanitizedHtml, options.pageMotionBlur);
const hasShaderTransitions = detectShaderTransitionUsage(sanitizedHtml);
// Detected BEFORE inlineExternalScripts: GSAP's own source contains
@@ -0,0 +1,363 @@
import { describe, expect, it } from "vitest";
import {
countCompositionSrcMounts,
countTagsWithAttr,
countedTagNames,
firstCompositionIdTag,
inlineScriptBlocks,
remoteSrcUrls,
removeDelimitedRuns,
removeScriptAndStyleBlocks,
replaceGsapDistScripts,
stripCrossoriginAttr,
stripPreloadNone,
} from "./htmlTagScans.js";
// The regexes each scan replaces, kept here as the reference the scans must agree with.
const reference = {
inlineScripts: (html: string) =>
[...html.matchAll(/<script\b([^>]*)>([\s\S]*?)<\/script>/gi)].map((m) => ({
attrs: m[1],
body: m[2],
})),
blockComments: (source: string) => source.replace(/\/\*[\s\S]*?\*\//g, ""),
gsap: (html: string) =>
html.replace(
/(<script\b[^>]*\bsrc=["'])([^"']*gsap[^"']*\/dist\/([^"']+))(["'][^>]*>)/gi,
(...parts: string[]) => `[${parts.slice(0, 5).join("|")}]`,
),
crossorigin: (html: string, tag: string) =>
html.replace(new RegExp(`(<${tag}\\b[^>]*)\\s+crossorigin(?:=["'][^"']*["'])?`, "gi"), "$1"),
preload: (html: string) =>
html.replace(/(<(?:video|audio)\b[^>]*?)\s+preload\s*=\s*["']none["']/gi, "$1"),
remote: {
media: /<(?:video|audio)\b[^>]*?\bsrc\s*=\s*["'](https?:\/\/[^"']+)["'][^>]*>/gi,
source: /<source\b[^>]*?\bsrc\s*=\s*["'](https?:\/\/[^"']+)["'][^>]*>/gi,
img: /<img\b[^>]*?(?<![\w-])src\s*=\s*["'](https?:\/\/[^"']+)["'][^>]*>/gi,
},
scriptAndStyle: (markup: string) => markup.replace(/<(script|style)\b[^>]*>[\s\S]*?<\/\1>/gi, ""),
tagNames: (markup: string) =>
[
...markup.matchAll(
/<\/([a-zA-Z][-a-zA-Z0-9]*)|<(img|br|hr|input|source|track|area|base|col|embed|link|meta|param|wbr)\b|<([a-zA-Z][-a-zA-Z0-9]*)\b[^>]*\/>/gi,
),
].map((m) => m[1] ?? m[2] ?? m[3]),
aroll: (markup: string) =>
markup.match(/<video\b[^>]*\bdata-aroll=["']true["'][^>]*>/gi)?.length ?? 0,
heygen: (markup: string) =>
markup.match(/<video\b[^>]*\bdata-media-source=["']heygen["'][^>]*>/gi)?.length ?? 0,
subComps: (markup: string) =>
markup.match(/<[a-zA-Z][-a-zA-Z0-9]*\b[^>]*\bdata-composition-src=["'][^"']*["']/gi)?.length ??
0,
rootTag: (html: string) =>
html.match(/<[a-zA-Z][-a-zA-Z0-9]*\b[^>]*\bdata-composition-id=["'][^"']*["'][^>]*>/i)?.[0],
};
const CASES = 20_000;
const TOKENS = [
"<",
">",
"/",
"/>",
"</",
'"',
"'",
"=",
" ",
" ",
"\t",
"\n",
" ",
"_",
"-",
"x",
"1",
"<script",
"<SCRIPT",
"</script>",
"</Script>",
"<style",
"</style>",
"<scr",
"ipt>",
"<video",
"<VIDEO",
"<audio",
"<img",
"<source",
"<div",
"<a_b",
"<x-",
"<br",
"<col",
"<circle",
"src",
"src=",
" src=",
"data-src=",
"SRC = ",
"crossorigin",
" crossorigin",
"=anonymous",
" preload",
"preload = ",
"none",
"http://",
"https://",
"gsap",
"/dist/",
"dist/",
".js",
"data-composition-id=",
" data-composition-id=",
"data-composition-src=",
"data-aroll=",
"true",
" data-media-source=",
"heygen",
"/*",
"*/",
"MOUNT{",
"}MOUNT",
];
function mulberry32(seed: number): () => number {
let state = seed;
return () => {
state = (state + 0x6d2b79f5) | 0;
let t = Math.imul(state ^ (state >>> 15), 1 | state);
t = (t + Math.imul(t ^ (t >>> 7), 61 | t)) ^ t;
return ((t ^ (t >>> 14)) >>> 0) / 4294967296;
};
}
// Smaller alphabets per scan, so most inputs come close to a match and many do match.
const FOCUSED = {
scripts: [
"<script",
"<STYLE",
"<style",
"</script>",
"</style>",
"</STYLE>",
">",
" ",
"x",
"<",
"/*",
"*/",
],
gsap: [
"<script",
"<script ",
' src="',
" src='",
"-src='",
"gsap",
"/dist/",
"gsap/dist/x.js",
'"gsap/dist/',
"GSAP/DIST/a",
'"',
"'",
">",
"x",
" ",
"<",
"/",
],
attrs: [
"<video",
"<img",
"<audio",
" ",
" ",
"\t",
" crossorigin",
' crossorigin="a"',
" CROSSORIGIN='x'",
"crossorigin",
"=",
'"',
"'",
"x",
">",
"<",
' preload="none"',
" preload = 'NONE'",
" preload=none",
' preload="no',
"preload",
],
remote: [
"<video",
"<audio",
"<source",
"<img",
" ",
' src="http://a',
" src='https://b",
' data-src="http://c',
" -src='http://",
' src = "https://',
' src="',
'"',
"'",
"x",
">",
"<",
"http://",
],
telemetry: [
"<video",
"<video ",
"<a",
"<a ",
"<a_b",
"<x-",
"<a1",
" ",
' data-aroll="true"',
" data-aroll='true",
"data-aroll=",
' data-media-source="heygen"',
" data-media-source='heygen",
'"',
"'",
">",
"/>",
"<",
' data-composition-src="',
" data-composition-src='a'",
' data-composition-id="',
"data-composition-id='r'",
"x",
"-",
"</",
"_",
"<script",
"</script>",
"<style",
"</style>",
],
};
function* randomInputs(seed: number, count: number, tokens = TOKENS): Generator<string> {
const random = mulberry32(seed);
for (let i = 0; i < count; i++) {
const length = 1 + Math.floor(random() * 40);
let text = "";
for (let j = 0; j < length; j++) text += tokens[Math.floor(random() * tokens.length)];
yield text;
}
}
const both = (seed: number, focused: readonly string[]) => [
...randomInputs(seed, CASES),
...randomInputs(seed + 100, CASES, focused),
];
// Each case runs tens of thousands of inputs through the old regexes too, which takes seconds on a CI runner.
describe("htmlTagScans agree with the regexes they replace", { timeout: 60_000 }, () => {
it("inline script blocks", () => {
for (const html of both(1, FOCUSED.scripts)) {
expect([...inlineScriptBlocks(html)], html).toEqual(reference.inlineScripts(html));
}
});
it("delimited runs", () => {
for (const text of both(2, FOCUSED.scripts)) {
expect(removeDelimitedRuns(text, "/*", "*/"), text).toBe(reference.blockComments(text));
expect(removeDelimitedRuns(text, "MOUNT{", "}MOUNT"), text).toBe(
text.replace(/MOUNT\{[\s\S]*?\}MOUNT/g, ""),
);
}
});
it("gsap dist script rewrite", () => {
for (const html of both(3, FOCUSED.gsap)) {
const ours = replaceGsapDistScripts(html, (...parts) => `[${parts.join("|")}]`);
expect(ours, html).toBe(reference.gsap(html));
}
});
it("crossorigin and preload strips", () => {
for (const html of both(4, FOCUSED.attrs)) {
for (const tag of ["video", "img", "audio"] as const) {
expect(stripCrossoriginAttr(html, tag), html).toBe(reference.crossorigin(html, tag));
}
expect(stripPreloadNone(html), html).toBe(reference.preload(html));
}
});
it("remote src urls", () => {
for (const html of both(5, FOCUSED.remote)) {
for (const tag of ["media", "source", "img"] as const) {
const expected = [...html.matchAll(reference.remote[tag])].map((m) => m[1]);
expect(remoteSrcUrls(html, tag), html).toEqual(expected);
}
}
});
it("script and style removal, tag names and attribute counts", () => {
for (const markup of both(6, FOCUSED.telemetry)) {
expect(removeScriptAndStyleBlocks(markup), markup).toBe(reference.scriptAndStyle(markup));
expect([...countedTagNames(markup)], markup).toEqual(reference.tagNames(markup));
expect(countTagsWithAttr(markup, "video", `\\bdata-aroll=["']true["']`), markup).toBe(
reference.aroll(markup),
);
expect(
countTagsWithAttr(markup, "video", `\\bdata-media-source=["']heygen["']`),
markup,
).toBe(reference.heygen(markup));
expect(countCompositionSrcMounts(markup), markup).toBe(reference.subComps(markup));
expect(firstCompositionIdTag(markup), markup).toBe(reference.rootTag(markup));
}
});
});
const BUDGET_MS = 400;
function elapsedMs(run: () => unknown): number {
const start = performance.now();
run();
return performance.now() - start;
}
// About 1 MB each: openers that never close, and one tag whose whitespace never reaches the attribute.
const hostile = (opener: string) => opener.repeat(Math.ceil(1_000_000 / opener.length));
const longGap = (tag: string, tail: string) => `${tag}${" ".repeat(1_000_000)}${tail}`;
describe("htmlTagScans run in linear time", () => {
const scans: Array<[string, (html: string) => unknown, string[]]> = [
["inline scripts", (h) => [...inlineScriptBlocks(h)], ["<script ", "<script>"]],
["block comments", (h) => removeDelimitedRuns(h, "/*", "*/"), ["/*"]],
[
"gsap rewrite",
(h) => replaceGsapDistScripts(h, (full) => full),
["<script ", '<script src="gsap/dist/x ', "<script src='a'>"],
],
["crossorigin", (h) => stripCrossoriginAttr(h, "video"), ["<video ", "<video crossorigin "]],
["preload", stripPreloadNone, ["<video ", '<video preload="none" ']],
["remote media", (h) => remoteSrcUrls(h, "media"), ["<video ", '<video src="http://a" ']],
["remote img", (h) => remoteSrcUrls(h, "img"), ["<img ", "<img src='https://a "]],
["script/style removal", removeScriptAndStyleBlocks, ["<script ", "<style>", "<script>"]],
["tag names", (h) => [...countedTagNames(h)], ["<a ", "<circle "]],
["aroll count", (h) => countTagsWithAttr(h, "video", "\\bdata-aroll"), ["<video "]],
["sub-composition count", countCompositionSrcMounts, ["<a ", '<a data-composition-src="']],
["root tag", firstCompositionIdTag, ["<a ", '<a data-composition-id="x" ']],
];
for (const [name, scan, openers] of scans) {
it(`${name}: 1 MB of unclosed openers and one tag with a 1 MB whitespace run`, () => {
for (const opener of openers)
expect(elapsedMs(() => scan(hostile(opener)))).toBeLessThan(BUDGET_MS);
for (const tag of ["<video", "<img", "<script", "<a"]) {
for (const tail of ["x>", 'crossorigin="a">', "preload", "src=", "/>", ""]) {
expect(elapsedMs(() => scan(longGap(tag, tail)))).toBeLessThan(BUDGET_MS);
}
}
});
}
});
@@ -0,0 +1,414 @@
/**
* Forward scans that return exactly what the compiler's and the render telemetry's tag regexes
* return, in time linear in the HTML. Each of those regexes rescans to the end of the page from
* every tag opener that never closes, or from every character of a long whitespace run in one tag.
*
* Two facts make the scans below exact. A `[^>]*` after an opener can only reach the first `>`
* after it, so every opener inside that span sees the same span, minus a prefix. And the parts
* matched after an attribute (a quoted value, the closing `>`) do not depend on where the tag began.
*/
type Finder = (from: number) => RegExpExecArray | null;
/** The first match of the global `pattern` at or after `from`. Cheap when `from` only grows. */
function forwardFinder(text: string, pattern: RegExp): Finder {
let start = -1;
let found: RegExpExecArray | null = null;
return (from) => {
if (start < 0 || from < start || (found !== null && from > found.index)) {
start = from;
pattern.lastIndex = from;
found = pattern.exec(text);
}
return found;
};
}
function indexFinder(text: string, pattern: RegExp): (from: number) => number {
const find = forwardFinder(text, pattern);
return (from) => find(from)?.index ?? -1;
}
const isLetter = (code: number) => (code >= 65 && code <= 90) || (code >= 97 && code <= 122);
const isDigit = (code: number) => code >= 48 && code <= 57;
const isNameChar = (code: number) => isLetter(code) || isDigit(code) || code === 45;
const isWordChar = (code: number) => isLetter(code) || isDigit(code) || code === 95;
/** The end of the `[-a-zA-Z0-9]*` run that starts at `from`. */
function nameRunEnd(text: string, from: number): number {
let end = from;
while (isNameChar(text.charCodeAt(end))) end++;
return end;
}
/**
* Where `<[a-zA-Z][-a-zA-Z0-9]*\b` ends when it matches at `open` (a `<` before a letter), taking
* the longest name the `\b` allows. -1 when no name length leaves a word boundary after it.
*/
function tagNameEnd(text: string, open: number): number {
for (let end = nameRunEnd(text, open + 2); end >= open + 2; end--) {
if (isWordChar(text.charCodeAt(end - 1)) !== isWordChar(text.charCodeAt(end))) return end;
}
return -1;
}
/** The `attrs` and `body` groups of `/<script\b([^>]*)>([\s\S]*?)<\/script>/gi`, match by match. */
export function* inlineScriptBlocks(html: string): Generator<{ attrs: string; body: string }> {
const opener = /<script\b/gi;
const closer = /<\/script>/gi;
for (let open = opener.exec(html); open; open = opener.exec(html)) {
const attrsStart = open.index + open[0].length;
const gt = html.indexOf(">", attrsStart);
if (gt < 0) return;
closer.lastIndex = gt + 1;
const close = closer.exec(html);
if (!close) return;
yield { attrs: html.slice(attrsStart, gt), body: html.slice(gt + 1, close.index) };
opener.lastIndex = close.index + close[0].length;
}
}
/** `source.replace(/<start>[\s\S]*?<end>/g, "")` for the literal strings `start` and `end`. */
export function removeDelimitedRuns(source: string, start: string, end: string): string {
let out = "";
let kept = 0;
for (let open = source.indexOf(start); open >= 0; open = source.indexOf(start, kept)) {
const close = source.indexOf(end, open + start.length);
if (close < 0) break;
out += source.slice(kept, open);
kept = close + end.length;
}
return out + source.slice(kept);
}
/** What `[^"']*gsap[^"']*\/dist\/([^"']+)` captures over a whole attribute value, or undefined. */
function gsapDistFile(value: string): string | undefined {
let dist = -1;
const distPattern = /\/dist\//gi;
for (let m = distPattern.exec(value); m; m = distPattern.exec(value)) {
if (m.index + 6 < value.length) dist = m.index;
distPattern.lastIndex = m.index + 1;
}
const gsap = dist < 0 ? null : /gsap/i.exec(value);
return gsap && gsap.index + 4 <= dist ? value.slice(dist + 6) : undefined;
}
type GsapScriptReplacer = (
full: string,
prefix: string,
src: string,
file: string,
suffix: string,
) => string;
type GsapMatch = { valueStart: number; close: number; end: number; file: string };
/**
* The last `src=` in one `<script` tag that the gsap rewrite matches, which is the one the greedy
* `[^>]*` keeps. `laterTagsFail` is set when a value never closes or no `>` follows it, since then
* no later value can close or reach a `>` either.
*/
function lastGsapMatch(
html: string,
finders: {
nextSrc: Finder;
nextQuote: (from: number) => number;
nextGt: (from: number) => number;
},
tagStart: number,
gt: number,
): { match?: GsapMatch; laterTagsFail: boolean } {
const { nextSrc, nextQuote, nextGt } = finders;
let match: GsapMatch | undefined;
for (let src = nextSrc(tagStart); src && src.index < gt; src = nextSrc(src.index + 1)) {
const valueStart = src.index + 5;
const close = nextQuote(valueStart);
const end = close < 0 ? -1 : nextGt(close + 1);
if (end < 0) return { match, laterTagsFail: true };
const file = gsapDistFile(html.slice(valueStart, close));
if (file !== undefined) match = { valueStart, close, end, file };
}
return { match, laterTagsFail: false };
}
/** `html.replace(/(<script\b[^>]*\bsrc=["'])([^"']*gsap[^"']*\/dist\/([^"']+))(["'][^>]*>)/gi, replace)`. */
export function replaceGsapDistScripts(html: string, replace: GsapScriptReplacer): string {
const opener = /<script\b/gi;
const finders = {
nextSrc: forwardFinder(html, /\bsrc=["']/gi),
nextQuote: indexFinder(html, /["']/g),
nextGt: indexFinder(html, />/g),
};
let out = "";
let kept = 0;
for (let open = opener.exec(html); open; open = opener.exec(html)) {
const gt = finders.nextGt(open.index);
if (gt < 0) break;
const { match, laterTagsFail } = lastGsapMatch(html, finders, open.index, gt);
if (match) {
const { valueStart, close, end, file } = match;
const full = html.slice(open.index, end + 1);
const src = html.slice(valueStart, close);
out += html.slice(kept, open.index);
out += replace(
full,
html.slice(open.index, valueStart),
src,
file,
html.slice(close, end + 1),
);
kept = end + 1;
}
if (laterTagsFail) break;
opener.lastIndex = match ? kept : gt;
}
return out + html.slice(kept);
}
/**
* Removes one span from each start tag `opener` finds: the span `spanIn` returns for the text
* from the end of the tag name to the tag's first `>` (or to the end of the page when there is
* none). The span may run past that `>`. Scanning resumes after each removed span, as a global
* `replace` would.
*/
function removeSpanPerTag(
html: string,
opener: RegExp,
spanIn: (attrsStart: number, tagEnd: number) => [number, number] | undefined,
): string {
const nextGt = indexFinder(html, />/g);
let out = "";
let kept = 0;
for (let open = opener.exec(html); open; open = opener.exec(html)) {
const attrsStart = open.index + open[0].length;
const gt = nextGt(attrsStart);
const span = spanIn(attrsStart, gt < 0 ? html.length : gt);
if (span) {
out += html.slice(kept, span[0]);
kept = span[1];
} else if (gt < 0) break;
opener.lastIndex = span ? kept : gt;
}
return out + html.slice(kept);
}
/** `html.replace(/(<tagName\b[^>]*)\s+crossorigin(?:=["'][^"']*["'])?/gi, "$1")`. */
export function stripCrossoriginAttr(html: string, tagName: "video" | "img" | "audio"): string {
const nextQuote = indexFinder(html, /["']/g);
const nextAttr = forwardFinder(html, /\scrossorigin/gi);
return removeSpanPerTag(html, new RegExp(`<${tagName}\\b`, "gi"), (attrsStart, tagEnd) => {
// The greedy `[^>]*` leaves the last whitespace character before the last `crossorigin`.
let last = -1;
for (let at = nextAttr(attrsStart); at && at.index < tagEnd; at = nextAttr(at.index + 12)) {
last = at.index;
}
if (last < 0) return undefined;
const nameEnd = last + 12;
const quote = html[nameEnd + 1];
const quoted = html[nameEnd] === "=" && (quote === '"' || quote === "'");
const close = quoted ? nextQuote(nameEnd + 2) : -1;
return [last, close < 0 ? nameEnd : close + 1];
});
}
/** `html.replace(/(<(?:video|audio)\b[^>]*?)\s+preload\s*=\s*["']none["']/gi, "$1")`. */
export function stripPreloadNone(html: string): string {
// Starting only where a whitespace run starts finds the same first match without retrying
// `\s+` from every character of the run.
const nextAttr = forwardFinder(html, /(?<!\s)\s+preload\s*=\s*["']none["']/gi);
return removeSpanPerTag(html, /<(?:video|audio)\b/gi, (attrsStart, tagEnd) => {
const attr = nextAttr(attrsStart);
return attr && attr.index < tagEnd ? [attr.index, attr.index + attr[0].length] : undefined;
});
}
// For <img> only a real `src` counts, never `data-src` or `data-*-src` (lazy-loader placeholders
// whose URL is not what Chrome paints). `srcset` never counts, since `\s*=` must follow `src`.
const REMOTE_SRC_TAGS = {
media: { opener: "<(?:video|audio)\\b", attr: /\bsrc\s*=\s*["'](https?:\/\/)/gi },
source: { opener: "<source\\b", attr: /\bsrc\s*=\s*["'](https?:\/\/)/gi },
img: { opener: "<img\\b", attr: /(?<![\w-])src\s*=\s*["'](https?:\/\/)/gi },
} as const;
/**
* Group 1 of every match of `/<tag\b[^>]*?src\s*=\s*["'](https?:\/\/[^"']+)["'][^>]*>/gi`, in
* order: `<video>`/`<audio>` and `<source>` use `\bsrc`, `<img>` uses `(?<![\w-])src`.
*/
export function remoteSrcUrls(html: string, tag: keyof typeof REMOTE_SRC_TAGS): string[] {
const opener = new RegExp(REMOTE_SRC_TAGS[tag].opener, "gi");
const nextAttr = forwardFinder(html, new RegExp(REMOTE_SRC_TAGS[tag].attr));
const nextGt = indexFinder(html, />/g);
const nextQuote = indexFinder(html, /["']/g);
const urls: string[] = [];
for (let open = opener.exec(html); open; open = opener.exec(html)) {
const attrsStart = open.index + open[0].length;
const gt = nextGt(attrsStart);
if (gt < 0) break;
let resumeAt = gt;
// The lazy `[^>]*?` takes the first `src=` that leads to a full match.
for (
let attr = nextAttr(attrsStart);
attr && attr.index < gt;
attr = nextAttr(attr.index + 1)
) {
const urlRest = attr.index + attr[0].length;
const close = nextQuote(urlRest);
if (close < 0) return urls;
if (close === urlRest) continue;
const end = nextGt(close + 1);
if (end < 0) return urls;
urls.push(html.slice(urlRest - attr[1]!.length, close));
resumeAt = end + 1;
break;
}
opener.lastIndex = resumeAt;
}
return urls;
}
/** One pass of `markup.replace(/<(script|style)\b[^>]*>[\s\S]*?<\/\1>/gi, "")`. */
export function removeScriptAndStyleBlocks(markup: string): string {
const opener = /<(script|style)\b/gi;
const nextGt = indexFinder(markup, />/g);
const closers: Record<string, Finder> = {
script: forwardFinder(markup, /<\/script>/gi),
style: forwardFinder(markup, /<\/style>/gi),
};
let out = "";
let kept = 0;
for (let open = opener.exec(markup); open; open = opener.exec(markup)) {
const gt = nextGt(open.index + open[0].length);
if (gt < 0) break;
const close = closers[open[1]!.toLowerCase()]!(gt + 1);
if (!close) continue;
out += markup.slice(kept, open.index);
kept = close.index + close[0].length;
opener.lastIndex = kept;
}
return out + markup.slice(kept);
}
const VOID_TAG = /(img|br|hr|input|source|track|area|base|col|embed|link|meta|param|wbr)\b/iy;
/** The name the counting regex captures at the `<` at `open`, and where its match ends. */
function countedTagAt(
markup: string,
open: number,
nextGt: (from: number) => number,
): { name: string; end: number } | undefined {
const first = markup.charCodeAt(open + 1);
if (first === 47 && isLetter(markup.charCodeAt(open + 2))) {
const end = nameRunEnd(markup, open + 3);
return { name: markup.slice(open + 2, end), end };
}
if (!isLetter(first)) return undefined;
VOID_TAG.lastIndex = open + 1;
const known = VOID_TAG.exec(markup);
if (known) return { name: known[1]!, end: open + 1 + known[0].length };
const nameEnd = tagNameEnd(markup, open);
const gt = nameEnd < 0 ? -1 : nextGt(open);
if (gt < 0 || markup.charCodeAt(gt - 1) !== 47) return undefined;
return { name: markup.slice(open + 1, nameEnd), end: gt + 1 };
}
/**
* The tag name each match of
* `/<\/([a-zA-Z][-a-zA-Z0-9]*)|<(img|br|hr|input|source|track|area|base|col|embed|link|meta|param|wbr)\b|<([a-zA-Z][-a-zA-Z0-9]*)\b[^>]*\/>/gi`
* captures, in order and with its original case.
*/
export function* countedTagNames(markup: string): Generator<string> {
const nextGt = indexFinder(markup, />/g);
for (let open = markup.indexOf("<"); open >= 0; ) {
const tag = countedTagAt(markup, open, nextGt);
if (tag) yield tag.name;
open = markup.indexOf("<", tag ? tag.end : open + 1);
}
}
/** How many matches `/<tagName\b[^>]*ATTR[^>]*>/gi` has, for an `attr` source that matches no `>`. */
export function countTagsWithAttr(markup: string, tagName: string, attr: string): number {
const opener = new RegExp(`<${tagName}\\b`, "gi");
const nextGt = indexFinder(markup, />/g);
const nextAttr = forwardFinder(markup, new RegExp(attr, "gi"));
let count = 0;
for (let open = opener.exec(markup); open; open = opener.exec(markup)) {
const attrsStart = open.index + open[0].length;
const gt = nextGt(attrsStart);
if (gt < 0) break;
const found = nextAttr(attrsStart);
const inTag = found !== null && found.index < gt;
if (inTag) count++;
opener.lastIndex = inTag ? gt + 1 : gt;
}
return count;
}
/**
* The closing quotes of the last two quoted `name` values that start inside one tag, -1 when
* absent. `unclosed` is set when a value never closes, since no later value can close either.
*/
function lastQuotedValueCloses(
nextAttr: Finder,
nextQuote: (from: number) => number,
from: number,
tagEnd: number,
): { close: number; earlierClose: number; unclosed: boolean } {
let close = -1;
let earlierClose = -1;
for (let attr = nextAttr(from); attr && attr.index < tagEnd; attr = nextAttr(attr.index + 1)) {
const quote = nextQuote(attr.index + attr[0].length);
if (quote < 0) return { close, earlierClose, unclosed: true };
earlierClose = close;
close = quote;
}
return { close, earlierClose, unclosed: false };
}
/**
* Each start tag with a quoted `name` attribute, in order, as
* `/<[a-zA-Z][-a-zA-Z0-9]*\b[^>]*\bname=["'][^"']*["']/gi` matches it: the tag's start, the quote
* that closes the last such value in the tag, and the quote that closes the one before it. A
* value may run past the tag's first `>`.
*/
function* tagsWithQuotedAttr(
html: string,
name: string,
): Generator<{ start: number; close: number; earlierClose: number }> {
const opener = /<[a-zA-Z]/g;
const nextGt = indexFinder(html, />/g);
const nextQuote = indexFinder(html, /["']/g);
const nextAttr = forwardFinder(html, new RegExp(`\\b${name}=["']`, "gi"));
for (let open = opener.exec(html); open; open = opener.exec(html)) {
if (tagNameEnd(html, open.index) < 0) continue;
const gt = nextGt(open.index);
const tagEnd = gt < 0 ? html.length : gt;
const { close, earlierClose, unclosed } = lastQuotedValueCloses(
nextAttr,
nextQuote,
open.index + 2,
tagEnd,
);
if (close >= 0) yield { start: open.index, close, earlierClose };
if (unclosed || (close < 0 && gt < 0)) return;
opener.lastIndex = close >= 0 ? close + 1 : gt;
}
}
/** How many matches `/<[a-zA-Z][-a-zA-Z0-9]*\b[^>]*\bdata-composition-src=["'][^"']*["']/gi` has. */
export function countCompositionSrcMounts(markup: string): number {
let count = 0;
for (const _ of tagsWithQuotedAttr(markup, "data-composition-src")) count++;
return count;
}
/** What `html.match(/<[a-zA-Z][-a-zA-Z0-9]*\b[^>]*\bdata-composition-id=["'][^"']*["'][^>]*>/i)?.[0]` returns. */
export function firstCompositionIdTag(html: string): string | undefined {
for (const { start, close, earlierClose } of tagsWithQuotedAttr(html, "data-composition-id")) {
// With no `>` after the last value, the match falls back to the value before it, which ends
// before the tag's `>` when the tag has one. No later tag can match either way.
let end = html.indexOf(">", close + 1);
if (end < 0 && earlierClose >= 0) end = html.indexOf(">", earlierClose + 1);
return end < 0 ? undefined : html.slice(start, end + 1);
}
return undefined;
}
@@ -2467,6 +2467,34 @@ describe("shouldPreferSingleWorkerDrawElement (DE priority inversion)", () => {
});
describe("scanElementTags", () => {
it("takes linear time on 1 MB of tags that never close or nest re-formed scripts", () => {
const root = '<div data-composition-id="r" data-width="100" data-height="100">';
const megabyte = (piece: string) => piece.repeat(Math.ceil(1_000_000 / piece.length));
const nested = (depth: number) =>
`${"<scr".repeat(depth)}<script></script>${"ipt></script>".repeat(depth)}`;
const inputs = [
megabyte("<a "),
megabyte("<script "),
megabyte("<style>"),
megabyte('<video data-aroll="true" '),
megabyte('<a data-composition-src="'),
megabyte('<a data-composition-id="x" '),
megabyte('<a data-color-grading="'),
root + megabyte("html,body{width:"),
`<video${" ".repeat(1_000_000)}x>`,
nested(70_000),
];
for (const html of inputs) {
const start = performance.now();
scanElementTags(html);
expect(performance.now() - start).toBeLessThan(1000);
}
});
it("strips script blocks re-formed by an earlier pass", () => {
expect(scanElementTags("<scr<script></script>ipt><b></b></script><i></i>").total).toBe(1);
});
it("counts closing tags", () => {
expect(scanElementTags("<div><span>a</span></div>").total).toBe(2);
});
@@ -58,6 +58,13 @@ import {
} from "@hyperframes/core";
import { HF_AUDIO_GROUP_TAG } from "@hyperframes/core/audio-groups";
import type { CompileForRenderOptions } from "./htmlCompiler.js";
import {
countCompositionSrcMounts,
countTagsWithAttr,
countedTagNames,
firstCompositionIdTag,
removeScriptAndStyleBlocks,
} from "./htmlTagScans.js";
import { matchHtmlBodyCssHeightFirst, matchHtmlBodyCssWidthFirst } from "@hyperframes/parsers";
import {
type EngineConfig,
@@ -1739,13 +1746,10 @@ export interface ElementTagScan {
}
const MAX_REPORTED_ELEMENT_TAGS = 50;
const MAX_SCRIPT_STYLE_STRIP_PASSES = 8;
/** none < self < backdrop — mirrors @hyperframes/core vfx.ts's internal CAPTURE_RANK, duplicated here (not exported) to combine per-host captures across an entire composition's static scan. */
const VFX_CAPTURE_RANK: Record<HfVfxCapture, number> = { none: 0, self: 1, backdrop: 2 };
/** First element carrying data-composition-id, the same root marker other `[data-composition-id]` queries here use. */
const ROOT_COMPOSITION_TAG_RE =
/<[a-zA-Z][-a-zA-Z0-9]*\b[^>]*\bdata-composition-id=["'][^"']*["'][^>]*>/i;
function bucketPxDelta(delta: number): NonNullable<ElementTagScan["rootBodyDeltaPxBucket"]> {
if (delta === 0) return "0";
if (delta <= 10) return "1-10";
@@ -1759,7 +1763,8 @@ function bucketPxDelta(delta: number): NonNullable<ElementTagScan["rootBodyDelta
function detectRootBodySizeMismatch(
html: string,
): Pick<ElementTagScan, "rootBodyMismatch" | "rootBodyDeltaPxBucket"> {
const rootTag = html.match(ROOT_COMPOSITION_TAG_RE)?.[0];
// The first element carrying data-composition-id, the root marker other queries here use.
const rootTag = firstCompositionIdTag(html);
const dataWidth = rootTag && Number(rootTag.match(/\bdata-width=["'](\d+)["']/i)?.[1]);
const dataHeight = rootTag && Number(rootTag.match(/\bdata-height=["'](\d+)["']/i)?.[1]);
if (!dataWidth || !dataHeight) return {};
@@ -1836,20 +1841,24 @@ export function scanElementTags(html: string): ElementTagScan {
// CodeQL flags as incomplete multi-character sanitization. The impact here
// is nil — the stripped string is counted and discarded, never rendered —
// but the incompleteness is real, and a stray reformed tag would perturb
// the count this gate reads. Converges: every iteration strictly shortens
// the string or changes nothing and exits.
// the count this gate reads. Each pass takes time linear in the markup, and
// the loop stops after MAX_SCRIPT_STYLE_STRIP_PASSES passes. Only a tag
// re-formed from pieces nested more deeply than that stops short of the
// fixed point, which no real composition does.
let markup = html;
for (let previous = ""; markup !== previous; ) {
for (
let previous = "", pass = 0;
markup !== previous && pass < MAX_SCRIPT_STYLE_STRIP_PASSES;
pass++
) {
previous = markup;
markup = markup.replace(/<(script|style)\b[^>]*>[\s\S]*?<\/\1>/gi, "");
markup = removeScriptAndStyleBlocks(markup);
}
let total = 0;
const counts = new Map<string, number>();
for (const m of markup.matchAll(
/<\/([a-zA-Z][-a-zA-Z0-9]*)|<(img|br|hr|input|source|track|area|base|col|embed|link|meta|param|wbr)\b|<([a-zA-Z][-a-zA-Z0-9]*)\b[^>]*\/>/gi,
)) {
for (const name of countedTagNames(markup)) {
total++;
const tag = (m[1] ?? m[2] ?? m[3] ?? "").toLowerCase();
const tag = name.toLowerCase();
counts.set(tag, (counts.get(tag) ?? 0) + 1);
}
const ranked = [...counts.entries()].sort((a, b) => b[1] - a[1]);
@@ -1868,14 +1877,16 @@ export function scanElementTags(html: string): ElementTagScan {
// gate), but this count is scoped to <video> only, matching its name: a-roll
// is a video-editing term for primary-take footage, and every real usage in
// this codebase's fixtures stamps it on video elements.
const arollVideoCount =
markup.match(/<video\b[^>]*\bdata-aroll=["']true["'][^>]*>/gi)?.length ?? 0;
const arollVideoCount = countTagsWithAttr(markup, "video", `\\bdata-aroll=["']true["']`);
// `data-media-source="heygen"` is the media-use skill's own provenance
// stamp (see resolve.md), written only when the mounted video's ledger
// record traces to the "heygen.video" provider — never any other provider
// or value, so a plain presence check is exact, not a substring guess.
const heygenVideoCount =
markup.match(/<video\b[^>]*\bdata-media-source=["']heygen["'][^>]*>/gi)?.length ?? 0;
const heygenVideoCount = countTagsWithAttr(
markup,
"video",
`\\bdata-media-source=["']heygen["']`,
);
// Uncapped `counts` Map, not `byTag`: 50+ distinct tags could push these
// into the "other" bucket, zeroing a named field the cap shouldn't affect.
const audioCount = counts.get("audio") ?? 0;
@@ -1884,9 +1895,7 @@ export function scanElementTags(html: string): ElementTagScan {
// Not collectSubCompositionSrcs (@hyperframes/parsers/asset-resolution):
// that dedupes by src and drops placeholder/remote mounts, so its length is
// distinct resolvable sub-comps, not this field's mount-element count.
const subCompositionCount =
markup.match(/<[a-zA-Z][-a-zA-Z0-9]*\b[^>]*\bdata-composition-src=["'][^"']*["']/gi)?.length ??
0;
const subCompositionCount = countCompositionSrcMounts(markup);
let colorGradingCount = 0;
let hasLut = false;
for (const m of markup.matchAll(
+1 -1
View File
@@ -36,7 +36,7 @@
"unpacked": 541601
},
"@hyperframes/producer": {
"packed": 901000,
"packed": 921000,
"unpacked": 3290000
},
"@hyperframes/sdk": {