mirror of
https://github.com/heygen-com/hyperframes.git
synced 2026-10-30 09:16:55 +00:00
fix(cli): validate captured Lottie archives and previews (#3811)
* fix(cli): validate captured Lottie archives and previews * fix(cli): bound Lottie discovery before buffering * fix(cli): bound Lottie discovery lifecycle and archive paths * fix(cli): retain highest-priority Lottie candidates
This commit is contained in:
@@ -1,3 +1,4 @@
|
||||
import { LottieDiscovery } from "./lottieDiscovery.js";
|
||||
import { createCaptureDownloadBudget } from "./readBoundedResponse.js";
|
||||
/**
|
||||
* Website capture orchestrator.
|
||||
@@ -84,6 +85,7 @@ export async function captureWebsite(
|
||||
onPhase,
|
||||
} = opts;
|
||||
|
||||
const downloadByteBudget = createCaptureDownloadBudget();
|
||||
const warnings: string[] = [];
|
||||
const progress = (stage: string, detail?: string) => {
|
||||
onProgress?.(stage, detail);
|
||||
@@ -204,51 +206,20 @@ export async function captureWebsite(
|
||||
|
||||
// Intercept network responses to detect Lottie JSON files
|
||||
const discoveredLotties: DiscoveredLottie[] = [];
|
||||
const lottieDiscovery = new LottieDiscovery();
|
||||
// Layer 1 (passive video discovery): every direct-video URL the page fetches
|
||||
// over the whole session (load / scroll / carousel rotation), independent of
|
||||
// whether a <video> for it exists at snapshot time. captureVideoManifest
|
||||
// downloads these (guarded) and merges them into the manifest.
|
||||
const discoveredVideoUrls = new Set<string>();
|
||||
// fallow-ignore-next-line complexity
|
||||
page1.on("response", async (response) => {
|
||||
page1.on("response", (response) => {
|
||||
try {
|
||||
const responseUrl = response.url();
|
||||
if (/\.(mp4|webm|mov|m4v)(\?|#|$)/i.test(responseUrl)) {
|
||||
discoveredVideoUrls.add(responseUrl);
|
||||
}
|
||||
const contentType = response.headers()["content-type"] || "";
|
||||
const isJsonUrl = responseUrl.endsWith(".json");
|
||||
const isLottieUrl = responseUrl.endsWith(".lottie");
|
||||
const isJson =
|
||||
contentType.includes("application/json") || contentType.includes("text/plain");
|
||||
|
||||
if (isLottieUrl) {
|
||||
discoveredLotties.push({ url: responseUrl });
|
||||
return;
|
||||
}
|
||||
|
||||
if (isJsonUrl || isJson) {
|
||||
// Check Content-Length before downloading to avoid OOM on huge responses
|
||||
const cl = parseInt(response.headers()["content-length"] || "0", 10);
|
||||
if (cl > 5_000_000) return;
|
||||
const buffer = await response.buffer();
|
||||
if (buffer.length < 100 || buffer.length > 5_000_000) return; // Skip tiny or huge
|
||||
const text = buffer.toString("utf-8");
|
||||
const json = JSON.parse(text);
|
||||
// Validate Lottie structure: must have version, in/out points, layers, dimensions, framerate
|
||||
if (
|
||||
json &&
|
||||
typeof json === "object" &&
|
||||
["v", "ip", "op", "layers", "w", "h", "fr"].every((k: string) => k in json)
|
||||
) {
|
||||
discoveredLotties.push({
|
||||
url: responseUrl,
|
||||
data: json,
|
||||
dimensions: { w: json.w, h: json.h },
|
||||
frameRate: json.fr,
|
||||
});
|
||||
}
|
||||
}
|
||||
lottieDiscovery.collect(response);
|
||||
} catch {
|
||||
/* not JSON or parse error — skip */
|
||||
}
|
||||
@@ -382,10 +353,16 @@ export async function captureWebsite(
|
||||
/* DOM scan failed — non-critical */
|
||||
}
|
||||
|
||||
for (const found of await lottieDiscovery.run(downloadByteBudget, remainingMs)) {
|
||||
const existing = discoveredLotties.findIndex((item) => item.url === found.url);
|
||||
if (existing < 0) discoveredLotties.push(found);
|
||||
else discoveredLotties[existing] = found;
|
||||
}
|
||||
|
||||
if (discoveredLotties.length > 0 && remainingMs() > 0) {
|
||||
const lottieDir = join(outputDir, "assets", "lottie");
|
||||
mkdirSync(lottieDir, { recursive: true });
|
||||
const lottieBudget = { remainingMs };
|
||||
const lottieBudget = { remainingMs, byteBudget: downloadByteBudget };
|
||||
const savedCount = await saveLottieAnimations(discoveredLotties, lottieDir, lottieBudget);
|
||||
// Generate manifest + preview thumbnails so the agent can SEE what each animation is
|
||||
if (savedCount > 0 && remainingMs() > 0) {
|
||||
@@ -609,7 +586,6 @@ export async function captureWebsite(
|
||||
// `budget-exhausted` for every one of them replaces a warning string that could only ever
|
||||
// say "some". A zero budget means it breaks on the first url, so this costs no network.
|
||||
phase("fonts", "started");
|
||||
const downloadByteBudget = createCaptureDownloadBudget();
|
||||
const fontPass = await downloadAndRewriteFonts(extracted.headHtml, outputDir, {
|
||||
remainingMs,
|
||||
byteBudget: downloadByteBudget,
|
||||
|
||||
@@ -0,0 +1,19 @@
|
||||
import { describe, expect, it, vi } from "vitest";
|
||||
import { readLottieArchive } from "./lottieValidation.js";
|
||||
|
||||
const getEntries = vi.hoisted(() => vi.fn(() => []));
|
||||
vi.mock("adm-zip", () => ({
|
||||
default: class {
|
||||
getEntryCount() {
|
||||
return 1000;
|
||||
}
|
||||
getEntries = getEntries;
|
||||
},
|
||||
}));
|
||||
|
||||
describe("Lottie archive entry allocation", () => {
|
||||
it("rejects the EOCD count without materializing entries", () => {
|
||||
expect(readLottieArchive(Buffer.from("archive metadata supplied by test"))).toBeNull();
|
||||
expect(getEntries).not.toHaveBeenCalled();
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,59 @@
|
||||
/** Bound raw central-directory names before adm-zip synthesizes their parent folders. */
|
||||
export function preflightLottieArchive(bytes: Buffer): boolean {
|
||||
const end = findDirectoryEnd(bytes);
|
||||
if (end < 0) return false;
|
||||
if (!unambiguousFooter(bytes, end)) return false;
|
||||
const count = bytes.readUInt16LE(end + 10);
|
||||
if (count > 256 || bytes.readUInt16LE(end + 8) !== count) return false;
|
||||
if (bytes.readUInt32LE(end + 4) !== 0) return false; // no multi-disk archives
|
||||
let offset = bytes.readUInt32LE(end + 16);
|
||||
const directoryEnd = offset + bytes.readUInt32LE(end + 12);
|
||||
if (directoryEnd > end) return false;
|
||||
for (let i = 0; i < count; i++) {
|
||||
const next = nextBoundedEntry(bytes, offset, directoryEnd);
|
||||
if (next === null) return false;
|
||||
offset = next;
|
||||
}
|
||||
return offset === directoryEnd;
|
||||
}
|
||||
|
||||
function unambiguousFooter(bytes: Buffer, end: number): boolean {
|
||||
const signature = Buffer.from([0x50, 0x4b, 0x05, 0x06]);
|
||||
if (bytes.lastIndexOf(signature) !== end) return false;
|
||||
// adm-zip scans just before EOCD for alternate/ZIP64 records. This bounded
|
||||
// archive format does not need those; do not let it select another directory.
|
||||
const preceding = bytes.subarray(Math.max(0, end - 76), end);
|
||||
return ![
|
||||
signature,
|
||||
Buffer.from([0x50, 0x4b, 0x06, 0x06]),
|
||||
Buffer.from([0x50, 0x4b, 0x06, 0x07]),
|
||||
].some((marker) => preceding.includes(marker));
|
||||
}
|
||||
|
||||
function findDirectoryEnd(bytes: Buffer): number {
|
||||
for (let at = bytes.length - 22; at >= Math.max(0, bytes.length - 65557); at--) {
|
||||
if (
|
||||
bytes.readUInt32LE(at) === 0x06054b50 &&
|
||||
at + 22 + bytes.readUInt16LE(at + 20) === bytes.length
|
||||
)
|
||||
return at;
|
||||
}
|
||||
return -1;
|
||||
}
|
||||
|
||||
function nextBoundedEntry(bytes: Buffer, offset: number, end: number): number | null {
|
||||
if (offset + 46 > end || bytes.readUInt32LE(offset) !== 0x02014b50) return null;
|
||||
const length = bytes.readUInt16LE(offset + 28);
|
||||
const next =
|
||||
offset + 46 + length + bytes.readUInt16LE(offset + 30) + bytes.readUInt16LE(offset + 32);
|
||||
if (length > 512 || next > end) return null;
|
||||
const name = bytes.toString("utf8", offset + 46, offset + 46 + length);
|
||||
return safeLottieArchivePath(name) ? next : null;
|
||||
}
|
||||
|
||||
export function safeLottieArchivePath(path: string): boolean {
|
||||
if (path.length > 512 || path.split("/").length > 16) return false;
|
||||
if (path.startsWith("/") || /[\\:]/.test(path)) return false;
|
||||
if (Array.from(path).some((char) => char.charCodeAt(0) < 32)) return false;
|
||||
return path.split("/").every((segment) => segment !== ".." && segment !== ".");
|
||||
}
|
||||
@@ -0,0 +1,122 @@
|
||||
import { afterEach, describe, expect, it, vi } from "vitest";
|
||||
import { discoverLottieResponse, LottieDiscovery } from "./lottieDiscovery.js";
|
||||
|
||||
describe("bounded Lottie discovery", () => {
|
||||
it("awaits delayed candidates and ignores response events after its save boundary", async () => {
|
||||
const discovery = new LottieDiscovery();
|
||||
const response = { url: () => "https://public.example/a.json", headers: () => ({}) };
|
||||
discovery.collect(response);
|
||||
discovery.collect(response);
|
||||
let finish!: (response: Response) => void;
|
||||
const fetchMock = vi.fn(
|
||||
() =>
|
||||
new Promise<Response>((resolve) => {
|
||||
finish = resolve;
|
||||
}),
|
||||
);
|
||||
vi.stubGlobal("fetch", fetchMock);
|
||||
const budget = { remainingBytes: 1000 };
|
||||
let settled = false;
|
||||
const pending = discovery
|
||||
.run(budget, () => 10000)
|
||||
.then((result) => {
|
||||
settled = true;
|
||||
return result;
|
||||
});
|
||||
await Promise.resolve();
|
||||
expect(settled).toBe(false);
|
||||
finish(
|
||||
new Response(JSON.stringify({ v: "5", w: 100, h: 100, fr: 30, ip: 0, op: 30, layers: [] })),
|
||||
);
|
||||
expect(await pending).toHaveLength(1);
|
||||
const remaining = budget.remainingBytes;
|
||||
discovery.collect({ url: () => "https://public.example/late.json", headers: () => ({}) });
|
||||
expect(await discovery.run(budget, () => 10000)).toEqual([]);
|
||||
expect(fetchMock).toHaveBeenCalledTimes(1);
|
||||
expect(budget.remainingBytes).toBe(remaining);
|
||||
});
|
||||
|
||||
it("caps ordinary JSON candidates and their aggregate byte consumption", async () => {
|
||||
const discovery = new LottieDiscovery();
|
||||
for (let n = 0; n < 100; n++)
|
||||
discovery.collect({
|
||||
url: () => `https://public.example/api/${n}`,
|
||||
headers: () => ({ "content-type": "application/json" }),
|
||||
});
|
||||
const fetchMock = vi.fn(async (_url: string | URL | Request) => new Response("{}"));
|
||||
vi.stubGlobal("fetch", fetchMock);
|
||||
const budget = { remainingBytes: 1000 };
|
||||
expect(await discovery.run(budget, () => 10000)).toEqual([]);
|
||||
expect(fetchMock).toHaveBeenCalledTimes(32);
|
||||
expect(budget.remainingBytes).toBe(936);
|
||||
});
|
||||
it("retains explicit JSON when a later archive displaces a generic candidate", async () => {
|
||||
const discovery = new LottieDiscovery();
|
||||
discovery.collect({ url: () => "https://public.example/real.json", headers: () => ({}) });
|
||||
for (let n = 0; n < 31; n++)
|
||||
discovery.collect({
|
||||
url: () => `https://public.example/api/${n}`,
|
||||
headers: () => ({ "content-type": "application/json" }),
|
||||
});
|
||||
discovery.collect({ url: () => "https://public.example/anim.lottie", headers: () => ({}) });
|
||||
const fetchMock = vi.fn(async (_url: string | URL | Request) => new Response("{}"));
|
||||
vi.stubGlobal("fetch", fetchMock);
|
||||
await discovery.run({ remainingBytes: 1000 }, () => 10000);
|
||||
const urls = fetchMock.mock.calls.map((call) => call[0]);
|
||||
expect(urls).toContain("https://public.example/real.json");
|
||||
expect(urls).not.toContain("https://public.example/api/0");
|
||||
expect(urls).toHaveLength(31);
|
||||
});
|
||||
afterEach(() => vi.unstubAllGlobals());
|
||||
it.each([undefined, "1"])(
|
||||
"does not trust intercepted Content-Length %s or materialize Puppeteer bodies",
|
||||
async (length) => {
|
||||
const buffer = vi.fn(() => {
|
||||
throw new Error("unbounded body read");
|
||||
});
|
||||
const response = {
|
||||
url: () => "https://public.example/animation.json",
|
||||
headers: () => ({
|
||||
"content-type": "application/json",
|
||||
...(length ? { "content-length": length } : {}),
|
||||
}),
|
||||
buffer,
|
||||
};
|
||||
const cancel = vi.fn();
|
||||
vi.stubGlobal(
|
||||
"fetch",
|
||||
vi.fn(
|
||||
async () =>
|
||||
new Response(
|
||||
new ReadableStream({
|
||||
pull(controller) {
|
||||
controller.enqueue(new Uint8Array(4));
|
||||
},
|
||||
cancel,
|
||||
}),
|
||||
{ headers: length ? { "content-length": length } : {} },
|
||||
),
|
||||
),
|
||||
);
|
||||
expect(await discoverLottieResponse(response, { remainingBytes: 5 })).toBeNull();
|
||||
expect(buffer).not.toHaveBeenCalled();
|
||||
expect(cancel).toHaveBeenCalledOnce();
|
||||
},
|
||||
);
|
||||
it("retains valid discovered JSON with its existing byte accounting", async () => {
|
||||
const data = { v: "5.12.2", w: 100, h: 100, layers: [], fr: 30, ip: 0, op: 30 };
|
||||
const body = JSON.stringify(data);
|
||||
vi.stubGlobal(
|
||||
"fetch",
|
||||
vi.fn(async () => new Response(body)),
|
||||
);
|
||||
const budget = { remainingBytes: body.length };
|
||||
const found = await discoverLottieResponse(
|
||||
{ url: () => "https://public.example/animation.json?version=1", headers: () => ({}) },
|
||||
budget,
|
||||
);
|
||||
expect(found?.data).toEqual(data);
|
||||
expect(found?.dataBudget).toBe(budget);
|
||||
expect(budget.remainingBytes).toBe(0);
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,105 @@
|
||||
import type { HTTPResponse } from "puppeteer-core";
|
||||
import type { DiscoveredLottie } from "./mediaCapture.js";
|
||||
import { safeFetch } from "./assetDownloader.js";
|
||||
import { readBoundedResponse, type DownloadByteBudget } from "./readBoundedResponse.js";
|
||||
import { validLottieJson } from "./lottieValidation.js";
|
||||
import { CAPTURE_USER_AGENT } from "./userAgent.js";
|
||||
|
||||
/** Puppeteer exposes a fully buffered body; inspect candidates through a bounded stream instead. */
|
||||
export async function discoverLottieResponse(
|
||||
response: Pick<HTTPResponse, "url" | "headers">,
|
||||
budget: DownloadByteBudget,
|
||||
timeoutMs = 10_000,
|
||||
): Promise<DiscoveredLottie | null> {
|
||||
const url = response.url();
|
||||
const pathname = new URL(url).pathname;
|
||||
if (pathname.endsWith(".lottie")) return { url };
|
||||
const contentType = response.headers()["content-type"] ?? "";
|
||||
if (!pathname.endsWith(".json") && !/application\/json|text\/plain/i.test(contentType))
|
||||
return null;
|
||||
if (budget.remainingBytes <= 0) return null;
|
||||
const fetched = await safeFetch(url, {
|
||||
signal: AbortSignal.timeout(timeoutMs),
|
||||
headers: { "User-Agent": CAPTURE_USER_AGENT },
|
||||
});
|
||||
if (!fetched?.ok) return null;
|
||||
const bytes = await readBoundedResponse(fetched, 5_000_000, budget);
|
||||
if (!bytes) return null;
|
||||
const source = bytes.toString("utf8");
|
||||
if (!validLottieJson(source)) return null;
|
||||
const data: unknown = JSON.parse(source);
|
||||
if (!hasDiscoveryFields(data)) return null;
|
||||
return { url, data, dataBudget: budget };
|
||||
}
|
||||
|
||||
function hasDiscoveryFields(data: unknown): data is Record<string, unknown> {
|
||||
if (data === null || typeof data !== "object") return false;
|
||||
return ["v", "ip", "op", "layers", "w", "h", "fr"].every((key) => key in data);
|
||||
}
|
||||
|
||||
interface Candidate {
|
||||
url: string;
|
||||
contentType: string;
|
||||
priority: number;
|
||||
}
|
||||
|
||||
/** Own discovery work: response events only enqueue metadata, and run() joins every fetch. */
|
||||
export class LottieDiscovery {
|
||||
private candidates = new Map<string, Candidate>();
|
||||
private closed = false;
|
||||
|
||||
collect(response: Pick<HTTPResponse, "url" | "headers">): void {
|
||||
if (this.closed) return;
|
||||
const url = response.url();
|
||||
if (this.candidates.has(url)) return;
|
||||
const contentType = response.headers()["content-type"] ?? "";
|
||||
const priority = candidatePriority(url, contentType);
|
||||
if (priority === null) return;
|
||||
if (this.candidates.size >= 32) {
|
||||
const worst = [...this.candidates.values()]
|
||||
.sort((a, b) => b.priority - a.priority)
|
||||
.find((candidate) => candidate.priority > priority);
|
||||
if (!worst) return;
|
||||
this.candidates.delete(worst.url);
|
||||
}
|
||||
this.candidates.set(url, { url, contentType, priority });
|
||||
}
|
||||
|
||||
async run(
|
||||
sharedBudget: DownloadByteBudget,
|
||||
remainingMs: () => number,
|
||||
): Promise<DiscoveredLottie[]> {
|
||||
if (this.closed) return [];
|
||||
this.closed = true;
|
||||
const candidates = [...this.candidates.values()].sort((a, b) => a.priority - b.priority);
|
||||
this.candidates.clear();
|
||||
const deadline = Date.now() + Math.min(10_000, remainingMs());
|
||||
const budget = { remainingBytes: Math.min(20 * 1024 * 1024, sharedBudget.remainingBytes) };
|
||||
const found: DiscoveredLottie[] = [];
|
||||
for (const candidate of candidates) {
|
||||
const timeout = Math.floor(Math.min(deadline - Date.now(), remainingMs()));
|
||||
if (timeout <= 0 || budget.remainingBytes <= 0 || found.length >= 10) break;
|
||||
const before = budget.remainingBytes;
|
||||
try {
|
||||
const result = await discoverLottieResponse(
|
||||
{ url: () => candidate.url, headers: () => ({ "content-type": candidate.contentType }) },
|
||||
budget,
|
||||
timeout,
|
||||
);
|
||||
if (result) found.push({ ...result, dataBudget: sharedBudget });
|
||||
} catch {
|
||||
/* unavailable candidate */
|
||||
} finally {
|
||||
sharedBudget.remainingBytes -= before - budget.remainingBytes;
|
||||
}
|
||||
}
|
||||
return found;
|
||||
}
|
||||
}
|
||||
|
||||
function candidatePriority(url: string, contentType: string): number | null {
|
||||
const path = new URL(url).pathname;
|
||||
if (path.endsWith(".lottie")) return 0;
|
||||
if (path.endsWith(".json")) return 1;
|
||||
return /application\/json|text\/plain/i.test(contentType) ? 2 : null;
|
||||
}
|
||||
@@ -0,0 +1,23 @@
|
||||
import { describe, expect, it } from "vitest";
|
||||
import { allowedLottieRequest, LOTTIE_RUNTIME_URL } from "./lottiePreviewRequests.js";
|
||||
|
||||
describe("Lottie preview request policy", () => {
|
||||
it("allows the pinned runtime and public image/font assets", () => {
|
||||
expect(allowedLottieRequest(LOTTIE_RUNTIME_URL, "script")).toBe(true);
|
||||
expect(allowedLottieRequest("https://cdn.example/image.png", "image")).toBe(true);
|
||||
expect(allowedLottieRequest("https://cdn.example/font.woff2", "font")).toBe(true);
|
||||
expect(allowedLottieRequest("data:image/png;base64,AA==", "image")).toBe(true);
|
||||
});
|
||||
it.each([
|
||||
"http://127.0.0.1/a",
|
||||
"http://169.254.169.254/a",
|
||||
"http://[::1]/a",
|
||||
"file:///etc/passwd",
|
||||
])("blocks private and local targets %s", (url) => {
|
||||
expect(allowedLottieRequest(url, "image")).toBe(false);
|
||||
});
|
||||
it("blocks arbitrary scripts and navigations", () => {
|
||||
expect(allowedLottieRequest("https://cdn.example/script.js", "script")).toBe(false);
|
||||
expect(allowedLottieRequest("https://cdn.example/page", "document")).toBe(false);
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,24 @@
|
||||
import type { Page } from "puppeteer-core";
|
||||
import { isPrivateUrl } from "./assetDownloader.js";
|
||||
|
||||
export const LOTTIE_RUNTIME_URL =
|
||||
"https://cdnjs.cloudflare.com/ajax/libs/lottie-web/5.12.2/lottie.min.js";
|
||||
|
||||
/** Apply capture's URL policy to each preview request, including redirect targets. */
|
||||
export function allowedLottieRequest(url: string, resourceType: string): boolean {
|
||||
if (url === LOTTIE_RUNTIME_URL) return true;
|
||||
if (!["image", "font", "stylesheet"].includes(resourceType)) return false;
|
||||
if (url.startsWith("data:") || url.startsWith("blob:")) return true;
|
||||
return !isPrivateUrl(url);
|
||||
}
|
||||
|
||||
export async function guardLottiePreviewRequests(page: Page): Promise<void> {
|
||||
await page.setRequestInterception(true);
|
||||
page.on("request", (request) => {
|
||||
if (request.isInterceptResolutionHandled()) return;
|
||||
const action = allowedLottieRequest(request.url(), request.resourceType())
|
||||
? request.continue()
|
||||
: request.abort("blockedbyclient");
|
||||
void action.catch(() => {});
|
||||
});
|
||||
}
|
||||
@@ -0,0 +1,66 @@
|
||||
import AdmZip from "adm-zip";
|
||||
import { describe, expect, it } from "vitest";
|
||||
import { MAX_LOTTIE_BYTES, readLottieArchive, validLottieJson } from "./lottieValidation.js";
|
||||
import { preflightLottieArchive } from "./lottieArchivePreflight.js";
|
||||
|
||||
const animation = JSON.stringify({ w: 100, h: 100, fr: 30, ip: 0, op: 30, layers: [] });
|
||||
|
||||
function archive(path: string, content = animation): Buffer {
|
||||
const zip = new AdmZip();
|
||||
zip.addFile(path, Buffer.from(content));
|
||||
return zip.toBuffer();
|
||||
}
|
||||
|
||||
describe("Lottie archive validation", () => {
|
||||
it("rejects one deeply segmented name before constructing adm-zip entries", () => {
|
||||
const bytes = archive(`a/${"x/".repeat(5000)}demo.json`);
|
||||
expect(preflightLottieArchive(bytes)).toBe(false);
|
||||
expect(readLottieArchive(bytes)).toBeNull();
|
||||
});
|
||||
it.each([0x06054b50, 0x06064b50, 0x07064b50])("rejects ambiguous footer marker %s", (marker) => {
|
||||
const bytes = archive("a/demo.json");
|
||||
const end = bytes.length - 22;
|
||||
const injected = Buffer.alloc(4);
|
||||
injected.writeUInt32LE(marker);
|
||||
const ambiguous = Buffer.concat([bytes.subarray(0, end), injected, bytes.subarray(end)]);
|
||||
expect(preflightLottieArchive(ambiguous)).toBe(false);
|
||||
});
|
||||
it("rejects excessive EOCD entry counts before entry materialization", () => {
|
||||
const bytes = archive("a/demo.json");
|
||||
const end = bytes.lastIndexOf(Buffer.from([0x50, 0x4b, 0x05, 0x06]));
|
||||
bytes.writeUInt16LE(1000, end + 8);
|
||||
bytes.writeUInt16LE(1000, end + 10);
|
||||
expect(readLottieArchive(bytes)).toBeNull();
|
||||
});
|
||||
it.each(["a/demo.json", "animations/demo.json"])("preserves valid %s animation JSON", (path) => {
|
||||
expect(readLottieArchive(archive(path))).toBe(animation);
|
||||
});
|
||||
it("rejects corrupt archives and traversal names", () => {
|
||||
expect(readLottieArchive(Buffer.from("not a zip"))).toBeNull();
|
||||
expect(readLottieArchive(archive("animations/../demo.json"))).toBeNull();
|
||||
});
|
||||
it.each([0, MAX_LOTTIE_BYTES + 1])("rejects an unsafe declared expansion size %s", (size) => {
|
||||
const bytes = archive("a/demo.json");
|
||||
const central = bytes.indexOf(Buffer.from([0x50, 0x4b, 0x01, 0x02]));
|
||||
expect(central).toBeGreaterThan(0);
|
||||
bytes.writeUInt32LE(size, central + 24);
|
||||
expect(readLottieArchive(bytes)).toBeNull();
|
||||
});
|
||||
});
|
||||
|
||||
describe("Lottie JSON validation", () => {
|
||||
it("accepts the supported animation shape", () => {
|
||||
expect(validLottieJson(animation)).toBe(true);
|
||||
});
|
||||
it.each([
|
||||
'{"w":100,"h":100,"layers":true}',
|
||||
'{"w":100000,"h":100000,"layers":[]}',
|
||||
'{"w":100,"h":100,"layers":[],"__proto__":{}}',
|
||||
'{"w":100,"h":100,"layers":[],"fr":1e999}',
|
||||
'{"w":100,"h":100,"layers":[true,null,"x"],"fr":30,"ip":0,"op":30}',
|
||||
'{"w":100,"h":100,"layers":[],"fr":5e-324,"ip":0,"op":10000000}',
|
||||
'{"w":100,"h":100,"layers":[],"assets":[{"layers":[false]}]}',
|
||||
])("rejects invalid animation data %s", (source) => {
|
||||
expect(validLottieJson(source)).toBe(false);
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,107 @@
|
||||
import AdmZip from "adm-zip";
|
||||
import { preflightLottieArchive, safeLottieArchivePath } from "./lottieArchivePreflight.js";
|
||||
|
||||
export const MAX_LOTTIE_BYTES = 10 * 1024 * 1024;
|
||||
|
||||
/** Read animation JSON without extracting remote archive paths onto the filesystem. */
|
||||
export function readLottieArchive(bytes: Buffer): string | null {
|
||||
if (bytes.length > MAX_LOTTIE_BYTES) return null;
|
||||
if (!preflightLottieArchive(bytes)) return null;
|
||||
try {
|
||||
const zip = new AdmZip(bytes);
|
||||
if (zip.getEntryCount() > 256) return null;
|
||||
const entries = zip.getEntries();
|
||||
if (entries.length > 256) return null;
|
||||
if (!validArchiveEntries(entries)) return null;
|
||||
const animation = entries.find((entry) =>
|
||||
/^(?:a|animations)\/[^/]+\.json$/.test(entry.entryName),
|
||||
);
|
||||
// adm-zip uses a positive expected size as zlib's maxOutputLength.
|
||||
if (!animation || animation.header.size <= 0) return null;
|
||||
const data = animation.getData();
|
||||
if (data.length !== animation.header.size || data.length > MAX_LOTTIE_BYTES) return null;
|
||||
return data.toString("utf8");
|
||||
} catch {
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
function validArchiveEntries(entries: { entryName: string; header: { size: number } }[]): boolean {
|
||||
let total = 0;
|
||||
for (const entry of entries) {
|
||||
if (!safeLottieArchivePath(entry.entryName)) return false;
|
||||
const size = entry.header.size;
|
||||
if (!Number.isSafeInteger(size) || size < 0 || size > MAX_LOTTIE_BYTES) return false;
|
||||
total += size;
|
||||
if (total > 50 * 1024 * 1024) return false;
|
||||
}
|
||||
return true;
|
||||
}
|
||||
|
||||
function record(value: unknown): value is Record<string, unknown> {
|
||||
return typeof value === "object" && value !== null && !Array.isArray(value);
|
||||
}
|
||||
|
||||
function dimension(value: unknown): value is number {
|
||||
return typeof value === "number" && Number.isFinite(value) && value > 0 && value <= 16384;
|
||||
}
|
||||
|
||||
/** Keep original valid animation JSON while bounding the data consumed by lottie-web. */
|
||||
export function validLottieJson(source: string): boolean {
|
||||
if (Buffer.byteLength(source) > MAX_LOTTIE_BYTES) return false;
|
||||
try {
|
||||
const value: unknown = JSON.parse(source);
|
||||
if (!record(value) || !dimension(value.w) || !dimension(value.h)) return false;
|
||||
if (value.w * value.h > 40_000_000 || !Array.isArray(value.layers)) return false;
|
||||
return validLottieTiming(value) && boundedLottieTree(value);
|
||||
} catch {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
function validLottieTiming(value: Record<string, unknown>): boolean {
|
||||
if (value.fr !== undefined && (typeof value.fr !== "number" || value.fr <= 0 || value.fr > 240))
|
||||
return false;
|
||||
if (![value.ip, value.op].every(validOptionalFrame)) return false;
|
||||
const start = numberOr(value.ip, 0);
|
||||
const end = numberOr(value.op, 0);
|
||||
const rate = numberOr(value.fr, 30);
|
||||
const duration = (end - start) / rate;
|
||||
return Number.isFinite(duration) && duration >= 0 && duration <= 3600;
|
||||
}
|
||||
|
||||
function numberOr(value: unknown, fallback: number): number {
|
||||
return typeof value === "number" ? value : fallback;
|
||||
}
|
||||
|
||||
function validOptionalFrame(frame: unknown): boolean {
|
||||
return (
|
||||
frame === undefined ||
|
||||
(typeof frame === "number" && Number.isFinite(frame) && Math.abs(frame) <= 10_000_000)
|
||||
);
|
||||
}
|
||||
|
||||
function jsonChildren(value: unknown): [string, unknown][] {
|
||||
return value !== null && typeof value === "object" ? Object.entries(value) : [];
|
||||
}
|
||||
|
||||
function boundedLottieTree(root: unknown): boolean {
|
||||
const pending = [{ value: root, depth: 0 }];
|
||||
let count = 0;
|
||||
while (pending.length) {
|
||||
const item = pending.pop()!;
|
||||
if (++count > 100000 || item.depth > 128) return false;
|
||||
if (typeof item.value === "number" && !Number.isFinite(item.value)) return false;
|
||||
for (const [key, value] of jsonChildren(item.value)) {
|
||||
if (!validLottieEntry(key, value)) return false;
|
||||
if (count + pending.length >= 100000) return false;
|
||||
pending.push({ value, depth: item.depth + 1 });
|
||||
}
|
||||
}
|
||||
return true;
|
||||
}
|
||||
|
||||
function validLottieEntry(key: string, value: unknown): boolean {
|
||||
if (["__proto__", "constructor", "prototype"].includes(key)) return false;
|
||||
return key !== "layers" || (Array.isArray(value) && value.every(record));
|
||||
}
|
||||
@@ -1,4 +1,12 @@
|
||||
import { existsSync, mkdirSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from "node:fs";
|
||||
import {
|
||||
existsSync,
|
||||
mkdirSync,
|
||||
mkdtempSync,
|
||||
readFileSync,
|
||||
readdirSync,
|
||||
rmSync,
|
||||
writeFileSync,
|
||||
} from "node:fs";
|
||||
import { tmpdir } from "node:os";
|
||||
import { join } from "node:path";
|
||||
import type { Browser, Page } from "puppeteer-core";
|
||||
@@ -56,6 +64,8 @@ describe("Lottie capture budget", () => {
|
||||
|
||||
let remainingMs = 10_000;
|
||||
const previewPage = {
|
||||
setRequestInterception: vi.fn(async () => undefined),
|
||||
on: vi.fn(),
|
||||
setViewport: vi.fn(async () => undefined),
|
||||
setContent: vi.fn(async () => undefined),
|
||||
evaluate: vi.fn(async () => undefined),
|
||||
@@ -96,6 +106,8 @@ describe("Lottie capture budget", () => {
|
||||
|
||||
const screenshot = vi.fn(async () => undefined);
|
||||
const previewPage = {
|
||||
setRequestInterception: vi.fn(async () => undefined),
|
||||
on: vi.fn(),
|
||||
setViewport: vi.fn(async () => undefined),
|
||||
setContent: vi.fn(async () => undefined),
|
||||
evaluate: vi.fn(async () => undefined),
|
||||
@@ -185,3 +197,24 @@ describe("remainingVideoDownloadTimeoutMs", () => {
|
||||
expect(remainingVideoDownloadTimeoutMs(1_000, 300_000, 2_000)).toBe(120_000);
|
||||
});
|
||||
});
|
||||
|
||||
describe("Lottie capture rejects unsafe persistence", () => {
|
||||
it("does not write corrupt archives as raw Lottie files", async () => {
|
||||
const dir = tempDir();
|
||||
vi.stubGlobal(
|
||||
"fetch",
|
||||
vi.fn(async () => new Response("not a zip")),
|
||||
);
|
||||
expect(await saveLottieAnimations([{ url: "https://public.example/bad.lottie" }], dir)).toBe(0);
|
||||
expect(readdirSync(dir)).toEqual([]);
|
||||
});
|
||||
it("does not publish truthy non-array layers", async () => {
|
||||
const dir = tempDir();
|
||||
vi.stubGlobal(
|
||||
"fetch",
|
||||
vi.fn(async () => new Response('{"w":100,"h":100,"layers":true}')),
|
||||
);
|
||||
expect(await saveLottieAnimations([{ url: "https://public.example/bad.json" }], dir)).toBe(0);
|
||||
expect(readdirSync(dir)).toEqual([]);
|
||||
});
|
||||
});
|
||||
|
||||
@@ -12,16 +12,26 @@ import { mkdirSync, writeFileSync, readdirSync, readFileSync, statSync } from "n
|
||||
import { join, extname } from "node:path";
|
||||
import { isPrivateUrl, safeFetch } from "./assetDownloader.js";
|
||||
import { CAPTURE_USER_AGENT } from "./userAgent.js";
|
||||
import { MAX_LOTTIE_BYTES, readLottieArchive, validLottieJson } from "./lottieValidation.js";
|
||||
import { guardLottiePreviewRequests, LOTTIE_RUNTIME_URL } from "./lottiePreviewRequests.js";
|
||||
import {
|
||||
readBoundedResponse,
|
||||
createCaptureDownloadBudget,
|
||||
type DownloadByteBudget,
|
||||
} from "./readBoundedResponse.js";
|
||||
|
||||
/** Discovered Lottie item from network interception or DOM scan. */
|
||||
export interface DiscoveredLottie {
|
||||
url: string;
|
||||
data?: unknown;
|
||||
/** Internal discovery accounting: avoid charging the same buffered data twice. */
|
||||
dataBudget?: DownloadByteBudget;
|
||||
dimensions?: { w: number; h: number };
|
||||
frameRate?: number;
|
||||
}
|
||||
|
||||
interface RemainingBudget {
|
||||
byteBudget?: DownloadByteBudget;
|
||||
remainingMs?: () => number;
|
||||
}
|
||||
|
||||
@@ -41,6 +51,7 @@ export async function saveLottieAnimations(
|
||||
lottieDir: string,
|
||||
budget: RemainingBudget = {},
|
||||
): Promise<number> {
|
||||
const byteBudget = budget.byteBudget ?? createCaptureDownloadBudget();
|
||||
let savedCount = 0;
|
||||
const savedHashes = new Set<string>(); // Deduplicate by content
|
||||
|
||||
@@ -53,6 +64,12 @@ export async function saveLottieAnimations(
|
||||
if (lottieItem.data) {
|
||||
// Already have the JSON data from network interception
|
||||
jsonData = JSON.stringify(lottieItem.data);
|
||||
const size = Buffer.byteLength(jsonData);
|
||||
if (size > MAX_LOTTIE_BYTES) continue;
|
||||
if (lottieItem.dataBudget !== byteBudget) {
|
||||
if (size > byteBudget.remainingBytes) continue;
|
||||
byteBudget.remainingBytes -= size;
|
||||
}
|
||||
} else if (lottieItem.url) {
|
||||
const requestTimeoutMs = Math.min(10_000, liveRemainingMs(budget, 10_000));
|
||||
if (requestTimeoutMs <= 0) break;
|
||||
@@ -62,36 +79,11 @@ export async function saveLottieAnimations(
|
||||
headers: { "User-Agent": CAPTURE_USER_AGENT },
|
||||
});
|
||||
if (!res || !res.ok) continue;
|
||||
const buf = Buffer.from(await res.arrayBuffer());
|
||||
|
||||
if (lottieItem.url.endsWith(".lottie")) {
|
||||
// dotLottie is a ZIP — extract the animation JSON
|
||||
try {
|
||||
const AdmZip = (await import("adm-zip")).default;
|
||||
const zip = new AdmZip(buf);
|
||||
const entries = zip.getEntries();
|
||||
// Look for animation JSON in both v1 (animations/) and v2 (a/) paths
|
||||
const animEntry = entries.find(
|
||||
(e) =>
|
||||
(e.entryName.startsWith("a/") || e.entryName.startsWith("animations/")) &&
|
||||
e.entryName.endsWith(".json"),
|
||||
);
|
||||
if (animEntry) {
|
||||
jsonData = animEntry.getData().toString("utf-8");
|
||||
}
|
||||
} catch {
|
||||
// adm-zip not available or extraction failed — save raw .lottie
|
||||
const hash = buf.toString("base64").slice(0, 100);
|
||||
if (savedHashes.has(hash)) continue;
|
||||
savedHashes.add(hash);
|
||||
writeFileSync(join(lottieDir, `animation-${savedCount}.lottie`), buf);
|
||||
savedCount++;
|
||||
continue;
|
||||
}
|
||||
} else {
|
||||
// Plain JSON file
|
||||
jsonData = buf.toString("utf-8");
|
||||
}
|
||||
const buf = await readBoundedResponse(res, MAX_LOTTIE_BYTES, byteBudget);
|
||||
if (!buf) continue;
|
||||
jsonData = new URL(lottieItem.url).pathname.endsWith(".lottie")
|
||||
? (readLottieArchive(buf) ?? undefined)
|
||||
: buf.toString("utf8");
|
||||
}
|
||||
|
||||
if (jsonData) {
|
||||
@@ -100,13 +92,7 @@ export async function saveLottieAnimations(
|
||||
if (savedHashes.has(hash)) continue;
|
||||
savedHashes.add(hash);
|
||||
|
||||
// Validate it's actually Lottie
|
||||
try {
|
||||
const parsed = JSON.parse(jsonData);
|
||||
if (!parsed.layers || !parsed.w) continue;
|
||||
} catch {
|
||||
continue;
|
||||
}
|
||||
if (!validLottieJson(jsonData)) continue;
|
||||
|
||||
writeFileSync(join(lottieDir, `animation-${savedCount}.json`), jsonData, "utf-8");
|
||||
savedCount++;
|
||||
@@ -165,6 +151,7 @@ export async function renderLottiePreviews(
|
||||
if (liveRemainingMs(budget, 1) <= 0) break;
|
||||
previewPage = await chromeBrowser.newPage();
|
||||
if (liveRemainingMs(budget, 1) <= 0) break;
|
||||
await guardLottiePreviewRequests(previewPage);
|
||||
await previewPage.setViewport({ width: 400, height: 400 });
|
||||
const animData = JSON.parse(readFileSync(join(lottieDir, file), "utf-8"));
|
||||
const midFrame = Math.floor(((raw.op || 0) - (raw.ip || 0)) * 0.3);
|
||||
@@ -172,7 +159,7 @@ export async function renderLottiePreviews(
|
||||
await previewPage.setContent(
|
||||
`<!DOCTYPE html>
|
||||
<html><head>
|
||||
<script src="https://cdnjs.cloudflare.com/ajax/libs/lottie-web/5.12.2/lottie.min.js"></script>
|
||||
<script src="${LOTTIE_RUNTIME_URL}"></script>
|
||||
<style>*{margin:0;padding:0;background:transparent}#c{width:400px;height:400px}</style>
|
||||
</head><body><div id="c"></div></body></html>`,
|
||||
{ waitUntil: "load", timeout: 10000 },
|
||||
|
||||
Reference in New Issue
Block a user