fix(cli): fetch a page's assets as the same agent that loaded the page (#3726)

A capture is one session with two halves: Chrome navigates the page with a
browser User-Agent, then Node fetches the assets that page referenced. Those
halves sent three different identities — "HyperFrames/1.0" from the asset and
media downloaders, a bare "Mozilla/5.0" from the stylesheet inliner, and the
real Chrome UA from the navigation itself.

An origin is free to answer those differently, and anti-bot edges do. Capturing
one large site, GET /favicon.svg answers 403 text/html to "HyperFrames/1.0" and
200 image/svg+xml to the UA the very same capture had just navigated with. The
favicon ranker had already picked that SVG as the best declared icon; the 403
discarded it and the downloader fell through to the next candidate, so the icon
written to assets/ was chosen by the CDN's bot rules rather than by the ranker.
The capture reported it as one "unavailable" drop and carried on.

Hoist the navigation UA into CAPTURE_USER_AGENT and use it for every
out-of-band fetch the capture makes: favicons, images, og:image, fonts,
stylesheets, Lottie JSON and videos. One constant is what stops the two halves
drifting apart again.

Verified end to end against that site: before, assets/favicon.png (the
apple-touch icon) plus one unavailable drop; after, assets/favicon.svg, byte
identical to the file the site itself serves.

packages/cli/src/capture/assetDownloader.test.ts

@@ -10,6 +10,8 @@ import {
   toStandaloneSvg,
 } from "./assetDownloader.js";
 import type { DesignTokens } from "./types.js";
+import type { IconCandidate } from "./faviconRanker.js";
+import { CAPTURE_USER_AGENT } from "./userAgent.js";
 
 describe("isPrivateUrl — SSRF denylist (security: F-003)", () => {
   it("blocks loopback, private, and metadata IPv4", () => {
@@ -187,6 +189,16 @@ describe("downloadAndRewriteFonts — attempt caps", () => {
   });
 });
 
+function withTempDir<T>(run: (dir: string) => Promise<T>): Promise<T> {
+  const dir = mkdtempSync(join(tmpdir(), "hf-drops-"));
+  return run(dir).finally(() => rmSync(dir, { recursive: true, force: true }));
+}
+
+/** The two fields `downloadAssets` reads off the token bundle. */
+function tokensWithNoSvgs(): DesignTokens {
+  return { svgs: [], sections: [], ogImage: "" } as unknown as DesignTokens;
+}
+
 describe("drop counts — why a referenced asset is not in the capture", () => {
   afterEach(() => vi.unstubAllGlobals());
 
@@ -199,11 +211,6 @@ describe("drop counts — why a referenced asset is not in the capture", () => {
     ).join("\n");
   }
 
-  function withTempDir<T>(run: (dir: string) => Promise<T>): Promise<T> {
-    const dir = mkdtempSync(join(tmpdir(), "hf-drops-"));
-    return run(dir).finally(() => rmSync(dir, { recursive: true, force: true }));
-  }
-
   it("counts every face the budget never let it reach, not just the one it stopped on", async () => {
     // Four declared, zero budget: the honest number is four, and a warning string could only
     // ever have said "some".
@@ -269,11 +276,6 @@ describe("drop counts — why a referenced asset is not in the capture", () => {
     });
   });
 
-  /** The two fields `downloadAssets` reads off the token bundle. */
-  function tokensWithNoSvgs(): DesignTokens {
-    return { svgs: [], sections: [], ogImage: "" } as unknown as DesignTokens;
-  }
-
   it("counts an image dropped for being under the raster floor", async () => {
     // 9 KB is under the 10 KB floor. Nothing lands, and the reason is now on the record.
     await withTempDir(async (dir) => {
@@ -328,3 +330,75 @@ describe("drop counts — why a referenced asset is not in the capture", () => {
     });
   });
 });
+
+describe("asset fetches present the same identity as the page navigation", () => {
+  afterEach(() => vi.unstubAllGlobals());
+
+  /**
+   * openai.com's declared icons, in DOM order, with the attributes the page really carries.
+   * `rankIconCandidates` puts `favicon.svg` first, so that is the file a capture must land.
+   */
+  const OPENAI_ICONS: IconCandidate[] = [
+    {
+      rel: "icon",
+      href: "https://openai.example/favicon.svg",
+      sizes: null,
+      type: "image/svg+xml",
+    },
+    {
+      rel: "icon",
+      href: "https://openai.example/favicon.ico",
+      sizes: "48x48",
+      type: "image/x-icon",
+    },
+    {
+      rel: "apple-touch-icon",
+      href: "https://openai.example/apple-icon.png",
+      sizes: "180x180",
+      type: "image/png",
+    },
+  ];
+
+  /**
+   * Measured against the real origin: `GET /favicon.svg` answers `403 text/html` to
+   * `User-Agent: HyperFrames/1.0` and `200 image/svg+xml` to the browser UA the capture
+   * already navigates with. The other two icons are served to either agent.
+   */
+  function serveLikeAnAntiBotEdge() {
+    return vi.fn(async (url: string, init?: RequestInit) => {
+      const agent = new Headers(init?.headers).get("user-agent") ?? "";
+      if (url.endsWith("/favicon.svg") && !agent.startsWith("Mozilla/")) {
+        return new Response("<html>denied</html>", {
+          status: 403,
+          headers: { "content-type": "text/html" },
+        });
+      }
+      const body = new Uint8Array(4096);
+      return new Response(body, { status: 200, headers: { "content-type": "image/svg+xml" } });
+    });
+  }
+
+  it("lands the icon the ranker chose from an origin that refuses non-browser agents", async () => {
+    // With a bot UA the ranker's winner 403s and the capture silently falls through to the
+    // next candidate, so the file on disk is chosen by the CDN rather than by the ranker.
+    await withTempDir(async (dir) => {
+      vi.stubGlobal("fetch", serveLikeAnAntiBotEdge());
+      const { assets } = await downloadAssets(tokensWithNoSvgs(), dir, [], OPENAI_ICONS);
+      expect(assets.map((a) => a.localPath)).toEqual(["assets/favicon.svg"]);
+      expect(assets[0]?.url).toBe("https://openai.example/favicon.svg");
+    });
+  });
+
+  it("sends one User-Agent for every asset request", async () => {
+    await withTempDir(async (dir) => {
+      const fetchMock = serveLikeAnAntiBotEdge();
+      vi.stubGlobal("fetch", fetchMock);
+      await downloadAssets(tokensWithNoSvgs(), dir, [], OPENAI_ICONS);
+      const agents = fetchMock.mock.calls.map((call) =>
+        new Headers(call[1]?.headers).get("user-agent"),
+      );
+      expect(agents).not.toHaveLength(0);
+      expect(new Set(agents)).toEqual(new Set([CAPTURE_USER_AGENT]));
+    });
+  });
+});

packages/cli/src/capture/assetDownloader.ts

@@ -10,6 +10,7 @@ import { join, extname } from "node:path";
 import { createHash } from "node:crypto";
 import type { DesignTokens, DownloadedAsset } from "./types.js";
 import type { CatalogedAsset } from "./assetCataloger.js";
+import { CAPTURE_USER_AGENT } from "./userAgent.js";
 import { rankIconCandidates, type IconCandidate } from "./faviconRanker.js";
 
 interface DownloadBudgetOptions {
@@ -503,7 +504,7 @@ async function fetchBuffer(url: string, timeoutMs = 10_000): Promise<Buffer | nu
   try {
     const res = await safeFetch(url, {
       signal: AbortSignal.timeout(timeoutMs),
-      headers: { "User-Agent": "HyperFrames/1.0" },
+      headers: { "User-Agent": CAPTURE_USER_AGENT },
     });
     if (!res || !res.ok) return null;
     // Reject XML/HTML error pages disguised as 200 OK (common with S3/CloudFront)

packages/cli/src/capture/htmlExtractor.ts

@@ -8,6 +8,7 @@
 import type { Page } from "puppeteer-core";
 import type { ExtractedHtml } from "./types.js";
 import { isPrivateUrl } from "./assetDownloader.js";
+import { CAPTURE_USER_AGENT } from "./userAgent.js";
 
 const DEFAULT_SETTLE_TIME = 3000;
 
@@ -35,7 +36,7 @@ export async function extractHtml(
       if (isPrivateUrl(href)) continue;
       const res = await fetch(href, {
         signal: AbortSignal.timeout(10000),
-        headers: { "User-Agent": "Mozilla/5.0" },
+        headers: { "User-Agent": CAPTURE_USER_AGENT },
       });
       if (!res.ok) continue;
       let css = await res.text();

packages/cli/src/capture/index.ts

@@ -24,6 +24,7 @@ import {
   totalDrops,
 } from "./assetDownloader.js";
 import type { IconCandidate } from "./faviconRanker.js";
+import { CAPTURE_USER_AGENT } from "./userAgent.js";
 import { extractFontMetadata } from "./fontMetadataExtractor.js";
 import { normalizeErrorMessage } from "../utils/errorMessage.js";
 import { diag } from "../ui/diagnostics.js";
@@ -166,9 +167,7 @@ export async function captureWebsite(
 
     const page1 = await chromeBrowser.newPage();
     await page1.setViewport({ width: viewportWidth, height: viewportHeight });
-    await page1.setUserAgent(
-      "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/131.0.0.0 Safari/537.36",
-    );
+    await page1.setUserAgent(CAPTURE_USER_AGENT);
 
     // Set up hooks BEFORE navigation
     await setupAnimationCapture(page1);

packages/cli/src/capture/mediaCapture.ts

@@ -11,6 +11,7 @@ import type { Browser, Page } from "puppeteer-core";
 import { mkdirSync, writeFileSync, readdirSync, readFileSync, statSync } from "node:fs";
 import { join, extname } from "node:path";
 import { isPrivateUrl, safeFetch } from "./assetDownloader.js";
+import { CAPTURE_USER_AGENT } from "./userAgent.js";
 
 /** Discovered Lottie item from network interception or DOM scan. */
 export interface DiscoveredLottie {
@@ -58,7 +59,7 @@ export async function saveLottieAnimations(
         // SSRF guard — safeFetch re-checks the denylist on every redirect hop
         const res = await safeFetch(lottieItem.url, {
           signal: AbortSignal.timeout(requestTimeoutMs),
-          headers: { "User-Agent": "HyperFrames/1.0" },
+          headers: { "User-Agent": CAPTURE_USER_AGENT },
         });
         if (!res || !res.ok) continue;
         const buf = Buffer.from(await res.arrayBuffer());
@@ -278,7 +279,7 @@ async function downloadVideoBody(
     // Location hop, so a public URL cannot 30x to an internal/metadata host.
     const res = await safeFetch(srcUrl, {
       signal: AbortSignal.timeout(timeoutMs), // bounded by both per-request and aggregate capture budgets
-      headers: { "User-Agent": "HyperFrames/1.0" },
+      headers: { "User-Agent": CAPTURE_USER_AGENT },
     });
     if (!res || !res.ok || !res.body) return null;
     const ct = (res.headers.get("content-type") || "").toLowerCase();

packages/cli/src/capture/userAgent.ts

@@ -0,0 +1,17 @@
+/**
+ * The single identity a capture presents to the origin it is capturing.
+ *
+ * A capture is one session with two halves: Chrome navigates the page, then Node fetches the
+ * assets that page referenced. When those halves send different `User-Agent` headers the origin
+ * is free to answer them differently, and an anti-bot edge does exactly that — it serves the
+ * document to the browser and refuses the out-of-band asset fetch.
+ *
+ * Measured against openai.com: `GET /favicon.svg` answers `403 text/html` for `HyperFrames/1.0`
+ * and `200 image/svg+xml` for this string. The favicon ranker had already chosen that SVG as the
+ * best declared icon; the 403 discarded it and the capture silently fell through to the next
+ * candidate, so the icon on disk was decided by the CDN's bot rules rather than by the ranker.
+ *
+ * One constant, used by both halves, is what keeps that from being reintroduced.
+ */
+export const CAPTURE_USER_AGENT =
+  "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/131.0.0.0 Safari/537.36";

packages/cli/src/commands/capture/video.ts

@@ -3,6 +3,7 @@ import { createWriteStream, existsSync, mkdirSync, readFileSync, unlinkSync } fr
 import { resolve, join, basename } from "node:path";
 import { c } from "../../ui/colors.js";
 import { safeFetch } from "../../capture/assetDownloader.js";
+import { CAPTURE_USER_AGENT } from "../../capture/userAgent.js";
 
 const MAX_VIDEO_BYTES = 250 * 1024 * 1024;
 const VIDEO_CONTENT_TYPE_RE = /^(video\/|application\/(mp4|octet-stream|x-mpegurl))/i;
@@ -12,7 +13,7 @@ async function streamToFile(url: string, destPath: string): Promise<number> {
   // safeFetch re-validates redirect hops; bare redirect:"follow" leaks to private hosts.
   const r = await safeFetch(url, {
     signal: AbortSignal.timeout(120_000),
-    headers: { "User-Agent": "HyperFrames/1.0" },
+    headers: { "User-Agent": CAPTURE_USER_AGENT },
   });
   if (!r) {
     throw new Error(