mirror of
https://github.com/heygen-com/hyperframes.git
synced 2026-09-03 12:54:29 +00:00
feat(cli): capture reports why a referenced asset is not in the folder (#3598)
Capture drops assets for four reasons and reported none of them, so a folder
with thirty images and a folder truncated to thirty images were the same
object. Every drop site was a bare `continue`, `break`, `return null` or an
empty `catch`, and the only signals downstream were two hand-written warning
strings that fired when the budget was already gone before a download pass
started, which is the one case where the pass could not say how much it lost.
`downloadAssets` and `downloadAndRewriteFonts` now return an `AssetDropCounts`
tally beside their result, incremented at the single line that performs each
drop. `capture --json` carries it as `dropped`; the human summary prints a
`Dropped:` line when it is non-zero.
Four reasons, three decisions and one failure:
size-floor fetched, then judged too small to be a real asset
budget-exhausted the post-navigation clock ran out before this one
cap-reached 30 inline SVGs, 30 fonts, or 6 faces per family
unavailable the request or the write failed
A break now counts everything it did not reach rather than the one it stopped
on, because "how many did we lose" is the question and one is never the answer.
The two budget warnings are gone. Both existed only to cover the case where
the budget ran out before a pass was called, so both passes are now called
unconditionally: a zero budget makes each loop break on its first item and
record `budget-exhausted` for the rest, which costs no network and produces a
real number instead of the word "some". The single remaining warning is derived
from the tally, so the prose and the count cannot disagree.
Measured on a live capture of a large marketing site:
default budget 232 kept, 91 dropped (39 size-floor, 20 cap-reached,
32 unavailable)
15s budget 30 kept, 299 dropped (279 budget-exhausted, 20 cap-reached)
Same page, same command, and until now both runs described themselves the same
way.
This commit is contained in:
+27
-1
@@ -186,10 +186,35 @@ npx hyperframes capture https://example.com --json
|
||||
|
||||
Screenshots: 12
|
||||
Assets: 45
|
||||
Dropped: 9 (6 size-floor, 3 cap-reached)
|
||||
Sections: 15
|
||||
Fonts: sohne-var
|
||||
```
|
||||
|
||||
`Dropped` is how many assets the page referenced that are **not** in the
|
||||
folder, and why. It is printed only when it is non-zero, and `--json` always
|
||||
carries it as a `dropped` object. Without it, a capture of a spare page and a
|
||||
capture that a limit truncated are the same three-line summary, and the only
|
||||
way to tell them apart is to open the page yourself.
|
||||
|
||||
| Reason | What it means |
|
||||
| ------------------ | ------------------------------------------------------------------------------------------------------ |
|
||||
| `size-floor` | Fetched, then judged too small to be a real asset rather than a spacer or tracking pixel. |
|
||||
| `budget-exhausted` | `--capture-budget` ran out before this one was reached. Raise it, or pass `--skip-vision` to buy time. |
|
||||
| `cap-reached` | A per-run or per-family limit was already met: 30 inline SVGs, 30 fonts, 6 faces per family. |
|
||||
| `unavailable` | The request or the write failed: network error, timeout, refused address, bad status, disk. |
|
||||
|
||||
Three of those are decisions the capture made and one is a failure it hit, so a
|
||||
run that is thin with an all-zero `dropped` is a thin page, and a run that is
|
||||
thin with counts on it was cut short.
|
||||
|
||||
```json
|
||||
{
|
||||
"assets": 45,
|
||||
"dropped": { "size-floor": 6, "budget-exhausted": 0, "cap-reached": 3, "unavailable": 0 }
|
||||
}
|
||||
```
|
||||
|
||||
| Flag | Description |
|
||||
| ------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------- |
|
||||
| `--output, -o` | Output directory. Default `./capture`, then `./capture-2/`, `./capture-3/`, … if that name is taken. |
|
||||
@@ -207,7 +232,8 @@ metadata, and contact sheets — plus whatever Lottie, video, and WebGL context
|
||||
the page exposed. It is raw material for an agent, not a finished composition;
|
||||
the `/product-launch-video` workflow uses it when a real product has to appear
|
||||
on screen. Dynamic sites, protected pages, and unusual media loaders produce
|
||||
partial results, so read the warnings and contact sheets before you build.
|
||||
partial results, so read `dropped`, the warnings, and the contact sheets before
|
||||
you build.
|
||||
|
||||
For AI image descriptions, set `GEMINI_API_KEY` in a `.env` file
|
||||
(~$0.001/image), or `OPENROUTER_API_KEY` to route any vision model through
|
||||
|
||||
@@ -4,10 +4,12 @@ import { tmpdir } from "node:os";
|
||||
import { join } from "node:path";
|
||||
import {
|
||||
downloadAndRewriteFonts,
|
||||
downloadAssets,
|
||||
isPrivateUrl,
|
||||
safeFetch,
|
||||
toStandaloneSvg,
|
||||
} from "./assetDownloader.js";
|
||||
import type { DesignTokens } from "./types.js";
|
||||
|
||||
describe("isPrivateUrl — SSRF denylist (security: F-003)", () => {
|
||||
it("blocks loopback, private, and metadata IPv4", () => {
|
||||
@@ -184,3 +186,145 @@ describe("downloadAndRewriteFonts — attempt caps", () => {
|
||||
}
|
||||
});
|
||||
});
|
||||
|
||||
describe("drop counts — why a referenced asset is not in the capture", () => {
|
||||
afterEach(() => vi.unstubAllGlobals());
|
||||
|
||||
/** `n` @font-face rules, each naming a DIFFERENT family, so only the global cap can bite. */
|
||||
function fontCss(n: number): string {
|
||||
return Array.from(
|
||||
{ length: n },
|
||||
(_, i) =>
|
||||
`@font-face { font-family: Family${i}; src: url(https://fonts${i}.example/font-${i}.woff2); }`,
|
||||
).join("\n");
|
||||
}
|
||||
|
||||
function withTempDir<T>(run: (dir: string) => Promise<T>): Promise<T> {
|
||||
const dir = mkdtempSync(join(tmpdir(), "hf-drops-"));
|
||||
return run(dir).finally(() => rmSync(dir, { recursive: true, force: true }));
|
||||
}
|
||||
|
||||
it("counts every face the budget never let it reach, not just the one it stopped on", async () => {
|
||||
// Four declared, zero budget: the honest number is four, and a warning string could only
|
||||
// ever have said "some".
|
||||
await withTempDir(async (dir) => {
|
||||
const fetchMock = vi.fn();
|
||||
vi.stubGlobal("fetch", fetchMock);
|
||||
const { drops } = await downloadAndRewriteFonts(fontCss(4), dir, { remainingMs: () => 0 });
|
||||
expect(fetchMock).not.toHaveBeenCalled();
|
||||
expect(drops["budget-exhausted"]).toBe(4);
|
||||
expect(drops["cap-reached"]).toBe(0);
|
||||
expect(drops.unavailable).toBe(0);
|
||||
});
|
||||
});
|
||||
|
||||
it("separates the faces the global cap refused from the ones that failed", async () => {
|
||||
// 35 declared, cap 30: 30 are attempted and every attempt 503s, 5 are never reached.
|
||||
await withTempDir(async (dir) => {
|
||||
vi.stubGlobal(
|
||||
"fetch",
|
||||
vi.fn(async () => new Response("no", { status: 503 })),
|
||||
);
|
||||
const { drops } = await downloadAndRewriteFonts(fontCss(35), dir);
|
||||
expect(drops["cap-reached"]).toBe(5);
|
||||
expect(drops.unavailable).toBe(30);
|
||||
expect(drops["budget-exhausted"]).toBe(0);
|
||||
});
|
||||
});
|
||||
|
||||
it("counts the faces the per-family cap refused", async () => {
|
||||
// 10 rules, all one family, per-family cap 6: 6 attempted, 4 refused.
|
||||
await withTempDir(async (dir) => {
|
||||
vi.stubGlobal(
|
||||
"fetch",
|
||||
vi.fn(async () => new Response("no", { status: 503 })),
|
||||
);
|
||||
const css = Array.from(
|
||||
{ length: 10 },
|
||||
(_, i) =>
|
||||
`@font-face { font-family: Shared; src: url(https://fonts.example/f-${i}.woff2); }`,
|
||||
).join("\n");
|
||||
const { drops } = await downloadAndRewriteFonts(css, dir);
|
||||
expect(drops["cap-reached"]).toBe(4);
|
||||
expect(drops.unavailable).toBe(6);
|
||||
});
|
||||
});
|
||||
|
||||
it("reports all zeroes when nothing was refused, which is what makes thin readable", async () => {
|
||||
// The whole point of the tally: this page declared one face and we have it. A reader can now
|
||||
// tell this apart from a page that declared thirty and got truncated to one.
|
||||
await withTempDir(async (dir) => {
|
||||
vi.stubGlobal(
|
||||
"fetch",
|
||||
vi.fn(async () => new Response(new Uint8Array(2048), { status: 200 })),
|
||||
);
|
||||
const { css, drops } = await downloadAndRewriteFonts(fontCss(1), dir);
|
||||
expect(css).toContain("assets/fonts/font-0.woff2");
|
||||
expect(drops).toEqual({
|
||||
"size-floor": 0,
|
||||
"budget-exhausted": 0,
|
||||
"cap-reached": 0,
|
||||
unavailable: 0,
|
||||
});
|
||||
});
|
||||
});
|
||||
|
||||
/** The two fields `downloadAssets` reads off the token bundle. */
|
||||
function tokensWithNoSvgs(): DesignTokens {
|
||||
return { svgs: [], sections: [], ogImage: "" } as unknown as DesignTokens;
|
||||
}
|
||||
|
||||
it("counts an image dropped for being under the raster floor", async () => {
|
||||
// 9 KB is under the 10 KB floor. Nothing lands, and the reason is now on the record.
|
||||
await withTempDir(async (dir) => {
|
||||
vi.stubGlobal(
|
||||
"fetch",
|
||||
vi.fn(async () => new Response(new Uint8Array(9000), { status: 200 })),
|
||||
);
|
||||
const { assets, drops } = await downloadAssets(tokensWithNoSvgs(), dir, [
|
||||
{ type: "Image", url: "https://cdn.example/hero.png", contexts: ["img[src]"] },
|
||||
] as never);
|
||||
expect(assets).toEqual([]);
|
||||
expect(drops["size-floor"]).toBe(1);
|
||||
expect(drops.unavailable).toBe(0);
|
||||
});
|
||||
});
|
||||
|
||||
it("counts every catalogued image the budget never let it reach", async () => {
|
||||
await withTempDir(async (dir) => {
|
||||
const fetchMock = vi.fn();
|
||||
vi.stubGlobal("fetch", fetchMock);
|
||||
const catalog = Array.from({ length: 7 }, (_, i) => ({
|
||||
type: "Image",
|
||||
url: `https://cdn.example/img-${i}.png`,
|
||||
contexts: ["img[src]"],
|
||||
}));
|
||||
const { assets, drops } = await downloadAssets(
|
||||
tokensWithNoSvgs(),
|
||||
dir,
|
||||
catalog as never,
|
||||
[],
|
||||
{ remainingMs: () => 0 },
|
||||
);
|
||||
expect(fetchMock).not.toHaveBeenCalled();
|
||||
expect(assets).toEqual([]);
|
||||
expect(drops["budget-exhausted"]).toBe(7);
|
||||
});
|
||||
});
|
||||
|
||||
it("counts the inline SVGs the 30-per-run cap refused", async () => {
|
||||
// 34 inline SVGs on the page, 30 kept: the four the cap dropped are now countable.
|
||||
await withTempDir(async (dir) => {
|
||||
const svgs = Array.from({ length: 34 }, (_, i) => ({
|
||||
outerHTML: `<svg viewBox="0 0 ${i} 10"><rect width="10" height="10" fill="#abc"/></svg>`,
|
||||
isLogo: false,
|
||||
}));
|
||||
const { assets, drops } = await downloadAssets(
|
||||
{ svgs, sections: [], ogImage: "" } as unknown as DesignTokens,
|
||||
dir,
|
||||
);
|
||||
expect(assets).toHaveLength(30);
|
||||
expect(drops["cap-reached"]).toBe(4);
|
||||
});
|
||||
});
|
||||
});
|
||||
|
||||
@@ -15,6 +15,47 @@ interface DownloadBudgetOptions {
|
||||
remainingMs?: () => number;
|
||||
}
|
||||
|
||||
/**
|
||||
* Why an asset the page referenced is not in the capture.
|
||||
*
|
||||
* Three of these are DECISIONS this downloader made and one is a FAILURE it hit, which is the
|
||||
* split a reader actually needs: a capture that is thin because the page is thin looks exactly
|
||||
* like a capture that is thin because a limit truncated it, and neither used to say so.
|
||||
*
|
||||
* Every member is counted at the single line that performs the drop, so a count can never
|
||||
* disagree with the branch it describes.
|
||||
*/
|
||||
export type AssetDropReason =
|
||||
/** Fetched, then judged too small to be a real asset rather than a spacer or tracking pixel. */
|
||||
| "size-floor"
|
||||
/** The post-navigation clock ran out before this one was reached. */
|
||||
| "budget-exhausted"
|
||||
/** A per-run or per-family limit was already met. */
|
||||
| "cap-reached"
|
||||
/** The request or the write failed: network error, timeout, refused address, bad status, disk. */
|
||||
| "unavailable";
|
||||
|
||||
export type AssetDropCounts = Record<AssetDropReason, number>;
|
||||
|
||||
/** A tally with every reason at zero — the shape a caller merges into. */
|
||||
export function noDrops(): AssetDropCounts {
|
||||
return { "size-floor": 0, "budget-exhausted": 0, "cap-reached": 0, unavailable: 0 };
|
||||
}
|
||||
|
||||
/** Sum two tallies. Used to fold the font pass and the asset pass into one capture-wide count. */
|
||||
export function mergeDrops(a: AssetDropCounts, b: AssetDropCounts): AssetDropCounts {
|
||||
const total = noDrops();
|
||||
for (const reason of Object.keys(total) as AssetDropReason[]) {
|
||||
total[reason] = a[reason] + b[reason];
|
||||
}
|
||||
return total;
|
||||
}
|
||||
|
||||
/** How many assets were dropped in total, for a caller deciding whether to say anything at all. */
|
||||
export function totalDrops(drops: AssetDropCounts): number {
|
||||
return Object.values(drops).reduce((sum, n) => sum + n, 0);
|
||||
}
|
||||
|
||||
// SVGs: hash-of-bytes filename so it can't drift from content; label-derived names mis-assigned brands.
|
||||
function svgContentHashSlug(svgSource: string | Buffer, isLogo: boolean): string {
|
||||
const hash = createHash("sha1").update(svgSource).digest("hex").slice(0, 8);
|
||||
@@ -54,18 +95,24 @@ export async function downloadAssets(
|
||||
catalogedAssets?: CatalogedAsset[],
|
||||
faviconLinks?: Array<{ rel: string; href: string }>,
|
||||
options: DownloadBudgetOptions = {},
|
||||
): Promise<DownloadedAsset[]> {
|
||||
): Promise<{ assets: DownloadedAsset[]; drops: AssetDropCounts }> {
|
||||
const assetsDir = join(outputDir, "assets");
|
||||
mkdirSync(assetsDir, { recursive: true });
|
||||
|
||||
const assets: DownloadedAsset[] = [];
|
||||
const drops = noDrops();
|
||||
const downloadedUrls = new Set<string>();
|
||||
|
||||
mkdirSync(join(outputDir, "assets", "svgs"), { recursive: true });
|
||||
const usedSvgNames = new Set<string>();
|
||||
for (let i = 0; i < tokens.svgs.length && i < 30; i++) {
|
||||
const MAX_INLINE_SVGS = 30;
|
||||
drops["cap-reached"] += Math.max(0, tokens.svgs.length - MAX_INLINE_SVGS);
|
||||
for (let i = 0; i < tokens.svgs.length && i < MAX_INLINE_SVGS; i++) {
|
||||
const svg = tokens.svgs[i]!;
|
||||
if (!svg.outerHTML || svg.outerHTML.length < 50) continue;
|
||||
if (!svg.outerHTML || svg.outerHTML.length < 50) {
|
||||
drops["size-floor"]++;
|
||||
continue;
|
||||
}
|
||||
// Hash the bytes that actually land on disk, so the filename still can't drift from content.
|
||||
const svgFile = toStandaloneSvg(svg.outerHTML);
|
||||
const slug = svgContentHashSlug(svgFile, !!svg.isLogo);
|
||||
@@ -82,14 +129,18 @@ export async function downloadAssets(
|
||||
writeFileSync(join(outputDir, localPath), svgFile, "utf-8");
|
||||
assets.push({ url: "", localPath, type: "svg" });
|
||||
} catch {
|
||||
/* skip */
|
||||
drops.unavailable++;
|
||||
}
|
||||
}
|
||||
|
||||
// 2. Favicon
|
||||
for (const icon of faviconLinks || []) {
|
||||
const icons = faviconLinks || [];
|
||||
for (const [index, icon] of icons.entries()) {
|
||||
const remainingMs = options.remainingMs?.() ?? 10_000;
|
||||
if (remainingMs <= 0) break;
|
||||
if (remainingMs <= 0) {
|
||||
drops["budget-exhausted"] += icons.length - index;
|
||||
break;
|
||||
}
|
||||
if (!icon.href) continue;
|
||||
try {
|
||||
const ext = extname(new URL(icon.href).pathname) || ".ico";
|
||||
@@ -101,8 +152,9 @@ export async function downloadAssets(
|
||||
assets.push({ url: icon.href, localPath, type: "favicon" });
|
||||
break;
|
||||
}
|
||||
drops.unavailable++;
|
||||
} catch {
|
||||
/* skip */
|
||||
drops.unavailable++;
|
||||
}
|
||||
}
|
||||
|
||||
@@ -158,7 +210,10 @@ export async function downloadAssets(
|
||||
const usedNames = new Set<string>();
|
||||
for (let i = 0; i < toDownload.length; i += BATCH_SIZE) {
|
||||
const remainingMs = options.remainingMs?.() ?? 10_000;
|
||||
if (remainingMs <= 0) break;
|
||||
if (remainingMs <= 0) {
|
||||
drops["budget-exhausted"] += toDownload.length - i;
|
||||
break;
|
||||
}
|
||||
const batch = toDownload.slice(i, i + BATCH_SIZE);
|
||||
const results = await Promise.allSettled(
|
||||
batch.map(async ({ url, isPoster, catalog }) => {
|
||||
@@ -166,15 +221,27 @@ export async function downloadAssets(
|
||||
const pathExt = extname(parsedUrl.pathname);
|
||||
const ext = pathExt && pathExt.length <= 5 ? pathExt : ".jpg";
|
||||
const buffer = await fetchBuffer(url, Math.min(10_000, remainingMs));
|
||||
if (!buffer) return null;
|
||||
if (!buffer) {
|
||||
drops.unavailable++;
|
||||
return null;
|
||||
}
|
||||
const isSvg = ext === ".svg" || url.includes(".svg");
|
||||
const minSize = isSvg ? 200 : 10000;
|
||||
if (buffer.length < minSize) return null;
|
||||
if (buffer.length < minSize) {
|
||||
drops["size-floor"]++;
|
||||
return null;
|
||||
}
|
||||
return { url, isPoster, parsedUrl, ext, buffer, catalog };
|
||||
}),
|
||||
);
|
||||
for (const result of results) {
|
||||
if (result.status !== "fulfilled" || !result.value) continue;
|
||||
// A rejection never reached a drop site of its own, so it is counted here. A fulfilled
|
||||
// `null` already counted itself above; counting it again here would double it.
|
||||
if (result.status === "rejected") {
|
||||
drops.unavailable++;
|
||||
continue;
|
||||
}
|
||||
if (!result.value) continue;
|
||||
const { url, isPoster, parsedUrl, ext, buffer, catalog } = result.value;
|
||||
try {
|
||||
let slug: string;
|
||||
@@ -201,7 +268,7 @@ export async function downloadAssets(
|
||||
assets.push({ url, localPath, type: "image" });
|
||||
imgIdx++;
|
||||
} catch {
|
||||
/* skip */
|
||||
drops.unavailable++;
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -212,18 +279,25 @@ export async function downloadAssets(
|
||||
try {
|
||||
const ext = extname(new URL(tokens.ogImage).pathname) || ".jpg";
|
||||
const localPath = `assets/og-image${ext}`;
|
||||
const buffer =
|
||||
remainingMs > 0 ? await fetchBuffer(tokens.ogImage, Math.min(10_000, remainingMs)) : null;
|
||||
if (buffer && buffer.length > 5000) {
|
||||
writeFileSync(join(outputDir, localPath), buffer);
|
||||
assets.push({ url: tokens.ogImage, localPath, type: "image" });
|
||||
if (remainingMs <= 0) {
|
||||
drops["budget-exhausted"]++;
|
||||
} else {
|
||||
const buffer = await fetchBuffer(tokens.ogImage, Math.min(10_000, remainingMs));
|
||||
if (!buffer) {
|
||||
drops.unavailable++;
|
||||
} else if (buffer.length <= 5000) {
|
||||
drops["size-floor"]++;
|
||||
} else {
|
||||
writeFileSync(join(outputDir, localPath), buffer);
|
||||
assets.push({ url: tokens.ogImage, localPath, type: "image" });
|
||||
}
|
||||
}
|
||||
} catch {
|
||||
/* skip */
|
||||
drops.unavailable++;
|
||||
}
|
||||
}
|
||||
|
||||
return assets;
|
||||
return { assets, drops };
|
||||
}
|
||||
|
||||
/** Normalize URL for deduplication — unwrap Next.js image proxy, strip w/q params */
|
||||
@@ -251,9 +325,10 @@ export async function downloadAndRewriteFonts(
|
||||
css: string,
|
||||
outputDir: string,
|
||||
options: DownloadBudgetOptions = {},
|
||||
): Promise<string> {
|
||||
): Promise<{ css: string; drops: AssetDropCounts }> {
|
||||
const assetsDir = join(outputDir, "assets", "fonts");
|
||||
mkdirSync(assetsDir, { recursive: true });
|
||||
const drops = noDrops();
|
||||
|
||||
const fontUrlRegex = /url\(['"]?(https?:\/\/[^'")\s]+\.(?:woff2?|ttf|otf)[^'")\s]*?)['"]?\)/g;
|
||||
const fontUrls = new Set<string>();
|
||||
@@ -262,7 +337,7 @@ export async function downloadAndRewriteFonts(
|
||||
if (match[1]) fontUrls.add(match[1]);
|
||||
}
|
||||
|
||||
if (fontUrls.size === 0) return css;
|
||||
if (fontUrls.size === 0) return { css, drops };
|
||||
|
||||
// Limit font download attempts to bound worst-case egress and latency. Google Fonts serves
|
||||
// 20+ unicode-range subsets per weight, so successes alone cannot be the bound: six transient
|
||||
@@ -293,13 +368,22 @@ export async function downloadAndRewriteFonts(
|
||||
let rewritten = css;
|
||||
let count = 0;
|
||||
|
||||
for (const fontUrl of sortedUrls) {
|
||||
for (const [index, fontUrl] of sortedUrls.entries()) {
|
||||
const remainingMs = options.remainingMs?.() ?? 10_000;
|
||||
if (remainingMs <= 0) break;
|
||||
if (count >= MAX_TOTAL_FONTS) break;
|
||||
if (remainingMs <= 0) {
|
||||
drops["budget-exhausted"] += sortedUrls.length - index;
|
||||
break;
|
||||
}
|
||||
if (count >= MAX_TOTAL_FONTS) {
|
||||
drops["cap-reached"] += sortedUrls.length - index;
|
||||
break;
|
||||
}
|
||||
const family = getFamilyForUrl(fontUrl);
|
||||
const familyCount = familyCounts.get(family) || 0;
|
||||
if (familyCount >= MAX_FONTS_PER_FAMILY) continue;
|
||||
if (familyCount >= MAX_FONTS_PER_FAMILY) {
|
||||
drops["cap-reached"]++;
|
||||
continue;
|
||||
}
|
||||
familyCounts.set(family, familyCount + 1);
|
||||
count++;
|
||||
|
||||
@@ -313,13 +397,15 @@ export async function downloadAndRewriteFonts(
|
||||
if (buffer) {
|
||||
writeFileSync(localPath, buffer);
|
||||
rewritten = rewritten.split(fontUrl).join(relativePath);
|
||||
} else {
|
||||
drops.unavailable++;
|
||||
}
|
||||
} catch {
|
||||
/* skip */
|
||||
drops.unavailable++;
|
||||
}
|
||||
}
|
||||
|
||||
return rewritten;
|
||||
return { css: rewritten, drops };
|
||||
}
|
||||
|
||||
// Reserved/loopback/private IPv4 blocks as [firstOctet, secondOctetLo, secondOctetHi].
|
||||
|
||||
@@ -16,7 +16,13 @@ import { extractHtml } from "./htmlExtractor.js";
|
||||
// captureScreenshots removed — full-page screenshot replaces per-section shots
|
||||
import { extractTokens } from "./tokenExtractor.js";
|
||||
import { extractDesignStyles } from "./designStyleExtractor.js";
|
||||
import { downloadAssets, downloadAndRewriteFonts } from "./assetDownloader.js";
|
||||
import {
|
||||
downloadAssets,
|
||||
downloadAndRewriteFonts,
|
||||
mergeDrops,
|
||||
noDrops,
|
||||
totalDrops,
|
||||
} from "./assetDownloader.js";
|
||||
import { extractFontMetadata } from "./fontMetadataExtractor.js";
|
||||
import { normalizeErrorMessage } from "../utils/errorMessage.js";
|
||||
import { diag } from "../ui/diagnostics.js";
|
||||
@@ -578,23 +584,22 @@ export async function captureWebsite(
|
||||
|
||||
phase("core-extraction", "completed");
|
||||
|
||||
// Download fonts and rewrite URLs to local paths
|
||||
if (remainingMs() > 0) {
|
||||
phase("fonts", "started");
|
||||
extracted.headHtml = await downloadAndRewriteFonts(extracted.headHtml, outputDir, {
|
||||
remainingMs,
|
||||
});
|
||||
phase(
|
||||
"fonts",
|
||||
remainingMs() > 0 ? "completed" : "degraded",
|
||||
remainingMs() > 0 ? undefined : "budget-exhausted",
|
||||
);
|
||||
} else {
|
||||
warnings.push(
|
||||
"Capture budget exhausted before font downloads; extracted font tokens were preserved.",
|
||||
);
|
||||
phase("fonts", "degraded", "budget-exhausted");
|
||||
}
|
||||
// Download fonts and rewrite URLs to local paths.
|
||||
//
|
||||
// Called even with the budget already gone, which is the point: its own loop is the only
|
||||
// thing that knows how many faces the page declared, so letting it run and record
|
||||
// `budget-exhausted` for every one of them replaces a warning string that could only ever
|
||||
// say "some". A zero budget means it breaks on the first url, so this costs no network.
|
||||
phase("fonts", "started");
|
||||
const fontPass = await downloadAndRewriteFonts(extracted.headHtml, outputDir, {
|
||||
remainingMs,
|
||||
});
|
||||
extracted.headHtml = fontPass.css;
|
||||
phase(
|
||||
"fonts",
|
||||
remainingMs() > 0 ? "completed" : "degraded",
|
||||
remainingMs() > 0 ? undefined : "budget-exhausted",
|
||||
);
|
||||
|
||||
// Identify each downloaded font by reading its OpenType name table.
|
||||
// Modern frameworks hash font filenames; this manifest tells the
|
||||
@@ -651,25 +656,40 @@ export async function captureWebsite(
|
||||
|
||||
// Download assets — single pass using the catalog for best image quality
|
||||
let assets: CaptureResult["assets"] = [];
|
||||
let assetDrops = noDrops();
|
||||
if (!skipAssets) {
|
||||
if (remainingMs() > 0) {
|
||||
phase("assets", "started");
|
||||
progress("assets", "Downloading assets...");
|
||||
assets = await downloadAssets(tokens, outputDir, catalogedAssets, faviconLinks, {
|
||||
remainingMs,
|
||||
});
|
||||
phase(
|
||||
"assets",
|
||||
remainingMs() > 0 ? "completed" : "degraded",
|
||||
remainingMs() > 0 ? undefined : "budget-exhausted",
|
||||
);
|
||||
} else {
|
||||
warnings.push("Capture budget exhausted before asset downloads; extraction continued.");
|
||||
phase("assets", "degraded", "budget-exhausted");
|
||||
}
|
||||
// Called even with the budget already gone, for the reason the font pass is: the loop that
|
||||
// skips an asset is the only thing that can say how many it skipped.
|
||||
phase("assets", "started");
|
||||
progress("assets", "Downloading assets...");
|
||||
const assetPass = await downloadAssets(tokens, outputDir, catalogedAssets, faviconLinks, {
|
||||
remainingMs,
|
||||
});
|
||||
assets = assetPass.assets;
|
||||
assetDrops = assetPass.drops;
|
||||
phase(
|
||||
"assets",
|
||||
remainingMs() > 0 ? "completed" : "degraded",
|
||||
remainingMs() > 0 ? undefined : "budget-exhausted",
|
||||
);
|
||||
} else {
|
||||
phase("assets", "degraded", "disabled");
|
||||
}
|
||||
// One capture-wide tally, summed from the two passes that own the drops. The warning is
|
||||
// DERIVED from it rather than written alongside it, so the prose and the number cannot
|
||||
// disagree the way two separately-authored budget strings could.
|
||||
const dropped = mergeDrops(fontPass.drops, assetDrops);
|
||||
const droppedTotal = totalDrops(dropped);
|
||||
if (droppedTotal > 0) {
|
||||
const breakdown = Object.entries(dropped)
|
||||
.filter(([, n]) => n > 0)
|
||||
.map(([reason, n]) => `${n} ${reason}`)
|
||||
.join(", ");
|
||||
warnings.push(
|
||||
`${droppedTotal} referenced asset(s) are not in this capture (${breakdown}). ` +
|
||||
"A thin capture with no drops is a thin page; this one was truncated.",
|
||||
);
|
||||
}
|
||||
|
||||
// Join in-section media URLs → downloaded local paths, then re-write
|
||||
// tokens.json. Downstream page recreation MUST reference local files:
|
||||
@@ -885,6 +905,7 @@ export async function captureWebsite(
|
||||
screenshots,
|
||||
tokens,
|
||||
assets,
|
||||
dropped,
|
||||
animationCatalog,
|
||||
warnings,
|
||||
lastPhase,
|
||||
|
||||
@@ -79,6 +79,13 @@ export interface CaptureResult {
|
||||
tokens: DesignTokens;
|
||||
/** Downloaded asset paths (relative to projectDir) */
|
||||
assets: DownloadedAsset[];
|
||||
/**
|
||||
* How many referenced assets are NOT here, by reason.
|
||||
*
|
||||
* Without this, a capture of a page with three images and a capture truncated to three images
|
||||
* are the same object. All zeroes means the capture kept everything it was offered.
|
||||
*/
|
||||
dropped: import("./assetDownloader.js").AssetDropCounts;
|
||||
/** Animation catalog (captured during full-JS page load) */
|
||||
animationCatalog?: import("./animationCataloger.js").AnimationCatalog;
|
||||
/** Errors/warnings encountered during capture */
|
||||
|
||||
@@ -207,6 +207,7 @@ export default defineCommand({
|
||||
title: result.title,
|
||||
screenshots: result.screenshots.length,
|
||||
assets: result.assets.length,
|
||||
dropped: result.dropped,
|
||||
detectedSections: result.tokens.sections.length,
|
||||
fonts: result.tokens.fonts.map((f) => f.family),
|
||||
fontsDetailed: result.tokens.fonts,
|
||||
@@ -225,6 +226,18 @@ export default defineCommand({
|
||||
console.log();
|
||||
console.log(` ${c.dim("Screenshots:")} ${result.screenshots.length}`);
|
||||
console.log(` ${c.dim("Assets:")} ${result.assets.length}`);
|
||||
const droppedTotal = Object.values(result.dropped).reduce((sum, n) => sum + n, 0);
|
||||
if (droppedTotal > 0) {
|
||||
const breakdown = Object.entries(result.dropped)
|
||||
.filter(function (entry) {
|
||||
return entry[1] > 0;
|
||||
})
|
||||
.map(function (entry) {
|
||||
return entry[1] + " " + entry[0];
|
||||
})
|
||||
.join(", ");
|
||||
console.log(` ${c.dim("Dropped:")} ${droppedTotal} (${breakdown})`);
|
||||
}
|
||||
console.log(` ${c.dim("Sections:")} ${result.tokens.sections.length}`);
|
||||
console.log(
|
||||
` ${c.dim("Fonts:")} ${result.tokens.fonts
|
||||
|
||||
Reference in New Issue
Block a user