mirror of
https://github.com/heygen-com/hyperframes.git
synced 2026-10-30 09:16:55 +00:00
feat(cli): capture accepts a local HTML file again (#6498)
* feat(cli): capture accepts a local HTML file again Adds an explicit --local-file mode. The file's directory (or --local-root) is served on a loopback port that only Chrome may reach outside the capture proxy; all other traffic keeps the public-address policy. The URL argument stays public HTTP(S) only. * fix(cli): local capture output names the file, not the loopback server In --local-file mode, meta.json, AGENTS.md, extracted files and the returned result used the temporary 127.0.0.1 origin. One rewriter maps that origin to file:// URLs under the root, and meta.json records the file as its source. Root and served paths now resolve with the same native realpath, which Windows needs. * fix(cli): local capture reads its files from disk and resolves real paths natively The asset downloader read the local page server with Node's fetch, which an environment proxy (NODE_USE_ENV_PROXY with HTTP_PROXY) intercepts, so local images and fonts were dropped. It now reads the file from disk under the server's own checks. Root and served paths both use the native realpath. The JS realpath kept Windows 8.3 short names while fs/promises expanded them, so every file answered 404 on Windows. The default root is the directory the file was named in, so a symlinked entry can no longer widen it to wherever the link points. The local-file docs move below the capture policy text so their heading does not swallow it. * refactor(cli): capture settles a watchdog outcome in one place * fix(cli): escape every RegExp metacharacter in the local origin rewriter --------- Co-authored-by: Jerrai <jerrai@heygen.com>
This commit is contained in:
Xuanru Li
@@ -295,6 +295,8 @@ thin with counts on it was cut short.
|
||||
| `--skip-vision` | Do not run AI image captioning |
|
||||
| `--max-screenshots` | Maximum page-capture files, including the full-page plate; derived contact sheets are excluded. Positive integer; default 21. A value of 1 captures only the top viewport. |
|
||||
| `--json` | Structured output for programmatic use |
|
||||
| `--local-file` | Capture a local HTML file, or a directory's `index.html`, instead of a URL. See [Local HTML files](#local-html-files). |
|
||||
| `--local-root` | With `--local-file`: the directory the page may read local files from. Default: the file's directory. |
|
||||
| `--video <folder>` | Switch to video-download mode against a captured folder's manifest. Pair with `--list`, `--index`, or `--video-url`. |
|
||||
|
||||
You get `AGENTS.md`, `CLAUDE.md`, `meta.json`, scroll screenshots, extracted
|
||||
@@ -315,6 +317,33 @@ For AI image descriptions, set `GEMINI_API_KEY` in a `.env` file
|
||||
[OpenRouter](https://openrouter.ai) — it wins if both are set, and
|
||||
`HYPERFRAMES_OPENROUTER_MODEL` overrides the model.
|
||||
|
||||
#### Local HTML files
|
||||
|
||||
`--local-file` captures an HTML file on disk instead of a URL, for example a
|
||||
page someone uploaded. Pass a directory to open its `index.html`.
|
||||
|
||||
```bash
|
||||
npx hyperframes capture --local-file ./upload/index.html -o upload-capture --json
|
||||
npx hyperframes capture --local-file ./site/pages/about.html --local-root ./site
|
||||
```
|
||||
|
||||
The URL argument still accepts only public HTTP(S) URLs; a `file://` URL there
|
||||
is refused as before. With `--local-file`, the CLI serves the file's directory
|
||||
(or `--local-root`, which must contain the file) on a loopback port for the
|
||||
length of the capture, and Chrome loads the page from it. Only regular files
|
||||
whose real path is inside that root are served, the HTML file included: `../`
|
||||
paths and symlinks that leave it answer 404, a symlinked HTML file that points
|
||||
outside is refused, and the page cannot read `file://` URLs. Every other
|
||||
request, remote or private, goes through the capture proxy and the
|
||||
public-address policy above, so a reference to a private address stays refused
|
||||
and counts as `unavailable`. Local images, fonts and stylesheets are copied
|
||||
into the capture like remote ones; `--skip-assets` skips images and SVGs
|
||||
exactly as it does for a URL. The temporary server never appears in the
|
||||
output: `url` in the JSON result, `meta.json` (`source`, and an `id` named after
|
||||
the file or its folder), `AGENTS.md` and the extracted files all use `file://`
|
||||
URLs of the same paths inside the root. With `HYPERFRAMES_CAPTURE_PROXY` set,
|
||||
local files are still read directly; every other request goes to that proxy.
|
||||
|
||||
### `transcribe`
|
||||
|
||||
Turn audio or video into word-level timestamps, or import a transcript you
|
||||
|
||||
@@ -12,6 +12,7 @@
|
||||
|
||||
import { readdirSync, existsSync } from "node:fs";
|
||||
import { writeCaptureFileSync } from "./captureFile.js";
|
||||
import { captureSiteName } from "./localPage.js";
|
||||
import { join } from "node:path";
|
||||
import type { DesignTokens } from "./types.js";
|
||||
import type { AnimationCatalog } from "./animationCataloger.js";
|
||||
@@ -67,7 +68,7 @@ function buildPrompt(
|
||||
hasLottie?: boolean,
|
||||
hasShaders?: boolean,
|
||||
): string {
|
||||
const title = tokens.title || new URL(url).hostname.replace(/^www\./, "");
|
||||
const title = tokens.title || captureSiteName(url);
|
||||
|
||||
const colorSummary = tokens.colors
|
||||
.slice(0, 10)
|
||||
|
||||
@@ -7,6 +7,7 @@
|
||||
|
||||
import { isBlockedNetworkHost } from "@hyperframes/engine";
|
||||
import { publicRequest } from "./publicNetwork.js";
|
||||
import { fetchLocalPageUrl } from "./localPage.js";
|
||||
import { ensureCaptureDirSync, writeCaptureFileSync } from "./captureFile.js";
|
||||
import { join, extname } from "node:path";
|
||||
import { createHash } from "node:crypto";
|
||||
@@ -674,6 +675,8 @@ const MAX_FETCH_REDIRECTS = 5;
|
||||
* or on network error.
|
||||
*/
|
||||
export async function safeFetch(url: string, init?: RequestInit): Promise<Response | null> {
|
||||
const local = fetchLocalPageUrl(url, init);
|
||||
if (local) return local;
|
||||
let current = url;
|
||||
for (let hop = 0; hop <= MAX_FETCH_REDIRECTS; hop++) {
|
||||
if (isPrivateUrl(current)) return null;
|
||||
|
||||
@@ -63,3 +63,26 @@ describe("captureWebsiteAttempt URL admission", () => {
|
||||
expect(ensureBrowserMock).not.toHaveBeenCalled();
|
||||
});
|
||||
});
|
||||
|
||||
describe("captureWebsiteAttempt local origin admission", () => {
|
||||
it.each(["http://127.0.0.1:9124/index.html", "file:///etc/passwd", "https://example.com/"])(
|
||||
"rejects %s when it is not on the local page server",
|
||||
async (url) => {
|
||||
const root = mkdtempSync(join(tmpdir(), "hf-capture-local-"));
|
||||
onTestFinished(() => rmSync(root, { recursive: true, force: true }));
|
||||
const opts = { url, outputDir: root };
|
||||
await expect(
|
||||
captureWebsiteAttempt(
|
||||
opts,
|
||||
undefined,
|
||||
false,
|
||||
watchdog,
|
||||
createPartialCaptureState(opts),
|
||||
"http://127.0.0.1:9123",
|
||||
),
|
||||
).rejects.toThrow("Local capture URL must use its server");
|
||||
expect(readdirSync(root)).toEqual([]);
|
||||
expect(ensureBrowserMock).not.toHaveBeenCalled();
|
||||
},
|
||||
);
|
||||
});
|
||||
|
||||
@@ -41,6 +41,7 @@ export async function captureWebsiteAttempt(
|
||||
disableWebgl: boolean,
|
||||
watchdog: CaptureWatchdog,
|
||||
state: PartialCaptureState,
|
||||
localOrigin?: string,
|
||||
): Promise<CaptureResult> {
|
||||
const {
|
||||
url,
|
||||
@@ -57,7 +58,9 @@ export async function captureWebsiteAttempt(
|
||||
} = opts;
|
||||
|
||||
const initialUrl = new URL(url);
|
||||
if (
|
||||
if (localOrigin !== undefined) {
|
||||
if (initialUrl.origin !== localOrigin) throw new Error("Local capture URL must use its server");
|
||||
} else if (
|
||||
!["http:", "https:"].includes(initialUrl.protocol) ||
|
||||
initialUrl.username ||
|
||||
initialUrl.password
|
||||
@@ -125,6 +128,7 @@ export async function captureWebsiteAttempt(
|
||||
disableWebgl,
|
||||
width: viewportWidth,
|
||||
height: viewportHeight,
|
||||
localOrigin,
|
||||
});
|
||||
const chromeBrowser = captureBrowser.browser;
|
||||
watchdog.registerBrowser(captureBrowser);
|
||||
|
||||
@@ -2,6 +2,13 @@ import { createCaptureWatchdog, runWithWatchdog } from "./captureWatchdog.js";
|
||||
import { createPartialCaptureState, writePartialCaptureBundle } from "./partialCapture.js";
|
||||
import type { PartialCaptureState } from "./partialCapture.js";
|
||||
import { captureWebsiteAttempt } from "./captureAttempt.js";
|
||||
import {
|
||||
localUrlRewriter,
|
||||
resolveLocalCaptureTarget,
|
||||
rewriteLocalCaptureOutput,
|
||||
rewriteStrings,
|
||||
startLocalPageServer,
|
||||
} from "./localPage.js";
|
||||
import { NavigationDeadlineError } from "./captureErrors.js";
|
||||
import { CAPTURE_PHASE_SCHEMA } from "./types.js";
|
||||
import type { CaptureOptions, CaptureResult } from "./types.js";
|
||||
@@ -11,15 +18,40 @@ export type { CaptureOptions, CaptureResult } from "./types.js";
|
||||
export async function captureWebsite(
|
||||
opts: CaptureOptions,
|
||||
onProgress?: (stage: string, detail?: string) => void,
|
||||
): Promise<CaptureResult> {
|
||||
if (opts.localFile === undefined) {
|
||||
if (opts.localRoot !== undefined) throw new Error("localRoot requires localFile");
|
||||
return captureTarget(opts, onProgress);
|
||||
}
|
||||
const local = resolveLocalCaptureTarget(opts.localFile, opts.localRoot);
|
||||
const server = await startLocalPageServer(local);
|
||||
const rewrite = localUrlRewriter(server.origin, local.root);
|
||||
try {
|
||||
const result = await captureTarget({ ...opts, url: server.url }, onProgress, server.origin);
|
||||
return { ...rewriteStrings(result, rewrite), url: local.fileUrl };
|
||||
} finally {
|
||||
await server.close();
|
||||
rewriteLocalCaptureOutput(opts.outputDir, rewrite, local.fileUrl);
|
||||
}
|
||||
}
|
||||
|
||||
async function captureTarget(
|
||||
opts: CaptureOptions,
|
||||
onProgress: ((stage: string, detail?: string) => void) | undefined,
|
||||
localOrigin?: string,
|
||||
): Promise<CaptureResult> {
|
||||
const watchdog = createCaptureWatchdog(opts.captureDeadlineMs);
|
||||
const state = createPartialCaptureState(opts);
|
||||
const attempt = captureWebsiteAttempt(opts, onProgress, false, watchdog, state);
|
||||
const run = (disableWebgl: boolean) =>
|
||||
captureWebsiteAttempt(opts, onProgress, disableWebgl, watchdog, state, localOrigin);
|
||||
const settle = async (attempt: Promise<CaptureResult>): Promise<CaptureResult> => {
|
||||
const outcome = await runWithWatchdog(attempt, watchdog.promise);
|
||||
if (outcome.kind === "deadline") return deadlineResult(opts, state);
|
||||
if (outcome.kind === "error") throw outcome.error;
|
||||
return outcome.result;
|
||||
};
|
||||
try {
|
||||
const first = await runWithWatchdog(attempt, watchdog.promise);
|
||||
if (first.kind === "deadline") return deadlineResult(opts, state);
|
||||
if (first.kind === "error") throw first.error;
|
||||
return first.result;
|
||||
return await settle(run(false));
|
||||
} catch (err) {
|
||||
if (!(err instanceof NavigationDeadlineError) || watchdog.expired() || !err.webglObserved) {
|
||||
throw err;
|
||||
@@ -33,14 +65,8 @@ export async function captureWebsite(
|
||||
remainingMs: null,
|
||||
reason: "webgl-disabled-retry",
|
||||
});
|
||||
const retry = captureWebsiteAttempt(opts, onProgress, true, watchdog, state);
|
||||
const second = await runWithWatchdog(retry, watchdog.promise);
|
||||
if (second.kind === "deadline") return deadlineResult(opts, state);
|
||||
if (second.kind === "error") throw second.error;
|
||||
return {
|
||||
...second.result,
|
||||
warnings: [...new Set([...state.warnings, ...second.result.warnings])],
|
||||
};
|
||||
const second = await settle(run(true));
|
||||
return { ...second, warnings: [...new Set([...state.warnings, ...second.warnings])] };
|
||||
} finally {
|
||||
watchdog.dispose();
|
||||
}
|
||||
|
||||
@@ -72,6 +72,7 @@ describe("capture navigation retry launch", () => {
|
||||
true,
|
||||
expect.anything(),
|
||||
expect.anything(),
|
||||
undefined,
|
||||
);
|
||||
expect(actual.warnings).toContain("Navigation timed out; retrying once with WebGL disabled");
|
||||
expect(phases).toContain("webgl-disabled-retry");
|
||||
|
||||
@@ -51,6 +51,17 @@ describe("capture browser transport lifecycle", () => {
|
||||
mocks.browserClose.mock.invocationCallOrder[0] ?? 0,
|
||||
);
|
||||
});
|
||||
it("exempts only a local page server's exact host and port from the proxy", async () => {
|
||||
const session = await launchCaptureBrowser({
|
||||
...options,
|
||||
localOrigin: "http://127.0.0.1:9123",
|
||||
});
|
||||
expect(mocks.launch.mock.calls[0]?.[1].args).toContain(
|
||||
"--proxy-bypass-list=<-loopback>;127.0.0.1:9123",
|
||||
);
|
||||
await session.close();
|
||||
});
|
||||
|
||||
// A profile directory the OS refuses to empty stands in for Windows' locked Chrome lockfile.
|
||||
it.runIf(process.platform !== "win32" && process.getuid?.() !== 0)(
|
||||
"finishes a close whose profile directory cannot be removed, with a warning",
|
||||
|
||||
@@ -18,6 +18,7 @@ export async function launchCaptureBrowser(options: {
|
||||
disableWebgl: boolean;
|
||||
width: number;
|
||||
height: number;
|
||||
localOrigin?: string;
|
||||
}): Promise<{ browser: Browser; close(): Promise<void> }> {
|
||||
const proxy = await startCaptureProxy();
|
||||
let profile: string | undefined;
|
||||
@@ -60,7 +61,7 @@ export async function launchCaptureBrowser(options: {
|
||||
args: [
|
||||
...captureBrowserArgs(options.disableWebgl, options.width, options.height),
|
||||
`--proxy-server=${proxy.url}`,
|
||||
"--proxy-bypass-list=<-loopback>",
|
||||
`--proxy-bypass-list=<-loopback>${options.localOrigin ? `;${new URL(options.localOrigin).host}` : ""}`,
|
||||
"--disable-quic",
|
||||
"--force-webrtc-ip-handling-policy=disable_non_proxied_udp",
|
||||
],
|
||||
|
||||
@@ -0,0 +1,128 @@
|
||||
import { createServer, type IncomingMessage, type RequestOptions } from "node:http";
|
||||
import { existsSync, mkdirSync, mkdtempSync, realpathSync, rmSync, symlinkSync } from "node:fs";
|
||||
import { writeFileSync } from "node:fs";
|
||||
import { tmpdir } from "node:os";
|
||||
import { join } from "node:path";
|
||||
import { pathToFileURL } from "node:url";
|
||||
import { afterAll, beforeAll, describe, expect, it, vi } from "vitest";
|
||||
|
||||
const fixture = vi.hoisted(() => ({ port: 0, hits: [] as string[], dialed: [] as string[] }));
|
||||
vi.mock("node:dns", async (importOriginal) => {
|
||||
const real = await importOriginal<typeof import("node:dns")>();
|
||||
return {
|
||||
...real,
|
||||
promises: {
|
||||
...real.promises,
|
||||
Resolver: class {
|
||||
async resolve4(host: string) {
|
||||
if (host === "capture-fixture.example.com") return ["93.184.216.34"];
|
||||
throw new Error("Fixture has no such DNS name");
|
||||
}
|
||||
cancel() {}
|
||||
},
|
||||
},
|
||||
};
|
||||
});
|
||||
vi.mock("node:child_process", async (importOriginal) => {
|
||||
const real = await importOriginal<typeof import("node:child_process")>();
|
||||
return {
|
||||
...real,
|
||||
execFile: (...args: Parameters<typeof real.execFile>) => {
|
||||
if (!(args[1] as string[] | undefined)?.includes("-e")) return real.execFile(...args);
|
||||
const callback = args.at(-1) as (error: Error, stdout: string, stderr: string) => void;
|
||||
callback(new Error("Fixture has no system DNS"), "", "ENOTFOUND");
|
||||
return { once() {}, kill() {} } as unknown as ReturnType<typeof real.execFile>;
|
||||
},
|
||||
};
|
||||
});
|
||||
vi.mock("node:http", async (importOriginal) => {
|
||||
const original = await importOriginal<typeof import("node:http")>();
|
||||
return {
|
||||
...original,
|
||||
request(options: RequestOptions, listener?: (response: IncomingMessage) => void) {
|
||||
if (options.hostname === "93.184.216.34") {
|
||||
fixture.dialed.push(String(options.path));
|
||||
options = { ...options, hostname: "127.0.0.1" };
|
||||
}
|
||||
return original.request(options, listener);
|
||||
},
|
||||
};
|
||||
});
|
||||
import { launchCaptureBrowser } from "./launchCaptureBrowser.js";
|
||||
import { resolveLocalCaptureTarget, startLocalPageServer } from "./localPage.js";
|
||||
|
||||
const chrome = process.env.HYPERFRAMES_TEST_BROWSER_PATH;
|
||||
describe.skipIf(!chrome || !existsSync(chrome))("real browser local page capture", () => {
|
||||
const remote = createServer((req, res) => {
|
||||
fixture.hits.push(req.url ?? "");
|
||||
res.end("remote");
|
||||
});
|
||||
let base: string;
|
||||
let local: Awaited<ReturnType<typeof startLocalPageServer>>;
|
||||
let capture: Awaited<ReturnType<typeof launchCaptureBrowser>>;
|
||||
|
||||
beforeAll(async () => {
|
||||
await new Promise<void>((resolve) => remote.listen(0, "127.0.0.1", resolve));
|
||||
fixture.port = (remote.address() as { port: number }).port;
|
||||
base = realpathSync.native(mkdtempSync(join(tmpdir(), "hf-local-browser-")));
|
||||
const site = join(base, "site");
|
||||
mkdirSync(site);
|
||||
writeFileSync(join(base, "secret.png"), "outside");
|
||||
symlinkSync(join(base, "secret.png"), join(site, "link.png"));
|
||||
writeFileSync(join(site, "logo.png"), "logo");
|
||||
writeFileSync(join(site, "font.woff2"), "font");
|
||||
writeFileSync(
|
||||
join(site, "style.css"),
|
||||
`@font-face { font-family: LocalFixture; src: url(font.woff2); } body { font-family: LocalFixture; }`,
|
||||
);
|
||||
const remoteOrigin = `http://capture-fixture.example.com:${fixture.port}`;
|
||||
writeFileSync(
|
||||
join(site, "index.html"),
|
||||
`<html><head><link rel="stylesheet" href="style.css"></head><body>local fixture
|
||||
<img src="logo.png"><img src="../secret.png"><img src="link.png">
|
||||
<img src="${pathToFileURL(join(base, "secret.png")).href}">
|
||||
<img src="${remoteOrigin}/remote.png">
|
||||
<img src="http://127.0.0.1:${fixture.port}/private.png"></body></html>`,
|
||||
);
|
||||
local = await startLocalPageServer(resolveLocalCaptureTarget(site));
|
||||
capture = await launchCaptureBrowser({
|
||||
executablePath: chrome!,
|
||||
protocolTimeout: 15_000,
|
||||
disableWebgl: true,
|
||||
width: 800,
|
||||
height: 600,
|
||||
localOrigin: local.origin,
|
||||
});
|
||||
}, 20_000);
|
||||
|
||||
afterAll(async () => {
|
||||
await capture?.close();
|
||||
await local?.close();
|
||||
remote.closeAllConnections();
|
||||
await new Promise<void>((resolve) => remote.close(() => resolve()));
|
||||
rmSync(base, { recursive: true, force: true });
|
||||
});
|
||||
|
||||
it("loads local files inside the root, refuses the rest and proxies remote ones", async () => {
|
||||
const page = await capture.browser.newPage();
|
||||
const statuses = new Map<string, number>();
|
||||
page.on("response", (response) => statuses.set(response.url(), response.status()));
|
||||
try {
|
||||
await page.goto(local.url, { waitUntil: "networkidle0", timeout: 10_000 });
|
||||
expect(await page.$eval("body", (body) => body.textContent)).toContain("local fixture");
|
||||
await page.evaluate(() => document.fonts.ready);
|
||||
|
||||
expect(statuses.get(`${local.origin}/logo.png`)).toBe(200);
|
||||
expect(statuses.get(`${local.origin}/style.css`)).toBe(200);
|
||||
expect(statuses.get(`${local.origin}/font.woff2`)).toBe(200);
|
||||
expect(statuses.get(`${local.origin}/secret.png`)).toBe(404);
|
||||
expect(statuses.get(`${local.origin}/link.png`)).toBe(404);
|
||||
expect([...statuses.keys()].some((url) => url.startsWith("file:"))).toBe(false);
|
||||
|
||||
expect(fixture.dialed).toContain("/remote.png");
|
||||
expect(fixture.hits).toEqual(["/remote.png"]);
|
||||
} finally {
|
||||
await page.close();
|
||||
}
|
||||
}, 20_000);
|
||||
});
|
||||
@@ -0,0 +1,134 @@
|
||||
import { mkdirSync, mkdtempSync, readFileSync, realpathSync, rmSync, writeFileSync } from "node:fs";
|
||||
import { tmpdir } from "node:os";
|
||||
import { join } from "node:path";
|
||||
import { pathToFileURL } from "node:url";
|
||||
import { afterEach, beforeEach, describe, expect, it, vi } from "vitest";
|
||||
import { createPartialCaptureState } from "./partialCapture.js";
|
||||
import { generateProjectScaffold } from "./scaffolding.js";
|
||||
import { CAPTURE_PHASE_SCHEMA } from "./types.js";
|
||||
import type { CaptureOptions, CaptureResult } from "./types.js";
|
||||
|
||||
const { attemptMock } = vi.hoisted(() => ({ attemptMock: vi.fn() }));
|
||||
vi.mock("./captureAttempt.js", () => ({ captureWebsiteAttempt: attemptMock }));
|
||||
import { captureWebsite } from "./index.js";
|
||||
|
||||
const REMOTE = "https://cdn.example.com/hero.png";
|
||||
|
||||
// Writes what a real attempt writes, through the real scaffold, with whatever URL it navigated.
|
||||
async function fakeAttempt(opts: CaptureOptions): Promise<CaptureResult> {
|
||||
const pageUrl = new URL(opts.url);
|
||||
const localAsset = new URL("img/logo%20mark.png?v=2", pageUrl).href;
|
||||
const partial = createPartialCaptureState(opts);
|
||||
mkdirSync(join(opts.outputDir, "extracted"), { recursive: true });
|
||||
writeFileSync(
|
||||
join(opts.outputDir, "extracted", "assets-manifest.json"),
|
||||
JSON.stringify({ page: opts.url, assets: [{ url: localAsset }, { url: REMOTE }] }, null, 2),
|
||||
);
|
||||
writeFileSync(
|
||||
join(opts.outputDir, "extracted", "page.html"),
|
||||
`<img src="${localAsset}"><img src="${REMOTE}">`,
|
||||
);
|
||||
await generateProjectScaffold(
|
||||
opts.outputDir,
|
||||
opts.url,
|
||||
partial.tokens,
|
||||
undefined,
|
||||
false,
|
||||
false,
|
||||
false,
|
||||
[],
|
||||
() => {},
|
||||
[],
|
||||
);
|
||||
return {
|
||||
ok: true,
|
||||
projectDir: opts.outputDir,
|
||||
url: opts.url,
|
||||
httpStatus: 200,
|
||||
title: "",
|
||||
extracted: partial.extracted,
|
||||
screenshots: [],
|
||||
tokens: partial.tokens,
|
||||
assets: [
|
||||
{ url: localAsset, localPath: "assets/logo.png", type: "image" },
|
||||
{ url: REMOTE, localPath: "assets/hero.png", type: "image" },
|
||||
],
|
||||
dropped: partial.dropped,
|
||||
warnings: [],
|
||||
lastPhase: {
|
||||
schema: CAPTURE_PHASE_SCHEMA,
|
||||
phase: "complete",
|
||||
status: "completed",
|
||||
remainingMs: 1,
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
let base: string;
|
||||
let site: string;
|
||||
let outputDir: string;
|
||||
|
||||
beforeEach(() => {
|
||||
base = realpathSync.native(mkdtempSync(join(tmpdir(), "hf-local-output-")));
|
||||
site = join(base, "Client Upload");
|
||||
outputDir = join(base, "out");
|
||||
mkdirSync(site);
|
||||
writeFileSync(join(site, "index.html"), "<h1>upload</h1>");
|
||||
attemptMock.mockImplementation(fakeAttempt);
|
||||
});
|
||||
|
||||
afterEach(() => {
|
||||
attemptMock.mockReset();
|
||||
rmSync(base, { recursive: true, force: true });
|
||||
});
|
||||
|
||||
function written(path: string): string {
|
||||
return readFileSync(join(outputDir, path), "utf8");
|
||||
}
|
||||
|
||||
describe("local capture output", () => {
|
||||
it("never mentions the temporary loopback server", async () => {
|
||||
const result = await captureWebsite({ url: "", localFile: site, outputDir });
|
||||
const outputs = [
|
||||
written("meta.json"),
|
||||
written("AGENTS.md"),
|
||||
written("CLAUDE.md"),
|
||||
written("extracted/assets-manifest.json"),
|
||||
written("extracted/page.html"),
|
||||
JSON.stringify(result),
|
||||
];
|
||||
for (const text of outputs) expect(text).not.toContain("127.0.0.1");
|
||||
|
||||
const entry = pathToFileURL(join(site, "index.html")).href;
|
||||
const logo = `${pathToFileURL(join(site, "img", "logo mark.png")).href}?v=2`;
|
||||
expect(JSON.parse(written("meta.json"))).toEqual({
|
||||
id: "client-upload-video",
|
||||
name: "client-upload",
|
||||
source: entry,
|
||||
});
|
||||
expect(written("AGENTS.md")).toContain(`Source: ${entry}`);
|
||||
expect(JSON.parse(written("extracted/assets-manifest.json"))).toEqual({
|
||||
page: entry,
|
||||
assets: [{ url: logo }, { url: REMOTE }],
|
||||
});
|
||||
expect(result.url).toBe(entry);
|
||||
expect(result.assets.map((asset) => asset.url)).toEqual([logo, REMOTE]);
|
||||
});
|
||||
|
||||
it("leaves URL-mode output as it was", async () => {
|
||||
const result = await captureWebsite({ url: "https://www.example.com/", outputDir });
|
||||
expect(JSON.parse(written("meta.json"))).toEqual({
|
||||
id: "example.com-video",
|
||||
name: "example.com",
|
||||
});
|
||||
expect(written("AGENTS.md")).toContain("Source: https://www.example.com/");
|
||||
expect(JSON.parse(written("extracted/assets-manifest.json")).assets).toEqual([
|
||||
{ url: "https://www.example.com/img/logo%20mark.png?v=2" },
|
||||
{ url: REMOTE },
|
||||
]);
|
||||
expect(result.assets.map((asset) => asset.url)).toEqual([
|
||||
"https://www.example.com/img/logo%20mark.png?v=2",
|
||||
REMOTE,
|
||||
]);
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,188 @@
|
||||
import { mkdirSync, mkdtempSync, realpathSync, rmSync, symlinkSync, writeFileSync } from "node:fs";
|
||||
import { tmpdir } from "node:os";
|
||||
import { join } from "node:path";
|
||||
import { pathToFileURL } from "node:url";
|
||||
import { afterAll, beforeAll, describe, expect, it, vi } from "vitest";
|
||||
import { safeFetch } from "./assetDownloader.js";
|
||||
import {
|
||||
captureSiteName,
|
||||
fetchLocalPageUrl,
|
||||
localUrlRewriter,
|
||||
rewriteStrings,
|
||||
resolveLocalCaptureTarget,
|
||||
startLocalPageServer,
|
||||
type LocalPageServer,
|
||||
} from "./localPage.js";
|
||||
|
||||
let base: string;
|
||||
let site: string;
|
||||
|
||||
beforeAll(() => {
|
||||
base = realpathSync.native(mkdtempSync(join(tmpdir(), "hf-local-page-")));
|
||||
site = join(base, "site");
|
||||
mkdirSync(join(site, "pages"), { recursive: true });
|
||||
writeFileSync(join(site, "index.html"), "<h1>home</h1>");
|
||||
writeFileSync(join(site, "pages", "about page.html"), "<h1>about</h1>");
|
||||
writeFileSync(join(site, "logo.png"), "png-bytes");
|
||||
writeFileSync(join(base, "secret.txt"), "outside the root");
|
||||
writeFileSync(join(base, "outside.html"), "<h1>outside</h1>");
|
||||
if (process.platform !== "win32") {
|
||||
symlinkSync(join(base, "secret.txt"), join(site, "link.txt"));
|
||||
symlinkSync(join(base, "outside.html"), join(site, "linked.html"));
|
||||
}
|
||||
});
|
||||
|
||||
afterAll(() => rmSync(base, { recursive: true, force: true }));
|
||||
|
||||
describe("resolveLocalCaptureTarget", () => {
|
||||
it("accepts a path, a file:// URL or a directory, rooted at the file's directory", () => {
|
||||
const entry = join(site, "index.html");
|
||||
const expected = { root: site, entry, fileUrl: pathToFileURL(entry).href };
|
||||
expect(resolveLocalCaptureTarget(entry)).toEqual(expected);
|
||||
expect(resolveLocalCaptureTarget(pathToFileURL(entry).href)).toEqual(expected);
|
||||
expect(resolveLocalCaptureTarget(site)).toEqual(expected);
|
||||
});
|
||||
|
||||
it("takes an explicit root that contains the file", () => {
|
||||
const entry = join(site, "pages", "about page.html");
|
||||
expect(resolveLocalCaptureTarget(entry, site).root).toBe(site);
|
||||
expect(resolveLocalCaptureTarget(entry).root).toBe(join(site, "pages"));
|
||||
});
|
||||
|
||||
it("refuses a file outside the root and a missing file", () => {
|
||||
expect(() => resolveLocalCaptureTarget(join(site, "index.html"), join(site, "pages"))).toThrow(
|
||||
"outside the root",
|
||||
);
|
||||
expect(() => resolveLocalCaptureTarget(join(site, "missing.html"))).toThrow(
|
||||
"Local file not found",
|
||||
);
|
||||
expect(() => resolveLocalCaptureTarget("https://example.com/")).toThrow("Local file not found");
|
||||
});
|
||||
|
||||
it.skipIf(process.platform === "win32")(
|
||||
"does not widen the default root to where a symlinked entry points",
|
||||
() => {
|
||||
expect(() => resolveLocalCaptureTarget(join(site, "linked.html"))).toThrow(
|
||||
"outside the root",
|
||||
);
|
||||
},
|
||||
);
|
||||
});
|
||||
|
||||
describe("startLocalPageServer", () => {
|
||||
let server: LocalPageServer;
|
||||
|
||||
beforeAll(async () => {
|
||||
const target = resolveLocalCaptureTarget(join(site, "pages", "about page.html"), site);
|
||||
server = await startLocalPageServer(target);
|
||||
});
|
||||
|
||||
afterAll(() => server.close());
|
||||
|
||||
it("serves the entry page and files inside the root", async () => {
|
||||
expect(server.url).toBe(`${server.origin}/pages/about%20page.html`);
|
||||
const page = await fetch(server.url);
|
||||
expect(page.headers.get("content-type")).toContain("text/html");
|
||||
expect(await page.text()).toBe("<h1>about</h1>");
|
||||
expect(await (await fetch(`${server.origin}/logo.png`)).text()).toBe("png-bytes");
|
||||
});
|
||||
|
||||
it.each([
|
||||
"/%2e%2e/secret.txt",
|
||||
"/..%2fsecret.txt",
|
||||
"/pages/..%2f..%2fsecret.txt",
|
||||
"/missing.png",
|
||||
])("answers 404 for %s", async (path) => {
|
||||
const response = await fetch(`${server.origin}${path}`);
|
||||
expect(response.status).toBe(404);
|
||||
expect(await response.text()).toBe("");
|
||||
});
|
||||
|
||||
it.skipIf(process.platform === "win32")("refuses a symlink that leaves the root", async () => {
|
||||
expect((await fetch(`${server.origin}/link.txt`)).status).toBe(404);
|
||||
});
|
||||
|
||||
it("refuses methods other than GET and HEAD", async () => {
|
||||
expect((await fetch(server.url, { method: "POST" })).status).toBe(405);
|
||||
});
|
||||
|
||||
it("lets the asset downloader read the active local origin, and only that origin", async () => {
|
||||
const response = await safeFetch(`${server.origin}/logo.png`);
|
||||
expect(await response?.text()).toBe("png-bytes");
|
||||
expect((await safeFetch(`${server.origin}/%2e%2e/secret.txt`))?.status).toBe(404);
|
||||
expect(fetchLocalPageUrl("http://127.0.0.1:1/logo.png")).toBeNull();
|
||||
expect(await safeFetch("http://127.0.0.1:1/logo.png")).toBeNull();
|
||||
});
|
||||
|
||||
it("reads the active local origin from disk, so an environment proxy cannot intercept it", async () => {
|
||||
const networkFetch = vi.fn(() => Promise.reject(new Error("network")));
|
||||
vi.stubGlobal("fetch", networkFetch);
|
||||
try {
|
||||
const response = await safeFetch(`${server.origin}/pages/about%20page.html`);
|
||||
expect(response?.headers.get("content-type")).toContain("text/html");
|
||||
expect(await response?.text()).toBe("<h1>about</h1>");
|
||||
const head = await safeFetch(`${server.origin}/logo.png`, { method: "HEAD" });
|
||||
expect(head?.headers.get("content-type")).toBe("image/png");
|
||||
expect(await head?.text()).toBe("");
|
||||
expect(networkFetch).not.toHaveBeenCalled();
|
||||
} finally {
|
||||
vi.unstubAllGlobals();
|
||||
}
|
||||
});
|
||||
|
||||
it("stops answering for the origin once closed", async () => {
|
||||
const target = resolveLocalCaptureTarget(site);
|
||||
const closed = await startLocalPageServer(target);
|
||||
await closed.close();
|
||||
expect(fetchLocalPageUrl(`${closed.origin}/index.html`)).toBeNull();
|
||||
expect(await safeFetch(`${closed.origin}/index.html`)).toBeNull();
|
||||
});
|
||||
});
|
||||
|
||||
describe("localUrlRewriter", () => {
|
||||
const rewrite = localUrlRewriter("http://127.0.0.1:4100", "/srv/site");
|
||||
|
||||
it("maps local-server URLs to file:// URLs of the same path in the root", () => {
|
||||
expect(rewrite(`url(http://127.0.0.1:4100/fonts/a%20b.woff2) "http://127.0.0.1:4100"`)).toBe(
|
||||
`url(${pathToFileURL("/srv/site/fonts/a b.woff2").href}) "${pathToFileURL("/srv/site/").href}"`,
|
||||
);
|
||||
expect(rewrite("http://127.0.0.1:4100/p.html?x=1#top")).toBe(
|
||||
`${pathToFileURL("/srv/site/p.html").href}?x=1#top`,
|
||||
);
|
||||
});
|
||||
|
||||
it("leaves other origins alone, including a longer port with the same prefix", () => {
|
||||
const text = "http://127.0.0.1:41000/a https://example.com/a http://127.0.0.1:4101/a";
|
||||
expect(rewrite(text)).toBe(text);
|
||||
});
|
||||
|
||||
it("matches the origin literally, not as a pattern", () => {
|
||||
expect(rewrite("http://127x0x0x1:4100/a")).toBe("http://127x0x0x1:4100/a");
|
||||
const plus = localUrlRewriter("http://a+b.example:4100", "/srv/site");
|
||||
expect(plus("http://aab.example:4100/x http://ab.example:4100/x")).toBe(
|
||||
"http://aab.example:4100/x http://ab.example:4100/x",
|
||||
);
|
||||
expect(plus("http://a+b.example:4100/x")).toBe(pathToFileURL("/srv/site/x").href);
|
||||
});
|
||||
|
||||
it("rewrites every string inside a nested value", () => {
|
||||
expect(
|
||||
rewriteStrings({ a: ["http://127.0.0.1:4100/x.png", 3], b: { c: null } }, rewrite),
|
||||
).toEqual({ a: [pathToFileURL("/srv/site/x.png").href, 3], b: { c: null } });
|
||||
});
|
||||
});
|
||||
|
||||
describe("captureSiteName", () => {
|
||||
it("names an active local server after its page, and a website after its host", async () => {
|
||||
const server = await startLocalPageServer(
|
||||
resolveLocalCaptureTarget(join(site, "pages", "about page.html")),
|
||||
);
|
||||
try {
|
||||
expect(captureSiteName(server.url)).toBe("about-page");
|
||||
} finally {
|
||||
await server.close();
|
||||
}
|
||||
expect(captureSiteName(server.url)).toBe("127.0.0.1");
|
||||
expect(captureSiteName("https://www.example.com/x")).toBe("example.com");
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,283 @@
|
||||
import {
|
||||
createReadStream,
|
||||
readdirSync,
|
||||
readFileSync,
|
||||
realpathSync,
|
||||
statSync,
|
||||
type Dirent,
|
||||
} from "node:fs";
|
||||
import { writeCaptureFileSync } from "./captureFile.js";
|
||||
import { realpath, stat } from "node:fs/promises";
|
||||
import { createServer, type ServerResponse } from "node:http";
|
||||
import type { Socket } from "node:net";
|
||||
import { basename, dirname, extname, isAbsolute, join, relative, resolve, sep } from "node:path";
|
||||
import { Readable } from "node:stream";
|
||||
import { fileURLToPath, pathToFileURL } from "node:url";
|
||||
|
||||
export interface LocalCaptureTarget {
|
||||
/** Real path of the directory the page may read from. */
|
||||
root: string;
|
||||
/** Real path of the HTML file to open. */
|
||||
entry: string;
|
||||
/** `file://` URL of the entry, reported as the capture source. */
|
||||
fileUrl: string;
|
||||
}
|
||||
|
||||
export interface LocalPageServer {
|
||||
origin: string;
|
||||
/** Loopback URL of the entry page. */
|
||||
url: string;
|
||||
close(): Promise<void>;
|
||||
}
|
||||
|
||||
const CONTENT_TYPES: Record<string, string> = {
|
||||
".html": "text/html; charset=utf-8",
|
||||
".htm": "text/html; charset=utf-8",
|
||||
".css": "text/css; charset=utf-8",
|
||||
".js": "text/javascript; charset=utf-8",
|
||||
".mjs": "text/javascript; charset=utf-8",
|
||||
".json": "application/json",
|
||||
".svg": "image/svg+xml",
|
||||
".png": "image/png",
|
||||
".jpg": "image/jpeg",
|
||||
".jpeg": "image/jpeg",
|
||||
".gif": "image/gif",
|
||||
".webp": "image/webp",
|
||||
".avif": "image/avif",
|
||||
".ico": "image/x-icon",
|
||||
".woff": "font/woff",
|
||||
".woff2": "font/woff2",
|
||||
".ttf": "font/ttf",
|
||||
".otf": "font/otf",
|
||||
".mp4": "video/mp4",
|
||||
".webm": "video/webm",
|
||||
".mov": "video/quicktime",
|
||||
".mp3": "audio/mpeg",
|
||||
".wav": "audio/wav",
|
||||
".txt": "text/plain; charset=utf-8",
|
||||
};
|
||||
|
||||
const activeSites = new Map<string, { root: string; name: string }>();
|
||||
|
||||
function localSiteName(entry: string): string {
|
||||
const stem = basename(entry, extname(entry));
|
||||
const name = stem.toLowerCase() === "index" ? basename(dirname(entry)) : stem;
|
||||
return (
|
||||
name
|
||||
.toLowerCase()
|
||||
.replace(/[^a-z0-9._-]+/g, "-")
|
||||
.replace(/^-+|-+$/g, "") || "local"
|
||||
);
|
||||
}
|
||||
|
||||
/** Name a capture is filed under: the local page's name for an active local server, else the host. */
|
||||
export function captureSiteName(url: string): string {
|
||||
const parsed = new URL(url);
|
||||
return activeSites.get(parsed.origin)?.name ?? parsed.hostname.replace(/^www\./, "");
|
||||
}
|
||||
|
||||
function isInside(root: string, path: string): boolean {
|
||||
const rel = relative(root, path);
|
||||
return rel === "" || (!!rel && !rel.startsWith(`..${sep}`) && rel !== ".." && !isAbsolute(rel));
|
||||
}
|
||||
|
||||
/**
|
||||
* Resolves a local HTML file (path or `file://` URL) or a directory's `index.html` to a capture
|
||||
* target. The entry must lie inside `root` (default: the directory it was named in).
|
||||
*/
|
||||
export function resolveLocalCaptureTarget(target: string, root?: string): LocalCaptureTarget {
|
||||
const path = resolve(target.startsWith("file:") ? fileURLToPath(target) : target);
|
||||
let entry: string;
|
||||
let directory: string;
|
||||
try {
|
||||
// Native realpath, as fs/promises uses for served files, so both agree on case and short names.
|
||||
entry = realpathSync.native(path);
|
||||
directory = realpathSync.native(dirname(path));
|
||||
if (statSync(entry).isDirectory()) {
|
||||
directory = entry;
|
||||
entry = realpathSync.native(join(entry, "index.html"));
|
||||
}
|
||||
} catch {
|
||||
throw new Error(`Local file not found: ${target}`);
|
||||
}
|
||||
if (!statSync(entry).isFile()) throw new Error(`Local capture target is not a file: ${target}`);
|
||||
let rootPath: string;
|
||||
try {
|
||||
rootPath = root === undefined ? directory : realpathSync.native(resolve(root));
|
||||
} catch {
|
||||
throw new Error(`Local capture root not found: ${root}`);
|
||||
}
|
||||
if (!statSync(rootPath).isDirectory()) {
|
||||
throw new Error(`Local capture root is not a directory: ${root}`);
|
||||
}
|
||||
if (!isInside(rootPath, entry)) {
|
||||
throw new Error(`Local capture target ${entry} is outside the root ${rootPath}`);
|
||||
}
|
||||
return { root: rootPath, entry, fileUrl: pathToFileURL(entry).href };
|
||||
}
|
||||
|
||||
async function servedFile(root: string, pathname: string): Promise<string | null> {
|
||||
let decoded: string;
|
||||
try {
|
||||
decoded = decodeURIComponent(pathname);
|
||||
} catch {
|
||||
return null;
|
||||
}
|
||||
if (decoded.includes("\0")) return null;
|
||||
const candidate = resolve(root, `.${decoded}`);
|
||||
if (!isInside(root, candidate)) return null;
|
||||
try {
|
||||
// Symlinks are followed, so the containment check must run on the real path.
|
||||
const real = await realpath(candidate);
|
||||
if (!isInside(root, real) || !(await stat(real)).isFile()) return null;
|
||||
return real;
|
||||
} catch {
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
function contentType(file: string): string {
|
||||
return CONTENT_TYPES[extname(file).toLowerCase()] ?? "application/octet-stream";
|
||||
}
|
||||
|
||||
function end(response: ServerResponse, status: number): void {
|
||||
response.writeHead(status, { "Content-Length": 0, Connection: "close" });
|
||||
response.end();
|
||||
}
|
||||
|
||||
/** Serves only regular files whose real path lies inside `target.root`, on a loopback port. */
|
||||
export async function startLocalPageServer(target: LocalCaptureTarget): Promise<LocalPageServer> {
|
||||
const sockets = new Set<Socket>();
|
||||
const server = createServer((request, response) => {
|
||||
void (async () => {
|
||||
if (request.method !== "GET" && request.method !== "HEAD") return end(response, 405);
|
||||
const pathname = new URL(request.url ?? "/", "http://local").pathname;
|
||||
const file = await servedFile(target.root, pathname);
|
||||
if (!file) return end(response, 404);
|
||||
response.writeHead(200, { "Content-Type": contentType(file), "Cache-Control": "no-store" });
|
||||
if (request.method === "HEAD") return response.end();
|
||||
const stream = createReadStream(file);
|
||||
stream.on("error", () => response.destroy());
|
||||
stream.pipe(response);
|
||||
})().catch(() => response.destroy());
|
||||
});
|
||||
server.on("connection", (socket) => {
|
||||
sockets.add(socket);
|
||||
socket.once("close", () => sockets.delete(socket));
|
||||
});
|
||||
await new Promise<void>((resolveListen, reject) => {
|
||||
server.once("error", reject);
|
||||
server.listen(0, "127.0.0.1", resolveListen);
|
||||
});
|
||||
const address = server.address();
|
||||
if (!address || typeof address === "string") throw new Error("Local page server did not bind");
|
||||
const origin = `http://127.0.0.1:${address.port}`;
|
||||
const entryPath = relative(target.root, target.entry).split(sep).map(encodeURIComponent);
|
||||
activeSites.set(origin, { root: target.root, name: localSiteName(target.entry) });
|
||||
let closing: Promise<void> | undefined;
|
||||
return {
|
||||
origin,
|
||||
url: `${origin}/${entryPath.join("/")}`,
|
||||
close() {
|
||||
closing ??= new Promise<void>((resolveClose) => {
|
||||
activeSites.delete(origin);
|
||||
server.close(() => resolveClose());
|
||||
for (const socket of sockets) socket.destroy();
|
||||
});
|
||||
return closing;
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
async function localResponse(root: string, pathname: string, method: string): Promise<Response> {
|
||||
if (method !== "GET" && method !== "HEAD") return new Response(null, { status: 405 });
|
||||
const file = await servedFile(root, pathname);
|
||||
if (!file) return new Response(null, { status: 404 });
|
||||
const body =
|
||||
method === "HEAD"
|
||||
? null
|
||||
: (Readable.toWeb(createReadStream(file)) as ReadableStream<Uint8Array>);
|
||||
return new Response(body, { status: 200, headers: { "Content-Type": contentType(file) } });
|
||||
}
|
||||
|
||||
/**
|
||||
* Answers a URL on an active local page server from disk, with the server's checks; null for every
|
||||
* other URL. It never touches the network, so an environment proxy cannot intercept it.
|
||||
*/
|
||||
export function fetchLocalPageUrl(url: string, init?: RequestInit): Promise<Response> | null {
|
||||
let parsed: URL;
|
||||
try {
|
||||
parsed = new URL(url);
|
||||
} catch {
|
||||
return null;
|
||||
}
|
||||
const site = activeSites.get(parsed.origin);
|
||||
if (site === undefined) return null;
|
||||
return localResponse(site.root, parsed.pathname, (init?.method ?? "GET").toUpperCase());
|
||||
}
|
||||
|
||||
const TEXT_OUTPUTS = new Set([".json", ".md", ".html", ".htm", ".css", ".txt", ".svg", ".js", ""]);
|
||||
|
||||
/** Maps every URL on the local server's origin to the `file://` URL of the same path in the root. */
|
||||
export function localUrlRewriter(origin: string, root: string): (text: string) => string {
|
||||
const escapedOrigin = origin.replace(/[.*+?^${}()|[\]\\]/g, "\\$&");
|
||||
const pattern = new RegExp(`${escapedOrigin}(?![0-9])(/[^\\s"'<>()\\\\]*)?`, "g");
|
||||
return (text) =>
|
||||
text.replace(pattern, (_match, path: string | undefined) => {
|
||||
const url = new URL(path ?? "/", origin);
|
||||
let decoded: string;
|
||||
try {
|
||||
decoded = decodeURIComponent(url.pathname);
|
||||
} catch {
|
||||
decoded = url.pathname;
|
||||
}
|
||||
return pathToFileURL(join(root, decoded)).href + url.search + url.hash;
|
||||
});
|
||||
}
|
||||
|
||||
/** Applies `rewrite` to every string inside a JSON-like value. */
|
||||
export function rewriteStrings<T>(value: T, rewrite: (text: string) => string): T {
|
||||
if (typeof value === "string") return rewrite(value) as T;
|
||||
if (Array.isArray(value)) return value.map((item) => rewriteStrings(item, rewrite)) as T;
|
||||
if (value && typeof value === "object") {
|
||||
return Object.fromEntries(
|
||||
Object.entries(value).map(([key, item]) => [key, rewriteStrings(item, rewrite)]),
|
||||
) as T;
|
||||
}
|
||||
return value;
|
||||
}
|
||||
|
||||
/** Rewrites local-server URLs in every text file the capture wrote, and records the source. */
|
||||
export function rewriteLocalCaptureOutput(
|
||||
outputDir: string,
|
||||
rewrite: (text: string) => string,
|
||||
sourceUrl: string,
|
||||
): void {
|
||||
const walk = (dir: string): void => {
|
||||
let entries: Dirent[];
|
||||
try {
|
||||
entries = readdirSync(dir, { withFileTypes: true });
|
||||
} catch {
|
||||
return;
|
||||
}
|
||||
for (const entry of entries) {
|
||||
const path = join(dir, entry.name);
|
||||
if (entry.isDirectory()) walk(path);
|
||||
else if (entry.isFile() && TEXT_OUTPUTS.has(extname(entry.name).toLowerCase())) {
|
||||
const text = readFileSync(path, "utf8");
|
||||
const next = rewrite(text);
|
||||
if (next !== text) writeCaptureFileSync(path, next, "utf-8");
|
||||
}
|
||||
}
|
||||
};
|
||||
walk(outputDir);
|
||||
const metaPath = join(outputDir, "meta.json");
|
||||
try {
|
||||
const meta: unknown = JSON.parse(readFileSync(metaPath, "utf8"));
|
||||
if (meta && typeof meta === "object" && !Array.isArray(meta)) {
|
||||
writeCaptureFileSync(metaPath, JSON.stringify({ ...meta, source: sourceUrl }, null, 2));
|
||||
}
|
||||
} catch {
|
||||
/* no meta.json to annotate */
|
||||
}
|
||||
}
|
||||
@@ -1,4 +1,5 @@
|
||||
import { ensureCaptureDirSync, writeCaptureFileSync } from "./captureFile.js";
|
||||
import { captureSiteName } from "./localPage.js";
|
||||
import { join } from "node:path";
|
||||
import { noDrops } from "./assetDownloader.js";
|
||||
import type {
|
||||
@@ -72,7 +73,7 @@ export function writePartialCaptureBundle(
|
||||
state: PartialCaptureState,
|
||||
lastPhase: CaptureResult["lastPhase"],
|
||||
): CaptureResult {
|
||||
const hostname = new URL(opts.url).hostname.replace(/^www\./, "");
|
||||
const hostname = captureSiteName(opts.url);
|
||||
const extractedDir = ensureCaptureDirSync(opts.outputDir, join(opts.outputDir, "extracted"));
|
||||
writeCaptureFileSync(join(extractedDir, "tokens.json"), serializeTokensForCapture(state.tokens));
|
||||
writeCaptureFileSync(
|
||||
|
||||
@@ -5,6 +5,7 @@
|
||||
* (index.html, meta.json, AGENTS.md, CLAUDE.md).
|
||||
*/
|
||||
|
||||
import { captureSiteName } from "./localPage.js";
|
||||
import { existsSync, readFileSync } from "node:fs";
|
||||
import { join, resolve } from "node:path";
|
||||
import { applyDotEnv, CAPTURE_ENV_KEYS } from "../utils/dotEnv.js";
|
||||
@@ -61,7 +62,7 @@ export async function generateProjectScaffold(
|
||||
// valid compositions, playing two audio tracks offset in time.
|
||||
const metaPath = join(outputDir, "meta.json");
|
||||
if (!existsSync(metaPath)) {
|
||||
const hostname = new URL(url).hostname.replace(/^www\./, "");
|
||||
const hostname = captureSiteName(url);
|
||||
writeNewFileSync(
|
||||
metaPath,
|
||||
JSON.stringify({ id: hostname + "-video", name: tokens.title || hostname }, null, 2),
|
||||
|
||||
@@ -42,8 +42,10 @@ export interface CapturePhaseProgress {
|
||||
}
|
||||
|
||||
export interface CaptureOptions {
|
||||
/** URL to capture */
|
||||
/** Public http(s) URL to capture. With `localFile`, the result reports the file's `file://` URL instead. */
|
||||
url: string;
|
||||
localFile?: string;
|
||||
localRoot?: string;
|
||||
/** Output directory */
|
||||
outputDir: string;
|
||||
/** Viewport width (default: 1920) */
|
||||
|
||||
@@ -3,6 +3,7 @@ import {
|
||||
lstatSync,
|
||||
mkdirSync,
|
||||
mkdtempSync,
|
||||
realpathSync,
|
||||
readFileSync,
|
||||
rmSync,
|
||||
symlinkSync,
|
||||
@@ -10,6 +11,7 @@ import {
|
||||
} from "node:fs";
|
||||
import { tmpdir } from "node:os";
|
||||
import { join } from "node:path";
|
||||
import { pathToFileURL } from "node:url";
|
||||
import { CliRuntimeError, registerRootExitRequester } from "../utils/commandResult.js";
|
||||
import { CAPTURE_PHASE_SCHEMA } from "../capture/types.js";
|
||||
|
||||
@@ -189,6 +191,48 @@ describe("capture command — vision control", () => {
|
||||
},
|
||||
);
|
||||
|
||||
it("passes a local HTML file and its root into capture", async () => {
|
||||
vi.spyOn(console, "log").mockImplementation(() => {});
|
||||
const site = mkdtempSync(join(tmpdir(), "hf-capture-local-cmd-"));
|
||||
try {
|
||||
mkdirSync(join(site, "pages"));
|
||||
writeFileSync(join(site, "pages", "about.html"), "<h1>about</h1>");
|
||||
await captureCommand.run!({
|
||||
args: {
|
||||
"local-file": join(site, "pages", "about.html"),
|
||||
"local-root": site,
|
||||
output: "/tmp/hf-local-cmd",
|
||||
json: true,
|
||||
},
|
||||
} as never);
|
||||
expect(captureWebsiteMock).toHaveBeenCalledWith(
|
||||
expect.objectContaining({
|
||||
url: pathToFileURL(realpathSync.native(join(site, "pages", "about.html"))).href,
|
||||
localFile: join(site, "pages", "about.html"),
|
||||
localRoot: site,
|
||||
}),
|
||||
undefined,
|
||||
);
|
||||
} finally {
|
||||
rmSync(site, { recursive: true, force: true });
|
||||
}
|
||||
});
|
||||
|
||||
it.each([
|
||||
[{ "local-file": "/definitely/missing/page.html" }, "Local file not found"],
|
||||
[{ url: "https://example.com", "local-file": "/tmp" }, "either a URL or --local-file"],
|
||||
[{ url: "https://example.com", "local-root": "/tmp" }, "--local-root requires --local-file"],
|
||||
])("refuses %o before capturing", async (target, message) => {
|
||||
const error = vi.spyOn(console, "error").mockImplementation(() => {});
|
||||
await expect(
|
||||
captureCommand.run!({
|
||||
args: { ...target, output: "/tmp/hf-local-refused", json: true },
|
||||
} as never),
|
||||
).rejects.toThrow(CliRuntimeError);
|
||||
expect(error.mock.calls.flat().join("\n")).toContain(message);
|
||||
expect(captureWebsiteMock).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it("passes a validated screenshot limit into capture", async () => {
|
||||
vi.spyOn(console, "log").mockImplementation(() => {});
|
||||
await captureCommand.run!({
|
||||
|
||||
@@ -32,6 +32,11 @@ export const examples: Example[] = [
|
||||
["Capture a website into ./capture/", "hyperframes capture https://stripe.com"],
|
||||
["Capture to a different directory", "hyperframes capture https://linear.app -o linear-video"],
|
||||
["JSON output for AI agents", "hyperframes capture https://example.com --json"],
|
||||
["Capture a local HTML file", "hyperframes capture --local-file ./site/index.html -o site-video"],
|
||||
[
|
||||
"Let a local page read files from a parent directory",
|
||||
"hyperframes capture --local-file ./site/pages/about.html --local-root ./site",
|
||||
],
|
||||
[
|
||||
"Pull a video from the captured manifest by index",
|
||||
"hyperframes capture --video ./linear-video --index 0",
|
||||
@@ -54,9 +59,19 @@ export default defineCommand({
|
||||
args: {
|
||||
url: {
|
||||
type: "positional",
|
||||
description: "Website URL to capture (omit when using --video)",
|
||||
description: "Website URL to capture (omit when using --local-file or --video)",
|
||||
required: false,
|
||||
},
|
||||
"local-file": {
|
||||
type: "string",
|
||||
description:
|
||||
"Capture a local HTML file (or a directory's index.html) instead of a URL; remote subresources still go through the public-address policy",
|
||||
},
|
||||
"local-root": {
|
||||
type: "string",
|
||||
description:
|
||||
"(--local-file) Directory the page may read local files from (default: the HTML file's directory)",
|
||||
},
|
||||
output: {
|
||||
type: "string",
|
||||
description: "Output directory name (default: ./capture, then ./capture-2/, ./capture-3/, …)",
|
||||
@@ -130,19 +145,38 @@ export default defineCommand({
|
||||
return;
|
||||
}
|
||||
|
||||
const url = args.url as string | undefined;
|
||||
if (!url) {
|
||||
console.error(
|
||||
"Missing URL. Pass a website URL, or use --video <project> for video download.",
|
||||
);
|
||||
failCommand();
|
||||
}
|
||||
|
||||
try {
|
||||
new URL(url);
|
||||
} catch {
|
||||
console.error(`Invalid URL: ${url}`);
|
||||
failCommand();
|
||||
const localFile = args["local-file"] as string | undefined;
|
||||
const localRoot = args["local-root"] as string | undefined;
|
||||
let url = args.url as string | undefined;
|
||||
if (localFile !== undefined) {
|
||||
if (url) {
|
||||
console.error("Pass either a URL or --local-file, not both.");
|
||||
failCommand();
|
||||
}
|
||||
const { resolveLocalCaptureTarget } = await import("../capture/localPage.js");
|
||||
try {
|
||||
url = resolveLocalCaptureTarget(localFile, localRoot).fileUrl;
|
||||
} catch (err) {
|
||||
console.error(normalizeErrorMessage(err));
|
||||
failCommand();
|
||||
}
|
||||
} else {
|
||||
if (localRoot !== undefined) {
|
||||
console.error("--local-root requires --local-file.");
|
||||
failCommand();
|
||||
}
|
||||
if (!url) {
|
||||
console.error(
|
||||
"Missing URL. Pass a website URL, --local-file <path>, or --video <project> for video download.",
|
||||
);
|
||||
failCommand();
|
||||
}
|
||||
try {
|
||||
new URL(url);
|
||||
} catch {
|
||||
console.error(`Invalid URL: ${url}`);
|
||||
failCommand();
|
||||
}
|
||||
}
|
||||
|
||||
const captureBudgetMs = parsePositiveInteger(
|
||||
@@ -195,6 +229,8 @@ export default defineCommand({
|
||||
const result = await captureWebsite(
|
||||
{
|
||||
url,
|
||||
localFile,
|
||||
localRoot,
|
||||
outputDir,
|
||||
skipAssets: args["skip-assets"] as boolean,
|
||||
skipVision: args["skip-vision"] as boolean,
|
||||
|
||||
Reference in New Issue
Block a user