fix(cli): compare and keyframes --shot serve only the files a bare HTML file names (#8006)

* fix(cli): compare and keyframes --shot serve only the files a bare HTML file names

A bare .html file given to compare or keyframes --shot is staged with
only the files its markup, stylesheets and sub-compositions name inside
its own folder, and only types a composition loads. Private files stay
out. A scene its project's index.html mounts is staged from the
project, so it keeps its shell.

* fix(cli): stage root-relative names, string imports, SVG links and grading tables

A bare HTML file's staging now also copies the files named by a
leading-slash URL (resolved against the staged folder), a string-form
CSS @import, an SVG xlink:href and the LUT of data-color-grading.
compare and keyframes --shot print one line naming any file they left
out. The CLI docs describe the single-file rule.
This commit is contained in:
Michael Wang
2026-11-17 03:27:12 +00:00
committed by GitHub
parent f880693cf1
commit 346579b6b9
9 changed files with 639 additions and 224 deletions
+13
View File
@@ -1605,6 +1605,13 @@ npx hyperframes keyframes [dir] --selector "#card" --shot card-motion.png
Use `--ghost` for motion that happens *inside* a canvas, where the marker onion
has nothing to draw.
With a single `.html` file, `--shot` serves only the files that file's markup,
stylesheets and sub-compositions name in its own folder, and only the types a
composition loads. A scene that its project's `index.html` mounts is served from
the project instead. Files the page builds names for in script are not served,
and the command prints a line naming any file it left out. Pass the project
folder to keep everything.
### `compare` and `grade-compare`
`compare` renders two or more independent variants into one labelled PNG.
@@ -1622,6 +1629,12 @@ npx hyperframes compare ./variant-a ./variant-b --labels "A,B"
| `--timeout` | Render-ready timeout per variant in ms; must be a positive integer (default 5000) |
| `--json` | Machine-readable results |
A variant can also be a single `.html` file. Then only the files its markup,
stylesheets and sub-compositions name in its own folder are served, and only the
types a composition loads. Files the page builds names for in script are not
served, and the command prints a line naming any file it left out. Pass a
project folder to keep everything.
`grade-compare` does the same for colour: candidate grades or LUTs applied to
one reference frame.
+48 -19
View File
@@ -1,4 +1,12 @@
import { existsSync, mkdirSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from "node:fs";
import {
existsSync,
mkdirSync,
mkdtempSync,
readdirSync,
readFileSync,
rmSync,
writeFileSync,
} from "node:fs";
import { tmpdir } from "node:os";
import { basename, join } from "node:path";
import { describe, expect, it, vi } from "vitest";
@@ -117,7 +125,7 @@ describe("prepareCompareVariantProjects", () => {
const secondStage = prepared[1];
if (!firstStage || !secondStage) throw new Error("expected two staged variants");
expect(existsSync(join(secondStage.projectDir, basename(firstStage.projectDir)))).toBe(false);
expect(readFileSync(join(secondStage.projectDir, "first.html"), "utf8")).toContain("First");
expect(readdirSync(secondStage.projectDir)).toEqual(["index.html"]);
expect(readFileSync(join(secondStage.projectDir, "index.html"), "utf8")).toContain("Second");
} finally {
temporaryRoot.path = "";
@@ -126,15 +134,17 @@ describe("prepareCompareVariantProjects", () => {
});
it.each(["direct", "nested"])(
"stages siblings when the temporary output is %s inside the source directory",
"stages only what the page names when the temporary output is %s inside the source directory",
(placement) => {
const dir = tempDir();
const htmlFile = join(dir, "candidate.html");
mkdirSync(join(dir, "assets"));
writeFileSync(htmlFile, "<!doctype html><title>Temporary candidate</title>");
writeFileSync(join(dir, "assets", "asset.txt"), "sibling asset");
mkdirSync(join(dir, "node_modules"));
writeFileSync(join(dir, "node_modules", "ignored.txt"), "ignored");
writeFileSync(
htmlFile,
'<!doctype html><title>Temporary candidate</title><img src="assets/asset.png">',
);
writeFileSync(join(dir, "assets", "asset.png"), "sibling asset");
writeFileSync(join(dir, "assets", "unnamed.png"), "not named");
temporaryRoot.path = placement === "direct" ? dir : join(dir, "temporary");
mkdirSync(temporaryRoot.path, { recursive: true });
writeFileSync(join(temporaryRoot.path, "sibling.txt"), "temporary-root sibling");
@@ -149,17 +159,11 @@ describe("prepareCompareVariantProjects", () => {
expect(readFileSync(join(staged.projectDir, "index.html"), "utf8")).toContain(
"Temporary candidate",
);
expect(readFileSync(join(staged.projectDir, "assets", "asset.txt"), "utf8")).toBe(
expect(readFileSync(join(staged.projectDir, "assets", "asset.png"), "utf8")).toBe(
"sibling asset",
);
expect(
readFileSync(
join(staged.projectDir, placement === "direct" ? "" : "temporary", "sibling.txt"),
"utf8",
),
).toBe("temporary-root sibling");
expect(existsSync(join(staged.projectDir, "candidate.html"))).toBe(false);
expect(existsSync(join(staged.projectDir, "node_modules"))).toBe(false);
expect(readdirSync(staged.projectDir).sort()).toEqual(["assets", "index.html"]);
expect(readdirSync(join(staged.projectDir, "assets"))).toEqual(["asset.png"]);
expect(readFileSync(htmlFile, "utf8")).toContain("Temporary candidate");
} finally {
temporaryRoot.path = "";
@@ -168,6 +172,29 @@ describe("prepareCompareVariantProjects", () => {
},
);
it("leaves out a file the page names that no composition loads", () => {
const dir = tempDir();
const htmlFile = join(dir, "sent.html");
writeFileSync(
htmlFile,
`<img src="taxes.pdf"><img src="notes.txt"><link href=".env"><img src="id_ed25519">
<script>fetch("statement.png").then((r) => r.blob())</script>`,
);
for (const name of ["taxes.pdf", "notes.txt", ".env", "id_ed25519", "statement.png"]) {
writeFileSync(join(dir, name), "private");
}
const prepared = prepareCompareVariantProjects([
{ label: "Sent", inputPath: htmlFile, displayPath: "sent.html" },
]);
try {
expect(readdirSync(prepared[0]!.projectDir)).toEqual(["index.html"]);
} finally {
rmSync(prepared[0]!.projectDir, { recursive: true, force: true });
rmSync(dir, { recursive: true, force: true });
}
});
it("uses project directories directly and stages standalone html files as index.html", () => {
const dir = tempDir();
try {
@@ -178,8 +205,10 @@ describe("prepareCompareVariantProjects", () => {
mkdirSync(projectDir, { recursive: true });
mkdirSync(htmlDir, { recursive: true });
writeFileSync(projectIndex, "<!doctype html><title>A</title>", { flag: "wx" });
writeFileSync(htmlFile, "<!doctype html><title>B</title>", { flag: "wx" });
writeFileSync(join(htmlDir, "asset.txt"), "asset");
writeFileSync(htmlFile, '<!doctype html><title>B</title><img src="asset.png">', {
flag: "wx",
});
writeFileSync(join(htmlDir, "asset.png"), "asset");
const prepared = prepareCompareVariantProjects([
{ label: "A", inputPath: projectDir, displayPath: "variant-a" },
@@ -200,7 +229,7 @@ describe("prepareCompareVariantProjects", () => {
expect(readFileSync(join(prepared[1]!.projectDir, "index.html"), "utf-8")).toContain(
"<title>B</title>",
);
expect(readFileSync(join(prepared[1]!.projectDir, "asset.txt"), "utf-8")).toBe("asset");
expect(readFileSync(join(prepared[1]!.projectDir, "asset.png"), "utf-8")).toBe("asset");
expect(existsSync(join(prepared[1]!.projectDir, "candidate.html"))).toBe(false);
} finally {
for (const variant of prepared) {
+11 -48
View File
@@ -1,14 +1,4 @@
import {
cpSync,
existsSync,
mkdirSync,
mkdtempSync,
readdirSync,
realpathSync,
renameSync,
rmSync,
statSync,
} from "node:fs";
import { existsSync, mkdtempSync, realpathSync, renameSync, rmSync, statSync } from "node:fs";
import { makeOutputFolder } from "../utils/outputFolder.js";
import { tmpdir } from "node:os";
import { basename, dirname, extname, join } from "node:path";
@@ -24,9 +14,9 @@ import { normalizeErrorMessage } from "../utils/errorMessage.js";
import { displayPathFromInput, readOptionalString, resolveFromBase } from "../utils/pathArgs.js";
import { trackCompareSheet } from "../telemetry/events.js";
import { serveStaticProjectHtml } from "../utils/staticProjectServer.js";
import { leftOutNotice, stageReferencedFiles } from "../utils/stageReferencedFiles.js";
import { withMeta } from "../utils/updateCheck.js";
import type { Example } from "./_examples.js";
import { isPathInside } from "@hyperframes/parsers/asset-paths";
import { failCaught } from "../utils/jsonError.js";
import { CliUsageError } from "../utils/commandResult.js";
@@ -221,40 +211,16 @@ function inputError(variant: CompareVariantSpec): Error {
);
}
function copyVariantSiblings(
sourceDir: string,
destinationDir: string,
stagingPaths: readonly string[],
): void {
mkdirSync(destinationDir, { recursive: true });
for (const entry of readdirSync(sourceDir)) {
if (entry === "node_modules" || entry === ".git") continue;
const src = join(sourceDir, entry);
const dest = join(destinationDir, entry);
if (stagingPaths.includes(src)) continue;
if (stagingPaths.some((path) => isPathInside(path, src))) {
copyVariantSiblings(src, dest, stagingPaths);
continue;
}
cpSync(src, dest, {
recursive: true,
filter: (path) => basename(path) !== "node_modules" && basename(path) !== ".git",
});
}
}
function stageHtmlVariant(
variant: CompareVariantSpec,
existingStagedDirs: readonly string[],
): PreparedCompareVariant {
function stageHtmlVariant(variant: CompareVariantSpec): PreparedCompareVariant {
const stagedDir = mkdtempSync(join(tmpdir(), "hf-compare-variant-"));
try {
// Copy the composition's sibling files but skip heavy/irrelevant trees — a
// variant sitting next to node_modules or .git shouldn't drag them into tmp.
const sourceDir = realpathSync(dirname(variant.inputPath));
const stagingPaths = [stagedDir, ...existingStagedDirs].map((path) => realpathSync(path));
copyVariantSiblings(sourceDir, realpathSync(stagedDir), stagingPaths);
const sourceName = basename(variant.inputPath);
const source = realpathSync(variant.inputPath);
const notice = leftOutNotice(
`compare ${variant.displayPath}`,
stageReferencedFiles(dirname(source), [source], stagedDir),
);
if (notice) console.warn(c.dim(notice));
const sourceName = basename(source);
if (sourceName !== "index.html") {
renameSync(join(stagedDir, sourceName), join(stagedDir, "index.html"));
}
@@ -287,10 +253,7 @@ export function prepareCompareVariantProjects(
continue;
}
if (stat.isFile() && extname(variant.inputPath).toLowerCase() === ".html") {
const stagedDirs = prepared.flatMap((project) =>
project.stagedDir ? [project.stagedDir] : [],
);
prepared.push(stageHtmlVariant(variant, stagedDirs));
prepared.push(stageHtmlVariant(variant));
continue;
}
throw inputError(variant);
@@ -18,6 +18,7 @@ import {
collectCompositions,
collectShotSelectors,
resolveScope,
singleFileStage,
surfaceComposition,
} from "./keyframes.js";
import { ensureShotOutputDir } from "./motionShot.js";
@@ -109,6 +110,47 @@ describe("keyframes direct composition scope", () => {
expect(scope.projectDir).toBe(projectDir);
expect(scope.entryFile).toBe("compositions/scene.html");
expect(scope.singleFile).toBe(scenePath);
});
});
describe("keyframes --shot on a bare HTML file", () => {
const scene = `<template id="scene-template"><div data-composition-id="scene"></div></template>`;
const mountingIndex = wrap("").replace(
'<div id="dot" class="clip"></div>',
'<div data-composition-id="scene" data-composition-src="compositions/scene.html"></div>',
);
function project(index: string) {
const projectDir = mkdtempSync(join(tmpdir(), "hf-keyframes-stage-"));
onTestFinished(() => rmSync(projectDir, { recursive: true, force: true }));
mkdirSync(join(projectDir, "compositions"));
writeFileSync(join(projectDir, "index.html"), index);
return projectDir;
}
it("stages a scene its project mounts from the project, so the scene keeps its shell", async () => {
const projectDir = project(mountingIndex);
const scenePath = join(projectDir, "compositions", "scene.html");
writeFileSync(scenePath, scene);
expect(await singleFileStage(scenePath, projectDir)).toEqual({
root: projectDir,
entries: [join(projectDir, "index.html"), scenePath],
entryFile: "compositions/scene.html",
});
});
it("stages any other file from its own folder, even under a folder holding index.html", async () => {
const projectDir = project(wrap(""));
const sentPath = join(projectDir, "compositions", "sent.html");
writeFileSync(sentPath, scene);
expect(await singleFileStage(sentPath, projectDir)).toEqual({
root: join(projectDir, "compositions"),
entries: [sentPath],
entryFile: "sent.html",
});
});
});
+67 -15
View File
@@ -1,6 +1,7 @@
import { terminalText, terminalTextDeep } from "@hyperframes/core";
import { defineCommand } from "../utils/defineCommand.js";
import { existsSync, readFileSync } from "node:fs";
import { existsSync, mkdtempSync, readFileSync, rmSync } from "node:fs";
import { tmpdir } from "node:os";
import { isWithinProjectRoot, readProjectFile } from "@hyperframes/parsers/asset-resolution";
import { resolve, dirname, basename, join, relative, sep } from "node:path";
import {
@@ -12,6 +13,7 @@ import type { Example } from "./_examples.js";
import { c } from "../ui/colors.js";
import { ensureDOMParser } from "../utils/dom.js";
import { resolveProject } from "../utils/project.js";
import { leftOutNotice, stageReferencedFiles } from "../utils/stageReferencedFiles.js";
import { withMeta } from "../utils/updateCheck.js";
export const examples: Example[] = [
@@ -896,42 +898,90 @@ function printOnionShotSaved(saved: string, elementCount: number): void {
async function runOnionShot(
comps: SurfacedComposition[],
allComps: SurfacedComposition[],
projectDir: string | undefined,
entryFile: string | undefined,
scope: ShotScope,
args: ShotArgs & { selector?: string },
): Promise<boolean> {
const { captureMotionPathShot } = await import("./motionShot.js");
// With --selector, sample from the FULL animated set and let the browser scope
// to the selector (or its animated descendants when the selector is a static
// wrapper like `.clip`). Without it, only the (already-filtered) comps qualify.
const requests = collectShotSelectors(args.selector ? allComps : comps);
const guardError = onionShotGuardError(projectDir, requests, args.ghost ?? false);
const guardError = onionShotGuardError(scope.projectDir, requests, args.ghost ?? false);
if (guardError) {
console.log(c.dim(guardError));
return true;
}
const saved = await captureMotionPathShot(projectDir!, requests, resolve(args.shot!), {
...onionShotOptions(args),
entryFile,
});
const saved = await captureOnionShot(scope, requests, args);
printOnionShotSaved(saved, requests.length);
return false;
}
async function captureOnionShot(
scope: ShotScope,
requests: Array<{ selector: string }>,
args: ShotArgs & { selector?: string },
): Promise<string> {
const { captureMotionPathShot, ensureShotOutputDir } = await import("./motionShot.js");
const outPath = resolve(args.shot!);
const options = onionShotOptions(args);
if (!scope.singleFile) {
return captureMotionPathShot(scope.projectDir!, requests, outPath, {
...options,
entryFile: scope.entryFile,
});
}
ensureShotOutputDir(outPath, scope.singleFile);
const stage = await singleFileStage(scope.singleFile, scope.projectDir!);
const stagedDir = mkdtempSync(join(tmpdir(), "hf-keyframes-shot-"));
try {
const notice = leftOutNotice(
`keyframes --shot ${stage.entryFile}`,
stageReferencedFiles(stage.root, stage.entries, stagedDir),
);
if (notice) console.warn(c.dim(notice));
return await captureMotionPathShot(stagedDir, requests, outPath, {
...options,
entryFile: stage.entryFile,
});
} finally {
rmSync(stagedDir, { recursive: true, force: true });
}
}
export async function singleFileStage(file: string, projectDir: string) {
const { extractStandaloneEntryFromIndex, isTemplateEntry } =
await import("@hyperframes/core/compiler");
const index = join(projectDir, "index.html");
const entryFile = relative(projectDir, file).split(sep).join("/");
const scene = readFileSync(file, "utf8");
const mounted =
file !== index &&
existsSync(index) &&
isTemplateEntry(scene) &&
extractStandaloneEntryFromIndex(readFileSync(index, "utf8"), entryFile, scene) !== null;
return mounted
? { root: projectDir, entries: [index, file], entryFile }
: { root: dirname(file), entries: [file], entryFile: basename(file) };
}
interface ShotScope {
projectDir: string | undefined;
entryFile: string | undefined;
singleFile?: string;
}
// Resolve the command target (a project dir or a single .html) into surfaced
// compositions, applying the optional --selector filter.
export function resolveScope(args: { target?: string; selector?: string }): {
export function resolveScope(args: { target?: string; selector?: string }): ShotScope & {
comps: SurfacedComposition[];
allComps: SurfacedComposition[];
projectName: string;
projectDir: string | undefined;
entryFile: string | undefined;
} {
const raw = args.target?.trim();
let comps: SurfacedComposition[];
let projectName: string;
let projectDir: string | undefined;
let entryFile: string | undefined;
let singleFile: string | undefined;
const entry = raw?.endsWith(".html") ? readProjectFile(raw) : undefined;
if (raw && entry?.kind === "file") {
const entryPath = resolve(raw);
@@ -939,6 +989,7 @@ export function resolveScope(args: { target?: string; selector?: string }): {
projectName = basename(entryPath);
projectDir = findProjectRoot(entryPath);
entryFile = relative(projectDir, entryPath).split(sep).join("/");
singleFile = entryPath;
} else {
const project = resolveProject(raw);
comps = collectCompositions(project.indexPath);
@@ -968,7 +1019,7 @@ export function resolveScope(args: { target?: string; selector?: string }): {
cmp.anime.length > 0,
);
}
return { comps, allComps, projectName, projectDir, entryFile };
return { comps, allComps, projectName, projectDir, entryFile, singleFile };
}
function findProjectRoot(entryPath: string): string {
@@ -1116,12 +1167,13 @@ function createKeyframesCommand(options: Partial<KeyframesCommandOptions> = {})
async run({ args }) {
ensureDOMParser();
const runtime = normalizeRuntime(args.runtime, commandOptions.defaultRuntime);
const { comps: rawComps, allComps, projectName, projectDir, entryFile } = resolveScope(args);
const scope = resolveScope(args);
const { comps: rawComps, allComps, projectName } = scope;
const comps = filterCompositionsByRuntime(rawComps, runtime);
// --shot: 3D onion-skin self-verify screenshot. Returns true when the command
// should stop (guard failure) so run() stays small.
if (args.shot && (await runOnionShot(comps, allComps, projectDir, entryFile, args))) return;
if (args.shot && (await runOnionShot(comps, allComps, scope, args))) return;
if (args.json) {
console.log(
+156
View File
@@ -0,0 +1,156 @@
import { closeSync, openSync, readSync, realpathSync, statSync } from "node:fs";
import { dirname, isAbsolute, resolve } from "node:path";
import { fileURLToPath } from "node:url";
import { parseHTML } from "linkedom";
import { isNonRelativeUrl } from "@hyperframes/core";
import { resolveCompositionSrc } from "@hyperframes/parsers/asset-resolution";
import { HF_COLOR_GRADING_ATTR } from "@hyperframes/core/color-grading";
const REFERENCE_ATTRS = ["src", "href", "poster", "xlink:href"];
const MAX_SCANNED_BYTES = 2 * 1024 * 1024;
export interface Reference {
raw: string;
composition: boolean;
follow: boolean;
colorGradingLut?: boolean;
}
function unquote(value: string): string {
const first = value[0];
return value.length >= 2 && (first === '"' || first === "'") && value.at(-1) === first
? value.slice(1, -1).trim()
: value;
}
export function cssReferences(css: string): Reference[] {
const refs: Reference[] = [];
for (let at = css.indexOf("url("); at !== -1; ) {
const close = css.indexOf(")", at + 4);
if (close === -1) break;
const raw = unquote(css.slice(at + 4, close).trim());
if (raw) refs.push({ raw, composition: false, follow: true });
at = css.indexOf("url(", close + 1);
}
for (const [, , raw] of css.matchAll(/@import\s+(["'])(.+?)\1/g)) {
if (raw?.trim()) refs.push({ raw: raw.trim(), composition: false, follow: true });
}
return refs;
}
function colorGradingLut(value: string | null): string | undefined {
try {
const parsed: unknown = JSON.parse(value ?? "");
const lut: unknown = typeof parsed === "object" && parsed ? Reflect.get(parsed, "lut") : null;
const src: unknown = typeof lut === "object" && lut ? Reflect.get(lut, "src") : lut;
return typeof src === "string" && src.trim() ? src.trim() : undefined;
} catch {
return undefined;
}
}
function elementReferences(el: Element): Reference[] {
const follow = el.localName === "link";
const refs: Reference[] = REFERENCE_ATTRS.flatMap((attr) => {
const raw = el.getAttribute(attr)?.trim();
return raw ? [{ raw, composition: false, follow }] : [];
});
for (const candidate of el.getAttribute("srcset")?.split(",") ?? []) {
const raw = candidate.trim().split(/\s+/)[0];
if (raw) refs.push({ raw, composition: false, follow: false });
}
const composition = el.getAttribute("data-composition-src")?.trim();
if (composition) refs.push({ raw: composition, composition: true, follow: true });
const lut = colorGradingLut(el.getAttribute(HF_COLOR_GRADING_ATTR));
if (lut) refs.push({ raw: lut, composition: false, follow: false, colorGradingLut: true });
return refs;
}
export function htmlReferences(html: string): Reference[] {
const { document } = parseHTML(html);
const hosts = document.querySelectorAll(
`[src], [href], [poster], [xlink\\:href], [srcset], [data-composition-src], [${HF_COLOR_GRADING_ATTR}]`,
);
const refs = [...hosts].flatMap(elementReferences);
for (const style of document.querySelectorAll("style")) {
refs.push(...cssReferences(style.textContent ?? ""));
}
for (const el of document.querySelectorAll("[style]")) {
refs.push(...cssReferences(el.getAttribute("style") ?? ""));
}
for (const template of document.querySelectorAll("template")) {
refs.push(...htmlReferences(template.innerHTML));
}
return refs;
}
const WINDOWS_DEVICE_PREFIX = /^[\\/]{2}[?.][\\/]/;
function withoutQuery(raw: string): string {
const device = WINDOWS_DEVICE_PREFIX.exec(raw)?.[0] ?? "";
return device + raw.slice(device.length).replace(/[?#].*$/, "");
}
export function localPath(raw: string, fromDir: string): string | null {
if (raw.startsWith("file:")) {
try {
return fileURLToPath(raw);
} catch {
return null;
}
}
const path = withoutQuery(raw);
if (isAbsolute(path) && !path.startsWith("//")) return path;
if (path === "" || isNonRelativeUrl(path)) return null;
try {
return resolve(fromDir, decodeURIComponent(path));
} catch {
return resolve(fromDir, path);
}
}
export function isFile(path: string): boolean {
try {
return statSync(path, { throwIfNoEntry: false })?.isFile() === true;
} catch {
return false;
}
}
export function realOrNull(path: string): string | null {
try {
return realpathSync(path);
} catch {
return null;
}
}
export function readHead(file: string): { text: string; whole: boolean } | null {
let fd: number | undefined;
try {
fd = openSync(file, "r");
const size = statSync(file).size;
const buffer = Buffer.alloc(Math.min(size, MAX_SCANNED_BYTES));
const read = readSync(fd, buffer, 0, buffer.length, 0);
return {
text: buffer.toString("utf-8", 0, read),
whole: size <= MAX_SCANNED_BYTES,
};
} catch {
return null;
} finally {
if (fd !== undefined) closeSync(fd);
}
}
export const isPageOrStylesheet = (path: string) => /\.(?:html?|css)$/i.test(path);
// A composition resolves as the render compiler resolves it; one that leaves the project resolves as written.
export function referencePath(ref: Reference, file: string, projectDir: string): string | null {
if (!ref.composition) return localPath(ref.raw, dirname(file));
try {
return resolveCompositionSrc(projectDir, ref.raw) ?? localPath(ref.raw, projectDir);
} catch {
return null;
}
}
+14 -142
View File
@@ -1,156 +1,28 @@
import {
closeSync,
openSync,
readdirSync,
readSync,
realpathSync,
statSync,
type Dirent,
} from "node:fs";
import { dirname, extname, isAbsolute, join, relative, resolve } from "node:path";
import { fileURLToPath } from "node:url";
import { parseHTML } from "linkedom";
import { isNonRelativeUrl, isPathInside, terminalText } from "@hyperframes/core";
import {
namesAnotherNetworkRoot,
resolveCompositionSrc,
} from "@hyperframes/parsers/asset-resolution";
import { closeSync, openSync, readdirSync, readSync, statSync, type Dirent } from "node:fs";
import { extname, join, relative, resolve } from "node:path";
import { isPathInside, terminalText } from "@hyperframes/core";
import { namesAnotherNetworkRoot } from "@hyperframes/parsers/asset-resolution";
import { linksOutside, receivedRecordStatus } from "./privateProjectFile.js";
import { walkBudget } from "./walkBudget.js";
import {
htmlReferences,
cssReferences,
isFile,
isPageOrStylesheet,
readHead,
realOrNull,
referencePath,
type Reference,
} from "./fileReferences.js";
const REFERENCE_ATTRS = ["src", "href", "poster"];
const MAX_SCANNED_BYTES = 2 * 1024 * 1024;
const MAX_SCANNED_FILES = 500;
interface Reference {
raw: string;
composition: boolean;
follow: boolean;
}
function unquote(value: string): string {
const first = value[0];
return value.length >= 2 && (first === '"' || first === "'") && value.at(-1) === first
? value.slice(1, -1).trim()
: value;
}
function cssReferences(css: string): Reference[] {
const refs: Reference[] = [];
for (let at = css.indexOf("url("); at !== -1; ) {
const close = css.indexOf(")", at + 4);
if (close === -1) break;
const raw = unquote(css.slice(at + 4, close).trim());
if (raw) refs.push({ raw, composition: false, follow: true });
at = css.indexOf("url(", close + 1);
}
return refs;
}
function elementReferences(el: Element): Reference[] {
const follow = el.localName === "link";
const refs: Reference[] = REFERENCE_ATTRS.flatMap((attr) => {
const raw = el.getAttribute(attr)?.trim();
return raw ? [{ raw, composition: false, follow }] : [];
});
const composition = el.getAttribute("data-composition-src")?.trim();
if (composition) refs.push({ raw: composition, composition: true, follow: true });
return refs;
}
function htmlReferences(html: string): Reference[] {
const { document } = parseHTML(html);
const hosts = document.querySelectorAll("[src], [href], [poster], [data-composition-src]");
const refs = [...hosts].flatMap(elementReferences);
for (const style of document.querySelectorAll("style")) {
refs.push(...cssReferences(style.textContent ?? ""));
}
for (const el of document.querySelectorAll("[style]")) {
refs.push(...cssReferences(el.getAttribute("style") ?? ""));
}
for (const template of document.querySelectorAll("template")) {
refs.push(...htmlReferences(template.innerHTML));
}
return refs;
}
const WINDOWS_DEVICE_PREFIX = /^[\\/]{2}[?.][\\/]/;
function withoutQuery(raw: string): string {
const device = WINDOWS_DEVICE_PREFIX.exec(raw)?.[0] ?? "";
return device + raw.slice(device.length).replace(/[?#].*$/, "");
}
function localPath(raw: string, fromDir: string): string | null {
if (raw.startsWith("file:")) {
try {
return fileURLToPath(raw);
} catch {
return null;
}
}
const path = withoutQuery(raw);
if (isAbsolute(path) && !path.startsWith("//")) return path;
if (path === "" || isNonRelativeUrl(path)) return null;
try {
return resolve(fromDir, decodeURIComponent(path));
} catch {
return resolve(fromDir, path);
}
}
function isFile(path: string): boolean {
try {
return statSync(path, { throwIfNoEntry: false })?.isFile() === true;
} catch {
return false;
}
}
function realOrNull(path: string): string | null {
try {
return realpathSync(path);
} catch {
return null;
}
}
function readHead(file: string): { text: string; whole: boolean } | null {
let fd: number | undefined;
try {
fd = openSync(file, "r");
const size = statSync(file).size;
const buffer = Buffer.alloc(Math.min(size, MAX_SCANNED_BYTES));
const read = readSync(fd, buffer, 0, buffer.length, 0);
return {
text: buffer.toString("utf-8", 0, read),
whole: size <= MAX_SCANNED_BYTES,
};
} catch {
return null;
} finally {
if (fd !== undefined) closeSync(fd);
}
}
const isPageOrStylesheet = (path: string) => /\.(?:html?|css)$/i.test(path);
interface OutsideReferences {
assets: Set<string>;
compositions: Set<string>;
complete: boolean;
}
// A composition resolves as the render compiler resolves it; one that leaves the project resolves as written.
function referencePath(ref: Reference, file: string, projectDir: string): string | null {
if (!ref.composition) return localPath(ref.raw, dirname(file));
try {
return resolveCompositionSrc(projectDir, ref.raw) ?? localPath(ref.raw, projectDir);
} catch {
return null;
}
}
function referencesIn(found: OutsideReferences, file: string): Reference[] {
const read = readHead(file);
if (read === null || !read.whole) found.complete = false;
@@ -0,0 +1,170 @@
import { mkdirSync, mkdtempSync, readdirSync, rmSync, symlinkSync, writeFileSync } from "node:fs";
import { tmpdir } from "node:os";
import { join, relative } from "node:path";
import { afterEach, beforeEach, describe, expect, it } from "vitest";
import { leftOutNotice, stageReferencedFiles } from "./stageReferencedFiles.js";
let root: string;
let folder: string;
let staged: string;
function write(path: string, text = "x"): void {
mkdirSync(join(path, ".."), { recursive: true });
writeFileSync(path, text);
}
function stagedFiles(): string[] {
const files: string[] = [];
const walk = (dir: string) => {
for (const entry of readdirSync(dir, { withFileTypes: true })) {
const path = join(dir, entry.name);
if (entry.isDirectory()) walk(path);
else files.push(relative(staged, path).split("\\").join("/"));
}
};
walk(staged);
return files.sort();
}
beforeEach(() => {
root = mkdtempSync(join(tmpdir(), "hf-stage-referenced-"));
folder = join(root, "sent");
staged = join(root, "staged");
mkdirSync(folder);
});
afterEach(() => {
rmSync(root, { recursive: true, force: true });
});
describe("stageReferencedFiles", () => {
it("copies the page and every file its markup, stylesheets and compositions name", () => {
write(
join(folder, "page.html"),
`<link rel="stylesheet" href="css/look.css?v=2">
<img src="img/a.png" srcset="img/a.png 1x, img/a@2x.png 2x">
<video src="media/clip.mp4" poster="img/poster.jpg"></video>
<div style="background: url('img/bg.webp')"></div>
<div data-composition-src="compositions/scene.html"></div>
<script src="js/run.js"></script>`,
);
write(join(folder, "css", "look.css"), `@font-face { src: url("../fonts/brand.woff2"); }`);
write(
join(folder, "compositions", "scene.html"),
`<template><img src="../img/scene.png"></template>`,
);
for (const file of ["img/a.png", "img/a@2x.png", "img/poster.jpg", "img/bg.webp"])
write(join(folder, file));
for (const file of ["img/scene.png", "media/clip.mp4", "fonts/brand.woff2", "js/run.js"])
write(join(folder, file));
write(join(folder, "img", "unnamed.png"));
stageReferencedFiles(folder, [join(folder, "page.html")], staged);
expect(stagedFiles()).toEqual([
"compositions/scene.html",
"css/look.css",
"fonts/brand.woff2",
"img/a.png",
"img/a@2x.png",
"img/bg.webp",
"img/poster.jpg",
"img/scene.png",
"js/run.js",
"media/clip.mp4",
"page.html",
]);
});
it("copies nothing from outside the page's folder, through a path or a link", () => {
write(join(root, "Pictures", "IMG_0001.png"));
symlinkSync(join(root, "Pictures"), join(folder, "pictures"));
write(
join(folder, "page.html"),
`<img src="../Pictures/IMG_0001.png"><img src="${join(root, "Pictures", "IMG_0001.png")}">
<img src="pictures/IMG_0001.png">`,
);
stageReferencedFiles(folder, [join(folder, "page.html")], staged);
expect(stagedFiles()).toEqual(["page.html"]);
});
it("copies no private file, whatever type its name gives", () => {
write(
join(folder, "page.html"),
`<img src=".git/logo.png"><img src=".env.png"><img src=".hyperframes/thumb.png">
<img src="secrets/.npmrc">`,
);
for (const file of [".git/logo.png", ".env.png", ".hyperframes/thumb.png", "secrets/.npmrc"])
write(join(folder, file));
stageReferencedFiles(folder, [join(folder, "page.html")], staged);
expect(stagedFiles()).toEqual(["page.html"]);
});
it("copies root-relative names, string imports, SVG links and the grading table", () => {
write(
join(folder, "page.html"),
`<style>@import "css/theme.css";</style>
<img src="/img/logo.png?v=1">
<svg><image xlink:href="img/art.png"/><use xlink:href="icons.svg#star"/></svg>
<video src="clip.mp4" data-color-grading='{"lut":{"src":"looks/warm.cube"}}'></video>
<img src="looks/cold.cube">`,
);
write(
join(folder, "css", "theme.css"),
`@import 'more.css'; body { background: url(/img/bg.png) }`,
);
write(join(folder, "css", "more.css"), "p {}");
for (const file of ["img/logo.png", "img/art.png", "icons.svg", "img/bg.png", "clip.mp4"])
write(join(folder, file));
write(join(folder, "looks", "warm.cube"));
write(join(folder, "looks", "cold.cube"));
const result = stageReferencedFiles(folder, [join(folder, "page.html")], staged);
expect(stagedFiles()).toEqual([
"clip.mp4",
"css/more.css",
"css/theme.css",
"icons.svg",
"img/art.png",
"img/bg.png",
"img/logo.png",
"looks/warm.cube",
"page.html",
]);
expect(result).toEqual({ leftOut: ["looks/cold.cube"], complete: true });
});
it("names what it left out, and nothing that does not exist", () => {
write(join(root, "Pictures", "IMG_0001.png"));
write(join(folder, "notes.txt"));
write(
join(folder, "page.html"),
`<img src="../Pictures/IMG_0001.png"><img src="notes.txt"><img src="missing.png">`,
);
const result = stageReferencedFiles(folder, [join(folder, "page.html")], staged);
expect(result).toEqual({ leftOut: ["../Pictures/IMG_0001.png", "notes.txt"], complete: true });
});
});
describe("leftOutNotice", () => {
it("says nothing when nothing was left out", () => {
expect(leftOutNotice("compare a.html", { leftOut: [], complete: true })).toBeNull();
});
it("names the first five escaped, counts the rest and says how to keep them", () => {
const leftOut = ["a\x1b[2J.png", "b.txt", "c.txt", "d.txt", "e.txt", "f.txt", "g.txt"];
const notice = leftOutNotice("compare a.html", { leftOut, complete: false });
expect(notice).not.toContain("\x1b");
expect(notice).toContain("a\\x1b[2J.png, b.txt, c.txt, d.txt, e.txt, and 2 more.");
expect(notice).toContain("too large or too many");
expect(notice).toContain("pass the project folder");
});
});
@@ -0,0 +1,118 @@
import { copyFileSync, mkdirSync } from "node:fs";
import { dirname, join, relative } from "node:path";
import { isPathInside, terminalText } from "@hyperframes/core";
import { isPrivateFileUnder, isStudioPrivatePath } from "@hyperframes/core/safe-path";
import { isCopiedOutsideFileType } from "@hyperframes/studio-server/outside-file";
import {
cssReferences,
htmlReferences,
isFile,
isPageOrStylesheet,
localPath,
readHead,
realOrNull,
referencePath,
type Reference,
} from "./fileReferences.js";
import { isPrivateCompositionFile, linksOutside } from "./privateProjectFile.js";
const MAX_FOLLOWED_FILES = 500;
const NAMES_SHOWN = 5;
export interface StagedFiles {
leftOut: string[];
complete: boolean;
}
interface NamedFile {
raw: string;
path: string;
colorGradingLut: boolean;
}
/** A file the page may load: a composition, a type a local render copies in, or the grading table its LUT names. */
function stageable(root: string, file: NamedFile): boolean {
const { path } = file;
if (linksOutside(root, path)) return false;
if (isPrivateFileUnder(root, path, isStudioPrivatePath) || isPrivateCompositionFile(root, path))
return false;
if (file.colorGradingLut && /\.cube$/i.test(path)) return true;
return /\.html?$/i.test(path) || isCopiedOutsideFileType(path);
}
// A leading slash names the served root, as the local server reads it; compositions resolve as the compiler resolves them.
function namedPath(ref: Reference, file: string, root: string): string | null {
if (!ref.composition && /^\/(?![/\\])/.test(ref.raw)) return localPath(ref.raw.slice(1), root);
return referencePath(ref, file, root);
}
function namedFiles(file: string, root: string, result: StagedFiles): NamedFile[] {
const read = readHead(file);
if (read === null || !read.whole) result.complete = false;
if (read === null) return [];
const refs = /\.css$/i.test(file) ? cssReferences(read.text) : htmlReferences(read.text);
return refs.flatMap((ref) => {
const path = namedPath(ref, file, root);
return path === null ? [] : [{ raw: ref.raw, path, colorGradingLut: !!ref.colorGradingLut }];
});
}
function realEntries(root: string, entries: readonly string[]): string[] {
return entries.map((entry) => {
const real = realOrNull(entry);
if (real === null || !isPathInside(real, root)) throw new Error(`Cannot read ${entry}`);
return real;
});
}
/**
* Copies `entries` into `stagedDir` with the files their markup, sub-compositions and stylesheets name under `root`.
* A page someone sent runs on the origin that serves `stagedDir`, so nothing else under `root` is put there: not
* private files, not other file types, not names the page builds in script.
*/
export function stageReferencedFiles(
root: string,
entries: readonly string[],
stagedDir: string,
): StagedFiles {
const realRoot = realOrNull(root);
if (realRoot === null) throw new Error(`Cannot read ${root}`);
const result: StagedFiles = { leftOut: [], complete: true };
const staged = new Set<string>();
const pending: string[] = [];
const stage = (path: string) => {
const real = realOrNull(path);
if (real === null || staged.has(real)) return;
staged.add(real);
const target = join(stagedDir, relative(realRoot, path));
mkdirSync(dirname(target), { recursive: true });
copyFileSync(real, target);
if (!isPageOrStylesheet(path)) return;
if (staged.size <= MAX_FOLLOWED_FILES) pending.push(path);
else result.complete = false;
};
const leaveOut = (raw: string) => {
if (!result.leftOut.includes(raw)) result.leftOut.push(raw);
};
realEntries(realRoot, entries).forEach(stage);
for (let file = pending.pop(); file !== undefined; file = pending.pop()) {
for (const named of namedFiles(file, realRoot, result)) {
if (!isPathInside(named.path, realRoot)) leaveOut(named.raw);
else if (!isFile(named.path)) continue;
else if (stageable(realRoot, named)) stage(named.path);
else leaveOut(named.raw);
}
}
return result;
}
/** One line naming what a bare HTML file lost, or null when nothing was left out. */
export function leftOutNotice(command: string, result: StagedFiles): string | null {
if (result.leftOut.length === 0 && result.complete) return null;
const shown = result.leftOut.slice(0, NAMES_SHOWN).map((raw) => terminalText(raw));
const more =
result.leftOut.length > NAMES_SHOWN ? `, and ${result.leftOut.length - NAMES_SHOWN} more` : "";
const named = shown.length > 0 ? ` left out ${shown.join(", ")}${more}.` : "";
const partial = result.complete ? "" : " Some files were too large or too many to read in full.";
return `${command}:${named}${partial} Only the files the page names in its own folder are served; pass the project folder to keep the rest.`;
}