diff --git a/docs/packages/cli.mdx b/docs/packages/cli.mdx index 0d479c04f9..7fd0660503 100644 --- a/docs/packages/cli.mdx +++ b/docs/packages/cli.mdx @@ -1605,6 +1605,13 @@ npx hyperframes keyframes [dir] --selector "#card" --shot card-motion.png Use `--ghost` for motion that happens *inside* a canvas, where the marker onion has nothing to draw. +With a single `.html` file, `--shot` serves only the files that file's markup, +stylesheets and sub-compositions name in its own folder, and only the types a +composition loads. A scene that its project's `index.html` mounts is served from +the project instead. Files the page builds names for in script are not served, +and the command prints a line naming any file it left out. Pass the project +folder to keep everything. + ### `compare` and `grade-compare` `compare` renders two or more independent variants into one labelled PNG. @@ -1622,6 +1629,12 @@ npx hyperframes compare ./variant-a ./variant-b --labels "A,B" | `--timeout` | Render-ready timeout per variant in ms; must be a positive integer (default 5000) | | `--json` | Machine-readable results | +A variant can also be a single `.html` file. Then only the files its markup, +stylesheets and sub-compositions name in its own folder are served, and only the +types a composition loads. Files the page builds names for in script are not +served, and the command prints a line naming any file it left out. Pass a +project folder to keep everything. + `grade-compare` does the same for colour: candidate grades or LUTs applied to one reference frame. diff --git a/packages/cli/src/commands/compare.test.ts b/packages/cli/src/commands/compare.test.ts index fc19b3135a..b2402f979c 100644 --- a/packages/cli/src/commands/compare.test.ts +++ b/packages/cli/src/commands/compare.test.ts @@ -1,4 +1,12 @@ -import { existsSync, mkdirSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from "node:fs"; +import { + existsSync, + mkdirSync, + mkdtempSync, + readdirSync, + readFileSync, + rmSync, + writeFileSync, +} from "node:fs"; import { tmpdir } from "node:os"; import { basename, join } from "node:path"; import { describe, expect, it, vi } from "vitest"; @@ -117,7 +125,7 @@ describe("prepareCompareVariantProjects", () => { const secondStage = prepared[1]; if (!firstStage || !secondStage) throw new Error("expected two staged variants"); expect(existsSync(join(secondStage.projectDir, basename(firstStage.projectDir)))).toBe(false); - expect(readFileSync(join(secondStage.projectDir, "first.html"), "utf8")).toContain("First"); + expect(readdirSync(secondStage.projectDir)).toEqual(["index.html"]); expect(readFileSync(join(secondStage.projectDir, "index.html"), "utf8")).toContain("Second"); } finally { temporaryRoot.path = ""; @@ -126,15 +134,17 @@ describe("prepareCompareVariantProjects", () => { }); it.each(["direct", "nested"])( - "stages siblings when the temporary output is %s inside the source directory", + "stages only what the page names when the temporary output is %s inside the source directory", (placement) => { const dir = tempDir(); const htmlFile = join(dir, "candidate.html"); mkdirSync(join(dir, "assets")); - writeFileSync(htmlFile, "Temporary candidate"); - writeFileSync(join(dir, "assets", "asset.txt"), "sibling asset"); - mkdirSync(join(dir, "node_modules")); - writeFileSync(join(dir, "node_modules", "ignored.txt"), "ignored"); + writeFileSync( + htmlFile, + 'Temporary candidate', + ); + writeFileSync(join(dir, "assets", "asset.png"), "sibling asset"); + writeFileSync(join(dir, "assets", "unnamed.png"), "not named"); temporaryRoot.path = placement === "direct" ? dir : join(dir, "temporary"); mkdirSync(temporaryRoot.path, { recursive: true }); writeFileSync(join(temporaryRoot.path, "sibling.txt"), "temporary-root sibling"); @@ -149,17 +159,11 @@ describe("prepareCompareVariantProjects", () => { expect(readFileSync(join(staged.projectDir, "index.html"), "utf8")).toContain( "Temporary candidate", ); - expect(readFileSync(join(staged.projectDir, "assets", "asset.txt"), "utf8")).toBe( + expect(readFileSync(join(staged.projectDir, "assets", "asset.png"), "utf8")).toBe( "sibling asset", ); - expect( - readFileSync( - join(staged.projectDir, placement === "direct" ? "" : "temporary", "sibling.txt"), - "utf8", - ), - ).toBe("temporary-root sibling"); - expect(existsSync(join(staged.projectDir, "candidate.html"))).toBe(false); - expect(existsSync(join(staged.projectDir, "node_modules"))).toBe(false); + expect(readdirSync(staged.projectDir).sort()).toEqual(["assets", "index.html"]); + expect(readdirSync(join(staged.projectDir, "assets"))).toEqual(["asset.png"]); expect(readFileSync(htmlFile, "utf8")).toContain("Temporary candidate"); } finally { temporaryRoot.path = ""; @@ -168,6 +172,29 @@ describe("prepareCompareVariantProjects", () => { }, ); + it("leaves out a file the page names that no composition loads", () => { + const dir = tempDir(); + const htmlFile = join(dir, "sent.html"); + writeFileSync( + htmlFile, + ` + `, + ); + for (const name of ["taxes.pdf", "notes.txt", ".env", "id_ed25519", "statement.png"]) { + writeFileSync(join(dir, name), "private"); + } + + const prepared = prepareCompareVariantProjects([ + { label: "Sent", inputPath: htmlFile, displayPath: "sent.html" }, + ]); + try { + expect(readdirSync(prepared[0]!.projectDir)).toEqual(["index.html"]); + } finally { + rmSync(prepared[0]!.projectDir, { recursive: true, force: true }); + rmSync(dir, { recursive: true, force: true }); + } + }); + it("uses project directories directly and stages standalone html files as index.html", () => { const dir = tempDir(); try { @@ -178,8 +205,10 @@ describe("prepareCompareVariantProjects", () => { mkdirSync(projectDir, { recursive: true }); mkdirSync(htmlDir, { recursive: true }); writeFileSync(projectIndex, "A", { flag: "wx" }); - writeFileSync(htmlFile, "B", { flag: "wx" }); - writeFileSync(join(htmlDir, "asset.txt"), "asset"); + writeFileSync(htmlFile, 'B', { + flag: "wx", + }); + writeFileSync(join(htmlDir, "asset.png"), "asset"); const prepared = prepareCompareVariantProjects([ { label: "A", inputPath: projectDir, displayPath: "variant-a" }, @@ -200,7 +229,7 @@ describe("prepareCompareVariantProjects", () => { expect(readFileSync(join(prepared[1]!.projectDir, "index.html"), "utf-8")).toContain( "B", ); - expect(readFileSync(join(prepared[1]!.projectDir, "asset.txt"), "utf-8")).toBe("asset"); + expect(readFileSync(join(prepared[1]!.projectDir, "asset.png"), "utf-8")).toBe("asset"); expect(existsSync(join(prepared[1]!.projectDir, "candidate.html"))).toBe(false); } finally { for (const variant of prepared) { diff --git a/packages/cli/src/commands/compare.ts b/packages/cli/src/commands/compare.ts index a37f945164..d54f78a306 100644 --- a/packages/cli/src/commands/compare.ts +++ b/packages/cli/src/commands/compare.ts @@ -1,14 +1,4 @@ -import { - cpSync, - existsSync, - mkdirSync, - mkdtempSync, - readdirSync, - realpathSync, - renameSync, - rmSync, - statSync, -} from "node:fs"; +import { existsSync, mkdtempSync, realpathSync, renameSync, rmSync, statSync } from "node:fs"; import { makeOutputFolder } from "../utils/outputFolder.js"; import { tmpdir } from "node:os"; import { basename, dirname, extname, join } from "node:path"; @@ -24,9 +14,9 @@ import { normalizeErrorMessage } from "../utils/errorMessage.js"; import { displayPathFromInput, readOptionalString, resolveFromBase } from "../utils/pathArgs.js"; import { trackCompareSheet } from "../telemetry/events.js"; import { serveStaticProjectHtml } from "../utils/staticProjectServer.js"; +import { leftOutNotice, stageReferencedFiles } from "../utils/stageReferencedFiles.js"; import { withMeta } from "../utils/updateCheck.js"; import type { Example } from "./_examples.js"; -import { isPathInside } from "@hyperframes/parsers/asset-paths"; import { failCaught } from "../utils/jsonError.js"; import { CliUsageError } from "../utils/commandResult.js"; @@ -221,40 +211,16 @@ function inputError(variant: CompareVariantSpec): Error { ); } -function copyVariantSiblings( - sourceDir: string, - destinationDir: string, - stagingPaths: readonly string[], -): void { - mkdirSync(destinationDir, { recursive: true }); - for (const entry of readdirSync(sourceDir)) { - if (entry === "node_modules" || entry === ".git") continue; - const src = join(sourceDir, entry); - const dest = join(destinationDir, entry); - if (stagingPaths.includes(src)) continue; - if (stagingPaths.some((path) => isPathInside(path, src))) { - copyVariantSiblings(src, dest, stagingPaths); - continue; - } - cpSync(src, dest, { - recursive: true, - filter: (path) => basename(path) !== "node_modules" && basename(path) !== ".git", - }); - } -} - -function stageHtmlVariant( - variant: CompareVariantSpec, - existingStagedDirs: readonly string[], -): PreparedCompareVariant { +function stageHtmlVariant(variant: CompareVariantSpec): PreparedCompareVariant { const stagedDir = mkdtempSync(join(tmpdir(), "hf-compare-variant-")); try { - // Copy the composition's sibling files but skip heavy/irrelevant trees — a - // variant sitting next to node_modules or .git shouldn't drag them into tmp. - const sourceDir = realpathSync(dirname(variant.inputPath)); - const stagingPaths = [stagedDir, ...existingStagedDirs].map((path) => realpathSync(path)); - copyVariantSiblings(sourceDir, realpathSync(stagedDir), stagingPaths); - const sourceName = basename(variant.inputPath); + const source = realpathSync(variant.inputPath); + const notice = leftOutNotice( + `compare ${variant.displayPath}`, + stageReferencedFiles(dirname(source), [source], stagedDir), + ); + if (notice) console.warn(c.dim(notice)); + const sourceName = basename(source); if (sourceName !== "index.html") { renameSync(join(stagedDir, sourceName), join(stagedDir, "index.html")); } @@ -287,10 +253,7 @@ export function prepareCompareVariantProjects( continue; } if (stat.isFile() && extname(variant.inputPath).toLowerCase() === ".html") { - const stagedDirs = prepared.flatMap((project) => - project.stagedDir ? [project.stagedDir] : [], - ); - prepared.push(stageHtmlVariant(variant, stagedDirs)); + prepared.push(stageHtmlVariant(variant)); continue; } throw inputError(variant); diff --git a/packages/cli/src/commands/keyframes.test.ts b/packages/cli/src/commands/keyframes.test.ts index 57b00f780a..75cc7f4718 100644 --- a/packages/cli/src/commands/keyframes.test.ts +++ b/packages/cli/src/commands/keyframes.test.ts @@ -18,6 +18,7 @@ import { collectCompositions, collectShotSelectors, resolveScope, + singleFileStage, surfaceComposition, } from "./keyframes.js"; import { ensureShotOutputDir } from "./motionShot.js"; @@ -109,6 +110,47 @@ describe("keyframes direct composition scope", () => { expect(scope.projectDir).toBe(projectDir); expect(scope.entryFile).toBe("compositions/scene.html"); + expect(scope.singleFile).toBe(scenePath); + }); +}); + +describe("keyframes --shot on a bare HTML file", () => { + const scene = ``; + const mountingIndex = wrap("").replace( + '
', + '
', + ); + + function project(index: string) { + const projectDir = mkdtempSync(join(tmpdir(), "hf-keyframes-stage-")); + onTestFinished(() => rmSync(projectDir, { recursive: true, force: true })); + mkdirSync(join(projectDir, "compositions")); + writeFileSync(join(projectDir, "index.html"), index); + return projectDir; + } + + it("stages a scene its project mounts from the project, so the scene keeps its shell", async () => { + const projectDir = project(mountingIndex); + const scenePath = join(projectDir, "compositions", "scene.html"); + writeFileSync(scenePath, scene); + + expect(await singleFileStage(scenePath, projectDir)).toEqual({ + root: projectDir, + entries: [join(projectDir, "index.html"), scenePath], + entryFile: "compositions/scene.html", + }); + }); + + it("stages any other file from its own folder, even under a folder holding index.html", async () => { + const projectDir = project(wrap("")); + const sentPath = join(projectDir, "compositions", "sent.html"); + writeFileSync(sentPath, scene); + + expect(await singleFileStage(sentPath, projectDir)).toEqual({ + root: join(projectDir, "compositions"), + entries: [sentPath], + entryFile: "sent.html", + }); }); }); diff --git a/packages/cli/src/commands/keyframes.ts b/packages/cli/src/commands/keyframes.ts index afcd875d54..2aa8aed550 100644 --- a/packages/cli/src/commands/keyframes.ts +++ b/packages/cli/src/commands/keyframes.ts @@ -1,6 +1,7 @@ import { terminalText, terminalTextDeep } from "@hyperframes/core"; import { defineCommand } from "../utils/defineCommand.js"; -import { existsSync, readFileSync } from "node:fs"; +import { existsSync, mkdtempSync, readFileSync, rmSync } from "node:fs"; +import { tmpdir } from "node:os"; import { isWithinProjectRoot, readProjectFile } from "@hyperframes/parsers/asset-resolution"; import { resolve, dirname, basename, join, relative, sep } from "node:path"; import { @@ -12,6 +13,7 @@ import type { Example } from "./_examples.js"; import { c } from "../ui/colors.js"; import { ensureDOMParser } from "../utils/dom.js"; import { resolveProject } from "../utils/project.js"; +import { leftOutNotice, stageReferencedFiles } from "../utils/stageReferencedFiles.js"; import { withMeta } from "../utils/updateCheck.js"; export const examples: Example[] = [ @@ -896,42 +898,90 @@ function printOnionShotSaved(saved: string, elementCount: number): void { async function runOnionShot( comps: SurfacedComposition[], allComps: SurfacedComposition[], - projectDir: string | undefined, - entryFile: string | undefined, + scope: ShotScope, args: ShotArgs & { selector?: string }, ): Promise { - const { captureMotionPathShot } = await import("./motionShot.js"); // With --selector, sample from the FULL animated set and let the browser scope // to the selector (or its animated descendants when the selector is a static // wrapper like `.clip`). Without it, only the (already-filtered) comps qualify. const requests = collectShotSelectors(args.selector ? allComps : comps); - const guardError = onionShotGuardError(projectDir, requests, args.ghost ?? false); + const guardError = onionShotGuardError(scope.projectDir, requests, args.ghost ?? false); if (guardError) { console.log(c.dim(guardError)); return true; } - const saved = await captureMotionPathShot(projectDir!, requests, resolve(args.shot!), { - ...onionShotOptions(args), - entryFile, - }); + const saved = await captureOnionShot(scope, requests, args); printOnionShotSaved(saved, requests.length); return false; } +async function captureOnionShot( + scope: ShotScope, + requests: Array<{ selector: string }>, + args: ShotArgs & { selector?: string }, +): Promise { + const { captureMotionPathShot, ensureShotOutputDir } = await import("./motionShot.js"); + const outPath = resolve(args.shot!); + const options = onionShotOptions(args); + if (!scope.singleFile) { + return captureMotionPathShot(scope.projectDir!, requests, outPath, { + ...options, + entryFile: scope.entryFile, + }); + } + ensureShotOutputDir(outPath, scope.singleFile); + const stage = await singleFileStage(scope.singleFile, scope.projectDir!); + const stagedDir = mkdtempSync(join(tmpdir(), "hf-keyframes-shot-")); + try { + const notice = leftOutNotice( + `keyframes --shot ${stage.entryFile}`, + stageReferencedFiles(stage.root, stage.entries, stagedDir), + ); + if (notice) console.warn(c.dim(notice)); + return await captureMotionPathShot(stagedDir, requests, outPath, { + ...options, + entryFile: stage.entryFile, + }); + } finally { + rmSync(stagedDir, { recursive: true, force: true }); + } +} + +export async function singleFileStage(file: string, projectDir: string) { + const { extractStandaloneEntryFromIndex, isTemplateEntry } = + await import("@hyperframes/core/compiler"); + const index = join(projectDir, "index.html"); + const entryFile = relative(projectDir, file).split(sep).join("/"); + const scene = readFileSync(file, "utf8"); + const mounted = + file !== index && + existsSync(index) && + isTemplateEntry(scene) && + extractStandaloneEntryFromIndex(readFileSync(index, "utf8"), entryFile, scene) !== null; + return mounted + ? { root: projectDir, entries: [index, file], entryFile } + : { root: dirname(file), entries: [file], entryFile: basename(file) }; +} + +interface ShotScope { + projectDir: string | undefined; + entryFile: string | undefined; + singleFile?: string; +} + // Resolve the command target (a project dir or a single .html) into surfaced // compositions, applying the optional --selector filter. -export function resolveScope(args: { target?: string; selector?: string }): { +export function resolveScope(args: { target?: string; selector?: string }): ShotScope & { comps: SurfacedComposition[]; allComps: SurfacedComposition[]; projectName: string; - projectDir: string | undefined; - entryFile: string | undefined; } { const raw = args.target?.trim(); let comps: SurfacedComposition[]; let projectName: string; let projectDir: string | undefined; let entryFile: string | undefined; + let singleFile: string | undefined; const entry = raw?.endsWith(".html") ? readProjectFile(raw) : undefined; if (raw && entry?.kind === "file") { const entryPath = resolve(raw); @@ -939,6 +989,7 @@ export function resolveScope(args: { target?: string; selector?: string }): { projectName = basename(entryPath); projectDir = findProjectRoot(entryPath); entryFile = relative(projectDir, entryPath).split(sep).join("/"); + singleFile = entryPath; } else { const project = resolveProject(raw); comps = collectCompositions(project.indexPath); @@ -968,7 +1019,7 @@ export function resolveScope(args: { target?: string; selector?: string }): { cmp.anime.length > 0, ); } - return { comps, allComps, projectName, projectDir, entryFile }; + return { comps, allComps, projectName, projectDir, entryFile, singleFile }; } function findProjectRoot(entryPath: string): string { @@ -1116,12 +1167,13 @@ function createKeyframesCommand(options: Partial = {}) async run({ args }) { ensureDOMParser(); const runtime = normalizeRuntime(args.runtime, commandOptions.defaultRuntime); - const { comps: rawComps, allComps, projectName, projectDir, entryFile } = resolveScope(args); + const scope = resolveScope(args); + const { comps: rawComps, allComps, projectName } = scope; const comps = filterCompositionsByRuntime(rawComps, runtime); // --shot: 3D onion-skin self-verify screenshot. Returns true when the command // should stop (guard failure) so run() stays small. - if (args.shot && (await runOnionShot(comps, allComps, projectDir, entryFile, args))) return; + if (args.shot && (await runOnionShot(comps, allComps, scope, args))) return; if (args.json) { console.log( diff --git a/packages/cli/src/utils/fileReferences.ts b/packages/cli/src/utils/fileReferences.ts new file mode 100644 index 0000000000..145bf0fb05 --- /dev/null +++ b/packages/cli/src/utils/fileReferences.ts @@ -0,0 +1,156 @@ +import { closeSync, openSync, readSync, realpathSync, statSync } from "node:fs"; +import { dirname, isAbsolute, resolve } from "node:path"; +import { fileURLToPath } from "node:url"; +import { parseHTML } from "linkedom"; +import { isNonRelativeUrl } from "@hyperframes/core"; +import { resolveCompositionSrc } from "@hyperframes/parsers/asset-resolution"; +import { HF_COLOR_GRADING_ATTR } from "@hyperframes/core/color-grading"; + +const REFERENCE_ATTRS = ["src", "href", "poster", "xlink:href"]; +const MAX_SCANNED_BYTES = 2 * 1024 * 1024; + +export interface Reference { + raw: string; + composition: boolean; + follow: boolean; + colorGradingLut?: boolean; +} + +function unquote(value: string): string { + const first = value[0]; + return value.length >= 2 && (first === '"' || first === "'") && value.at(-1) === first + ? value.slice(1, -1).trim() + : value; +} + +export function cssReferences(css: string): Reference[] { + const refs: Reference[] = []; + for (let at = css.indexOf("url("); at !== -1; ) { + const close = css.indexOf(")", at + 4); + if (close === -1) break; + const raw = unquote(css.slice(at + 4, close).trim()); + if (raw) refs.push({ raw, composition: false, follow: true }); + at = css.indexOf("url(", close + 1); + } + for (const [, , raw] of css.matchAll(/@import\s+(["'])(.+?)\1/g)) { + if (raw?.trim()) refs.push({ raw: raw.trim(), composition: false, follow: true }); + } + return refs; +} + +function colorGradingLut(value: string | null): string | undefined { + try { + const parsed: unknown = JSON.parse(value ?? ""); + const lut: unknown = typeof parsed === "object" && parsed ? Reflect.get(parsed, "lut") : null; + const src: unknown = typeof lut === "object" && lut ? Reflect.get(lut, "src") : lut; + return typeof src === "string" && src.trim() ? src.trim() : undefined; + } catch { + return undefined; + } +} + +function elementReferences(el: Element): Reference[] { + const follow = el.localName === "link"; + const refs: Reference[] = REFERENCE_ATTRS.flatMap((attr) => { + const raw = el.getAttribute(attr)?.trim(); + return raw ? [{ raw, composition: false, follow }] : []; + }); + for (const candidate of el.getAttribute("srcset")?.split(",") ?? []) { + const raw = candidate.trim().split(/\s+/)[0]; + if (raw) refs.push({ raw, composition: false, follow: false }); + } + const composition = el.getAttribute("data-composition-src")?.trim(); + if (composition) refs.push({ raw: composition, composition: true, follow: true }); + const lut = colorGradingLut(el.getAttribute(HF_COLOR_GRADING_ATTR)); + if (lut) refs.push({ raw: lut, composition: false, follow: false, colorGradingLut: true }); + return refs; +} + +export function htmlReferences(html: string): Reference[] { + const { document } = parseHTML(html); + const hosts = document.querySelectorAll( + `[src], [href], [poster], [xlink\\:href], [srcset], [data-composition-src], [${HF_COLOR_GRADING_ATTR}]`, + ); + const refs = [...hosts].flatMap(elementReferences); + for (const style of document.querySelectorAll("style")) { + refs.push(...cssReferences(style.textContent ?? "")); + } + for (const el of document.querySelectorAll("[style]")) { + refs.push(...cssReferences(el.getAttribute("style") ?? "")); + } + for (const template of document.querySelectorAll("template")) { + refs.push(...htmlReferences(template.innerHTML)); + } + return refs; +} + +const WINDOWS_DEVICE_PREFIX = /^[\\/]{2}[?.][\\/]/; + +function withoutQuery(raw: string): string { + const device = WINDOWS_DEVICE_PREFIX.exec(raw)?.[0] ?? ""; + return device + raw.slice(device.length).replace(/[?#].*$/, ""); +} + +export function localPath(raw: string, fromDir: string): string | null { + if (raw.startsWith("file:")) { + try { + return fileURLToPath(raw); + } catch { + return null; + } + } + const path = withoutQuery(raw); + if (isAbsolute(path) && !path.startsWith("//")) return path; + if (path === "" || isNonRelativeUrl(path)) return null; + try { + return resolve(fromDir, decodeURIComponent(path)); + } catch { + return resolve(fromDir, path); + } +} + +export function isFile(path: string): boolean { + try { + return statSync(path, { throwIfNoEntry: false })?.isFile() === true; + } catch { + return false; + } +} + +export function realOrNull(path: string): string | null { + try { + return realpathSync(path); + } catch { + return null; + } +} + +export function readHead(file: string): { text: string; whole: boolean } | null { + let fd: number | undefined; + try { + fd = openSync(file, "r"); + const size = statSync(file).size; + const buffer = Buffer.alloc(Math.min(size, MAX_SCANNED_BYTES)); + const read = readSync(fd, buffer, 0, buffer.length, 0); + return { + text: buffer.toString("utf-8", 0, read), + whole: size <= MAX_SCANNED_BYTES, + }; + } catch { + return null; + } finally { + if (fd !== undefined) closeSync(fd); + } +} + +export const isPageOrStylesheet = (path: string) => /\.(?:html?|css)$/i.test(path); + +// A composition resolves as the render compiler resolves it; one that leaves the project resolves as written. +export function referencePath(ref: Reference, file: string, projectDir: string): string | null { + if (!ref.composition) return localPath(ref.raw, dirname(file)); + try { + return resolveCompositionSrc(projectDir, ref.raw) ?? localPath(ref.raw, projectDir); + } catch { + return null; + } +} diff --git a/packages/cli/src/utils/receivedRender.ts b/packages/cli/src/utils/receivedRender.ts index 1e9dd5b586..bfaa75e097 100644 --- a/packages/cli/src/utils/receivedRender.ts +++ b/packages/cli/src/utils/receivedRender.ts @@ -1,156 +1,28 @@ -import { - closeSync, - openSync, - readdirSync, - readSync, - realpathSync, - statSync, - type Dirent, -} from "node:fs"; -import { dirname, extname, isAbsolute, join, relative, resolve } from "node:path"; -import { fileURLToPath } from "node:url"; -import { parseHTML } from "linkedom"; -import { isNonRelativeUrl, isPathInside, terminalText } from "@hyperframes/core"; -import { - namesAnotherNetworkRoot, - resolveCompositionSrc, -} from "@hyperframes/parsers/asset-resolution"; +import { closeSync, openSync, readdirSync, readSync, statSync, type Dirent } from "node:fs"; +import { extname, join, relative, resolve } from "node:path"; +import { isPathInside, terminalText } from "@hyperframes/core"; +import { namesAnotherNetworkRoot } from "@hyperframes/parsers/asset-resolution"; import { linksOutside, receivedRecordStatus } from "./privateProjectFile.js"; import { walkBudget } from "./walkBudget.js"; +import { + htmlReferences, + cssReferences, + isFile, + isPageOrStylesheet, + readHead, + realOrNull, + referencePath, + type Reference, +} from "./fileReferences.js"; -const REFERENCE_ATTRS = ["src", "href", "poster"]; -const MAX_SCANNED_BYTES = 2 * 1024 * 1024; const MAX_SCANNED_FILES = 500; -interface Reference { - raw: string; - composition: boolean; - follow: boolean; -} - -function unquote(value: string): string { - const first = value[0]; - return value.length >= 2 && (first === '"' || first === "'") && value.at(-1) === first - ? value.slice(1, -1).trim() - : value; -} - -function cssReferences(css: string): Reference[] { - const refs: Reference[] = []; - for (let at = css.indexOf("url("); at !== -1; ) { - const close = css.indexOf(")", at + 4); - if (close === -1) break; - const raw = unquote(css.slice(at + 4, close).trim()); - if (raw) refs.push({ raw, composition: false, follow: true }); - at = css.indexOf("url(", close + 1); - } - return refs; -} - -function elementReferences(el: Element): Reference[] { - const follow = el.localName === "link"; - const refs: Reference[] = REFERENCE_ATTRS.flatMap((attr) => { - const raw = el.getAttribute(attr)?.trim(); - return raw ? [{ raw, composition: false, follow }] : []; - }); - const composition = el.getAttribute("data-composition-src")?.trim(); - if (composition) refs.push({ raw: composition, composition: true, follow: true }); - return refs; -} - -function htmlReferences(html: string): Reference[] { - const { document } = parseHTML(html); - const hosts = document.querySelectorAll("[src], [href], [poster], [data-composition-src]"); - const refs = [...hosts].flatMap(elementReferences); - for (const style of document.querySelectorAll("style")) { - refs.push(...cssReferences(style.textContent ?? "")); - } - for (const el of document.querySelectorAll("[style]")) { - refs.push(...cssReferences(el.getAttribute("style") ?? "")); - } - for (const template of document.querySelectorAll("template")) { - refs.push(...htmlReferences(template.innerHTML)); - } - return refs; -} - -const WINDOWS_DEVICE_PREFIX = /^[\\/]{2}[?.][\\/]/; - -function withoutQuery(raw: string): string { - const device = WINDOWS_DEVICE_PREFIX.exec(raw)?.[0] ?? ""; - return device + raw.slice(device.length).replace(/[?#].*$/, ""); -} - -function localPath(raw: string, fromDir: string): string | null { - if (raw.startsWith("file:")) { - try { - return fileURLToPath(raw); - } catch { - return null; - } - } - const path = withoutQuery(raw); - if (isAbsolute(path) && !path.startsWith("//")) return path; - if (path === "" || isNonRelativeUrl(path)) return null; - try { - return resolve(fromDir, decodeURIComponent(path)); - } catch { - return resolve(fromDir, path); - } -} - -function isFile(path: string): boolean { - try { - return statSync(path, { throwIfNoEntry: false })?.isFile() === true; - } catch { - return false; - } -} - -function realOrNull(path: string): string | null { - try { - return realpathSync(path); - } catch { - return null; - } -} - -function readHead(file: string): { text: string; whole: boolean } | null { - let fd: number | undefined; - try { - fd = openSync(file, "r"); - const size = statSync(file).size; - const buffer = Buffer.alloc(Math.min(size, MAX_SCANNED_BYTES)); - const read = readSync(fd, buffer, 0, buffer.length, 0); - return { - text: buffer.toString("utf-8", 0, read), - whole: size <= MAX_SCANNED_BYTES, - }; - } catch { - return null; - } finally { - if (fd !== undefined) closeSync(fd); - } -} - -const isPageOrStylesheet = (path: string) => /\.(?:html?|css)$/i.test(path); - interface OutsideReferences { assets: Set; compositions: Set; complete: boolean; } -// A composition resolves as the render compiler resolves it; one that leaves the project resolves as written. -function referencePath(ref: Reference, file: string, projectDir: string): string | null { - if (!ref.composition) return localPath(ref.raw, dirname(file)); - try { - return resolveCompositionSrc(projectDir, ref.raw) ?? localPath(ref.raw, projectDir); - } catch { - return null; - } -} - function referencesIn(found: OutsideReferences, file: string): Reference[] { const read = readHead(file); if (read === null || !read.whole) found.complete = false; diff --git a/packages/cli/src/utils/stageReferencedFiles.test.ts b/packages/cli/src/utils/stageReferencedFiles.test.ts new file mode 100644 index 0000000000..51bd885652 --- /dev/null +++ b/packages/cli/src/utils/stageReferencedFiles.test.ts @@ -0,0 +1,170 @@ +import { mkdirSync, mkdtempSync, readdirSync, rmSync, symlinkSync, writeFileSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join, relative } from "node:path"; +import { afterEach, beforeEach, describe, expect, it } from "vitest"; +import { leftOutNotice, stageReferencedFiles } from "./stageReferencedFiles.js"; + +let root: string; +let folder: string; +let staged: string; + +function write(path: string, text = "x"): void { + mkdirSync(join(path, ".."), { recursive: true }); + writeFileSync(path, text); +} + +function stagedFiles(): string[] { + const files: string[] = []; + const walk = (dir: string) => { + for (const entry of readdirSync(dir, { withFileTypes: true })) { + const path = join(dir, entry.name); + if (entry.isDirectory()) walk(path); + else files.push(relative(staged, path).split("\\").join("/")); + } + }; + walk(staged); + return files.sort(); +} + +beforeEach(() => { + root = mkdtempSync(join(tmpdir(), "hf-stage-referenced-")); + folder = join(root, "sent"); + staged = join(root, "staged"); + mkdirSync(folder); +}); + +afterEach(() => { + rmSync(root, { recursive: true, force: true }); +}); + +describe("stageReferencedFiles", () => { + it("copies the page and every file its markup, stylesheets and compositions name", () => { + write( + join(folder, "page.html"), + ` + + +
+
+ `, + ); + write(join(folder, "css", "look.css"), `@font-face { src: url("../fonts/brand.woff2"); }`); + write( + join(folder, "compositions", "scene.html"), + ``, + ); + for (const file of ["img/a.png", "img/a@2x.png", "img/poster.jpg", "img/bg.webp"]) + write(join(folder, file)); + for (const file of ["img/scene.png", "media/clip.mp4", "fonts/brand.woff2", "js/run.js"]) + write(join(folder, file)); + write(join(folder, "img", "unnamed.png")); + + stageReferencedFiles(folder, [join(folder, "page.html")], staged); + + expect(stagedFiles()).toEqual([ + "compositions/scene.html", + "css/look.css", + "fonts/brand.woff2", + "img/a.png", + "img/a@2x.png", + "img/bg.webp", + "img/poster.jpg", + "img/scene.png", + "js/run.js", + "media/clip.mp4", + "page.html", + ]); + }); + + it("copies nothing from outside the page's folder, through a path or a link", () => { + write(join(root, "Pictures", "IMG_0001.png")); + symlinkSync(join(root, "Pictures"), join(folder, "pictures")); + write( + join(folder, "page.html"), + ` + `, + ); + + stageReferencedFiles(folder, [join(folder, "page.html")], staged); + + expect(stagedFiles()).toEqual(["page.html"]); + }); + + it("copies no private file, whatever type its name gives", () => { + write( + join(folder, "page.html"), + ` + `, + ); + for (const file of [".git/logo.png", ".env.png", ".hyperframes/thumb.png", "secrets/.npmrc"]) + write(join(folder, file)); + + stageReferencedFiles(folder, [join(folder, "page.html")], staged); + + expect(stagedFiles()).toEqual(["page.html"]); + }); + + it("copies root-relative names, string imports, SVG links and the grading table", () => { + write( + join(folder, "page.html"), + ` + + + + `, + ); + write( + join(folder, "css", "theme.css"), + `@import 'more.css'; body { background: url(/img/bg.png) }`, + ); + write(join(folder, "css", "more.css"), "p {}"); + for (const file of ["img/logo.png", "img/art.png", "icons.svg", "img/bg.png", "clip.mp4"]) + write(join(folder, file)); + write(join(folder, "looks", "warm.cube")); + write(join(folder, "looks", "cold.cube")); + + const result = stageReferencedFiles(folder, [join(folder, "page.html")], staged); + + expect(stagedFiles()).toEqual([ + "clip.mp4", + "css/more.css", + "css/theme.css", + "icons.svg", + "img/art.png", + "img/bg.png", + "img/logo.png", + "looks/warm.cube", + "page.html", + ]); + expect(result).toEqual({ leftOut: ["looks/cold.cube"], complete: true }); + }); + + it("names what it left out, and nothing that does not exist", () => { + write(join(root, "Pictures", "IMG_0001.png")); + write(join(folder, "notes.txt")); + write( + join(folder, "page.html"), + ``, + ); + + const result = stageReferencedFiles(folder, [join(folder, "page.html")], staged); + + expect(result).toEqual({ leftOut: ["../Pictures/IMG_0001.png", "notes.txt"], complete: true }); + }); +}); + +describe("leftOutNotice", () => { + it("says nothing when nothing was left out", () => { + expect(leftOutNotice("compare a.html", { leftOut: [], complete: true })).toBeNull(); + }); + + it("names the first five escaped, counts the rest and says how to keep them", () => { + const leftOut = ["a\x1b[2J.png", "b.txt", "c.txt", "d.txt", "e.txt", "f.txt", "g.txt"]; + const notice = leftOutNotice("compare a.html", { leftOut, complete: false }); + + expect(notice).not.toContain("\x1b"); + expect(notice).toContain("a\\x1b[2J.png, b.txt, c.txt, d.txt, e.txt, and 2 more."); + expect(notice).toContain("too large or too many"); + expect(notice).toContain("pass the project folder"); + }); +}); diff --git a/packages/cli/src/utils/stageReferencedFiles.ts b/packages/cli/src/utils/stageReferencedFiles.ts new file mode 100644 index 0000000000..7b7476b49c --- /dev/null +++ b/packages/cli/src/utils/stageReferencedFiles.ts @@ -0,0 +1,118 @@ +import { copyFileSync, mkdirSync } from "node:fs"; +import { dirname, join, relative } from "node:path"; +import { isPathInside, terminalText } from "@hyperframes/core"; +import { isPrivateFileUnder, isStudioPrivatePath } from "@hyperframes/core/safe-path"; +import { isCopiedOutsideFileType } from "@hyperframes/studio-server/outside-file"; +import { + cssReferences, + htmlReferences, + isFile, + isPageOrStylesheet, + localPath, + readHead, + realOrNull, + referencePath, + type Reference, +} from "./fileReferences.js"; +import { isPrivateCompositionFile, linksOutside } from "./privateProjectFile.js"; + +const MAX_FOLLOWED_FILES = 500; +const NAMES_SHOWN = 5; + +export interface StagedFiles { + leftOut: string[]; + complete: boolean; +} + +interface NamedFile { + raw: string; + path: string; + colorGradingLut: boolean; +} + +/** A file the page may load: a composition, a type a local render copies in, or the grading table its LUT names. */ +function stageable(root: string, file: NamedFile): boolean { + const { path } = file; + if (linksOutside(root, path)) return false; + if (isPrivateFileUnder(root, path, isStudioPrivatePath) || isPrivateCompositionFile(root, path)) + return false; + if (file.colorGradingLut && /\.cube$/i.test(path)) return true; + return /\.html?$/i.test(path) || isCopiedOutsideFileType(path); +} + +// A leading slash names the served root, as the local server reads it; compositions resolve as the compiler resolves them. +function namedPath(ref: Reference, file: string, root: string): string | null { + if (!ref.composition && /^\/(?![/\\])/.test(ref.raw)) return localPath(ref.raw.slice(1), root); + return referencePath(ref, file, root); +} + +function namedFiles(file: string, root: string, result: StagedFiles): NamedFile[] { + const read = readHead(file); + if (read === null || !read.whole) result.complete = false; + if (read === null) return []; + const refs = /\.css$/i.test(file) ? cssReferences(read.text) : htmlReferences(read.text); + return refs.flatMap((ref) => { + const path = namedPath(ref, file, root); + return path === null ? [] : [{ raw: ref.raw, path, colorGradingLut: !!ref.colorGradingLut }]; + }); +} + +function realEntries(root: string, entries: readonly string[]): string[] { + return entries.map((entry) => { + const real = realOrNull(entry); + if (real === null || !isPathInside(real, root)) throw new Error(`Cannot read ${entry}`); + return real; + }); +} + +/** + * Copies `entries` into `stagedDir` with the files their markup, sub-compositions and stylesheets name under `root`. + * A page someone sent runs on the origin that serves `stagedDir`, so nothing else under `root` is put there: not + * private files, not other file types, not names the page builds in script. + */ +export function stageReferencedFiles( + root: string, + entries: readonly string[], + stagedDir: string, +): StagedFiles { + const realRoot = realOrNull(root); + if (realRoot === null) throw new Error(`Cannot read ${root}`); + const result: StagedFiles = { leftOut: [], complete: true }; + const staged = new Set(); + const pending: string[] = []; + const stage = (path: string) => { + const real = realOrNull(path); + if (real === null || staged.has(real)) return; + staged.add(real); + const target = join(stagedDir, relative(realRoot, path)); + mkdirSync(dirname(target), { recursive: true }); + copyFileSync(real, target); + if (!isPageOrStylesheet(path)) return; + if (staged.size <= MAX_FOLLOWED_FILES) pending.push(path); + else result.complete = false; + }; + const leaveOut = (raw: string) => { + if (!result.leftOut.includes(raw)) result.leftOut.push(raw); + }; + realEntries(realRoot, entries).forEach(stage); + for (let file = pending.pop(); file !== undefined; file = pending.pop()) { + for (const named of namedFiles(file, realRoot, result)) { + if (!isPathInside(named.path, realRoot)) leaveOut(named.raw); + else if (!isFile(named.path)) continue; + else if (stageable(realRoot, named)) stage(named.path); + else leaveOut(named.raw); + } + } + return result; +} + +/** One line naming what a bare HTML file lost, or null when nothing was left out. */ +export function leftOutNotice(command: string, result: StagedFiles): string | null { + if (result.leftOut.length === 0 && result.complete) return null; + const shown = result.leftOut.slice(0, NAMES_SHOWN).map((raw) => terminalText(raw)); + const more = + result.leftOut.length > NAMES_SHOWN ? `, and ${result.leftOut.length - NAMES_SHOWN} more` : ""; + const named = shown.length > 0 ? ` left out ${shown.join(", ")}${more}.` : ""; + const partial = result.complete ? "" : " Some files were too large or too many to read in full."; + return `${command}:${named}${partial} Only the files the page names in its own folder are served; pass the project folder to keep the rest.`; +}