diff --git a/docs/packages/cli.mdx b/docs/packages/cli.mdx index 87a8d0829b..75c861e7da 100644 --- a/docs/packages/cli.mdx +++ b/docs/packages/cli.mdx @@ -2901,6 +2901,18 @@ imports projects for you can tell the CLI more by pointing - The state applies to the project, its subfolders, and any folder that holds it. - If the variable is set but the file is missing or cannot be parsed, every file counts as received. +- The app can keep its own copy of an `"expected"` project's record in + `received-records/.json` beside the state file, as + `{ "project": "", "files": [...] }` with `path` exactly as the state file + names it. A file that copy lists still counts as received while something is at + its path, even if the project's record no longer lists it. For a project found + only by `dev` and `ino`, the copy counts only when its `"identity"` is + `"::"` of the project folder. A copy with + `"files": null` or `"doubted": true`, or one that is a link or cannot be read, + counts every file as received. +- A file the copy lists that is no longer at its path is not added back. A + received file that is renamed after its record entry is removed is judged by + the project's record alone until the app reads the record again. A file from outside the project folder that would be copied (each file a composition points at, and each project file that is a link to a file outside diff --git a/packages/cli/src/utils/privateProjectFile.test.ts b/packages/cli/src/utils/privateProjectFile.test.ts index 6c588ba945..8a35e45393 100644 --- a/packages/cli/src/utils/privateProjectFile.test.ts +++ b/packages/cli/src/utils/privateProjectFile.test.ts @@ -6,6 +6,7 @@ import { renameSync, mkdtempSync, readFileSync, + realpathSync, rmSync, symlinkSync, utimesSync, @@ -14,13 +15,14 @@ import { import { spawn } from "node:child_process"; import { createHash } from "node:crypto"; import { tmpdir } from "node:os"; -import { dirname, join } from "node:path"; +import { basename, dirname, join } from "node:path"; import { isCredentialOrToolStatePath } from "@hyperframes/core/safe-path"; import { RECEIVED_STATE_ENV, isEnvOrGitPath, isPrivateCompositionFile, receivedFiles, + receivedLinesAtOrAbove, receivedRecordStatus, recordReceivedFiles, } from "./privateProjectFile.js"; @@ -337,6 +339,133 @@ describe("a host's received state file", () => { hostSays([{ path: join(tmpdir(), "hf-some-other-project"), received: "all" }]); expect(receivedFiles(app)("index.html")).toBe(false); }); + + describe("and the host's kept copy of a record", () => { + const keptFile = (path: string) => + join( + dir as string, + "received-records", + `${createHash("sha256").update(path).digest("hex")}.json`, + ); + + function hostKeeps(path: string, kept: unknown): void { + mkdirSync(join(dir as string, "received-records"), { recursive: true }); + writeFileSync(keptFile(path), typeof kept === "string" ? kept : JSON.stringify(kept)); + } + + const keeps = (path: string, files: string[] | null) => + hostKeeps(path, { project: path, identity: null, files, seenAt: 1 }); + + function emptiedProject(): string { + const app = project(JSON.stringify({ files: [] })); + mkdirSync(join(app, "assets")); + writeFileSync(join(app, "assets", "x.js"), ""); + return app; + } + + it("counts entries an emptied record lost as received while their files are there", () => { + const app = emptiedProject(); + const named = realpathSync(app); + hostSays([{ path: named, received: "expected" }]); + keeps(named, ["index.html", "assets/x.js"]); + const received = receivedFiles(app); + expect(received("index.html")).toBe(true); + expect(received("assets/x.js")).toBe(true); + expect(received("compositions/intro.html")).toBe(false); + const status = receivedRecordStatus(app); + expect(status !== "unchecked" && status?.listed?.map((path) => basename(path))).toEqual([ + "index.html", + "x.js", + ]); + }); + + it("leaves off a lost entry whose file is gone", () => { + const app = emptiedProject(); + hostSays([{ path: app, received: "expected" }]); + keeps(app, ["index.html", "assets/x.js"]); + rmSync(join(app, "assets", "x.js")); + const received = receivedFiles(app); + expect(received("index.html")).toBe(true); + expect(received("assets/x.js")).toBe(false); + expect(received.gone).toBeUndefined(); + }); + + it("finds the kept copy of a project named under its old path by its folder's identity", () => { + const app = emptiedProject(); + const { dev, ino, birthtimeNs } = lstatSync(app, { bigint: true }); + hostSays([{ path: app, dev: String(dev), ino: String(ino), received: "expected" }]); + const kept = { project: app, files: ["index.html"], seenAt: 1 }; + hostKeeps(app, { ...kept, identity: `${dev}:${ino}:${birthtimeNs}` }); + const renamed = `${app}-renamed`; + renameSync(app, renamed); + expect(receivedFiles(renamed)("index.html")).toBe(true); + expect(receivedFiles(renamed)("assets/x.js")).toBe(false); + hostKeeps(app, { ...kept, identity: `${dev}:${ino}:1` }); + expect(receivedFiles(renamed)("index.html")).toBe(false); + }); + + it("counts every file as received when the kept copy holds no list, and keeps the record's lines", () => { + const app = emptiedProject(); + writeFileSync( + join(app, ".hyperframes", "received-files.json"), + JSON.stringify({ files: ["index.html"] }), + ); + hostSays([{ path: app, received: "expected" }]); + keeps(app, null); + expect(receivedFiles(app)("compositions/intro.html")).toBe(true); + expect(receivedRecordStatus(app)).toMatchObject({ path: keptFile(app), listed: null }); + expect(receivedLinesAtOrAbove(app).map(({ lines }) => lines)).toEqual([["index.html"]]); + }); + + it("counts every file as received when the kept copy is doubted, unreadable or for another project", () => { + const app = emptiedProject(); + hostSays([{ path: app, received: "expected" }]); + for (const kept of [ + { project: app, identity: null, files: [], seenAt: 1, doubted: true }, + { project: `${app}-other`, identity: null, files: [], seenAt: 1 }, + { project: app, identity: null, files: [1], seenAt: 1 }, + "{not json", + ]) { + hostKeeps(app, kept); + expect(receivedFiles(app)("compositions/intro.html")).toBe(true); + } + }); + + it("trusts the record when no state file is set or the state file does not name the project", () => { + const app = emptiedProject(); + hostSays([{ path: join(tmpdir(), "hf-some-other-project"), received: "expected" }]); + keeps(app, ["index.html", "assets/x.js"]); + expect(receivedFiles(app)("index.html")).toBe(false); + vi.stubEnv(RECEIVED_STATE_ENV, ""); + expect(receivedFiles(app)("index.html")).toBe(false); + }); + + it.skipIf(process.platform === "win32")( + "does not follow a link in place of the kept copy", + () => { + const app = emptiedProject(); + hostSays([{ path: app, received: "expected" }]); + const elsewhere = join(dir as string, "elsewhere.json"); + writeFileSync( + elsewhere, + JSON.stringify({ project: app, identity: null, files: ["index.html"], seenAt: 1 }), + ); + mkdirSync(join(dir as string, "received-records")); + symlinkSync(elsewhere, keptFile(app)); + expect(receivedFiles(app)("compositions/intro.html")).toBe(true); + }, + ); + + it("reads the person's own project as before when the host keeps no copy of its record", () => { + const app = emptiedProject(); + hostSays([{ path: app, received: "expected" }]); + keeps(join(tmpdir(), "hf-some-other-project"), ["index.html"]); + const received = receivedFiles(app); + expect(received("index.html")).toBe(false); + expect(received("assets/x.js")).toBe(false); + expect(receivedRecordStatus(app)).toMatchObject({ listed: [] }); + }); + }); }); describe("recordReceivedFiles", () => { diff --git a/packages/cli/src/utils/privateProjectFile.ts b/packages/cli/src/utils/privateProjectFile.ts index eb3678bad1..4853cda8c3 100644 --- a/packages/cli/src/utils/privateProjectFile.ts +++ b/packages/cli/src/utils/privateProjectFile.ts @@ -9,6 +9,7 @@ import { rmSync, statSync, } from "node:fs"; +import { createHash } from "node:crypto"; import { readdir } from "node:fs/promises"; import { basename, dirname, isAbsolute, join, relative, resolve, sep } from "node:path"; import { @@ -24,7 +25,7 @@ import { realpath, } from "@hyperframes/core/safe-path"; import { registryRoot } from "../registry/publication.js"; -import { recordFolder, writeRecord } from "./projectRecords.js"; +import { readPlainTail, recordFolder, writeRecord } from "./projectRecords.js"; import { walkBudget } from "./walkBudget.js"; import { contentHash, @@ -52,6 +53,12 @@ interface ReceivedRecord { root: string; projectDirFromRoot: string; listed: unknown; + unreadKeptCopy?: string; +} + +interface ReadRecord { + listed: unknown; + unreadKeptCopy?: string; } type RecordKind = "absent" | "file" | "other" | "unknown"; @@ -76,15 +83,27 @@ export const RECEIVED_STATE_ENV = "HYPERFRAMES_RECEIVED_STATE_FILE"; type HostState = "expected" | "all"; +interface Named { + received: HostState; + paths: string[]; +} + interface HostStates { + file: string; unreadable: boolean; - byPath: Map; - byIdentity: Map; + byPath: Map; + byIdentity: Map; } let hostStatesCache: { key: string; states: HostStates } | undefined; -const strongest = (a: HostState | undefined, b: HostState): HostState => (a === "all" ? a : b); +function addNamed(map: Map, key: string, received: HostState, path: string): void { + const known = map.get(key); + map.set(key, { + received: known?.received === "all" ? "all" : received, + paths: [...(known?.paths ?? []), path], + }); +} interface HostEntry { path: string; @@ -104,18 +123,15 @@ function hostEntry(value: unknown): HostEntry { }; } -function parseHostStates(text: string): HostStates { +function parseHostStates(file: string, text: string): HostStates { const projects: unknown = JSON.parse(text)?.projects; if (!Array.isArray(projects)) throw new Error("projects is not a list"); - const states: HostStates = { unreadable: false, byPath: new Map(), byIdentity: new Map() }; + const states: HostStates = { file, unreadable: false, byPath: new Map(), byIdentity: new Map() }; for (const value of projects) { const { path, received, dev, ino } = hostEntry(value); - const key = walkStart(path); - states.byPath.set(key, strongest(states.byPath.get(key), received)); - if (dev !== undefined && ino !== undefined) { - const identity = `${dev}:${ino}`; - states.byIdentity.set(identity, strongest(states.byIdentity.get(identity), received)); - } + addNamed(states.byPath, walkStart(path), received, path); + if (dev !== undefined && ino !== undefined) + addNamed(states.byIdentity, `${dev}:${ino}`, received, path); } return states; } @@ -129,23 +145,100 @@ function hostStates(): HostStates | null { const stat = fstatSync(handle, { bigint: true }); const key = `${file}\0${stat.dev}:${stat.ino}:${stat.size}:${stat.mtimeNs}:${stat.ctimeNs}`; if (hostStatesCache?.key !== key) - hostStatesCache = { key, states: parseHostStates(readFileSync(handle, "utf-8")) }; + hostStatesCache = { key, states: parseHostStates(file, readFileSync(handle, "utf-8")) }; return hostStatesCache.states; } finally { closeSync(handle); } } catch { - return { unreadable: true, byPath: new Map(), byIdentity: new Map() }; + return { file, unreadable: true, byPath: new Map(), byIdentity: new Map() }; } } function hostState(dir: string, states = hostStates()): HostState | undefined { if (!states) return undefined; if (states.unreadable) return "all"; - const named = states.byPath.get(dir); + const named = states.byPath.get(dir)?.received; if (named || states.byIdentity.size === 0) return named; const identity = fileIdentity(dir); - return identity === null ? undefined : states.byIdentity.get(identity); + return identity === null ? undefined : states.byIdentity.get(identity)?.received; +} + +interface Naming { + path: string; + identity?: string; +} + +function folderBirth(dir: string): string { + try { + return String(lstatSync(dir, { bigint: true }).birthtimeNs); + } catch { + return ""; + } +} + +function namings(dir: string, states: HostStates): Naming[] { + const byPath = (states.byPath.get(dir)?.paths ?? []).map((path) => ({ path })); + const identity = states.byIdentity.size === 0 ? null : fileIdentity(dir); + const named = identity === null ? undefined : states.byIdentity.get(identity); + if (!named) return byPath; + const kept = `${identity}:${folderBirth(dir)}`; + return [...byPath, ...named.paths.map((path) => ({ path, identity: kept }))]; +} + +const KEPT_RECORDS = "received-records"; +const MAX_KEPT_BYTES = 16 * 1024 * 1024; + +type KeptFiles = string[] | "all" | null; + +function keptFilesIn(kept: Record, { path, identity }: Naming): KeptFiles { + if (kept.project !== path) return "all"; + if (identity !== undefined && kept.identity !== identity) return null; + const { files } = kept; + if (kept.doubted === true || !Array.isArray(files)) return "all"; + return files.every((entry) => typeof entry === "string") ? files : "all"; +} + +function keptFilesFor(naming: Naming, text: string): KeptFiles { + try { + const kept: unknown = JSON.parse(text); + return keptFilesIn((kept ?? {}) as Record, naming); + } catch { + return "all"; + } +} + +function hostKeptRecord( + stateFile: string, + naming: Naming, +): { file: string; files: string[] | "all" } | null { + const name = `${createHash("sha256").update(naming.path).digest("hex")}.json`; + const file = join(dirname(stateFile), KEPT_RECORDS, name); + try { + if (!lstatSync(file, { throwIfNoEntry: false })) return null; + } catch (error) { + return (error as NodeJS.ErrnoException).code === "ENOTDIR" ? null : { file, files: "all" }; + } + const read = readPlainTail(file, MAX_KEPT_BYTES); + if (!read?.whole) return onDisk(file) ? { file, files: "all" } : null; + const files = keptFilesFor(naming, read.text); + return files === null ? null : { file, files }; +} + +function withEntriesHostKept(dir: string, listed: unknown[]): ReadRecord { + const states = hostStates(); + if (!states || hostState(dir, states) !== "expected") return { listed }; + const copies = namings(dir, states).flatMap( + (naming) => hostKeptRecord(states.file, naming) ?? [], + ); + const unread = copies.find((copy) => copy.files === "all"); + if (unread) return { listed, unreadKeptCopy: unread.file }; + const present = new Set(listed.map(String)); + const kept = copies.flatMap((copy) => (copy.files === "all" ? [] : copy.files)); + const back = [...new Set(kept)].filter( + (entry) => !present.has(entry) && onDisk(join(dir, entry)), + ); + return { listed: back.length === 0 ? listed : [...listed, ...back] }; } function allReceivedAtOrAbove(start: string, states: HostStates): boolean { @@ -160,12 +253,13 @@ function allReceivedAround(start: string): boolean { if (!states) return false; if (states.unreadable) return true; const overlaps = [...states.byPath].some( - ([path, state]) => state === "all" && (isAtOrUnder(path, start) || isAtOrUnder(start, path)), + ([path, { received }]) => + received === "all" && (isAtOrUnder(path, start) || isAtOrUnder(start, path)), ); return overlaps || allReceivedAtOrAbove(start, states); } -function readRecord(dir: string, ownFolder: boolean): { listed: unknown } | null { +function readRecordAsWritten(dir: string, ownFolder: boolean): { listed: unknown } | null { const kind = recordKind(dir); if (kind === "absent" && hostState(dir) !== undefined) return { listed: undefined }; if (kind === "absent" || (kind === "unknown" && !ownFolder)) return null; @@ -177,6 +271,11 @@ function readRecord(dir: string, ownFolder: boolean): { listed: unknown } | null } } +function readRecord(dir: string, ownFolder: boolean): ReadRecord | null { + const record = readRecordAsWritten(dir, ownFolder); + return record && Array.isArray(record.listed) ? withEntriesHostKept(dir, record.listed) : record; +} + function sleepSync(ms: number): void { Atomics.wait(new Int32Array(new SharedArrayBuffer(4)), 0, 0, ms); } @@ -219,7 +318,7 @@ function recordLock(projectDir: string): { root: string; lock: string } { } function applyRecordChange(root: string, change: RecordChange): void { - const record = readRecord(root, true); + const record = readRecordAsWritten(root, true); if (record && !Array.isArray(record.listed)) return; const files = new Set(((record?.listed ?? []) as unknown[]).map(String)); const content = readReceivedContent(root); @@ -316,12 +415,13 @@ function receivedRecordsAtOrAbove(start: string): ReceivedRecord[] { } } -function nestedRecord(start: string, dir: string, record: { listed: unknown }): ReceivedRecord { +function nestedRecord(start: string, dir: string, record: ReadRecord): ReceivedRecord { const folder = relative(start, dir).split(sep).join("/"); const listed = Array.isArray(record.listed) ? record.listed.map((entry) => (folder ? `${folder}/${String(entry)}` : String(entry))) : undefined; - return { file: join(dir, RECEIVED_RECORD), root: start, projectDirFromRoot: "", listed }; + const nested = { file: join(dir, RECEIVED_RECORD), root: start, projectDirFromRoot: "", listed }; + return record.unreadKeptCopy ? { ...nested, unreadKeptCopy: record.unreadKeptCopy } : nested; } const GIT_AND_PACKAGE_FOLDERS = new Set([".git", "node_modules"]); @@ -389,13 +489,14 @@ function receivedRecords(projectDir: string): ReceivedRecord[] | null { const below = receivedRecordsBelow(start); if (!below) return null; const records = [...receivedRecordsAtOrAbove(start), ...below]; - if (allReceivedAround(start)) - records.push({ - file: process.env[RECEIVED_STATE_ENV] ?? "", - root: start, - projectDirFromRoot: "", - listed: undefined, - }); + const everyFile = (file: string) => ({ + file, + root: start, + projectDirFromRoot: "", + listed: undefined, + }); + records.push(...records.flatMap(({ unreadKeptCopy }) => unreadKeptCopy ?? []).map(everyFile)); + if (allReceivedAround(start)) records.push(everyFile(process.env[RECEIVED_STATE_ENV] ?? "")); return records; } diff --git a/packages/cli/src/utils/projectRecords.ts b/packages/cli/src/utils/projectRecords.ts index 6a7b73dff7..403952ee74 100644 --- a/packages/cli/src/utils/projectRecords.ts +++ b/packages/cli/src/utils/projectRecords.ts @@ -49,7 +49,10 @@ function plainFileAt(fd: number, path: string) { /** The last `maxBytes` of a plain file, and whether that is all of it; null for a link, a device, or a missing file. * The file is opened first and read only when `lstat` then finds the same plain file at the path, so nothing * path-based follows the check. */ -function readPlainTail(path: string, maxBytes: number): { text: string; whole: boolean } | null { +export function readPlainTail( + path: string, + maxBytes: number, +): { text: string; whole: boolean } | null { let fd: number; try { fd = openSync(path, constants.O_RDONLY | NO_FOLLOW | NON_BLOCK); diff --git a/packages/cli/src/utils/receivedRender.test.ts b/packages/cli/src/utils/receivedRender.test.ts index b7def2a9df..643bc1e8e9 100644 --- a/packages/cli/src/utils/receivedRender.test.ts +++ b/packages/cli/src/utils/receivedRender.test.ts @@ -239,6 +239,23 @@ describe("outsideAssetsAllowed", () => { expect(decide(project(received())).allowed).toBe(true); }); + it("refuses outside files for a received entry that an emptied record lost, as the host's kept copy shows", () => { + const dir = project(received()); + const state = join(root as string, "state.json"); + writeFileSync(state, JSON.stringify({ projects: [{ path: dir, received: "expected" }] })); + const kept = { project: dir, identity: null, files: ["index.html", "style.css"], seenAt: 1 }; + const name = `${createHash("sha256").update(dir).digest("hex")}.json`; + write(join(root as string, "received-records", name), JSON.stringify(kept)); + vi.stubEnv("HYPERFRAMES_RECEIVED_STATE_FILE", state); + try { + const { allowed, line } = decide(dir); + expect(allowed).toBe(false); + expect(line).toContain(`index.html ${SOMEONE_ELSE}`); + } finally { + vi.unstubAllEnvs(); + } + }); + it("keeps outside files for an empty record in a project with very many files", () => { const dir = project(received()); const many = join(dir, "node_modules");