diff --git a/docs/packages/cli.mdx b/docs/packages/cli.mdx index 75c861e7da..b95f1522de 100644 --- a/docs/packages/cli.mdx +++ b/docs/packages/cli.mdx @@ -2906,13 +2906,15 @@ imports projects for you can tell the CLI more by pointing `{ "project": "", "files": [...] }` with `path` exactly as the state file names it. A file that copy lists still counts as received while something is at its path, even if the project's record no longer lists it. For a project found - only by `dev` and `ino`, the copy counts only when its `"identity"` is - `"::"` of the project folder. A copy with - `"files": null` or `"doubted": true`, or one that is a link or cannot be read, - counts every file as received. -- A file the copy lists that is no longer at its path is not added back. A - received file that is renamed after its record entry is removed is judged by - the project's record alone until the app reads the record again. + only by `dev` and `ino`, the copy's `"identity"` must be + `"::"` of the project folder; a copy with another + identity counts every file as received. A copy that names another project, + has `"files": null` or `"doubted": true`, or is a link, larger than 16 MB or + unreadable also counts every file as received. +- A file the copy lists that is no longer at its path is not added back. When + the app restores lost entries to a project's record, it uses the same rule, so + a received file that is renamed after its record entry is removed is judged by + the project's record alone. A file from outside the project folder that would be copied (each file a composition points at, and each project file that is a link to a file outside diff --git a/packages/cli/src/media-use/lib/received-env.mjs b/packages/cli/src/media-use/lib/received-env.mjs index f761c33a1c..bfee878e02 100644 --- a/packages/cli/src/media-use/lib/received-env.mjs +++ b/packages/cli/src/media-use/lib/received-env.mjs @@ -20,6 +20,7 @@ const RECEIVED_RECORD = join(".hyperframes", "received-files.json"); const RECEIVED_CONTENT = "received-content.json"; const RECEIVED_STATE_ENV = "HYPERFRAMES_RECEIVED_STATE_FILE"; const MAX_RECORD_BYTES = 16 * 1024 * 1024; +const KEPT_RECORDS = "received-records"; const SHA256 = /^[0-9a-f]{64}$/; const WALK_MAX_ENTRIES = 20_000; const WALK_MAX_MS = 2_000; @@ -90,6 +91,14 @@ function recordKind(dir) { const strongest = (a, b) => (a === "all" ? a : b); +function addNamed(map, key, received, path) { + const known = map.get(key); + map.set(key, { + received: strongest(known?.received, received), + paths: [...(known?.paths ?? []), path], + }); +} + function hostEntry(value) { const { path, received, dev, ino } = value ?? {}; if (typeof path !== "string" || !isAbsolute(path)) throw new Error("path is not absolute"); @@ -98,18 +107,15 @@ function hostEntry(value) { return { path, received, ...(identity ? { dev, ino } : {}) }; } -function parseHostStates(text) { +function parseHostStates(file, text) { const projects = JSON.parse(text)?.projects; if (!Array.isArray(projects)) throw new Error("projects is not a list"); - const states = { unreadable: false, byPath: new Map(), byIdentity: new Map() }; + const states = { file, unreadable: false, byPath: new Map(), byIdentity: new Map() }; for (const value of projects) { const { path, received, dev, ino } = hostEntry(value); - const key = walkStart(path); - states.byPath.set(key, strongest(states.byPath.get(key), received)); - if (dev !== undefined && ino !== undefined) { - const identity = `${dev}:${ino}`; - states.byIdentity.set(identity, strongest(states.byIdentity.get(identity), received)); - } + addNamed(states.byPath, walkStart(path), received, path); + if (dev !== undefined && ino !== undefined) + addNamed(states.byIdentity, `${dev}:${ino}`, received, path); } return states; } @@ -118,19 +124,84 @@ function hostStates() { const file = process.env[RECEIVED_STATE_ENV]; if (!file) return null; try { - return parseHostStates(readFileSync(file, "utf-8")); + return parseHostStates(file, readFileSync(file, "utf-8")); } catch { - return { unreadable: true, byPath: new Map(), byIdentity: new Map() }; + return { file, unreadable: true, byPath: new Map(), byIdentity: new Map() }; } } function hostState(dir, states) { if (!states) return undefined; if (states.unreadable) return "all"; - const named = states.byPath.get(dir); + const named = states.byPath.get(dir)?.received; if (named || states.byIdentity.size === 0) return named; const identity = fileIdentity(dir); - return identity === null ? undefined : states.byIdentity.get(identity); + return identity === null ? undefined : states.byIdentity.get(identity)?.received; +} + +function folderBirth(dir) { + try { + return String(lstatSync(dir, { bigint: true }).birthtimeNs); + } catch { + return ""; + } +} + +// Each path the state file names the project by; one found only by dev and ino carries the folder's identity. +function namings(dir, states) { + const byPath = new Set(states.byPath.get(dir)?.paths); + const found = [...byPath].map((path) => ({ path })); + const identity = states.byIdentity.size === 0 ? null : fileIdentity(dir); + const named = identity === null ? undefined : states.byIdentity.get(identity); + if (!named) return found; + const kept = `${identity}:${folderBirth(dir)}`; + const moved = [...new Set(named.paths)].filter((path) => !byPath.has(path)); + return [...found, ...moved.map((path) => ({ path, identity: kept }))]; +} + +function keptFilesIn(kept, { path, identity }) { + if (kept.project !== path) return "all"; + if (identity !== undefined && kept.identity !== identity) return "all"; + const { files } = kept; + if (kept.doubted === true || !Array.isArray(files)) return "all"; + return files.every((entry) => typeof entry === "string") ? files : "all"; +} + +function keptFilesFor(naming, text) { + try { + return keptFilesIn(JSON.parse(text) ?? {}, naming); + } catch { + return "all"; + } +} + +// The host's kept copy of the record for one naming: its files, "all" when it cannot be trusted, null when absent. +function hostKeptRecord(stateFile, naming) { + const name = `${createHash("sha256").update(naming.path).digest("hex")}.json`; + const file = join(dirname(stateFile), KEPT_RECORDS, name); + try { + if (!lstatSync(file, { throwIfNoEntry: false })) return null; + } catch (error) { + return error.code === "ENOTDIR" ? null : { file, files: "all" }; + } + const read = readPlainTail(file, MAX_RECORD_BYTES) ?? readPlainTail(file, MAX_RECORD_BYTES); + if (!read?.whole) return onDisk(file) ? { file, files: "all" } : null; + return { file, files: keptFilesFor(naming, read.text) }; +} + +function withEntriesHostKept(dir, listed, states) { + if (!states || hostState(dir, states) !== "expected") return { listed }; + const copies = namings(dir, states).flatMap( + (naming) => hostKeptRecord(states.file, naming) ?? [], + ); + const unread = copies.find((copy) => copy.files === "all"); + if (unread) return { listed, unreadKeptCopy: unread.file }; + const present = new Set(listed.map(String)); + const kept = copies.flatMap((copy) => copy.files); + const back = [...new Set(kept)].filter( + (entry) => !present.has(entry) && onDisk(join(dir, entry)), + ); + return { listed: back.length === 0 ? listed : [...listed, ...back] }; } function allReceivedAtOrAbove(start, states) { @@ -144,12 +215,13 @@ function allReceivedAround(start, states) { if (!states) return false; if (states.unreadable) return true; const overlaps = [...states.byPath].some( - ([path, state]) => state === "all" && (isAtOrUnder(path, start) || isAtOrUnder(start, path)), + ([path, { received }]) => + received === "all" && (isAtOrUnder(path, start) || isAtOrUnder(start, path)), ); return overlaps || allReceivedAtOrAbove(start, states); } -function readRecord(dir, ownFolder, states) { +function readRecordAsWritten(dir, ownFolder, states) { const kind = recordKind(dir); if (kind === "absent" && hostState(dir, states) !== undefined) return { listed: undefined }; if (kind === "absent" || (kind === "unknown" && !ownFolder)) return null; @@ -161,6 +233,13 @@ function readRecord(dir, ownFolder, states) { } } +function readRecord(dir, ownFolder, states) { + const record = readRecordAsWritten(dir, ownFolder, states); + return record && Array.isArray(record.listed) + ? withEntriesHostKept(dir, record.listed, states) + : record; +} + function recordsAtOrAbove(start, states) { const records = []; for (let dir = start; ; ) { @@ -183,7 +262,8 @@ function nestedRecord(start, dir, record) { const listed = Array.isArray(record.listed) ? record.listed.map((entry) => (folder ? `${folder}/${String(entry)}` : String(entry))) : undefined; - return { file: join(dir, RECEIVED_RECORD), root: start, projectDirFromRoot: "", listed }; + const nested = { file: join(dir, RECEIVED_RECORD), root: start, projectDirFromRoot: "", listed }; + return record.unreadKeptCopy ? { ...nested, unreadKeptCopy: record.unreadKeptCopy } : nested; } // The parsers' projectPathSegments: a segment as a case-insensitive file system or Windows opens it. @@ -339,11 +419,12 @@ function receivedRecords(start, atOrAboveOnly) { const below = atOrAboveOnly ? [] : recordsBelow(start, states); if (!below) return null; const records = [...recordsAtOrAbove(start, states), ...below]; + const everyFile = (file) => ({ file, root: start, listed: undefined }); + records.push(...records.flatMap(({ unreadKeptCopy }) => unreadKeptCopy ?? []).map(everyFile)); const all = atOrAboveOnly ? states !== null && (states.unreadable || allReceivedAtOrAbove(start, states)) : allReceivedAround(start, states); - if (all) - records.push({ file: process.env[RECEIVED_STATE_ENV] ?? "", root: start, listed: undefined }); + if (all) records.push(everyFile(process.env[RECEIVED_STATE_ENV] ?? "")); return records; } diff --git a/packages/cli/src/media-use/receivedEnv.parity.test.ts b/packages/cli/src/media-use/receivedEnv.parity.test.ts index 4b7d0da57f..67681371e2 100644 --- a/packages/cli/src/media-use/receivedEnv.parity.test.ts +++ b/packages/cli/src/media-use/receivedEnv.parity.test.ts @@ -10,6 +10,24 @@ import { } from "../utils/privateProjectFile.js"; import { receivedFileState } from "./lib/received-env.mjs"; +const opens = vi.hoisted(() => ({ path: "", seen: 0, replacing: 0 })); + +vi.mock("node:fs", async (importOriginal) => { + const actual = await importOriginal(); + const openSync = ((...args: Parameters) => { + const fd = actual.openSync(...args); + if (args[0] !== opens.path) return fd; + opens.seen += 1; + if (opens.replacing > 0) { + opens.replacing -= 1; + actual.copyFileSync(opens.path, `${opens.path}.new`); + actual.renameSync(`${opens.path}.new`, opens.path); + } + return fd; + }) as typeof actual.openSync; + return { ...actual, default: { ...actual, openSync }, openSync }; +}); + const ENV_TEXT = "ELEVENLABS_API_KEY=from-env\n"; const sha256 = (text: string) => createHash("sha256").update(text).digest("hex"); @@ -26,6 +44,30 @@ const hostSays = (root: string, projects: unknown) => { vi.stubEnv(RECEIVED_STATE_ENV, file); }; +const keptFile = (root: string, path: string) => + join(root, "received-records", `${sha256(path)}.json`); + +const hostKeeps = (root: string, path: string, kept: Record) => { + mkdirSync(join(root, "received-records"), { recursive: true }); + writeFileSync(keptFile(root, path), JSON.stringify({ project: path, seenAt: 1, ...kept })); +}; + +const expectedWithEmptyRecord = (root: string, project: string, extra: object = {}) => { + record(project, []); + hostSays(root, [{ path: project, received: "expected", ...extra }]); +}; + +function movedProject(root: string, project: string, identity: (birth: bigint) => string) { + const { dev, ino, birthtimeNs } = lstatSync(project, { bigint: true }); + const old = join(root, "old-name"); + record(project, []); + hostSays(root, [{ path: old, dev: String(dev), ino: String(ino), received: "expected" }]); + hostKeeps(root, old, { + files: ["index.html"], + identity: `${dev}:${ino}:${identity(birthtimeNs)}`, + }); +} + const unreadableRecord = (folder: string) => { mkdirSync(join(folder, ".hyperframes"), { recursive: true }); writeFileSync(join(folder, ".hyperframes", "received-files.json"), "{oops"); @@ -152,6 +194,83 @@ const FIXTURES: [string, Arrange, "received" | "unchecked" | null][] = [ }, "received", ], + [ + "an emptied record whose kept copy lists the .env", + (root, project) => { + expectedWithEmptyRecord(root, project); + hostKeeps(root, project, { files: [".env", "index.html"] }); + }, + "received", + ], + [ + "an emptied record whose kept copy lists the .env, which is gone", + (root, project) => { + expectedWithEmptyRecord(root, project); + hostKeeps(root, project, { files: [".env"] }); + rmSync(join(project, ".env")); + }, + null, + ], + [ + "a kept copy with no list", + (root, project) => { + expectedWithEmptyRecord(root, project); + hostKeeps(root, project, { files: null }); + }, + "received", + ], + [ + "a doubted kept copy", + (root, project) => { + expectedWithEmptyRecord(root, project); + hostKeeps(root, project, { files: ["index.html"], doubted: true }); + }, + "received", + ], + [ + "a kept copy for another project", + (root, project) => { + expectedWithEmptyRecord(root, project); + hostKeeps(root, project, { project: join(root, "other"), files: [] }); + }, + "received", + ], + [ + "an unparsable kept copy", + (root, project) => { + expectedWithEmptyRecord(root, project); + hostKeeps(root, project, {}); + writeFileSync(keptFile(root, project), "{not json"); + }, + "received", + ], + [ + "no kept copy", + (root, project) => { + expectedWithEmptyRecord(root, project); + hostKeeps(root, join(root, "other"), { files: [".env"] }); + }, + null, + ], + [ + "a moved project's kept copy with its folder's identity", + (root, project) => movedProject(root, project, (birth) => String(birth)), + null, + ], + [ + "a moved project's kept copy with another folder's identity", + (root, project) => movedProject(root, project, () => "1"), + "received", + ], + [ + "a nested project's kept copy with no list", + (root, project) => { + record(project, []); + expectedWithEmptyRecord(root, join(project, "scenes")); + hostKeeps(root, join(project, "scenes"), { files: null }); + }, + "received", + ], ["an unreadable host state file", (root) => hostSays(root, "{not json"), "received"], [ "a host state file that is missing", @@ -174,6 +293,7 @@ const dirs: string[] = []; afterEach(() => { vi.unstubAllEnvs(); + Object.assign(opens, { path: "", seen: 0, replacing: 0 }); for (const dir of dirs.splice(0)) rmSync(dir, { recursive: true, force: true }); }); @@ -212,3 +332,81 @@ it.skipIf(process.platform === "win32")( expect(receivedFileState(project, ".env")).toBe("received"); }, ); + +function projectWithEmptyRecord(): { root: string; project: string } { + const root = mkdtempSync(join(tmpdir(), "hf-received-env-")); + dirs.push(root); + const project = join(root, "app"); + mkdirSync(project); + writeFileSync(join(project, ".env"), ENV_TEXT); + return { root, project }; +} + +it.skipIf(process.platform === "win32")( + "the media-use copy reads a link in place of the kept copy as the CLI does", + () => { + const { root, project } = projectWithEmptyRecord(); + expectedWithEmptyRecord(root, project); + hostKeeps(root, join(root, "elsewhere"), { project, files: ["index.html"] }); + symlinkSync(keptFile(root, join(root, "elsewhere")), keptFile(root, project)); + expect(cliState(project)).toBe("received"); + expect(receivedFileState(project, ".env")).toBe("received"); + }, +); + +it.skipIf(process.platform === "win32")( + "the media-use copy leaves off a gone kept entry as the CLI does", + () => { + const { root, project } = projectWithEmptyRecord(); + rmSync(join(project, ".env")); + writeFileSync(join(root, "outside.env"), ENV_TEXT); + symlinkSync(join(root, "outside.env"), join(project, ".env")); + expectedWithEmptyRecord(root, project); + hostKeeps(root, project, { files: ["gone.html"] }); + expect(cliState(project)).toBe(null); + expect(receivedFileState(project, ".env")).toBe(null); + }, +); + +it("the media-use copy reads the kept copy once more when the host replaces it, as the CLI does", () => { + const { root, project } = projectWithEmptyRecord(); + expectedWithEmptyRecord(root, project); + hostKeeps(root, project, { files: ["index.html"] }); + for (const state of [ + (dir: string) => receivedFiles(dir)(".env"), + (dir: string) => receivedFileState(dir, ".env"), + ]) { + Object.assign(opens, { path: keptFile(root, project), seen: 0, replacing: 1 }); + expect(state(project)).toBeFalsy(); + expect(opens.seen).toBe(2); + } +}); + +it("the media-use copy reads a kept copy named by path and identity once, as the CLI does", () => { + const { root, project } = projectWithEmptyRecord(); + const { dev, ino } = lstatSync(project, { bigint: true }); + expectedWithEmptyRecord(root, project, { dev: String(dev), ino: String(ino) }); + hostKeeps(root, project, { files: ["index.html"], identity: null }); + for (const state of [ + (dir: string) => receivedFiles(dir)(".env"), + (dir: string) => receivedFileState(dir, ".env"), + ]) { + Object.assign(opens, { path: keptFile(root, project), seen: 0, replacing: 0 }); + expect(state(project)).toBeFalsy(); + expect(opens.seen).toBe(1); + } +}); + +it.each([ + [[".env"], "received"], + [null, "received"], + [["index.html"], null], +])( + "the media-use copy reads a kept copy listing %j when it looks only at and above the folder", + (files, expected) => { + const { root, project } = projectWithEmptyRecord(); + expectedWithEmptyRecord(root, project); + hostKeeps(root, project, { files }); + expect(receivedFileState(project, ".env", { atOrAboveOnly: true })).toBe(expected); + }, +); diff --git a/packages/cli/src/utils/privateProjectFile.test.ts b/packages/cli/src/utils/privateProjectFile.test.ts index 8a35e45393..d270bf4031 100644 --- a/packages/cli/src/utils/privateProjectFile.test.ts +++ b/packages/cli/src/utils/privateProjectFile.test.ts @@ -28,6 +28,7 @@ import { } from "./privateProjectFile.js"; const zeroFileIds = vi.hoisted(() => ({ on: false })); +const opens = vi.hoisted(() => ({ path: "", seen: 0, replacing: 0 })); vi.mock("node:fs", async (importOriginal) => { const actual = await importOriginal(); @@ -35,13 +36,25 @@ vi.mock("node:fs", async (importOriginal) => { const stat = actual.lstatSync(path, options as never); return zeroFileIds.on && stat ? Object.assign(stat, { dev: 0n, ino: 0n }) : stat; }) as typeof actual.lstatSync; - return { ...actual, default: { ...actual, lstatSync }, lstatSync }; + const openSync = ((...args: Parameters) => { + const fd = actual.openSync(...args); + if (args[0] !== opens.path) return fd; + opens.seen += 1; + if (opens.replacing > 0) { + opens.replacing -= 1; + actual.copyFileSync(opens.path, `${opens.path}.new`); + actual.renameSync(`${opens.path}.new`, opens.path); + } + return fd; + }) as typeof actual.openSync; + return { ...actual, default: { ...actual, lstatSync, openSync }, lstatSync, openSync }; }); let dir: string | undefined; afterEach(() => { zeroFileIds.on = false; + Object.assign(opens, { path: "", seen: 0, replacing: 0 }); vi.unstubAllEnvs(); if (dir) rmSync(dir, { recursive: true, force: true }); dir = undefined; @@ -400,8 +413,45 @@ describe("a host's received state file", () => { renameSync(app, renamed); expect(receivedFiles(renamed)("index.html")).toBe(true); expect(receivedFiles(renamed)("assets/x.js")).toBe(false); - hostKeeps(app, { ...kept, identity: `${dev}:${ino}:1` }); - expect(receivedFiles(renamed)("index.html")).toBe(false); + }); + + it("counts every file as received when a moved project's kept copy names another folder", () => { + const app = emptiedProject(); + const { dev, ino } = lstatSync(app, { bigint: true }); + hostSays([{ path: app, dev: String(dev), ino: String(ino), received: "expected" }]); + hostKeeps(app, { + project: app, + files: ["index.html"], + seenAt: 1, + identity: `${dev}:${ino}:1`, + }); + const renamed = `${app}-renamed`; + renameSync(app, renamed); + expect(receivedFiles(renamed)("compositions/intro.html")).toBe(true); + expect(receivedRecordStatus(renamed)).toMatchObject({ path: keptFile(app), listed: null }); + }); + + it("reads the kept copy once more when the host replaces it during the first read", () => { + const app = emptiedProject(); + hostSays([{ path: app, received: "expected" }]); + keeps(app, ["index.html"]); + Object.assign(opens, { path: keptFile(app), seen: 0, replacing: 1 }); + const received = receivedFiles(app); + expect(opens.seen).toBe(2); + expect(received("index.html")).toBe(true); + expect(received("compositions/intro.html")).toBe(false); + }); + + it("reads the kept copy once when the state file names the project by path and identity", () => { + const app = emptiedProject(); + const { dev, ino } = lstatSync(app, { bigint: true }); + hostSays([{ path: app, dev: String(dev), ino: String(ino), received: "expected" }]); + keeps(app, ["index.html"]); + Object.assign(opens, { path: keptFile(app), seen: 0, replacing: 0 }); + const received = receivedFiles(app); + expect(opens.seen).toBe(1); + expect(received("index.html")).toBe(true); + expect(received("compositions/intro.html")).toBe(false); }); it("counts every file as received when the kept copy holds no list, and keeps the record's lines", () => { diff --git a/packages/cli/src/utils/privateProjectFile.ts b/packages/cli/src/utils/privateProjectFile.ts index cb4ee055df..2f21bb02ac 100644 --- a/packages/cli/src/utils/privateProjectFile.ts +++ b/packages/cli/src/utils/privateProjectFile.ts @@ -178,22 +178,24 @@ function folderBirth(dir: string): string { } function namings(dir: string, states: HostStates): Naming[] { - const byPath = (states.byPath.get(dir)?.paths ?? []).map((path) => ({ path })); + const byPath = new Set(states.byPath.get(dir)?.paths); + const found = [...byPath].map((path) => ({ path })); const identity = states.byIdentity.size === 0 ? null : fileIdentity(dir); const named = identity === null ? undefined : states.byIdentity.get(identity); - if (!named) return byPath; + if (!named) return found; const kept = `${identity}:${folderBirth(dir)}`; - return [...byPath, ...named.paths.map((path) => ({ path, identity: kept }))]; + const moved = [...new Set(named.paths)].filter((path) => !byPath.has(path)); + return [...found, ...moved.map((path) => ({ path, identity: kept }))]; } const KEPT_RECORDS = "received-records"; const MAX_KEPT_BYTES = 16 * 1024 * 1024; -type KeptFiles = string[] | "all" | null; +type KeptFiles = string[] | "all"; function keptFilesIn(kept: Record, { path, identity }: Naming): KeptFiles { if (kept.project !== path) return "all"; - if (identity !== undefined && kept.identity !== identity) return null; + if (identity !== undefined && kept.identity !== identity) return "all"; const { files } = kept; if (kept.doubted === true || !Array.isArray(files)) return "all"; return files.every((entry) => typeof entry === "string") ? files : "all"; @@ -211,7 +213,7 @@ function keptFilesFor(naming: Naming, text: string): KeptFiles { function hostKeptRecord( stateFile: string, naming: Naming, -): { file: string; files: string[] | "all" } | null { +): { file: string; files: KeptFiles } | null { const name = `${createHash("sha256").update(naming.path).digest("hex")}.json`; const file = join(dirname(stateFile), KEPT_RECORDS, name); try { @@ -219,10 +221,9 @@ function hostKeptRecord( } catch (error) { return (error as NodeJS.ErrnoException).code === "ENOTDIR" ? null : { file, files: "all" }; } - const read = readPlainTail(file, MAX_KEPT_BYTES); + const read = readPlainTail(file, MAX_KEPT_BYTES) ?? readPlainTail(file, MAX_KEPT_BYTES); if (!read?.whole) return onDisk(file) ? { file, files: "all" } : null; - const files = keptFilesFor(naming, read.text); - return files === null ? null : { file, files }; + return { file, files: keptFilesFor(naming, read.text) }; } function withEntriesHostKept(dir: string, listed: unknown[]): ReadRecord { diff --git a/skills-manifest.json b/skills-manifest.json index 6d5e4578a0..998130fc73 100644 --- a/skills-manifest.json +++ b/skills-manifest.json @@ -18,7 +18,7 @@ "files": 13 }, "media-use": { - "hash": "abbb676657aa5be4", + "hash": "9740d923a54e63ba", "files": 130 }, "remotion-to-hyperframes": { diff --git a/skills/media-use/scripts/lib/received-env.mjs b/skills/media-use/scripts/lib/received-env.mjs index f761c33a1c..bfee878e02 100644 --- a/skills/media-use/scripts/lib/received-env.mjs +++ b/skills/media-use/scripts/lib/received-env.mjs @@ -20,6 +20,7 @@ const RECEIVED_RECORD = join(".hyperframes", "received-files.json"); const RECEIVED_CONTENT = "received-content.json"; const RECEIVED_STATE_ENV = "HYPERFRAMES_RECEIVED_STATE_FILE"; const MAX_RECORD_BYTES = 16 * 1024 * 1024; +const KEPT_RECORDS = "received-records"; const SHA256 = /^[0-9a-f]{64}$/; const WALK_MAX_ENTRIES = 20_000; const WALK_MAX_MS = 2_000; @@ -90,6 +91,14 @@ function recordKind(dir) { const strongest = (a, b) => (a === "all" ? a : b); +function addNamed(map, key, received, path) { + const known = map.get(key); + map.set(key, { + received: strongest(known?.received, received), + paths: [...(known?.paths ?? []), path], + }); +} + function hostEntry(value) { const { path, received, dev, ino } = value ?? {}; if (typeof path !== "string" || !isAbsolute(path)) throw new Error("path is not absolute"); @@ -98,18 +107,15 @@ function hostEntry(value) { return { path, received, ...(identity ? { dev, ino } : {}) }; } -function parseHostStates(text) { +function parseHostStates(file, text) { const projects = JSON.parse(text)?.projects; if (!Array.isArray(projects)) throw new Error("projects is not a list"); - const states = { unreadable: false, byPath: new Map(), byIdentity: new Map() }; + const states = { file, unreadable: false, byPath: new Map(), byIdentity: new Map() }; for (const value of projects) { const { path, received, dev, ino } = hostEntry(value); - const key = walkStart(path); - states.byPath.set(key, strongest(states.byPath.get(key), received)); - if (dev !== undefined && ino !== undefined) { - const identity = `${dev}:${ino}`; - states.byIdentity.set(identity, strongest(states.byIdentity.get(identity), received)); - } + addNamed(states.byPath, walkStart(path), received, path); + if (dev !== undefined && ino !== undefined) + addNamed(states.byIdentity, `${dev}:${ino}`, received, path); } return states; } @@ -118,19 +124,84 @@ function hostStates() { const file = process.env[RECEIVED_STATE_ENV]; if (!file) return null; try { - return parseHostStates(readFileSync(file, "utf-8")); + return parseHostStates(file, readFileSync(file, "utf-8")); } catch { - return { unreadable: true, byPath: new Map(), byIdentity: new Map() }; + return { file, unreadable: true, byPath: new Map(), byIdentity: new Map() }; } } function hostState(dir, states) { if (!states) return undefined; if (states.unreadable) return "all"; - const named = states.byPath.get(dir); + const named = states.byPath.get(dir)?.received; if (named || states.byIdentity.size === 0) return named; const identity = fileIdentity(dir); - return identity === null ? undefined : states.byIdentity.get(identity); + return identity === null ? undefined : states.byIdentity.get(identity)?.received; +} + +function folderBirth(dir) { + try { + return String(lstatSync(dir, { bigint: true }).birthtimeNs); + } catch { + return ""; + } +} + +// Each path the state file names the project by; one found only by dev and ino carries the folder's identity. +function namings(dir, states) { + const byPath = new Set(states.byPath.get(dir)?.paths); + const found = [...byPath].map((path) => ({ path })); + const identity = states.byIdentity.size === 0 ? null : fileIdentity(dir); + const named = identity === null ? undefined : states.byIdentity.get(identity); + if (!named) return found; + const kept = `${identity}:${folderBirth(dir)}`; + const moved = [...new Set(named.paths)].filter((path) => !byPath.has(path)); + return [...found, ...moved.map((path) => ({ path, identity: kept }))]; +} + +function keptFilesIn(kept, { path, identity }) { + if (kept.project !== path) return "all"; + if (identity !== undefined && kept.identity !== identity) return "all"; + const { files } = kept; + if (kept.doubted === true || !Array.isArray(files)) return "all"; + return files.every((entry) => typeof entry === "string") ? files : "all"; +} + +function keptFilesFor(naming, text) { + try { + return keptFilesIn(JSON.parse(text) ?? {}, naming); + } catch { + return "all"; + } +} + +// The host's kept copy of the record for one naming: its files, "all" when it cannot be trusted, null when absent. +function hostKeptRecord(stateFile, naming) { + const name = `${createHash("sha256").update(naming.path).digest("hex")}.json`; + const file = join(dirname(stateFile), KEPT_RECORDS, name); + try { + if (!lstatSync(file, { throwIfNoEntry: false })) return null; + } catch (error) { + return error.code === "ENOTDIR" ? null : { file, files: "all" }; + } + const read = readPlainTail(file, MAX_RECORD_BYTES) ?? readPlainTail(file, MAX_RECORD_BYTES); + if (!read?.whole) return onDisk(file) ? { file, files: "all" } : null; + return { file, files: keptFilesFor(naming, read.text) }; +} + +function withEntriesHostKept(dir, listed, states) { + if (!states || hostState(dir, states) !== "expected") return { listed }; + const copies = namings(dir, states).flatMap( + (naming) => hostKeptRecord(states.file, naming) ?? [], + ); + const unread = copies.find((copy) => copy.files === "all"); + if (unread) return { listed, unreadKeptCopy: unread.file }; + const present = new Set(listed.map(String)); + const kept = copies.flatMap((copy) => copy.files); + const back = [...new Set(kept)].filter( + (entry) => !present.has(entry) && onDisk(join(dir, entry)), + ); + return { listed: back.length === 0 ? listed : [...listed, ...back] }; } function allReceivedAtOrAbove(start, states) { @@ -144,12 +215,13 @@ function allReceivedAround(start, states) { if (!states) return false; if (states.unreadable) return true; const overlaps = [...states.byPath].some( - ([path, state]) => state === "all" && (isAtOrUnder(path, start) || isAtOrUnder(start, path)), + ([path, { received }]) => + received === "all" && (isAtOrUnder(path, start) || isAtOrUnder(start, path)), ); return overlaps || allReceivedAtOrAbove(start, states); } -function readRecord(dir, ownFolder, states) { +function readRecordAsWritten(dir, ownFolder, states) { const kind = recordKind(dir); if (kind === "absent" && hostState(dir, states) !== undefined) return { listed: undefined }; if (kind === "absent" || (kind === "unknown" && !ownFolder)) return null; @@ -161,6 +233,13 @@ function readRecord(dir, ownFolder, states) { } } +function readRecord(dir, ownFolder, states) { + const record = readRecordAsWritten(dir, ownFolder, states); + return record && Array.isArray(record.listed) + ? withEntriesHostKept(dir, record.listed, states) + : record; +} + function recordsAtOrAbove(start, states) { const records = []; for (let dir = start; ; ) { @@ -183,7 +262,8 @@ function nestedRecord(start, dir, record) { const listed = Array.isArray(record.listed) ? record.listed.map((entry) => (folder ? `${folder}/${String(entry)}` : String(entry))) : undefined; - return { file: join(dir, RECEIVED_RECORD), root: start, projectDirFromRoot: "", listed }; + const nested = { file: join(dir, RECEIVED_RECORD), root: start, projectDirFromRoot: "", listed }; + return record.unreadKeptCopy ? { ...nested, unreadKeptCopy: record.unreadKeptCopy } : nested; } // The parsers' projectPathSegments: a segment as a case-insensitive file system or Windows opens it. @@ -339,11 +419,12 @@ function receivedRecords(start, atOrAboveOnly) { const below = atOrAboveOnly ? [] : recordsBelow(start, states); if (!below) return null; const records = [...recordsAtOrAbove(start, states), ...below]; + const everyFile = (file) => ({ file, root: start, listed: undefined }); + records.push(...records.flatMap(({ unreadKeptCopy }) => unreadKeptCopy ?? []).map(everyFile)); const all = atOrAboveOnly ? states !== null && (states.unreadable || allReceivedAtOrAbove(start, states)) : allReceivedAround(start, states); - if (all) - records.push({ file: process.env[RECEIVED_STATE_ENV] ?? "", root: start, listed: undefined }); + if (all) records.push(everyFile(process.env[RECEIVED_STATE_ENV] ?? "")); return records; }