From c27adb35ffa400a2d526c9c83a5b3d184d1080f8 Mon Sep 17 00:00:00 2001 From: Michael Wang Date: Tue, 17 Nov 2026 23:57:13 +0000 Subject: [PATCH] fix(cli): media-use scripts read the host's kept record copy as the CLI does (#8119) The media-use copy of the received-files rule now reads the host app's kept copy of an expected project's record, as the CLI does. It adds back each kept entry whose path still holds something, checks the folder identity for a project found only by dev and ino, and counts every file as received when the copy has no list, is doubted, a link, oversized, unparsable or another project's. In both the CLI and the copy: - a kept copy whose identity names another folder counts every file as received instead of being ignored; - a kept copy that fails its first read, for example because the app replaced it mid-read, is read once more before every file counts as received; - a kept copy named both by path and by dev and ino is read once. The docs no longer suggest that the app's next read restores a kept entry whose file is gone. --- docs/packages/cli.mdx | 16 +- .../cli/src/media-use/lib/received-env.mjs | 115 ++++++++-- .../src/media-use/receivedEnv.parity.test.ts | 198 ++++++++++++++++++ .../cli/src/utils/privateProjectFile.test.ts | 56 ++++- packages/cli/src/utils/privateProjectFile.ts | 19 +- skills-manifest.json | 2 +- skills/media-use/scripts/lib/received-env.mjs | 115 ++++++++-- 7 files changed, 467 insertions(+), 54 deletions(-) diff --git a/docs/packages/cli.mdx b/docs/packages/cli.mdx index 75c861e7da..b95f1522de 100644 --- a/docs/packages/cli.mdx +++ b/docs/packages/cli.mdx @@ -2906,13 +2906,15 @@ imports projects for you can tell the CLI more by pointing `{ "project": "", "files": [...] }` with `path` exactly as the state file names it. A file that copy lists still counts as received while something is at its path, even if the project's record no longer lists it. For a project found - only by `dev` and `ino`, the copy counts only when its `"identity"` is - `"::"` of the project folder. A copy with - `"files": null` or `"doubted": true`, or one that is a link or cannot be read, - counts every file as received. -- A file the copy lists that is no longer at its path is not added back. A - received file that is renamed after its record entry is removed is judged by - the project's record alone until the app reads the record again. + only by `dev` and `ino`, the copy's `"identity"` must be + `"::"` of the project folder; a copy with another + identity counts every file as received. A copy that names another project, + has `"files": null` or `"doubted": true`, or is a link, larger than 16 MB or + unreadable also counts every file as received. +- A file the copy lists that is no longer at its path is not added back. When + the app restores lost entries to a project's record, it uses the same rule, so + a received file that is renamed after its record entry is removed is judged by + the project's record alone. A file from outside the project folder that would be copied (each file a composition points at, and each project file that is a link to a file outside diff --git a/packages/cli/src/media-use/lib/received-env.mjs b/packages/cli/src/media-use/lib/received-env.mjs index f761c33a1c..bfee878e02 100644 --- a/packages/cli/src/media-use/lib/received-env.mjs +++ b/packages/cli/src/media-use/lib/received-env.mjs @@ -20,6 +20,7 @@ const RECEIVED_RECORD = join(".hyperframes", "received-files.json"); const RECEIVED_CONTENT = "received-content.json"; const RECEIVED_STATE_ENV = "HYPERFRAMES_RECEIVED_STATE_FILE"; const MAX_RECORD_BYTES = 16 * 1024 * 1024; +const KEPT_RECORDS = "received-records"; const SHA256 = /^[0-9a-f]{64}$/; const WALK_MAX_ENTRIES = 20_000; const WALK_MAX_MS = 2_000; @@ -90,6 +91,14 @@ function recordKind(dir) { const strongest = (a, b) => (a === "all" ? a : b); +function addNamed(map, key, received, path) { + const known = map.get(key); + map.set(key, { + received: strongest(known?.received, received), + paths: [...(known?.paths ?? []), path], + }); +} + function hostEntry(value) { const { path, received, dev, ino } = value ?? {}; if (typeof path !== "string" || !isAbsolute(path)) throw new Error("path is not absolute"); @@ -98,18 +107,15 @@ function hostEntry(value) { return { path, received, ...(identity ? { dev, ino } : {}) }; } -function parseHostStates(text) { +function parseHostStates(file, text) { const projects = JSON.parse(text)?.projects; if (!Array.isArray(projects)) throw new Error("projects is not a list"); - const states = { unreadable: false, byPath: new Map(), byIdentity: new Map() }; + const states = { file, unreadable: false, byPath: new Map(), byIdentity: new Map() }; for (const value of projects) { const { path, received, dev, ino } = hostEntry(value); - const key = walkStart(path); - states.byPath.set(key, strongest(states.byPath.get(key), received)); - if (dev !== undefined && ino !== undefined) { - const identity = `${dev}:${ino}`; - states.byIdentity.set(identity, strongest(states.byIdentity.get(identity), received)); - } + addNamed(states.byPath, walkStart(path), received, path); + if (dev !== undefined && ino !== undefined) + addNamed(states.byIdentity, `${dev}:${ino}`, received, path); } return states; } @@ -118,19 +124,84 @@ function hostStates() { const file = process.env[RECEIVED_STATE_ENV]; if (!file) return null; try { - return parseHostStates(readFileSync(file, "utf-8")); + return parseHostStates(file, readFileSync(file, "utf-8")); } catch { - return { unreadable: true, byPath: new Map(), byIdentity: new Map() }; + return { file, unreadable: true, byPath: new Map(), byIdentity: new Map() }; } } function hostState(dir, states) { if (!states) return undefined; if (states.unreadable) return "all"; - const named = states.byPath.get(dir); + const named = states.byPath.get(dir)?.received; if (named || states.byIdentity.size === 0) return named; const identity = fileIdentity(dir); - return identity === null ? undefined : states.byIdentity.get(identity); + return identity === null ? undefined : states.byIdentity.get(identity)?.received; +} + +function folderBirth(dir) { + try { + return String(lstatSync(dir, { bigint: true }).birthtimeNs); + } catch { + return ""; + } +} + +// Each path the state file names the project by; one found only by dev and ino carries the folder's identity. +function namings(dir, states) { + const byPath = new Set(states.byPath.get(dir)?.paths); + const found = [...byPath].map((path) => ({ path })); + const identity = states.byIdentity.size === 0 ? null : fileIdentity(dir); + const named = identity === null ? undefined : states.byIdentity.get(identity); + if (!named) return found; + const kept = `${identity}:${folderBirth(dir)}`; + const moved = [...new Set(named.paths)].filter((path) => !byPath.has(path)); + return [...found, ...moved.map((path) => ({ path, identity: kept }))]; +} + +function keptFilesIn(kept, { path, identity }) { + if (kept.project !== path) return "all"; + if (identity !== undefined && kept.identity !== identity) return "all"; + const { files } = kept; + if (kept.doubted === true || !Array.isArray(files)) return "all"; + return files.every((entry) => typeof entry === "string") ? files : "all"; +} + +function keptFilesFor(naming, text) { + try { + return keptFilesIn(JSON.parse(text) ?? {}, naming); + } catch { + return "all"; + } +} + +// The host's kept copy of the record for one naming: its files, "all" when it cannot be trusted, null when absent. +function hostKeptRecord(stateFile, naming) { + const name = `${createHash("sha256").update(naming.path).digest("hex")}.json`; + const file = join(dirname(stateFile), KEPT_RECORDS, name); + try { + if (!lstatSync(file, { throwIfNoEntry: false })) return null; + } catch (error) { + return error.code === "ENOTDIR" ? null : { file, files: "all" }; + } + const read = readPlainTail(file, MAX_RECORD_BYTES) ?? readPlainTail(file, MAX_RECORD_BYTES); + if (!read?.whole) return onDisk(file) ? { file, files: "all" } : null; + return { file, files: keptFilesFor(naming, read.text) }; +} + +function withEntriesHostKept(dir, listed, states) { + if (!states || hostState(dir, states) !== "expected") return { listed }; + const copies = namings(dir, states).flatMap( + (naming) => hostKeptRecord(states.file, naming) ?? [], + ); + const unread = copies.find((copy) => copy.files === "all"); + if (unread) return { listed, unreadKeptCopy: unread.file }; + const present = new Set(listed.map(String)); + const kept = copies.flatMap((copy) => copy.files); + const back = [...new Set(kept)].filter( + (entry) => !present.has(entry) && onDisk(join(dir, entry)), + ); + return { listed: back.length === 0 ? listed : [...listed, ...back] }; } function allReceivedAtOrAbove(start, states) { @@ -144,12 +215,13 @@ function allReceivedAround(start, states) { if (!states) return false; if (states.unreadable) return true; const overlaps = [...states.byPath].some( - ([path, state]) => state === "all" && (isAtOrUnder(path, start) || isAtOrUnder(start, path)), + ([path, { received }]) => + received === "all" && (isAtOrUnder(path, start) || isAtOrUnder(start, path)), ); return overlaps || allReceivedAtOrAbove(start, states); } -function readRecord(dir, ownFolder, states) { +function readRecordAsWritten(dir, ownFolder, states) { const kind = recordKind(dir); if (kind === "absent" && hostState(dir, states) !== undefined) return { listed: undefined }; if (kind === "absent" || (kind === "unknown" && !ownFolder)) return null; @@ -161,6 +233,13 @@ function readRecord(dir, ownFolder, states) { } } +function readRecord(dir, ownFolder, states) { + const record = readRecordAsWritten(dir, ownFolder, states); + return record && Array.isArray(record.listed) + ? withEntriesHostKept(dir, record.listed, states) + : record; +} + function recordsAtOrAbove(start, states) { const records = []; for (let dir = start; ; ) { @@ -183,7 +262,8 @@ function nestedRecord(start, dir, record) { const listed = Array.isArray(record.listed) ? record.listed.map((entry) => (folder ? `${folder}/${String(entry)}` : String(entry))) : undefined; - return { file: join(dir, RECEIVED_RECORD), root: start, projectDirFromRoot: "", listed }; + const nested = { file: join(dir, RECEIVED_RECORD), root: start, projectDirFromRoot: "", listed }; + return record.unreadKeptCopy ? { ...nested, unreadKeptCopy: record.unreadKeptCopy } : nested; } // The parsers' projectPathSegments: a segment as a case-insensitive file system or Windows opens it. @@ -339,11 +419,12 @@ function receivedRecords(start, atOrAboveOnly) { const below = atOrAboveOnly ? [] : recordsBelow(start, states); if (!below) return null; const records = [...recordsAtOrAbove(start, states), ...below]; + const everyFile = (file) => ({ file, root: start, listed: undefined }); + records.push(...records.flatMap(({ unreadKeptCopy }) => unreadKeptCopy ?? []).map(everyFile)); const all = atOrAboveOnly ? states !== null && (states.unreadable || allReceivedAtOrAbove(start, states)) : allReceivedAround(start, states); - if (all) - records.push({ file: process.env[RECEIVED_STATE_ENV] ?? "", root: start, listed: undefined }); + if (all) records.push(everyFile(process.env[RECEIVED_STATE_ENV] ?? "")); return records; } diff --git a/packages/cli/src/media-use/receivedEnv.parity.test.ts b/packages/cli/src/media-use/receivedEnv.parity.test.ts index 4b7d0da57f..67681371e2 100644 --- a/packages/cli/src/media-use/receivedEnv.parity.test.ts +++ b/packages/cli/src/media-use/receivedEnv.parity.test.ts @@ -10,6 +10,24 @@ import { } from "../utils/privateProjectFile.js"; import { receivedFileState } from "./lib/received-env.mjs"; +const opens = vi.hoisted(() => ({ path: "", seen: 0, replacing: 0 })); + +vi.mock("node:fs", async (importOriginal) => { + const actual = await importOriginal(); + const openSync = ((...args: Parameters) => { + const fd = actual.openSync(...args); + if (args[0] !== opens.path) return fd; + opens.seen += 1; + if (opens.replacing > 0) { + opens.replacing -= 1; + actual.copyFileSync(opens.path, `${opens.path}.new`); + actual.renameSync(`${opens.path}.new`, opens.path); + } + return fd; + }) as typeof actual.openSync; + return { ...actual, default: { ...actual, openSync }, openSync }; +}); + const ENV_TEXT = "ELEVENLABS_API_KEY=from-env\n"; const sha256 = (text: string) => createHash("sha256").update(text).digest("hex"); @@ -26,6 +44,30 @@ const hostSays = (root: string, projects: unknown) => { vi.stubEnv(RECEIVED_STATE_ENV, file); }; +const keptFile = (root: string, path: string) => + join(root, "received-records", `${sha256(path)}.json`); + +const hostKeeps = (root: string, path: string, kept: Record) => { + mkdirSync(join(root, "received-records"), { recursive: true }); + writeFileSync(keptFile(root, path), JSON.stringify({ project: path, seenAt: 1, ...kept })); +}; + +const expectedWithEmptyRecord = (root: string, project: string, extra: object = {}) => { + record(project, []); + hostSays(root, [{ path: project, received: "expected", ...extra }]); +}; + +function movedProject(root: string, project: string, identity: (birth: bigint) => string) { + const { dev, ino, birthtimeNs } = lstatSync(project, { bigint: true }); + const old = join(root, "old-name"); + record(project, []); + hostSays(root, [{ path: old, dev: String(dev), ino: String(ino), received: "expected" }]); + hostKeeps(root, old, { + files: ["index.html"], + identity: `${dev}:${ino}:${identity(birthtimeNs)}`, + }); +} + const unreadableRecord = (folder: string) => { mkdirSync(join(folder, ".hyperframes"), { recursive: true }); writeFileSync(join(folder, ".hyperframes", "received-files.json"), "{oops"); @@ -152,6 +194,83 @@ const FIXTURES: [string, Arrange, "received" | "unchecked" | null][] = [ }, "received", ], + [ + "an emptied record whose kept copy lists the .env", + (root, project) => { + expectedWithEmptyRecord(root, project); + hostKeeps(root, project, { files: [".env", "index.html"] }); + }, + "received", + ], + [ + "an emptied record whose kept copy lists the .env, which is gone", + (root, project) => { + expectedWithEmptyRecord(root, project); + hostKeeps(root, project, { files: [".env"] }); + rmSync(join(project, ".env")); + }, + null, + ], + [ + "a kept copy with no list", + (root, project) => { + expectedWithEmptyRecord(root, project); + hostKeeps(root, project, { files: null }); + }, + "received", + ], + [ + "a doubted kept copy", + (root, project) => { + expectedWithEmptyRecord(root, project); + hostKeeps(root, project, { files: ["index.html"], doubted: true }); + }, + "received", + ], + [ + "a kept copy for another project", + (root, project) => { + expectedWithEmptyRecord(root, project); + hostKeeps(root, project, { project: join(root, "other"), files: [] }); + }, + "received", + ], + [ + "an unparsable kept copy", + (root, project) => { + expectedWithEmptyRecord(root, project); + hostKeeps(root, project, {}); + writeFileSync(keptFile(root, project), "{not json"); + }, + "received", + ], + [ + "no kept copy", + (root, project) => { + expectedWithEmptyRecord(root, project); + hostKeeps(root, join(root, "other"), { files: [".env"] }); + }, + null, + ], + [ + "a moved project's kept copy with its folder's identity", + (root, project) => movedProject(root, project, (birth) => String(birth)), + null, + ], + [ + "a moved project's kept copy with another folder's identity", + (root, project) => movedProject(root, project, () => "1"), + "received", + ], + [ + "a nested project's kept copy with no list", + (root, project) => { + record(project, []); + expectedWithEmptyRecord(root, join(project, "scenes")); + hostKeeps(root, join(project, "scenes"), { files: null }); + }, + "received", + ], ["an unreadable host state file", (root) => hostSays(root, "{not json"), "received"], [ "a host state file that is missing", @@ -174,6 +293,7 @@ const dirs: string[] = []; afterEach(() => { vi.unstubAllEnvs(); + Object.assign(opens, { path: "", seen: 0, replacing: 0 }); for (const dir of dirs.splice(0)) rmSync(dir, { recursive: true, force: true }); }); @@ -212,3 +332,81 @@ it.skipIf(process.platform === "win32")( expect(receivedFileState(project, ".env")).toBe("received"); }, ); + +function projectWithEmptyRecord(): { root: string; project: string } { + const root = mkdtempSync(join(tmpdir(), "hf-received-env-")); + dirs.push(root); + const project = join(root, "app"); + mkdirSync(project); + writeFileSync(join(project, ".env"), ENV_TEXT); + return { root, project }; +} + +it.skipIf(process.platform === "win32")( + "the media-use copy reads a link in place of the kept copy as the CLI does", + () => { + const { root, project } = projectWithEmptyRecord(); + expectedWithEmptyRecord(root, project); + hostKeeps(root, join(root, "elsewhere"), { project, files: ["index.html"] }); + symlinkSync(keptFile(root, join(root, "elsewhere")), keptFile(root, project)); + expect(cliState(project)).toBe("received"); + expect(receivedFileState(project, ".env")).toBe("received"); + }, +); + +it.skipIf(process.platform === "win32")( + "the media-use copy leaves off a gone kept entry as the CLI does", + () => { + const { root, project } = projectWithEmptyRecord(); + rmSync(join(project, ".env")); + writeFileSync(join(root, "outside.env"), ENV_TEXT); + symlinkSync(join(root, "outside.env"), join(project, ".env")); + expectedWithEmptyRecord(root, project); + hostKeeps(root, project, { files: ["gone.html"] }); + expect(cliState(project)).toBe(null); + expect(receivedFileState(project, ".env")).toBe(null); + }, +); + +it("the media-use copy reads the kept copy once more when the host replaces it, as the CLI does", () => { + const { root, project } = projectWithEmptyRecord(); + expectedWithEmptyRecord(root, project); + hostKeeps(root, project, { files: ["index.html"] }); + for (const state of [ + (dir: string) => receivedFiles(dir)(".env"), + (dir: string) => receivedFileState(dir, ".env"), + ]) { + Object.assign(opens, { path: keptFile(root, project), seen: 0, replacing: 1 }); + expect(state(project)).toBeFalsy(); + expect(opens.seen).toBe(2); + } +}); + +it("the media-use copy reads a kept copy named by path and identity once, as the CLI does", () => { + const { root, project } = projectWithEmptyRecord(); + const { dev, ino } = lstatSync(project, { bigint: true }); + expectedWithEmptyRecord(root, project, { dev: String(dev), ino: String(ino) }); + hostKeeps(root, project, { files: ["index.html"], identity: null }); + for (const state of [ + (dir: string) => receivedFiles(dir)(".env"), + (dir: string) => receivedFileState(dir, ".env"), + ]) { + Object.assign(opens, { path: keptFile(root, project), seen: 0, replacing: 0 }); + expect(state(project)).toBeFalsy(); + expect(opens.seen).toBe(1); + } +}); + +it.each([ + [[".env"], "received"], + [null, "received"], + [["index.html"], null], +])( + "the media-use copy reads a kept copy listing %j when it looks only at and above the folder", + (files, expected) => { + const { root, project } = projectWithEmptyRecord(); + expectedWithEmptyRecord(root, project); + hostKeeps(root, project, { files }); + expect(receivedFileState(project, ".env", { atOrAboveOnly: true })).toBe(expected); + }, +); diff --git a/packages/cli/src/utils/privateProjectFile.test.ts b/packages/cli/src/utils/privateProjectFile.test.ts index 8a35e45393..d270bf4031 100644 --- a/packages/cli/src/utils/privateProjectFile.test.ts +++ b/packages/cli/src/utils/privateProjectFile.test.ts @@ -28,6 +28,7 @@ import { } from "./privateProjectFile.js"; const zeroFileIds = vi.hoisted(() => ({ on: false })); +const opens = vi.hoisted(() => ({ path: "", seen: 0, replacing: 0 })); vi.mock("node:fs", async (importOriginal) => { const actual = await importOriginal(); @@ -35,13 +36,25 @@ vi.mock("node:fs", async (importOriginal) => { const stat = actual.lstatSync(path, options as never); return zeroFileIds.on && stat ? Object.assign(stat, { dev: 0n, ino: 0n }) : stat; }) as typeof actual.lstatSync; - return { ...actual, default: { ...actual, lstatSync }, lstatSync }; + const openSync = ((...args: Parameters) => { + const fd = actual.openSync(...args); + if (args[0] !== opens.path) return fd; + opens.seen += 1; + if (opens.replacing > 0) { + opens.replacing -= 1; + actual.copyFileSync(opens.path, `${opens.path}.new`); + actual.renameSync(`${opens.path}.new`, opens.path); + } + return fd; + }) as typeof actual.openSync; + return { ...actual, default: { ...actual, lstatSync, openSync }, lstatSync, openSync }; }); let dir: string | undefined; afterEach(() => { zeroFileIds.on = false; + Object.assign(opens, { path: "", seen: 0, replacing: 0 }); vi.unstubAllEnvs(); if (dir) rmSync(dir, { recursive: true, force: true }); dir = undefined; @@ -400,8 +413,45 @@ describe("a host's received state file", () => { renameSync(app, renamed); expect(receivedFiles(renamed)("index.html")).toBe(true); expect(receivedFiles(renamed)("assets/x.js")).toBe(false); - hostKeeps(app, { ...kept, identity: `${dev}:${ino}:1` }); - expect(receivedFiles(renamed)("index.html")).toBe(false); + }); + + it("counts every file as received when a moved project's kept copy names another folder", () => { + const app = emptiedProject(); + const { dev, ino } = lstatSync(app, { bigint: true }); + hostSays([{ path: app, dev: String(dev), ino: String(ino), received: "expected" }]); + hostKeeps(app, { + project: app, + files: ["index.html"], + seenAt: 1, + identity: `${dev}:${ino}:1`, + }); + const renamed = `${app}-renamed`; + renameSync(app, renamed); + expect(receivedFiles(renamed)("compositions/intro.html")).toBe(true); + expect(receivedRecordStatus(renamed)).toMatchObject({ path: keptFile(app), listed: null }); + }); + + it("reads the kept copy once more when the host replaces it during the first read", () => { + const app = emptiedProject(); + hostSays([{ path: app, received: "expected" }]); + keeps(app, ["index.html"]); + Object.assign(opens, { path: keptFile(app), seen: 0, replacing: 1 }); + const received = receivedFiles(app); + expect(opens.seen).toBe(2); + expect(received("index.html")).toBe(true); + expect(received("compositions/intro.html")).toBe(false); + }); + + it("reads the kept copy once when the state file names the project by path and identity", () => { + const app = emptiedProject(); + const { dev, ino } = lstatSync(app, { bigint: true }); + hostSays([{ path: app, dev: String(dev), ino: String(ino), received: "expected" }]); + keeps(app, ["index.html"]); + Object.assign(opens, { path: keptFile(app), seen: 0, replacing: 0 }); + const received = receivedFiles(app); + expect(opens.seen).toBe(1); + expect(received("index.html")).toBe(true); + expect(received("compositions/intro.html")).toBe(false); }); it("counts every file as received when the kept copy holds no list, and keeps the record's lines", () => { diff --git a/packages/cli/src/utils/privateProjectFile.ts b/packages/cli/src/utils/privateProjectFile.ts index cb4ee055df..2f21bb02ac 100644 --- a/packages/cli/src/utils/privateProjectFile.ts +++ b/packages/cli/src/utils/privateProjectFile.ts @@ -178,22 +178,24 @@ function folderBirth(dir: string): string { } function namings(dir: string, states: HostStates): Naming[] { - const byPath = (states.byPath.get(dir)?.paths ?? []).map((path) => ({ path })); + const byPath = new Set(states.byPath.get(dir)?.paths); + const found = [...byPath].map((path) => ({ path })); const identity = states.byIdentity.size === 0 ? null : fileIdentity(dir); const named = identity === null ? undefined : states.byIdentity.get(identity); - if (!named) return byPath; + if (!named) return found; const kept = `${identity}:${folderBirth(dir)}`; - return [...byPath, ...named.paths.map((path) => ({ path, identity: kept }))]; + const moved = [...new Set(named.paths)].filter((path) => !byPath.has(path)); + return [...found, ...moved.map((path) => ({ path, identity: kept }))]; } const KEPT_RECORDS = "received-records"; const MAX_KEPT_BYTES = 16 * 1024 * 1024; -type KeptFiles = string[] | "all" | null; +type KeptFiles = string[] | "all"; function keptFilesIn(kept: Record, { path, identity }: Naming): KeptFiles { if (kept.project !== path) return "all"; - if (identity !== undefined && kept.identity !== identity) return null; + if (identity !== undefined && kept.identity !== identity) return "all"; const { files } = kept; if (kept.doubted === true || !Array.isArray(files)) return "all"; return files.every((entry) => typeof entry === "string") ? files : "all"; @@ -211,7 +213,7 @@ function keptFilesFor(naming: Naming, text: string): KeptFiles { function hostKeptRecord( stateFile: string, naming: Naming, -): { file: string; files: string[] | "all" } | null { +): { file: string; files: KeptFiles } | null { const name = `${createHash("sha256").update(naming.path).digest("hex")}.json`; const file = join(dirname(stateFile), KEPT_RECORDS, name); try { @@ -219,10 +221,9 @@ function hostKeptRecord( } catch (error) { return (error as NodeJS.ErrnoException).code === "ENOTDIR" ? null : { file, files: "all" }; } - const read = readPlainTail(file, MAX_KEPT_BYTES); + const read = readPlainTail(file, MAX_KEPT_BYTES) ?? readPlainTail(file, MAX_KEPT_BYTES); if (!read?.whole) return onDisk(file) ? { file, files: "all" } : null; - const files = keptFilesFor(naming, read.text); - return files === null ? null : { file, files }; + return { file, files: keptFilesFor(naming, read.text) }; } function withEntriesHostKept(dir: string, listed: unknown[]): ReadRecord { diff --git a/skills-manifest.json b/skills-manifest.json index 6d5e4578a0..998130fc73 100644 --- a/skills-manifest.json +++ b/skills-manifest.json @@ -18,7 +18,7 @@ "files": 13 }, "media-use": { - "hash": "abbb676657aa5be4", + "hash": "9740d923a54e63ba", "files": 130 }, "remotion-to-hyperframes": { diff --git a/skills/media-use/scripts/lib/received-env.mjs b/skills/media-use/scripts/lib/received-env.mjs index f761c33a1c..bfee878e02 100644 --- a/skills/media-use/scripts/lib/received-env.mjs +++ b/skills/media-use/scripts/lib/received-env.mjs @@ -20,6 +20,7 @@ const RECEIVED_RECORD = join(".hyperframes", "received-files.json"); const RECEIVED_CONTENT = "received-content.json"; const RECEIVED_STATE_ENV = "HYPERFRAMES_RECEIVED_STATE_FILE"; const MAX_RECORD_BYTES = 16 * 1024 * 1024; +const KEPT_RECORDS = "received-records"; const SHA256 = /^[0-9a-f]{64}$/; const WALK_MAX_ENTRIES = 20_000; const WALK_MAX_MS = 2_000; @@ -90,6 +91,14 @@ function recordKind(dir) { const strongest = (a, b) => (a === "all" ? a : b); +function addNamed(map, key, received, path) { + const known = map.get(key); + map.set(key, { + received: strongest(known?.received, received), + paths: [...(known?.paths ?? []), path], + }); +} + function hostEntry(value) { const { path, received, dev, ino } = value ?? {}; if (typeof path !== "string" || !isAbsolute(path)) throw new Error("path is not absolute"); @@ -98,18 +107,15 @@ function hostEntry(value) { return { path, received, ...(identity ? { dev, ino } : {}) }; } -function parseHostStates(text) { +function parseHostStates(file, text) { const projects = JSON.parse(text)?.projects; if (!Array.isArray(projects)) throw new Error("projects is not a list"); - const states = { unreadable: false, byPath: new Map(), byIdentity: new Map() }; + const states = { file, unreadable: false, byPath: new Map(), byIdentity: new Map() }; for (const value of projects) { const { path, received, dev, ino } = hostEntry(value); - const key = walkStart(path); - states.byPath.set(key, strongest(states.byPath.get(key), received)); - if (dev !== undefined && ino !== undefined) { - const identity = `${dev}:${ino}`; - states.byIdentity.set(identity, strongest(states.byIdentity.get(identity), received)); - } + addNamed(states.byPath, walkStart(path), received, path); + if (dev !== undefined && ino !== undefined) + addNamed(states.byIdentity, `${dev}:${ino}`, received, path); } return states; } @@ -118,19 +124,84 @@ function hostStates() { const file = process.env[RECEIVED_STATE_ENV]; if (!file) return null; try { - return parseHostStates(readFileSync(file, "utf-8")); + return parseHostStates(file, readFileSync(file, "utf-8")); } catch { - return { unreadable: true, byPath: new Map(), byIdentity: new Map() }; + return { file, unreadable: true, byPath: new Map(), byIdentity: new Map() }; } } function hostState(dir, states) { if (!states) return undefined; if (states.unreadable) return "all"; - const named = states.byPath.get(dir); + const named = states.byPath.get(dir)?.received; if (named || states.byIdentity.size === 0) return named; const identity = fileIdentity(dir); - return identity === null ? undefined : states.byIdentity.get(identity); + return identity === null ? undefined : states.byIdentity.get(identity)?.received; +} + +function folderBirth(dir) { + try { + return String(lstatSync(dir, { bigint: true }).birthtimeNs); + } catch { + return ""; + } +} + +// Each path the state file names the project by; one found only by dev and ino carries the folder's identity. +function namings(dir, states) { + const byPath = new Set(states.byPath.get(dir)?.paths); + const found = [...byPath].map((path) => ({ path })); + const identity = states.byIdentity.size === 0 ? null : fileIdentity(dir); + const named = identity === null ? undefined : states.byIdentity.get(identity); + if (!named) return found; + const kept = `${identity}:${folderBirth(dir)}`; + const moved = [...new Set(named.paths)].filter((path) => !byPath.has(path)); + return [...found, ...moved.map((path) => ({ path, identity: kept }))]; +} + +function keptFilesIn(kept, { path, identity }) { + if (kept.project !== path) return "all"; + if (identity !== undefined && kept.identity !== identity) return "all"; + const { files } = kept; + if (kept.doubted === true || !Array.isArray(files)) return "all"; + return files.every((entry) => typeof entry === "string") ? files : "all"; +} + +function keptFilesFor(naming, text) { + try { + return keptFilesIn(JSON.parse(text) ?? {}, naming); + } catch { + return "all"; + } +} + +// The host's kept copy of the record for one naming: its files, "all" when it cannot be trusted, null when absent. +function hostKeptRecord(stateFile, naming) { + const name = `${createHash("sha256").update(naming.path).digest("hex")}.json`; + const file = join(dirname(stateFile), KEPT_RECORDS, name); + try { + if (!lstatSync(file, { throwIfNoEntry: false })) return null; + } catch (error) { + return error.code === "ENOTDIR" ? null : { file, files: "all" }; + } + const read = readPlainTail(file, MAX_RECORD_BYTES) ?? readPlainTail(file, MAX_RECORD_BYTES); + if (!read?.whole) return onDisk(file) ? { file, files: "all" } : null; + return { file, files: keptFilesFor(naming, read.text) }; +} + +function withEntriesHostKept(dir, listed, states) { + if (!states || hostState(dir, states) !== "expected") return { listed }; + const copies = namings(dir, states).flatMap( + (naming) => hostKeptRecord(states.file, naming) ?? [], + ); + const unread = copies.find((copy) => copy.files === "all"); + if (unread) return { listed, unreadKeptCopy: unread.file }; + const present = new Set(listed.map(String)); + const kept = copies.flatMap((copy) => copy.files); + const back = [...new Set(kept)].filter( + (entry) => !present.has(entry) && onDisk(join(dir, entry)), + ); + return { listed: back.length === 0 ? listed : [...listed, ...back] }; } function allReceivedAtOrAbove(start, states) { @@ -144,12 +215,13 @@ function allReceivedAround(start, states) { if (!states) return false; if (states.unreadable) return true; const overlaps = [...states.byPath].some( - ([path, state]) => state === "all" && (isAtOrUnder(path, start) || isAtOrUnder(start, path)), + ([path, { received }]) => + received === "all" && (isAtOrUnder(path, start) || isAtOrUnder(start, path)), ); return overlaps || allReceivedAtOrAbove(start, states); } -function readRecord(dir, ownFolder, states) { +function readRecordAsWritten(dir, ownFolder, states) { const kind = recordKind(dir); if (kind === "absent" && hostState(dir, states) !== undefined) return { listed: undefined }; if (kind === "absent" || (kind === "unknown" && !ownFolder)) return null; @@ -161,6 +233,13 @@ function readRecord(dir, ownFolder, states) { } } +function readRecord(dir, ownFolder, states) { + const record = readRecordAsWritten(dir, ownFolder, states); + return record && Array.isArray(record.listed) + ? withEntriesHostKept(dir, record.listed, states) + : record; +} + function recordsAtOrAbove(start, states) { const records = []; for (let dir = start; ; ) { @@ -183,7 +262,8 @@ function nestedRecord(start, dir, record) { const listed = Array.isArray(record.listed) ? record.listed.map((entry) => (folder ? `${folder}/${String(entry)}` : String(entry))) : undefined; - return { file: join(dir, RECEIVED_RECORD), root: start, projectDirFromRoot: "", listed }; + const nested = { file: join(dir, RECEIVED_RECORD), root: start, projectDirFromRoot: "", listed }; + return record.unreadKeptCopy ? { ...nested, unreadKeptCopy: record.unreadKeptCopy } : nested; } // The parsers' projectPathSegments: a segment as a case-insensitive file system or Windows opens it. @@ -339,11 +419,12 @@ function receivedRecords(start, atOrAboveOnly) { const below = atOrAboveOnly ? [] : recordsBelow(start, states); if (!below) return null; const records = [...recordsAtOrAbove(start, states), ...below]; + const everyFile = (file) => ({ file, root: start, listed: undefined }); + records.push(...records.flatMap(({ unreadKeptCopy }) => unreadKeptCopy ?? []).map(everyFile)); const all = atOrAboveOnly ? states !== null && (states.unreadable || allReceivedAtOrAbove(start, states)) : allReceivedAround(start, states); - if (all) - records.push({ file: process.env[RECEIVED_STATE_ENV] ?? "", root: start, listed: undefined }); + if (all) records.push(everyFile(process.env[RECEIVED_STATE_ENV] ?? "")); return records; }