From 5ea86450a21fe80a9817420a2aab84261f41f3d7 Mon Sep 17 00:00:00 2001 From: Xuanru Li <157947275+xuanruli@users.noreply.github.com> Date: Tue, 17 Nov 2026 08:54:39 +0000 Subject: [PATCH] fix(cli): a host can name projects whose received files are expected or all received (#8025) * fix(cli): a host can name projects whose received files are expected or all received HYPERFRAMES_RECEIVED_STATE_FILE points at a JSON file a host app keeps, listing project folders (by path, and optionally by device and inode) as "expected" or "all". For an expected project, a missing received-files record counts every file as received and is not started afresh; for an all project, every file counts as received whatever the record lists. A state file that is named but cannot be read counts every file as received. A record lock dated in the future is taken over like a stale one. * fix(cli): read host provenance state through one file handle * fix(cli): canonicalize host state paths like record writes * test(cli): expect canonical received warning paths --- docs/packages/cli.mdx | 25 ++++ .../cli/src/utils/privateProjectFile.test.ts | 89 ++++++++++++++ packages/cli/src/utils/privateProjectFile.ts | 113 +++++++++++++++++- packages/cli/src/utils/receivedRender.test.ts | 8 +- 4 files changed, 227 insertions(+), 8 deletions(-) diff --git a/docs/packages/cli.mdx b/docs/packages/cli.mdx index c2ac44d3db..f8b308d473 100644 --- a/docs/packages/cli.mdx +++ b/docs/packages/cli.mdx @@ -2845,6 +2845,31 @@ it is. A full run also prints each as a `{"event":"left_out",…}` line on stderr, so a wrapper can ask with `--dry-run --json` before uploading and still hear about each one during the run. +Which files came in with an imported project is kept in the project's +`.hyperframes/received-files.json`. A project without that file is read as +having received nothing, which suits projects you made yourself. An app that +imports projects for you can tell the CLI more by pointing +`HYPERFRAMES_RECEIVED_STATE_FILE` at a JSON file it keeps: + +```json +{ + "projects": [ + { "path": "/Users/me/projects/shared", "dev": "16777230", "ino": "4521", "received": "expected" }, + { "path": "/Users/me/projects/draft", "received": "all" } + ] +} +``` + +- `"expected"`: the project must carry the record. If it is missing, every file + counts as received, and the CLI does not start a new record in its place. +- `"all"`: every file in the project counts as received, whatever its record + lists. +- `dev` and `ino` are optional. When given, they also match the project folder + after it is renamed or moved on the same disk. +- The state applies to the project, its subfolders, and any folder that holds it. +- If the variable is set but the file is missing or cannot be parsed, every file + counts as received. + A file from outside the project folder that would be copied (each file a composition points at, and each project file that is a link to a file outside the project) is uploaded only when you approve it. Run in a terminal, the diff --git a/packages/cli/src/utils/privateProjectFile.test.ts b/packages/cli/src/utils/privateProjectFile.test.ts index acf3ff00a9..6c588ba945 100644 --- a/packages/cli/src/utils/privateProjectFile.test.ts +++ b/packages/cli/src/utils/privateProjectFile.test.ts @@ -1,7 +1,9 @@ import { afterEach, describe, expect, it, vi } from "vitest"; import { existsSync, + lstatSync, mkdirSync, + renameSync, mkdtempSync, readFileSync, rmSync, @@ -15,9 +17,11 @@ import { tmpdir } from "node:os"; import { dirname, join } from "node:path"; import { isCredentialOrToolStatePath } from "@hyperframes/core/safe-path"; import { + RECEIVED_STATE_ENV, isEnvOrGitPath, isPrivateCompositionFile, receivedFiles, + receivedRecordStatus, recordReceivedFiles, } from "./privateProjectFile.js"; @@ -36,6 +40,7 @@ let dir: string | undefined; afterEach(() => { zeroFileIds.on = false; + vi.unstubAllEnvs(); if (dir) rmSync(dir, { recursive: true, force: true }); dir = undefined; }); @@ -258,6 +263,82 @@ describe("receivedFiles for a received project copied into a folder", () => { }); }); +describe("a host's received state file", () => { + function project(record?: string): string { + const root = mkdtempSync(join(tmpdir(), "hf-received-state-")); + mkdirSync(join(root, "app", "compositions"), { recursive: true }); + writeFileSync(join(root, "app", "index.html"), ""); + writeFileSync(join(root, "app", "compositions", "intro.html"), ""); + if (record !== undefined) { + mkdirSync(join(root, "app", ".hyperframes")); + writeFileSync(join(root, "app", ".hyperframes", "received-files.json"), record); + } + dir = root; + return join(root, "app"); + } + + function hostSays(projects: Record[] | string): void { + const file = join(dir as string, "state.json"); + writeFileSync(file, typeof projects === "string" ? projects : JSON.stringify({ projects })); + vi.stubEnv(RECEIVED_STATE_ENV, file); + } + + it("counts every file as received when an expected record is missing", () => { + const app = project(); + hostSays([{ path: app, received: "expected" }]); + expect(receivedFiles(app)("index.html")).toBe(true); + expect(receivedFiles(join(app, "compositions"))("intro.html")).toBe(true); + expect(receivedFiles(dirname(app))("other.html")).toBe(true); + expect(receivedRecordStatus(app)).toMatchObject({ listed: null }); + }); + + it("reads an expected record that is there as written", () => { + const app = project(JSON.stringify({ files: ["compositions/intro.html"] })); + hostSays([{ path: app, received: "expected" }]); + const received = receivedFiles(app); + expect(received("compositions/intro.html")).toBe(true); + expect(received("index.html")).toBe(false); + }); + + it("counts every file as received in a project named all, whatever its record lists", () => { + const app = project(JSON.stringify({ files: [] })); + hostSays([{ path: app, received: "all" }]); + expect(receivedFiles(app)("index.html")).toBe(true); + expect(receivedFiles(join(app, "compositions"))("intro.html")).toBe(true); + expect(receivedFiles(dirname(app))("other.html")).toBe(true); + }); + + it("finds a named project under a new name by its folder's identity", () => { + const app = project(); + const { dev, ino } = lstatSync(app, { bigint: true }); + hostSays([{ path: app, dev: String(dev), ino: String(ino), received: "expected" }]); + const renamed = `${app}-renamed`; + renameSync(app, renamed); + expect(receivedFiles(renamed)("index.html")).toBe(true); + }); + + it("counts every file as received when the state file cannot be read", () => { + const app = project(JSON.stringify({ files: [] })); + hostSays("{not json"); + expect(receivedFiles(app)("index.html")).toBe(true); + vi.stubEnv(RECEIVED_STATE_ENV, join(dir as string, "missing.json")); + expect(receivedFiles(app)("index.html")).toBe(true); + }); + + it("does not start a fresh record in place of an expected one", () => { + const app = project(); + hostSays([{ path: app, received: "expected" }]); + expect(recordReceivedFiles(app, ["index.html"])).toEqual([]); + expect(existsSync(join(app, ".hyperframes", "received-files.json"))).toBe(false); + }); + + it("leaves a project it does not name as having received nothing", () => { + const app = project(); + hostSays([{ path: join(tmpdir(), "hf-some-other-project"), received: "all" }]); + expect(receivedFiles(app)("index.html")).toBe(false); + }); +}); + describe("recordReceivedFiles", () => { const record = (root: string) => join(root, ".hyperframes", "received-files.json"); const lock = (root: string) => `${record(root)}.lock`; @@ -309,6 +390,14 @@ describe("recordReceivedFiles", () => { expect(listed(root)).toEqual(["hyperframes.json", "mine.html"]); }); + it("takes over a lock dated in the future", () => { + const root = project(); + writeFileSync(lock(root), ""); + const later = new Date(Date.now() + 60 * 60_000); + utimesSync(lock(root), later, later); + expect(recordReceivedFiles(root, ["mine.html"])).toEqual(["mine.html"]); + }); + it("fails rather than write past a lock that stays held", () => { const root = project(); writeFileSync(lock(root), ""); diff --git a/packages/cli/src/utils/privateProjectFile.ts b/packages/cli/src/utils/privateProjectFile.ts index bc0264312c..eb3678bad1 100644 --- a/packages/cli/src/utils/privateProjectFile.ts +++ b/packages/cli/src/utils/privateProjectFile.ts @@ -1,11 +1,11 @@ import { closeSync, + fstatSync, type Dirent, lstatSync, openSync, readdirSync, readFileSync, - realpathSync, rmSync, statSync, } from "node:fs"; @@ -21,6 +21,7 @@ import { isCredentialOrToolStatePath, isStudioPrivatePath, projectPathSegments, + realpath, } from "@hyperframes/core/safe-path"; import { registryRoot } from "../registry/publication.js"; import { recordFolder, writeRecord } from "./projectRecords.js"; @@ -44,6 +45,7 @@ const RECEIVED_RECORD = join(".hyperframes", "received-files.json"); const RECEIVED_RECORD_LOCK = `${RECEIVED_RECORD}.lock`; const LOCK_WAIT_MS = 2_000; const LOCK_STALE_MS = 10_000; +const LOCK_CLOCK_SLACK_MS = 1_000; interface ReceivedRecord { file: string; @@ -70,8 +72,102 @@ function recordKind(dir: string): RecordKind { } } +export const RECEIVED_STATE_ENV = "HYPERFRAMES_RECEIVED_STATE_FILE"; + +type HostState = "expected" | "all"; + +interface HostStates { + unreadable: boolean; + byPath: Map; + byIdentity: Map; +} + +let hostStatesCache: { key: string; states: HostStates } | undefined; + +const strongest = (a: HostState | undefined, b: HostState): HostState => (a === "all" ? a : b); + +interface HostEntry { + path: string; + received: HostState; + dev?: string; + ino?: string; +} + +function hostEntry(value: unknown): HostEntry { + const { path, received, dev, ino } = (value ?? {}) as Record; + if (typeof path !== "string" || !isAbsolute(path)) throw new Error("path is not absolute"); + if (received !== "expected" && received !== "all") throw new Error("unknown state"); + return { + path, + received, + ...(typeof dev === "string" && typeof ino === "string" && ino !== "0" ? { dev, ino } : {}), + }; +} + +function parseHostStates(text: string): HostStates { + const projects: unknown = JSON.parse(text)?.projects; + if (!Array.isArray(projects)) throw new Error("projects is not a list"); + const states: HostStates = { unreadable: false, byPath: new Map(), byIdentity: new Map() }; + for (const value of projects) { + const { path, received, dev, ino } = hostEntry(value); + const key = walkStart(path); + states.byPath.set(key, strongest(states.byPath.get(key), received)); + if (dev !== undefined && ino !== undefined) { + const identity = `${dev}:${ino}`; + states.byIdentity.set(identity, strongest(states.byIdentity.get(identity), received)); + } + } + return states; +} + +function hostStates(): HostStates | null { + const file = process.env[RECEIVED_STATE_ENV]; + if (!file) return null; + try { + const handle = openSync(file, "r"); + try { + const stat = fstatSync(handle, { bigint: true }); + const key = `${file}\0${stat.dev}:${stat.ino}:${stat.size}:${stat.mtimeNs}:${stat.ctimeNs}`; + if (hostStatesCache?.key !== key) + hostStatesCache = { key, states: parseHostStates(readFileSync(handle, "utf-8")) }; + return hostStatesCache.states; + } finally { + closeSync(handle); + } + } catch { + return { unreadable: true, byPath: new Map(), byIdentity: new Map() }; + } +} + +function hostState(dir: string, states = hostStates()): HostState | undefined { + if (!states) return undefined; + if (states.unreadable) return "all"; + const named = states.byPath.get(dir); + if (named || states.byIdentity.size === 0) return named; + const identity = fileIdentity(dir); + return identity === null ? undefined : states.byIdentity.get(identity); +} + +function allReceivedAtOrAbove(start: string, states: HostStates): boolean { + for (let dir = start; ; dir = dirname(dir)) { + if (hostState(dir, states) === "all") return true; + if (dirname(dir) === dir) return false; + } +} + +function allReceivedAround(start: string): boolean { + const states = hostStates(); + if (!states) return false; + if (states.unreadable) return true; + const overlaps = [...states.byPath].some( + ([path, state]) => state === "all" && (isAtOrUnder(path, start) || isAtOrUnder(start, path)), + ); + return overlaps || allReceivedAtOrAbove(start, states); +} + function readRecord(dir: string, ownFolder: boolean): { listed: unknown } | null { const kind = recordKind(dir); + if (kind === "absent" && hostState(dir) !== undefined) return { listed: undefined }; if (kind === "absent" || (kind === "unknown" && !ownFolder)) return null; if (kind !== "file") return { listed: undefined }; try { @@ -95,7 +191,7 @@ function tryRecordLock(lock: string, deadline: number): "taken" | "wait" { if ((error as NodeJS.ErrnoException).code !== "EEXIST") throw error; } const heldFor = Date.now() - (statSync(lock, { throwIfNoEntry: false })?.mtimeMs ?? Date.now()); - if (heldFor > LOCK_STALE_MS) rmSync(lock, { force: true }); + if (heldFor > LOCK_STALE_MS || heldFor < -LOCK_CLOCK_SLACK_MS) rmSync(lock, { force: true }); else if (Date.now() >= deadline) throw new Error(`Another program is changing ${RECEIVED_RECORD}; try again.`); return "wait"; @@ -282,7 +378,7 @@ async function receivedRecordsBelowAsync(start: string): Promise { const dir = withNested("{not json"); const { allowed, line } = decide(dir); expect(allowed).toBe(false); - const path = join(realpathSync(dir), NESTED, ".hyperframes", "received-files.json"); + const path = join(realpath(dir), NESTED, ".hyperframes", "received-files.json"); expect(line).toContain(`Could not read ${path}`); }); @@ -364,7 +364,7 @@ describe("outsideAssetsAllowed", () => { const { allowed, line } = decide(dir); expect(allowed).toBe(false); expect(line).toContain( - `Could not read ${join(realpathSync(dir), ".hyperframes", "received-content.json")}`, + `Could not read ${join(realpath(dir), ".hyperframes", "received-content.json")}`, ); }); @@ -404,7 +404,7 @@ describe("outsideAssetsAllowed", () => { expect(allowed).toBe(false); expect(warn).toHaveBeenCalledTimes(1); expect(line).toContain( - `Could not read ${join(realpathSync(dir), ".hyperframes", "received-files.json")}`, + `Could not read ${join(realpath(dir), ".hyperframes", "received-files.json")}`, ); expect(line).toContain("../outside/secret.js"); },