fix(cli): a host can name projects whose received files are expected or all received (#8025)

* fix(cli): a host can name projects whose received files are expected or all received

HYPERFRAMES_RECEIVED_STATE_FILE points at a JSON file a host app keeps,
listing project folders (by path, and optionally by device and inode) as
"expected" or "all". For an expected project, a missing received-files
record counts every file as received and is not started afresh; for an
all project, every file counts as received whatever the record lists. A
state file that is named but cannot be read counts every file as received.

A record lock dated in the future is taken over like a stale one.

* fix(cli): read host provenance state through one file handle

* fix(cli): canonicalize host state paths like record writes

* test(cli): expect canonical received warning paths
This commit is contained in:
Xuanru Li
2026-11-17 08:54:39 +00:00
committed by GitHub
parent 55291c00c2
commit 5ea86450a2
4 changed files with 227 additions and 8 deletions
+25
View File
@@ -2845,6 +2845,31 @@ it is. A full run also prints each as a `{"event":"left_out",…}`
line on stderr, so a wrapper can ask with `--dry-run --json` before uploading
and still hear about each one during the run.
Which files came in with an imported project is kept in the project's
`.hyperframes/received-files.json`. A project without that file is read as
having received nothing, which suits projects you made yourself. An app that
imports projects for you can tell the CLI more by pointing
`HYPERFRAMES_RECEIVED_STATE_FILE` at a JSON file it keeps:
```json
{
"projects": [
{ "path": "/Users/me/projects/shared", "dev": "16777230", "ino": "4521", "received": "expected" },
{ "path": "/Users/me/projects/draft", "received": "all" }
]
}
```
- `"expected"`: the project must carry the record. If it is missing, every file
counts as received, and the CLI does not start a new record in its place.
- `"all"`: every file in the project counts as received, whatever its record
lists.
- `dev` and `ino` are optional. When given, they also match the project folder
after it is renamed or moved on the same disk.
- The state applies to the project, its subfolders, and any folder that holds it.
- If the variable is set but the file is missing or cannot be parsed, every file
counts as received.
A file from outside the project folder that would be copied (each file a
composition points at, and each project file that is a link to a file outside
the project) is uploaded only when you approve it. Run in a terminal, the
@@ -1,7 +1,9 @@
import { afterEach, describe, expect, it, vi } from "vitest";
import {
existsSync,
lstatSync,
mkdirSync,
renameSync,
mkdtempSync,
readFileSync,
rmSync,
@@ -15,9 +17,11 @@ import { tmpdir } from "node:os";
import { dirname, join } from "node:path";
import { isCredentialOrToolStatePath } from "@hyperframes/core/safe-path";
import {
RECEIVED_STATE_ENV,
isEnvOrGitPath,
isPrivateCompositionFile,
receivedFiles,
receivedRecordStatus,
recordReceivedFiles,
} from "./privateProjectFile.js";
@@ -36,6 +40,7 @@ let dir: string | undefined;
afterEach(() => {
zeroFileIds.on = false;
vi.unstubAllEnvs();
if (dir) rmSync(dir, { recursive: true, force: true });
dir = undefined;
});
@@ -258,6 +263,82 @@ describe("receivedFiles for a received project copied into a folder", () => {
});
});
describe("a host's received state file", () => {
function project(record?: string): string {
const root = mkdtempSync(join(tmpdir(), "hf-received-state-"));
mkdirSync(join(root, "app", "compositions"), { recursive: true });
writeFileSync(join(root, "app", "index.html"), "");
writeFileSync(join(root, "app", "compositions", "intro.html"), "");
if (record !== undefined) {
mkdirSync(join(root, "app", ".hyperframes"));
writeFileSync(join(root, "app", ".hyperframes", "received-files.json"), record);
}
dir = root;
return join(root, "app");
}
function hostSays(projects: Record<string, unknown>[] | string): void {
const file = join(dir as string, "state.json");
writeFileSync(file, typeof projects === "string" ? projects : JSON.stringify({ projects }));
vi.stubEnv(RECEIVED_STATE_ENV, file);
}
it("counts every file as received when an expected record is missing", () => {
const app = project();
hostSays([{ path: app, received: "expected" }]);
expect(receivedFiles(app)("index.html")).toBe(true);
expect(receivedFiles(join(app, "compositions"))("intro.html")).toBe(true);
expect(receivedFiles(dirname(app))("other.html")).toBe(true);
expect(receivedRecordStatus(app)).toMatchObject({ listed: null });
});
it("reads an expected record that is there as written", () => {
const app = project(JSON.stringify({ files: ["compositions/intro.html"] }));
hostSays([{ path: app, received: "expected" }]);
const received = receivedFiles(app);
expect(received("compositions/intro.html")).toBe(true);
expect(received("index.html")).toBe(false);
});
it("counts every file as received in a project named all, whatever its record lists", () => {
const app = project(JSON.stringify({ files: [] }));
hostSays([{ path: app, received: "all" }]);
expect(receivedFiles(app)("index.html")).toBe(true);
expect(receivedFiles(join(app, "compositions"))("intro.html")).toBe(true);
expect(receivedFiles(dirname(app))("other.html")).toBe(true);
});
it("finds a named project under a new name by its folder's identity", () => {
const app = project();
const { dev, ino } = lstatSync(app, { bigint: true });
hostSays([{ path: app, dev: String(dev), ino: String(ino), received: "expected" }]);
const renamed = `${app}-renamed`;
renameSync(app, renamed);
expect(receivedFiles(renamed)("index.html")).toBe(true);
});
it("counts every file as received when the state file cannot be read", () => {
const app = project(JSON.stringify({ files: [] }));
hostSays("{not json");
expect(receivedFiles(app)("index.html")).toBe(true);
vi.stubEnv(RECEIVED_STATE_ENV, join(dir as string, "missing.json"));
expect(receivedFiles(app)("index.html")).toBe(true);
});
it("does not start a fresh record in place of an expected one", () => {
const app = project();
hostSays([{ path: app, received: "expected" }]);
expect(recordReceivedFiles(app, ["index.html"])).toEqual([]);
expect(existsSync(join(app, ".hyperframes", "received-files.json"))).toBe(false);
});
it("leaves a project it does not name as having received nothing", () => {
const app = project();
hostSays([{ path: join(tmpdir(), "hf-some-other-project"), received: "all" }]);
expect(receivedFiles(app)("index.html")).toBe(false);
});
});
describe("recordReceivedFiles", () => {
const record = (root: string) => join(root, ".hyperframes", "received-files.json");
const lock = (root: string) => `${record(root)}.lock`;
@@ -309,6 +390,14 @@ describe("recordReceivedFiles", () => {
expect(listed(root)).toEqual(["hyperframes.json", "mine.html"]);
});
it("takes over a lock dated in the future", () => {
const root = project();
writeFileSync(lock(root), "");
const later = new Date(Date.now() + 60 * 60_000);
utimesSync(lock(root), later, later);
expect(recordReceivedFiles(root, ["mine.html"])).toEqual(["mine.html"]);
});
it("fails rather than write past a lock that stays held", () => {
const root = project();
writeFileSync(lock(root), "");
+109 -4
View File
@@ -1,11 +1,11 @@
import {
closeSync,
fstatSync,
type Dirent,
lstatSync,
openSync,
readdirSync,
readFileSync,
realpathSync,
rmSync,
statSync,
} from "node:fs";
@@ -21,6 +21,7 @@ import {
isCredentialOrToolStatePath,
isStudioPrivatePath,
projectPathSegments,
realpath,
} from "@hyperframes/core/safe-path";
import { registryRoot } from "../registry/publication.js";
import { recordFolder, writeRecord } from "./projectRecords.js";
@@ -44,6 +45,7 @@ const RECEIVED_RECORD = join(".hyperframes", "received-files.json");
const RECEIVED_RECORD_LOCK = `${RECEIVED_RECORD}.lock`;
const LOCK_WAIT_MS = 2_000;
const LOCK_STALE_MS = 10_000;
const LOCK_CLOCK_SLACK_MS = 1_000;
interface ReceivedRecord {
file: string;
@@ -70,8 +72,102 @@ function recordKind(dir: string): RecordKind {
}
}
export const RECEIVED_STATE_ENV = "HYPERFRAMES_RECEIVED_STATE_FILE";
type HostState = "expected" | "all";
interface HostStates {
unreadable: boolean;
byPath: Map<string, HostState>;
byIdentity: Map<string, HostState>;
}
let hostStatesCache: { key: string; states: HostStates } | undefined;
const strongest = (a: HostState | undefined, b: HostState): HostState => (a === "all" ? a : b);
interface HostEntry {
path: string;
received: HostState;
dev?: string;
ino?: string;
}
function hostEntry(value: unknown): HostEntry {
const { path, received, dev, ino } = (value ?? {}) as Record<string, unknown>;
if (typeof path !== "string" || !isAbsolute(path)) throw new Error("path is not absolute");
if (received !== "expected" && received !== "all") throw new Error("unknown state");
return {
path,
received,
...(typeof dev === "string" && typeof ino === "string" && ino !== "0" ? { dev, ino } : {}),
};
}
function parseHostStates(text: string): HostStates {
const projects: unknown = JSON.parse(text)?.projects;
if (!Array.isArray(projects)) throw new Error("projects is not a list");
const states: HostStates = { unreadable: false, byPath: new Map(), byIdentity: new Map() };
for (const value of projects) {
const { path, received, dev, ino } = hostEntry(value);
const key = walkStart(path);
states.byPath.set(key, strongest(states.byPath.get(key), received));
if (dev !== undefined && ino !== undefined) {
const identity = `${dev}:${ino}`;
states.byIdentity.set(identity, strongest(states.byIdentity.get(identity), received));
}
}
return states;
}
function hostStates(): HostStates | null {
const file = process.env[RECEIVED_STATE_ENV];
if (!file) return null;
try {
const handle = openSync(file, "r");
try {
const stat = fstatSync(handle, { bigint: true });
const key = `${file}\0${stat.dev}:${stat.ino}:${stat.size}:${stat.mtimeNs}:${stat.ctimeNs}`;
if (hostStatesCache?.key !== key)
hostStatesCache = { key, states: parseHostStates(readFileSync(handle, "utf-8")) };
return hostStatesCache.states;
} finally {
closeSync(handle);
}
} catch {
return { unreadable: true, byPath: new Map(), byIdentity: new Map() };
}
}
function hostState(dir: string, states = hostStates()): HostState | undefined {
if (!states) return undefined;
if (states.unreadable) return "all";
const named = states.byPath.get(dir);
if (named || states.byIdentity.size === 0) return named;
const identity = fileIdentity(dir);
return identity === null ? undefined : states.byIdentity.get(identity);
}
function allReceivedAtOrAbove(start: string, states: HostStates): boolean {
for (let dir = start; ; dir = dirname(dir)) {
if (hostState(dir, states) === "all") return true;
if (dirname(dir) === dir) return false;
}
}
function allReceivedAround(start: string): boolean {
const states = hostStates();
if (!states) return false;
if (states.unreadable) return true;
const overlaps = [...states.byPath].some(
([path, state]) => state === "all" && (isAtOrUnder(path, start) || isAtOrUnder(start, path)),
);
return overlaps || allReceivedAtOrAbove(start, states);
}
function readRecord(dir: string, ownFolder: boolean): { listed: unknown } | null {
const kind = recordKind(dir);
if (kind === "absent" && hostState(dir) !== undefined) return { listed: undefined };
if (kind === "absent" || (kind === "unknown" && !ownFolder)) return null;
if (kind !== "file") return { listed: undefined };
try {
@@ -95,7 +191,7 @@ function tryRecordLock(lock: string, deadline: number): "taken" | "wait" {
if ((error as NodeJS.ErrnoException).code !== "EEXIST") throw error;
}
const heldFor = Date.now() - (statSync(lock, { throwIfNoEntry: false })?.mtimeMs ?? Date.now());
if (heldFor > LOCK_STALE_MS) rmSync(lock, { force: true });
if (heldFor > LOCK_STALE_MS || heldFor < -LOCK_CLOCK_SLACK_MS) rmSync(lock, { force: true });
else if (Date.now() >= deadline)
throw new Error(`Another program is changing ${RECEIVED_RECORD}; try again.`);
return "wait";
@@ -282,7 +378,7 @@ async function receivedRecordsBelowAsync(start: string): Promise<ReceivedRecord[
function walkStart(projectDir: string): string {
try {
return realpathSync(projectDir);
return realpath(projectDir);
} catch {
return resolve(projectDir);
}
@@ -291,7 +387,16 @@ function walkStart(projectDir: string): string {
function receivedRecords(projectDir: string): ReceivedRecord[] | null {
const start = walkStart(projectDir);
const below = receivedRecordsBelow(start);
return below && [...receivedRecordsAtOrAbove(start), ...below];
if (!below) return null;
const records = [...receivedRecordsAtOrAbove(start), ...below];
if (allReceivedAround(start))
records.push({
file: process.env[RECEIVED_STATE_ENV] ?? "",
root: start,
projectDirFromRoot: "",
listed: undefined,
});
return records;
}
function fileIdentity(path: string): string | null {
@@ -1,3 +1,4 @@
import { realpath } from "@hyperframes/core/safe-path";
import { afterEach, describe, expect, it, vi } from "vitest";
import {
chmodSync,
@@ -5,7 +6,6 @@ import {
linkSync,
mkdirSync,
mkdtempSync,
realpathSync,
rmSync,
symlinkSync,
writeFileSync,
@@ -291,7 +291,7 @@ describe("outsideAssetsAllowed", () => {
const dir = withNested("{not json");
const { allowed, line } = decide(dir);
expect(allowed).toBe(false);
const path = join(realpathSync(dir), NESTED, ".hyperframes", "received-files.json");
const path = join(realpath(dir), NESTED, ".hyperframes", "received-files.json");
expect(line).toContain(`Could not read ${path}`);
});
@@ -364,7 +364,7 @@ describe("outsideAssetsAllowed", () => {
const { allowed, line } = decide(dir);
expect(allowed).toBe(false);
expect(line).toContain(
`Could not read ${join(realpathSync(dir), ".hyperframes", "received-content.json")}`,
`Could not read ${join(realpath(dir), ".hyperframes", "received-content.json")}`,
);
});
@@ -404,7 +404,7 @@ describe("outsideAssetsAllowed", () => {
expect(allowed).toBe(false);
expect(warn).toHaveBeenCalledTimes(1);
expect(line).toContain(
`Could not read ${join(realpathSync(dir), ".hyperframes", "received-files.json")}`,
`Could not read ${join(realpath(dir), ".hyperframes", "received-files.json")}`,
);
expect(line).toContain("../outside/secret.js");
},