mirror of
https://github.com/heygen-com/hyperframes.git
synced 2026-11-18 16:08:56 +00:00
fix(cli): a host can name projects whose received files are expected or all received (#8025)
* fix(cli): a host can name projects whose received files are expected or all received HYPERFRAMES_RECEIVED_STATE_FILE points at a JSON file a host app keeps, listing project folders (by path, and optionally by device and inode) as "expected" or "all". For an expected project, a missing received-files record counts every file as received and is not started afresh; for an all project, every file counts as received whatever the record lists. A state file that is named but cannot be read counts every file as received. A record lock dated in the future is taken over like a stale one. * fix(cli): read host provenance state through one file handle * fix(cli): canonicalize host state paths like record writes * test(cli): expect canonical received warning paths
This commit is contained in:
@@ -2845,6 +2845,31 @@ it is. A full run also prints each as a `{"event":"left_out",…}`
|
||||
line on stderr, so a wrapper can ask with `--dry-run --json` before uploading
|
||||
and still hear about each one during the run.
|
||||
|
||||
Which files came in with an imported project is kept in the project's
|
||||
`.hyperframes/received-files.json`. A project without that file is read as
|
||||
having received nothing, which suits projects you made yourself. An app that
|
||||
imports projects for you can tell the CLI more by pointing
|
||||
`HYPERFRAMES_RECEIVED_STATE_FILE` at a JSON file it keeps:
|
||||
|
||||
```json
|
||||
{
|
||||
"projects": [
|
||||
{ "path": "/Users/me/projects/shared", "dev": "16777230", "ino": "4521", "received": "expected" },
|
||||
{ "path": "/Users/me/projects/draft", "received": "all" }
|
||||
]
|
||||
}
|
||||
```
|
||||
|
||||
- `"expected"`: the project must carry the record. If it is missing, every file
|
||||
counts as received, and the CLI does not start a new record in its place.
|
||||
- `"all"`: every file in the project counts as received, whatever its record
|
||||
lists.
|
||||
- `dev` and `ino` are optional. When given, they also match the project folder
|
||||
after it is renamed or moved on the same disk.
|
||||
- The state applies to the project, its subfolders, and any folder that holds it.
|
||||
- If the variable is set but the file is missing or cannot be parsed, every file
|
||||
counts as received.
|
||||
|
||||
A file from outside the project folder that would be copied (each file a
|
||||
composition points at, and each project file that is a link to a file outside
|
||||
the project) is uploaded only when you approve it. Run in a terminal, the
|
||||
|
||||
@@ -1,7 +1,9 @@
|
||||
import { afterEach, describe, expect, it, vi } from "vitest";
|
||||
import {
|
||||
existsSync,
|
||||
lstatSync,
|
||||
mkdirSync,
|
||||
renameSync,
|
||||
mkdtempSync,
|
||||
readFileSync,
|
||||
rmSync,
|
||||
@@ -15,9 +17,11 @@ import { tmpdir } from "node:os";
|
||||
import { dirname, join } from "node:path";
|
||||
import { isCredentialOrToolStatePath } from "@hyperframes/core/safe-path";
|
||||
import {
|
||||
RECEIVED_STATE_ENV,
|
||||
isEnvOrGitPath,
|
||||
isPrivateCompositionFile,
|
||||
receivedFiles,
|
||||
receivedRecordStatus,
|
||||
recordReceivedFiles,
|
||||
} from "./privateProjectFile.js";
|
||||
|
||||
@@ -36,6 +40,7 @@ let dir: string | undefined;
|
||||
|
||||
afterEach(() => {
|
||||
zeroFileIds.on = false;
|
||||
vi.unstubAllEnvs();
|
||||
if (dir) rmSync(dir, { recursive: true, force: true });
|
||||
dir = undefined;
|
||||
});
|
||||
@@ -258,6 +263,82 @@ describe("receivedFiles for a received project copied into a folder", () => {
|
||||
});
|
||||
});
|
||||
|
||||
describe("a host's received state file", () => {
|
||||
function project(record?: string): string {
|
||||
const root = mkdtempSync(join(tmpdir(), "hf-received-state-"));
|
||||
mkdirSync(join(root, "app", "compositions"), { recursive: true });
|
||||
writeFileSync(join(root, "app", "index.html"), "");
|
||||
writeFileSync(join(root, "app", "compositions", "intro.html"), "");
|
||||
if (record !== undefined) {
|
||||
mkdirSync(join(root, "app", ".hyperframes"));
|
||||
writeFileSync(join(root, "app", ".hyperframes", "received-files.json"), record);
|
||||
}
|
||||
dir = root;
|
||||
return join(root, "app");
|
||||
}
|
||||
|
||||
function hostSays(projects: Record<string, unknown>[] | string): void {
|
||||
const file = join(dir as string, "state.json");
|
||||
writeFileSync(file, typeof projects === "string" ? projects : JSON.stringify({ projects }));
|
||||
vi.stubEnv(RECEIVED_STATE_ENV, file);
|
||||
}
|
||||
|
||||
it("counts every file as received when an expected record is missing", () => {
|
||||
const app = project();
|
||||
hostSays([{ path: app, received: "expected" }]);
|
||||
expect(receivedFiles(app)("index.html")).toBe(true);
|
||||
expect(receivedFiles(join(app, "compositions"))("intro.html")).toBe(true);
|
||||
expect(receivedFiles(dirname(app))("other.html")).toBe(true);
|
||||
expect(receivedRecordStatus(app)).toMatchObject({ listed: null });
|
||||
});
|
||||
|
||||
it("reads an expected record that is there as written", () => {
|
||||
const app = project(JSON.stringify({ files: ["compositions/intro.html"] }));
|
||||
hostSays([{ path: app, received: "expected" }]);
|
||||
const received = receivedFiles(app);
|
||||
expect(received("compositions/intro.html")).toBe(true);
|
||||
expect(received("index.html")).toBe(false);
|
||||
});
|
||||
|
||||
it("counts every file as received in a project named all, whatever its record lists", () => {
|
||||
const app = project(JSON.stringify({ files: [] }));
|
||||
hostSays([{ path: app, received: "all" }]);
|
||||
expect(receivedFiles(app)("index.html")).toBe(true);
|
||||
expect(receivedFiles(join(app, "compositions"))("intro.html")).toBe(true);
|
||||
expect(receivedFiles(dirname(app))("other.html")).toBe(true);
|
||||
});
|
||||
|
||||
it("finds a named project under a new name by its folder's identity", () => {
|
||||
const app = project();
|
||||
const { dev, ino } = lstatSync(app, { bigint: true });
|
||||
hostSays([{ path: app, dev: String(dev), ino: String(ino), received: "expected" }]);
|
||||
const renamed = `${app}-renamed`;
|
||||
renameSync(app, renamed);
|
||||
expect(receivedFiles(renamed)("index.html")).toBe(true);
|
||||
});
|
||||
|
||||
it("counts every file as received when the state file cannot be read", () => {
|
||||
const app = project(JSON.stringify({ files: [] }));
|
||||
hostSays("{not json");
|
||||
expect(receivedFiles(app)("index.html")).toBe(true);
|
||||
vi.stubEnv(RECEIVED_STATE_ENV, join(dir as string, "missing.json"));
|
||||
expect(receivedFiles(app)("index.html")).toBe(true);
|
||||
});
|
||||
|
||||
it("does not start a fresh record in place of an expected one", () => {
|
||||
const app = project();
|
||||
hostSays([{ path: app, received: "expected" }]);
|
||||
expect(recordReceivedFiles(app, ["index.html"])).toEqual([]);
|
||||
expect(existsSync(join(app, ".hyperframes", "received-files.json"))).toBe(false);
|
||||
});
|
||||
|
||||
it("leaves a project it does not name as having received nothing", () => {
|
||||
const app = project();
|
||||
hostSays([{ path: join(tmpdir(), "hf-some-other-project"), received: "all" }]);
|
||||
expect(receivedFiles(app)("index.html")).toBe(false);
|
||||
});
|
||||
});
|
||||
|
||||
describe("recordReceivedFiles", () => {
|
||||
const record = (root: string) => join(root, ".hyperframes", "received-files.json");
|
||||
const lock = (root: string) => `${record(root)}.lock`;
|
||||
@@ -309,6 +390,14 @@ describe("recordReceivedFiles", () => {
|
||||
expect(listed(root)).toEqual(["hyperframes.json", "mine.html"]);
|
||||
});
|
||||
|
||||
it("takes over a lock dated in the future", () => {
|
||||
const root = project();
|
||||
writeFileSync(lock(root), "");
|
||||
const later = new Date(Date.now() + 60 * 60_000);
|
||||
utimesSync(lock(root), later, later);
|
||||
expect(recordReceivedFiles(root, ["mine.html"])).toEqual(["mine.html"]);
|
||||
});
|
||||
|
||||
it("fails rather than write past a lock that stays held", () => {
|
||||
const root = project();
|
||||
writeFileSync(lock(root), "");
|
||||
|
||||
@@ -1,11 +1,11 @@
|
||||
import {
|
||||
closeSync,
|
||||
fstatSync,
|
||||
type Dirent,
|
||||
lstatSync,
|
||||
openSync,
|
||||
readdirSync,
|
||||
readFileSync,
|
||||
realpathSync,
|
||||
rmSync,
|
||||
statSync,
|
||||
} from "node:fs";
|
||||
@@ -21,6 +21,7 @@ import {
|
||||
isCredentialOrToolStatePath,
|
||||
isStudioPrivatePath,
|
||||
projectPathSegments,
|
||||
realpath,
|
||||
} from "@hyperframes/core/safe-path";
|
||||
import { registryRoot } from "../registry/publication.js";
|
||||
import { recordFolder, writeRecord } from "./projectRecords.js";
|
||||
@@ -44,6 +45,7 @@ const RECEIVED_RECORD = join(".hyperframes", "received-files.json");
|
||||
const RECEIVED_RECORD_LOCK = `${RECEIVED_RECORD}.lock`;
|
||||
const LOCK_WAIT_MS = 2_000;
|
||||
const LOCK_STALE_MS = 10_000;
|
||||
const LOCK_CLOCK_SLACK_MS = 1_000;
|
||||
|
||||
interface ReceivedRecord {
|
||||
file: string;
|
||||
@@ -70,8 +72,102 @@ function recordKind(dir: string): RecordKind {
|
||||
}
|
||||
}
|
||||
|
||||
export const RECEIVED_STATE_ENV = "HYPERFRAMES_RECEIVED_STATE_FILE";
|
||||
|
||||
type HostState = "expected" | "all";
|
||||
|
||||
interface HostStates {
|
||||
unreadable: boolean;
|
||||
byPath: Map<string, HostState>;
|
||||
byIdentity: Map<string, HostState>;
|
||||
}
|
||||
|
||||
let hostStatesCache: { key: string; states: HostStates } | undefined;
|
||||
|
||||
const strongest = (a: HostState | undefined, b: HostState): HostState => (a === "all" ? a : b);
|
||||
|
||||
interface HostEntry {
|
||||
path: string;
|
||||
received: HostState;
|
||||
dev?: string;
|
||||
ino?: string;
|
||||
}
|
||||
|
||||
function hostEntry(value: unknown): HostEntry {
|
||||
const { path, received, dev, ino } = (value ?? {}) as Record<string, unknown>;
|
||||
if (typeof path !== "string" || !isAbsolute(path)) throw new Error("path is not absolute");
|
||||
if (received !== "expected" && received !== "all") throw new Error("unknown state");
|
||||
return {
|
||||
path,
|
||||
received,
|
||||
...(typeof dev === "string" && typeof ino === "string" && ino !== "0" ? { dev, ino } : {}),
|
||||
};
|
||||
}
|
||||
|
||||
function parseHostStates(text: string): HostStates {
|
||||
const projects: unknown = JSON.parse(text)?.projects;
|
||||
if (!Array.isArray(projects)) throw new Error("projects is not a list");
|
||||
const states: HostStates = { unreadable: false, byPath: new Map(), byIdentity: new Map() };
|
||||
for (const value of projects) {
|
||||
const { path, received, dev, ino } = hostEntry(value);
|
||||
const key = walkStart(path);
|
||||
states.byPath.set(key, strongest(states.byPath.get(key), received));
|
||||
if (dev !== undefined && ino !== undefined) {
|
||||
const identity = `${dev}:${ino}`;
|
||||
states.byIdentity.set(identity, strongest(states.byIdentity.get(identity), received));
|
||||
}
|
||||
}
|
||||
return states;
|
||||
}
|
||||
|
||||
function hostStates(): HostStates | null {
|
||||
const file = process.env[RECEIVED_STATE_ENV];
|
||||
if (!file) return null;
|
||||
try {
|
||||
const handle = openSync(file, "r");
|
||||
try {
|
||||
const stat = fstatSync(handle, { bigint: true });
|
||||
const key = `${file}\0${stat.dev}:${stat.ino}:${stat.size}:${stat.mtimeNs}:${stat.ctimeNs}`;
|
||||
if (hostStatesCache?.key !== key)
|
||||
hostStatesCache = { key, states: parseHostStates(readFileSync(handle, "utf-8")) };
|
||||
return hostStatesCache.states;
|
||||
} finally {
|
||||
closeSync(handle);
|
||||
}
|
||||
} catch {
|
||||
return { unreadable: true, byPath: new Map(), byIdentity: new Map() };
|
||||
}
|
||||
}
|
||||
|
||||
function hostState(dir: string, states = hostStates()): HostState | undefined {
|
||||
if (!states) return undefined;
|
||||
if (states.unreadable) return "all";
|
||||
const named = states.byPath.get(dir);
|
||||
if (named || states.byIdentity.size === 0) return named;
|
||||
const identity = fileIdentity(dir);
|
||||
return identity === null ? undefined : states.byIdentity.get(identity);
|
||||
}
|
||||
|
||||
function allReceivedAtOrAbove(start: string, states: HostStates): boolean {
|
||||
for (let dir = start; ; dir = dirname(dir)) {
|
||||
if (hostState(dir, states) === "all") return true;
|
||||
if (dirname(dir) === dir) return false;
|
||||
}
|
||||
}
|
||||
|
||||
function allReceivedAround(start: string): boolean {
|
||||
const states = hostStates();
|
||||
if (!states) return false;
|
||||
if (states.unreadable) return true;
|
||||
const overlaps = [...states.byPath].some(
|
||||
([path, state]) => state === "all" && (isAtOrUnder(path, start) || isAtOrUnder(start, path)),
|
||||
);
|
||||
return overlaps || allReceivedAtOrAbove(start, states);
|
||||
}
|
||||
|
||||
function readRecord(dir: string, ownFolder: boolean): { listed: unknown } | null {
|
||||
const kind = recordKind(dir);
|
||||
if (kind === "absent" && hostState(dir) !== undefined) return { listed: undefined };
|
||||
if (kind === "absent" || (kind === "unknown" && !ownFolder)) return null;
|
||||
if (kind !== "file") return { listed: undefined };
|
||||
try {
|
||||
@@ -95,7 +191,7 @@ function tryRecordLock(lock: string, deadline: number): "taken" | "wait" {
|
||||
if ((error as NodeJS.ErrnoException).code !== "EEXIST") throw error;
|
||||
}
|
||||
const heldFor = Date.now() - (statSync(lock, { throwIfNoEntry: false })?.mtimeMs ?? Date.now());
|
||||
if (heldFor > LOCK_STALE_MS) rmSync(lock, { force: true });
|
||||
if (heldFor > LOCK_STALE_MS || heldFor < -LOCK_CLOCK_SLACK_MS) rmSync(lock, { force: true });
|
||||
else if (Date.now() >= deadline)
|
||||
throw new Error(`Another program is changing ${RECEIVED_RECORD}; try again.`);
|
||||
return "wait";
|
||||
@@ -282,7 +378,7 @@ async function receivedRecordsBelowAsync(start: string): Promise<ReceivedRecord[
|
||||
|
||||
function walkStart(projectDir: string): string {
|
||||
try {
|
||||
return realpathSync(projectDir);
|
||||
return realpath(projectDir);
|
||||
} catch {
|
||||
return resolve(projectDir);
|
||||
}
|
||||
@@ -291,7 +387,16 @@ function walkStart(projectDir: string): string {
|
||||
function receivedRecords(projectDir: string): ReceivedRecord[] | null {
|
||||
const start = walkStart(projectDir);
|
||||
const below = receivedRecordsBelow(start);
|
||||
return below && [...receivedRecordsAtOrAbove(start), ...below];
|
||||
if (!below) return null;
|
||||
const records = [...receivedRecordsAtOrAbove(start), ...below];
|
||||
if (allReceivedAround(start))
|
||||
records.push({
|
||||
file: process.env[RECEIVED_STATE_ENV] ?? "",
|
||||
root: start,
|
||||
projectDirFromRoot: "",
|
||||
listed: undefined,
|
||||
});
|
||||
return records;
|
||||
}
|
||||
|
||||
function fileIdentity(path: string): string | null {
|
||||
|
||||
@@ -1,3 +1,4 @@
|
||||
import { realpath } from "@hyperframes/core/safe-path";
|
||||
import { afterEach, describe, expect, it, vi } from "vitest";
|
||||
import {
|
||||
chmodSync,
|
||||
@@ -5,7 +6,6 @@ import {
|
||||
linkSync,
|
||||
mkdirSync,
|
||||
mkdtempSync,
|
||||
realpathSync,
|
||||
rmSync,
|
||||
symlinkSync,
|
||||
writeFileSync,
|
||||
@@ -291,7 +291,7 @@ describe("outsideAssetsAllowed", () => {
|
||||
const dir = withNested("{not json");
|
||||
const { allowed, line } = decide(dir);
|
||||
expect(allowed).toBe(false);
|
||||
const path = join(realpathSync(dir), NESTED, ".hyperframes", "received-files.json");
|
||||
const path = join(realpath(dir), NESTED, ".hyperframes", "received-files.json");
|
||||
expect(line).toContain(`Could not read ${path}`);
|
||||
});
|
||||
|
||||
@@ -364,7 +364,7 @@ describe("outsideAssetsAllowed", () => {
|
||||
const { allowed, line } = decide(dir);
|
||||
expect(allowed).toBe(false);
|
||||
expect(line).toContain(
|
||||
`Could not read ${join(realpathSync(dir), ".hyperframes", "received-content.json")}`,
|
||||
`Could not read ${join(realpath(dir), ".hyperframes", "received-content.json")}`,
|
||||
);
|
||||
});
|
||||
|
||||
@@ -404,7 +404,7 @@ describe("outsideAssetsAllowed", () => {
|
||||
expect(allowed).toBe(false);
|
||||
expect(warn).toHaveBeenCalledTimes(1);
|
||||
expect(line).toContain(
|
||||
`Could not read ${join(realpathSync(dir), ".hyperframes", "received-files.json")}`,
|
||||
`Could not read ${join(realpath(dir), ".hyperframes", "received-files.json")}`,
|
||||
);
|
||||
expect(line).toContain("../outside/secret.js");
|
||||
},
|
||||
|
||||
Reference in New Issue
Block a user