mirror of
https://github.com/heygen-com/hyperframes.git
synced 2026-11-19 11:26:02 +00:00
feat(cli): HF_TOOL_CONTEXT artifact events and project lock for agent harnesses (#5520)
* feat(cli): HF_TOOL_CONTEXT artifact events and project lock for agent harnesses * fix(cli): serialize stale project lock takeover Use a short-lived break lock so only one waiter can remove a dead holder's lock. Cover the contention path with six real processes. * fix(cli): hold snapshot lock through finalization Keep the project lock while contact sheets, artifact events, and frame descriptions read captured files so another snapshot cannot replace them. * fix(cli): recover a project lock after the breaker crashes A killed takeover left project.lock.break in place, so later commands waited out the timeout even though both holders were dead. Reclaim that file only when its recorded process is dead, and compare artifact paths after the platform resolves them. * fix(cli): accept a killed breaker process with no signal name Windows ends the process without reporting SIGKILL, so the regression was failing before it checked that the leftover breaker can be reclaimed. * fix(cli): leave a killed project-lock breaker in place Reclaiming a dead breaker unlinks a name another process may just have created, so two commands can hold the lock. The timeout names that file and its pid and says to delete it to continue.
This commit is contained in:
@@ -0,0 +1,161 @@
|
||||
import { spawn } from "node:child_process";
|
||||
import { existsSync, mkdirSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from "node:fs";
|
||||
import { tmpdir } from "node:os";
|
||||
import { dirname, join } from "node:path";
|
||||
import { pathToFileURL } from "node:url";
|
||||
import { afterEach, describe, expect, it } from "vitest";
|
||||
import { ProjectLockTimeout, projectLockPath, withProjectLock } from "./projectLock.js";
|
||||
|
||||
const directories: string[] = [];
|
||||
const project = () => {
|
||||
const directory = mkdtempSync(join(tmpdir(), "hf-project-lock-"));
|
||||
directories.push(directory);
|
||||
return directory;
|
||||
};
|
||||
afterEach(() => {
|
||||
for (const directory of directories.splice(0))
|
||||
rmSync(directory, { recursive: true, force: true });
|
||||
});
|
||||
const locked = (waitMs: number) => ({ projectLock: { waitMs } });
|
||||
const hold = (dir: string, pid: number) => {
|
||||
mkdirSync(dirname(projectLockPath(dir)), { recursive: true });
|
||||
writeFileSync(projectLockPath(dir), JSON.stringify({ pid, command: "check", startedAt: 1 }));
|
||||
};
|
||||
|
||||
describe("withProjectLock", () => {
|
||||
it("runs without a lock file when the capability is absent", async () => {
|
||||
const dir = project();
|
||||
expect(
|
||||
await withProjectLock(dir, "check", async () => existsSync(projectLockPath(dir)), {}),
|
||||
).toBe(false);
|
||||
});
|
||||
|
||||
it("holds the lock while running and releases it afterwards, even on failure", async () => {
|
||||
const dir = project();
|
||||
const seen = await withProjectLock(
|
||||
dir,
|
||||
"snapshot",
|
||||
async () => JSON.parse(readFileSync(projectLockPath(dir), "utf8")),
|
||||
locked(0),
|
||||
);
|
||||
expect(seen).toMatchObject({ pid: process.pid, command: "snapshot" });
|
||||
expect(existsSync(projectLockPath(dir))).toBe(false);
|
||||
await expect(
|
||||
withProjectLock(dir, "check", async () => Promise.reject(new Error("boom")), locked(0)),
|
||||
).rejects.toThrow("boom");
|
||||
expect(existsSync(projectLockPath(dir))).toBe(false);
|
||||
});
|
||||
|
||||
it("serializes concurrent holders in one project", async () => {
|
||||
const dir = project();
|
||||
const order: string[] = [];
|
||||
const task = (name: string) =>
|
||||
withProjectLock(
|
||||
dir,
|
||||
name,
|
||||
async () => {
|
||||
order.push(`${name}:start`);
|
||||
await new Promise((done) => setTimeout(done, 50));
|
||||
order.push(`${name}:end`);
|
||||
},
|
||||
locked(5000),
|
||||
);
|
||||
await Promise.all([task("check"), task("snapshot")]);
|
||||
expect(order).toEqual(["check:start", "check:end", "snapshot:start", "snapshot:end"]);
|
||||
});
|
||||
|
||||
it("waits up to waitMs for a live holder, then refuses with who holds it", async () => {
|
||||
const dir = project();
|
||||
hold(dir, process.pid);
|
||||
const started = Date.now();
|
||||
await expect(withProjectLock(dir, "snapshot", async () => "ran", locked(300))).rejects.toThrow(
|
||||
ProjectLockTimeout,
|
||||
);
|
||||
expect(Date.now() - started).toBeGreaterThanOrEqual(300);
|
||||
await expect(withProjectLock(dir, "snapshot", async () => "ran", locked(0))).rejects.toThrow(
|
||||
`hyperframes check (pid ${process.pid}) is still using this project`,
|
||||
);
|
||||
});
|
||||
|
||||
it("takes over a lock whose process has exited", async () => {
|
||||
const dir = project();
|
||||
hold(dir, 2 ** 22 + 12345);
|
||||
expect(await withProjectLock(dir, "check", async () => "ran", locked(0))).toBe("ran");
|
||||
});
|
||||
|
||||
it("serializes processes that concurrently take over a dead holder", async () => {
|
||||
const dir = project();
|
||||
const events = join(dir, "events.ndjson");
|
||||
hold(dir, 2 ** 22 + 12345);
|
||||
const moduleUrl = pathToFileURL(join(import.meta.dirname, "projectLock.ts")).href;
|
||||
const source = `
|
||||
import { appendFileSync } from "node:fs";
|
||||
import { withProjectLock } from ${JSON.stringify(moduleUrl)};
|
||||
await withProjectLock(
|
||||
${JSON.stringify(dir)},
|
||||
"check",
|
||||
async () => {
|
||||
appendFileSync(${JSON.stringify(events)}, JSON.stringify({ type: "start", pid: process.pid }) + "\\n");
|
||||
await new Promise((done) => setTimeout(done, 50));
|
||||
appendFileSync(${JSON.stringify(events)}, JSON.stringify({ type: "end", pid: process.pid }) + "\\n");
|
||||
},
|
||||
{ projectLock: { waitMs: 5000 } },
|
||||
);
|
||||
`;
|
||||
const children = Array.from({ length: 6 }, () =>
|
||||
spawn(process.execPath, ["--import", "tsx", "--input-type=module", "--eval", source], {
|
||||
stdio: ["ignore", "ignore", "pipe"],
|
||||
}),
|
||||
);
|
||||
const outcomes = await Promise.all(
|
||||
children.map(
|
||||
(child) =>
|
||||
new Promise<{ code: number | null; error: string }>((resolve) => {
|
||||
let error = "";
|
||||
child.stderr.on("data", (chunk) => (error += String(chunk)));
|
||||
child.on("close", (code) => resolve({ code, error }));
|
||||
}),
|
||||
),
|
||||
);
|
||||
expect(outcomes).toEqual(outcomes.map(() => ({ code: 0, error: "" })));
|
||||
const records = readFileSync(events, "utf8")
|
||||
.trim()
|
||||
.split("\n")
|
||||
.map((line) => JSON.parse(line) as { type: "start" | "end"; pid: number });
|
||||
let active = 0;
|
||||
for (const record of records) {
|
||||
active += record.type === "start" ? 1 : -1;
|
||||
expect(active).toBeLessThanOrEqual(1);
|
||||
}
|
||||
expect(active).toBe(0);
|
||||
});
|
||||
|
||||
it("refuses a breaker left by a killed process and names the file", async () => {
|
||||
const dir = project();
|
||||
hold(dir, 2 ** 22 + 12345);
|
||||
const breakPath = `${projectLockPath(dir)}.break`;
|
||||
const source = `
|
||||
import { linkSync, writeFileSync } from "node:fs";
|
||||
const holder = { pid: process.pid, command: "check", startedAt: 1 };
|
||||
const draft = ${JSON.stringify(`${breakPath}.killed`)};
|
||||
writeFileSync(draft, JSON.stringify(holder), { flag: "wx" });
|
||||
linkSync(draft, ${JSON.stringify(breakPath)});
|
||||
process.kill(process.pid, "SIGKILL");
|
||||
`;
|
||||
const child = spawn(process.execPath, ["--input-type=module", "--eval", source], {
|
||||
stdio: "ignore",
|
||||
});
|
||||
const outcome = await new Promise<{ code: number | null; signal: NodeJS.Signals | null }>(
|
||||
(resolve) => {
|
||||
child.on("close", (code, signal) => resolve({ code, signal }));
|
||||
},
|
||||
);
|
||||
expect(outcome.signal === "SIGKILL" || outcome.code !== 0).toBe(true);
|
||||
expect(existsSync(breakPath)).toBe(true);
|
||||
await expect(withProjectLock(dir, "check", async () => "ran", locked(0))).rejects.toThrow(
|
||||
`hyperframes left ${breakPath} (pid ${child.pid}); delete it to continue`,
|
||||
);
|
||||
expect(existsSync(breakPath)).toBe(true);
|
||||
expect(existsSync(projectLockPath(dir))).toBe(true);
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,116 @@
|
||||
import { linkSync, mkdirSync, readFileSync, unlinkSync, writeFileSync } from "node:fs";
|
||||
import { dirname, join } from "node:path";
|
||||
import { setTimeout as sleep } from "node:timers/promises";
|
||||
import { readToolContext, type ToolContext } from "./toolContext.js";
|
||||
|
||||
export class ProjectLockTimeout extends Error {}
|
||||
|
||||
interface Holder {
|
||||
pid: number;
|
||||
command: string;
|
||||
startedAt: number;
|
||||
}
|
||||
|
||||
const POLL_MS = 250;
|
||||
|
||||
export const projectLockPath = (projectDir: string) =>
|
||||
join(projectDir, ".hyperframes", "project.lock");
|
||||
|
||||
function readHolder(path: string): Holder | null {
|
||||
try {
|
||||
return JSON.parse(readFileSync(path, "utf8")) as Holder;
|
||||
} catch {
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
function alive(pid: number): boolean {
|
||||
try {
|
||||
process.kill(pid, 0);
|
||||
return true;
|
||||
} catch (error) {
|
||||
return (error as NodeJS.ErrnoException).code === "EPERM";
|
||||
}
|
||||
}
|
||||
|
||||
const sameHolder = (a: Holder | null, b: Holder) =>
|
||||
a?.pid === b.pid && a.startedAt === b.startedAt && a.command === b.command;
|
||||
|
||||
function breakStaleHolder(path: string, current: Holder, holder: Holder): boolean {
|
||||
const breakPath = `${path}.break`;
|
||||
const breakDraft = `${breakPath}.${holder.pid}.${holder.startedAt}`;
|
||||
let acquired = false;
|
||||
writeFileSync(breakDraft, JSON.stringify(holder), { flag: "wx" });
|
||||
try {
|
||||
try {
|
||||
linkSync(breakDraft, breakPath);
|
||||
} catch (error) {
|
||||
if ((error as NodeJS.ErrnoException).code === "EEXIST") return false;
|
||||
throw error;
|
||||
}
|
||||
if (sameHolder(readHolder(path), current)) {
|
||||
try {
|
||||
unlinkSync(path);
|
||||
} catch (error) {
|
||||
if ((error as NodeJS.ErrnoException).code !== "ENOENT") throw error;
|
||||
}
|
||||
}
|
||||
acquired = tryAcquire(path, holder);
|
||||
} finally {
|
||||
unlinkSync(breakDraft);
|
||||
if (sameHolder(readHolder(breakPath), holder)) unlinkSync(breakPath);
|
||||
}
|
||||
return acquired;
|
||||
}
|
||||
|
||||
function tryAcquire(path: string, holder: Holder): boolean {
|
||||
const draft = `${path}.${holder.pid}.${holder.startedAt}`;
|
||||
writeFileSync(draft, JSON.stringify(holder), { flag: "wx" });
|
||||
try {
|
||||
linkSync(draft, path);
|
||||
return true;
|
||||
} catch (error) {
|
||||
if ((error as NodeJS.ErrnoException).code !== "EEXIST") throw error;
|
||||
} finally {
|
||||
unlinkSync(draft);
|
||||
}
|
||||
const current = readHolder(path);
|
||||
if (current && !alive(current.pid)) return breakStaleHolder(path, current, holder);
|
||||
return false;
|
||||
}
|
||||
|
||||
export async function withProjectLock<T>(
|
||||
projectDir: string,
|
||||
command: string,
|
||||
run: () => Promise<T>,
|
||||
context: ToolContext = readToolContext(),
|
||||
): Promise<T> {
|
||||
if (!context.projectLock) return run();
|
||||
const path = projectLockPath(projectDir);
|
||||
mkdirSync(dirname(path), { recursive: true });
|
||||
const holder: Holder = { pid: process.pid, command, startedAt: Date.now() };
|
||||
const deadline = holder.startedAt + context.projectLock.waitMs;
|
||||
while (!tryAcquire(path, holder)) {
|
||||
const remaining = deadline - Date.now();
|
||||
if (remaining <= 0) {
|
||||
const current = readHolder(path);
|
||||
const breakPath = `${path}.break`;
|
||||
const breaker = readHolder(breakPath);
|
||||
if ((!current || !alive(current.pid)) && breaker && !alive(breaker.pid)) {
|
||||
throw new ProjectLockTimeout(
|
||||
`hyperframes left ${breakPath} (pid ${breaker.pid}); delete it to continue`,
|
||||
);
|
||||
}
|
||||
throw new ProjectLockTimeout(
|
||||
`hyperframes ${current?.command ?? "another command"} (pid ${current?.pid ?? "unknown"}) ` +
|
||||
`is still using this project after ${Math.round(context.projectLock.waitMs / 1000)}s`,
|
||||
);
|
||||
}
|
||||
await sleep(Math.min(POLL_MS, remaining));
|
||||
}
|
||||
try {
|
||||
return await run();
|
||||
} finally {
|
||||
if (sameHolder(readHolder(path), holder)) unlinkSync(path);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,101 @@
|
||||
import { mkdtempSync, readFileSync, rmSync } from "node:fs";
|
||||
import { tmpdir } from "node:os";
|
||||
import { join, resolve } from "node:path";
|
||||
import { afterEach, describe, expect, it, vi } from "vitest";
|
||||
import { readToolContext, reportArtifacts } from "./toolContext.js";
|
||||
|
||||
const directories: string[] = [];
|
||||
const temporary = () => {
|
||||
const directory = mkdtempSync(join(tmpdir(), "hf-tool-context-"));
|
||||
directories.push(directory);
|
||||
return directory;
|
||||
};
|
||||
afterEach(() => {
|
||||
for (const directory of directories.splice(0))
|
||||
rmSync(directory, { recursive: true, force: true });
|
||||
});
|
||||
|
||||
const context = (capabilities: unknown, version: unknown = 1) =>
|
||||
JSON.stringify({ version, capabilities });
|
||||
const eventsAt = (path: string) => ({ transport: { type: "ndjson-file", path } });
|
||||
|
||||
describe("readToolContext", () => {
|
||||
it("is empty without a valid version 1 declaration", () => {
|
||||
for (const raw of [
|
||||
undefined,
|
||||
"",
|
||||
"not json",
|
||||
"[]",
|
||||
context({ events: eventsAt("/tmp/e") }, 2),
|
||||
context(["events"]),
|
||||
])
|
||||
expect(readToolContext(raw)).toEqual({});
|
||||
});
|
||||
|
||||
it("keeps each valid capability and ignores invalid or unknown ones", () => {
|
||||
const raw = context({
|
||||
events: eventsAt("/tmp/events.ndjson"),
|
||||
"project-lock": { waitMs: -1 },
|
||||
"future-capability": { anything: true },
|
||||
});
|
||||
expect(readToolContext(raw)).toEqual({ events: { path: "/tmp/events.ndjson" } });
|
||||
expect(readToolContext(context({ "project-lock": { waitMs: 500 } }))).toEqual({
|
||||
projectLock: { waitMs: 500 },
|
||||
});
|
||||
expect(readToolContext(context({ events: eventsAt("relative.ndjson") }))).toEqual({});
|
||||
expect(
|
||||
readToolContext(context({ events: { transport: { type: "http", path: "/x" } } })),
|
||||
).toEqual({});
|
||||
});
|
||||
});
|
||||
|
||||
describe("reportArtifacts", () => {
|
||||
it("appends one artifact.created line per path, with absolute paths and media types", () => {
|
||||
const path = join(temporary(), "events.ndjson");
|
||||
const events = { events: { path } };
|
||||
reportArtifacts("snapshot", "contact-sheet", ["/p/contact-sheet.jpg"], events);
|
||||
reportArtifacts("check", "finding", ["/p/crop.bin", "/p/crop.png"], events);
|
||||
const lines = readFileSync(path, "utf8")
|
||||
.trimEnd()
|
||||
.split("\n")
|
||||
.map((line) => JSON.parse(line));
|
||||
expect(lines).toEqual([
|
||||
{
|
||||
version: 1,
|
||||
type: "artifact.created",
|
||||
command: "snapshot",
|
||||
artifact: {
|
||||
role: "contact-sheet",
|
||||
path: resolve("/p/contact-sheet.jpg"),
|
||||
mediaType: "image/jpeg",
|
||||
},
|
||||
},
|
||||
{
|
||||
version: 1,
|
||||
type: "artifact.created",
|
||||
command: "check",
|
||||
artifact: {
|
||||
role: "finding",
|
||||
path: resolve("/p/crop.bin"),
|
||||
mediaType: "application/octet-stream",
|
||||
},
|
||||
},
|
||||
{
|
||||
version: 1,
|
||||
type: "artifact.created",
|
||||
command: "check",
|
||||
artifact: { role: "finding", path: resolve("/p/crop.png"), mediaType: "image/png" },
|
||||
},
|
||||
]);
|
||||
});
|
||||
|
||||
it("writes nothing without the events capability, and only warns when the file is unwritable", () => {
|
||||
expect(() => reportArtifacts("snapshot", "frame", ["/p/a.png"], {})).not.toThrow();
|
||||
const error = vi.spyOn(console, "error").mockImplementation(() => undefined);
|
||||
reportArtifacts("snapshot", "frame", ["/p/a.png"], {
|
||||
events: { path: join(temporary(), "missing", "events.ndjson") },
|
||||
});
|
||||
expect(error).toHaveBeenCalledOnce();
|
||||
error.mockRestore();
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,84 @@
|
||||
import { appendFileSync } from "node:fs";
|
||||
import { extname, isAbsolute, resolve } from "node:path";
|
||||
|
||||
const TOOL_CONTEXT_VERSION = 1;
|
||||
|
||||
export interface ToolContext {
|
||||
events?: { path: string };
|
||||
projectLock?: { waitMs: number };
|
||||
}
|
||||
|
||||
export type ArtifactRole = "contact-sheet" | "frame" | "finding";
|
||||
|
||||
interface ArtifactCreatedEvent {
|
||||
version: typeof TOOL_CONTEXT_VERSION;
|
||||
type: "artifact.created";
|
||||
command: string;
|
||||
artifact: { role: ArtifactRole; path: string; mediaType: string };
|
||||
}
|
||||
|
||||
const isRecord = (value: unknown): value is Record<string, unknown> =>
|
||||
typeof value === "object" && value !== null && !Array.isArray(value);
|
||||
|
||||
function parseEvents(value: unknown): ToolContext["events"] {
|
||||
if (!isRecord(value) || !isRecord(value.transport)) return undefined;
|
||||
const { type, path } = value.transport;
|
||||
if (type !== "ndjson-file" || typeof path !== "string" || !isAbsolute(path)) return undefined;
|
||||
return { path };
|
||||
}
|
||||
|
||||
function parseProjectLock(value: unknown): ToolContext["projectLock"] {
|
||||
if (!isRecord(value)) return undefined;
|
||||
const { waitMs } = value;
|
||||
if (typeof waitMs !== "number" || !Number.isFinite(waitMs) || waitMs < 0) return undefined;
|
||||
return { waitMs };
|
||||
}
|
||||
|
||||
export function readToolContext(raw = process.env.HF_TOOL_CONTEXT): ToolContext {
|
||||
if (!raw) return {};
|
||||
let value: unknown;
|
||||
try {
|
||||
value = JSON.parse(raw);
|
||||
} catch {
|
||||
return {};
|
||||
}
|
||||
if (!isRecord(value) || value.version !== TOOL_CONTEXT_VERSION || !isRecord(value.capabilities))
|
||||
return {};
|
||||
const events = parseEvents(value.capabilities.events);
|
||||
const projectLock = parseProjectLock(value.capabilities["project-lock"]);
|
||||
return { ...(events && { events }), ...(projectLock && { projectLock }) };
|
||||
}
|
||||
|
||||
const MEDIA_TYPES: Record<string, string> = {
|
||||
".png": "image/png",
|
||||
".jpg": "image/jpeg",
|
||||
".jpeg": "image/jpeg",
|
||||
};
|
||||
|
||||
export function reportArtifacts(
|
||||
command: string,
|
||||
role: ArtifactRole,
|
||||
paths: readonly string[],
|
||||
context = readToolContext(),
|
||||
): void {
|
||||
if (!context.events || paths.length === 0) return;
|
||||
const lines = paths.map((path) => {
|
||||
const absolute = resolve(path);
|
||||
const event: ArtifactCreatedEvent = {
|
||||
version: TOOL_CONTEXT_VERSION,
|
||||
type: "artifact.created",
|
||||
command,
|
||||
artifact: {
|
||||
role,
|
||||
path: absolute,
|
||||
mediaType: MEDIA_TYPES[extname(absolute).toLowerCase()] ?? "application/octet-stream",
|
||||
},
|
||||
};
|
||||
return JSON.stringify(event) + "\n";
|
||||
});
|
||||
try {
|
||||
appendFileSync(context.events.path, lines.join(""));
|
||||
} catch (error) {
|
||||
console.error(`HF_TOOL_CONTEXT events could not be written: ${(error as Error).message}`);
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user