mirror of
https://github.com/heygen-com/hyperframes.git
synced 2026-11-18 07:50:13 +00:00
fix(share): publish fallback locks with their owner (#7516)
* fix(share): publish fallback locks with their owner * fix(share): cover lock reaping races and retry transient renames
This commit is contained in:
@@ -45,7 +45,7 @@ Hosts can sync a renamed project with `client.updateTitle(id, { title, expected_
|
||||
|
||||
Identity remains in the project’s generated review.json metadata (or the existing backend-specific filename); pending export requests remain in the generated review-export.json metadata. No bearer is stored there. A copied folder must deliberately reset sharing identity when creating a new project; a renamed folder retains it. A malformed/unreadable record fails instead of silently registering a new project. The error identifies the metadata file and asks the owner to restore a backup or contact support to reconnect the existing share; deleting the file can discard recovery state or consume another slot.
|
||||
|
||||
Metadata mutations and version uploads use separate cross-process locks under `.hyperframes`. Locks normally publish complete owner metadata with a hard link. Volumes that reject hard links use exclusive file creation instead; contenders wait while the owner metadata is written. An interrupted empty/malformed fallback lock is not automatically stolen. A lock is reclaimed only when the same machine proves its owner process is dead; live, foreign-machine and malformed locks are not stolen. Interrupted upload operations resume with their saved idempotency keys. Concurrent identical uploads re-read the completed version under the lock and do not create another version.
|
||||
Metadata mutations and version uploads use separate cross-process locks under `.hyperframes`. Locks publish complete owner metadata with a hard link or, on volumes that reject hard links, an atomic rename of a directory containing the owner record. The kernel reclaims a recorded owner only when the same machine proves that process is dead. Older clients may report a filesystem error while a fallback directory is held. An empty legacy lock file still requires manual recovery after stopping all sharing processes that can access the project. Interrupted upload operations resume with their saved idempotency keys. Concurrent identical uploads re-read the completed version under the lock and do not create another version.
|
||||
|
||||
`snapshotProject` skips symlinks by default, retaining Desktop watcher, change-set and saved-render compatibility; `snapshotFiles` also skips symlink entries. Set `followSymlinks: true` for a link-aware full snapshot. The CLI-facing `projectFingerprint` opts into following linked media (with cycle detection), so edited linked files invalidate its cached render. Both comparisons ignore HTML ID stamps. Export/history metadata, caches, hidden folders and installed packages are excluded. Hosts must record the fingerprint of the actual rendered source and must not relabel an old render with a newer fingerprint.
|
||||
|
||||
|
||||
@@ -1,9 +1,19 @@
|
||||
// @ts-check
|
||||
import fs, { open, readFile, realpath, unlink, writeFile } from "node:fs/promises";
|
||||
import fs, {
|
||||
lstat,
|
||||
mkdir,
|
||||
open,
|
||||
readdir,
|
||||
readFile,
|
||||
realpath,
|
||||
rmdir,
|
||||
unlink,
|
||||
writeFile,
|
||||
} from "node:fs/promises";
|
||||
import { join } from "node:path";
|
||||
import { hostname } from "node:os";
|
||||
import { randomUUID } from "node:crypto";
|
||||
import { errorCode, mkdirInside } from "./storage.mjs";
|
||||
import { errorCode, isJsonObject, mkdirInside } from "./storage.mjs";
|
||||
|
||||
export class ShareBusyError extends Error {
|
||||
/** @param {string} path */
|
||||
@@ -15,19 +25,27 @@ export class ShareBusyError extends Error {
|
||||
}
|
||||
}
|
||||
/** @typedef {{ token: string, pid: number, hostname: string }} Owner */
|
||||
/** @param {unknown} value @returns {value is Owner} */
|
||||
function isOwner(value) {
|
||||
return (
|
||||
isJsonObject(value) &&
|
||||
typeof value.token === "string" &&
|
||||
/^[a-f0-9-]{36}$/.test(value.token) &&
|
||||
typeof value.pid === "number" &&
|
||||
Number.isInteger(value.pid) &&
|
||||
value.pid > 0 &&
|
||||
typeof value.hostname === "string"
|
||||
);
|
||||
}
|
||||
|
||||
/** @param {string} path @returns {Promise<Owner | null>} */
|
||||
async function ownerAt(path) {
|
||||
try {
|
||||
const value = JSON.parse(await readFile(path, "utf8"));
|
||||
return typeof value.token === "string" &&
|
||||
/^[a-f0-9-]{36}$/.test(value.token) &&
|
||||
Number.isInteger(value.pid) &&
|
||||
value.pid > 0 &&
|
||||
typeof value.hostname === "string"
|
||||
? value
|
||||
: null;
|
||||
return isOwner(value) ? value : null;
|
||||
} catch (error) {
|
||||
if (errorCode(error) === "ENOENT" || error instanceof SyntaxError) return null;
|
||||
if (missing(error) || errorCode(error) === "EISDIR" || error instanceof SyntaxError)
|
||||
return null;
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
@@ -41,8 +59,21 @@ function isDead(owner) {
|
||||
return errorCode(error) === "ESRCH";
|
||||
}
|
||||
}
|
||||
/** A claim scoped to the old owner prevents two stale readers deleting a replacement lock. @param {string} path */
|
||||
async function reap(path) {
|
||||
/** @param {unknown} error */
|
||||
function missing(error) {
|
||||
return errorCode(error) === "ENOENT" || errorCode(error) === "ENOTDIR";
|
||||
}
|
||||
|
||||
/** @param {string} path */
|
||||
function lockAt(path) {
|
||||
return lstat(path).catch((error) => {
|
||||
if (missing(error)) return null;
|
||||
throw error;
|
||||
});
|
||||
}
|
||||
|
||||
/** A claim scoped to the old owner prevents two stale readers deleting a replacement file. @param {string} path */
|
||||
async function reapFile(path) {
|
||||
const old = await ownerAt(path);
|
||||
if (!old || !isDead(old)) return;
|
||||
const claim = `${path}.reap-${old.token}`;
|
||||
@@ -58,26 +89,74 @@ async function reap(path) {
|
||||
await unlink(claim);
|
||||
}
|
||||
}
|
||||
/** @param {string} candidate @param {string} lock @param {Owner} owner */
|
||||
async function claim(candidate, lock, owner) {
|
||||
/** Remove only an empty directory: a replacement is published with its owner already inside. @param {string} path */
|
||||
async function removeEmptyDirectory(path) {
|
||||
await rmdir(path).catch((error) => {
|
||||
if (!missing(error) && !["ENOTEMPTY", "EEXIST"].includes(String(errorCode(error)))) throw error;
|
||||
});
|
||||
}
|
||||
|
||||
/** @param {string} path @param {string} marker */
|
||||
async function releaseDirectory(path, marker) {
|
||||
await unlink(join(path, marker)).catch((error) => {
|
||||
if (!missing(error)) throw error;
|
||||
});
|
||||
await removeEmptyDirectory(path);
|
||||
}
|
||||
|
||||
/** @param {string} path */
|
||||
async function reapDirectory(path) {
|
||||
const entries = await readdir(path, { withFileTypes: true }).catch((error) => {
|
||||
if (missing(error)) return [];
|
||||
throw error;
|
||||
});
|
||||
if (entries.length === 0) return removeEmptyDirectory(path);
|
||||
const [entry] = entries;
|
||||
if (entries.length !== 1 || !entry?.isFile()) return;
|
||||
const owner = await ownerAt(join(path, entry.name));
|
||||
if (owner && entry.name === `${owner.token}.json` && isDead(owner))
|
||||
await releaseDirectory(path, entry.name);
|
||||
}
|
||||
|
||||
/** @param {string} path */
|
||||
async function reap(path) {
|
||||
const stat = await lockAt(path);
|
||||
if (stat?.isFile()) return reapFile(path);
|
||||
if (stat?.isDirectory()) return reapDirectory(path);
|
||||
}
|
||||
|
||||
/** @param {string} draft @param {string} marker @param {string} lock @param {number} deadline */
|
||||
async function claim(draft, marker, lock, deadline) {
|
||||
try {
|
||||
await fs.link(candidate, lock);
|
||||
await fs.link(join(draft, marker), lock);
|
||||
return "file";
|
||||
} catch (error) {
|
||||
if (errorCode(error) === "EEXIST") return null;
|
||||
if (
|
||||
!["EPERM", "ENOTSUP", "EOPNOTSUPP", "EXDEV", "EINVAL", "EISDIR"].includes(
|
||||
!["EPERM", "ENOTSUP", "EOPNOTSUPP", "EXDEV", "EINVAL", "EISDIR", "EACCES"].includes(
|
||||
String(errorCode(error)),
|
||||
)
|
||||
)
|
||||
throw error;
|
||||
const handle = await open(lock, "wx", 0o600);
|
||||
let written = false;
|
||||
try {
|
||||
await handle.writeFile(JSON.stringify(owner));
|
||||
written = true;
|
||||
} finally {
|
||||
await handle.close();
|
||||
if (!written) await unlink(lock);
|
||||
}
|
||||
return claimDirectory(draft, lock, deadline);
|
||||
}
|
||||
|
||||
/** @param {string} draft @param {string} lock @param {number} deadline */
|
||||
async function claimDirectory(draft, lock, deadline) {
|
||||
// A directory rename publishes the owner atomically and cannot replace a file or a nonempty directory.
|
||||
try {
|
||||
await fs.rename(draft, lock);
|
||||
return "directory";
|
||||
} catch (error) {
|
||||
const code = String(errorCode(error));
|
||||
if (["EEXIST", "ENOTEMPTY", "ENOTDIR", "EISDIR"].includes(code)) return null;
|
||||
if (["EPERM", "EACCES", "EBUSY"].includes(code)) {
|
||||
if (await lockAt(lock)) return null;
|
||||
// Windows can deny the rename while another process has a handle open on the draft.
|
||||
if (process.platform === "win32" && Date.now() < deadline) return null;
|
||||
}
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
/** @template T @param {string} projectPath @param {"review" | "export"} lane @param {() => Promise<T>} run
|
||||
@@ -92,20 +171,16 @@ export async function withProjectShareLock(
|
||||
const folder = await mkdirInside(root, ".hyperframes");
|
||||
const lock = join(folder, lane === "review" ? "review.lock" : "review-export.lock");
|
||||
const owner = { token: randomUUID(), pid: process.pid, hostname: hostname() };
|
||||
const candidate = `${lock}.${owner.token}.tmp`;
|
||||
await writeFile(candidate, JSON.stringify(owner), { flag: "wx", mode: 0o600 });
|
||||
const draft = `${lock}.${owner.token}.tmp`;
|
||||
const marker = `${owner.token}.json`;
|
||||
await mkdir(draft, { mode: 0o700 });
|
||||
const deadline = Date.now() + timeoutMs;
|
||||
let held = false;
|
||||
let held = null;
|
||||
try {
|
||||
await writeFile(join(draft, marker), JSON.stringify(owner), { flag: "wx", mode: 0o600 });
|
||||
while (!held) {
|
||||
signal?.throwIfAborted();
|
||||
held = await claim(candidate, lock, owner).then(
|
||||
() => true,
|
||||
(error) => {
|
||||
if (errorCode(error) === "EEXIST") return false;
|
||||
throw error;
|
||||
},
|
||||
);
|
||||
held = await claim(draft, marker, lock, deadline);
|
||||
if (held) break;
|
||||
await reap(lock);
|
||||
if (Date.now() >= deadline) throw new ShareBusyError(lock);
|
||||
@@ -114,7 +189,11 @@ export async function withProjectShareLock(
|
||||
signal?.throwIfAborted();
|
||||
return await run();
|
||||
} finally {
|
||||
if (held && (await ownerAt(lock))?.token === owner.token) await unlink(lock);
|
||||
await unlink(candidate);
|
||||
try {
|
||||
if (held === "directory") await releaseDirectory(lock, marker);
|
||||
if (held === "file" && (await ownerAt(lock))?.token === owner.token) await unlink(lock);
|
||||
} finally {
|
||||
await releaseDirectory(draft, marker);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -335,8 +335,8 @@ test("linked media changes invalidate the rendered source and link cycles termin
|
||||
assert.notEqual(await projectFingerprint(dir), before);
|
||||
});
|
||||
|
||||
for (const code of ["EPERM", "ENOTSUP", "EXDEV", "EINVAL", "EISDIR"]) {
|
||||
test(`exclusive-create fallback serializes operations when hard links fail with ${code}`, async (t) => {
|
||||
for (const code of ["EPERM", "ENOTSUP", "EXDEV", "EINVAL", "EISDIR", "EACCES"]) {
|
||||
test(`directory fallback serializes operations when hard links fail with ${code}`, async (t) => {
|
||||
const dir = await folder(t);
|
||||
t.mock.method(fs, "link", async () => {
|
||||
throw Object.assign(new Error("unsupported"), { code });
|
||||
|
||||
@@ -0,0 +1,218 @@
|
||||
import assert from "node:assert/strict";
|
||||
import { spawn } from "node:child_process";
|
||||
import { randomUUID } from "node:crypto";
|
||||
import { once } from "node:events";
|
||||
import fs from "node:fs/promises";
|
||||
import { syncBuiltinESMExports } from "node:module";
|
||||
import { hostname, tmpdir } from "node:os";
|
||||
import { join } from "node:path";
|
||||
import { test } from "node:test";
|
||||
import { setTimeout } from "node:timers/promises";
|
||||
import { ShareBusyError, withProjectShareLock } from "../src/projectLock.mjs";
|
||||
|
||||
const lockModule = new URL("../src/projectLock.mjs", import.meta.url).href;
|
||||
const pausedAcquisition = `
|
||||
import fs from "node:fs/promises";
|
||||
import { syncBuiltinESMExports } from "node:module";
|
||||
const [moduleUrl, root, lock] = process.argv.slice(1);
|
||||
fs.link = async () => { throw Object.assign(new Error("unsupported"), { code: "ENOTSUP" }); };
|
||||
const pause = async () => {
|
||||
process.send("published");
|
||||
await new Promise((resolve) => process.once("message", resolve));
|
||||
};
|
||||
const open = fs.open;
|
||||
fs.open = async (...args) => {
|
||||
const handle = await open(...args);
|
||||
if (args[0] === lock) await pause();
|
||||
return handle;
|
||||
};
|
||||
const rename = fs.rename;
|
||||
fs.rename = async (...args) => {
|
||||
await rename(...args);
|
||||
if (args[1] === lock) await pause();
|
||||
};
|
||||
syncBuiltinESMExports();
|
||||
const { withProjectShareLock } = await import(moduleUrl);
|
||||
await withProjectShareLock(root, "review", () => { throw new Error("owner should still be paused"); });
|
||||
`;
|
||||
|
||||
async function project(t) {
|
||||
const root = await fs.realpath(await fs.mkdtemp(join(tmpdir(), "share-project-lock-")));
|
||||
t.after(() => fs.rm(root, { recursive: true, force: true }));
|
||||
t.mock.method(fs, "link", async () => {
|
||||
throw Object.assign(new Error("unsupported"), { code: "ENOTSUP" });
|
||||
});
|
||||
return root;
|
||||
}
|
||||
|
||||
async function exitedPid() {
|
||||
const child = spawn(process.execPath, ["-e", ""]);
|
||||
assert.deepEqual(await once(child, "exit"), [0, null]);
|
||||
return child.pid;
|
||||
}
|
||||
|
||||
async function ownerDirectory(path, owner) {
|
||||
await fs.mkdir(path, { recursive: true });
|
||||
const marker = join(path, `${owner.token}.json`);
|
||||
await fs.writeFile(marker, JSON.stringify(owner));
|
||||
return marker;
|
||||
}
|
||||
|
||||
test(
|
||||
"a paused fallback owner is exclusive, and a kill at publication allows recovery",
|
||||
{ timeout: 10_000 },
|
||||
async (t) => {
|
||||
const root = await project(t);
|
||||
const lock = join(root, ".hyperframes", "review.lock");
|
||||
const child = spawn(
|
||||
process.execPath,
|
||||
["--input-type=module", "--eval", pausedAcquisition, lockModule, root, lock],
|
||||
{
|
||||
stdio: ["ignore", "ignore", "inherit", "ipc"],
|
||||
},
|
||||
);
|
||||
const exited = once(child, "exit");
|
||||
try {
|
||||
const [message] = await Promise.race([
|
||||
once(child, "message"),
|
||||
exited.then(([code, signal]) => assert.fail(`lock owner exited early: ${code ?? signal}`)),
|
||||
]);
|
||||
assert.equal(message, "published");
|
||||
await assert.rejects(
|
||||
withProjectShareLock(
|
||||
root,
|
||||
"review",
|
||||
() => assert.fail("paused owner still holds the lock"),
|
||||
{
|
||||
timeoutMs: 0,
|
||||
},
|
||||
),
|
||||
ShareBusyError,
|
||||
);
|
||||
} finally {
|
||||
child.kill("SIGKILL");
|
||||
await exited;
|
||||
}
|
||||
|
||||
let active = 0;
|
||||
const run = () =>
|
||||
withProjectShareLock(
|
||||
root,
|
||||
"review",
|
||||
async () => {
|
||||
assert.equal(++active, 1);
|
||||
await setTimeout(25);
|
||||
active--;
|
||||
},
|
||||
{ timeoutMs: 3000 },
|
||||
);
|
||||
await Promise.all([run(), run()]);
|
||||
await assert.rejects(fs.lstat(lock), { code: "ENOENT" });
|
||||
},
|
||||
);
|
||||
|
||||
test("fallback leaves an empty legacy lock file alone", async (t) => {
|
||||
const root = await project(t);
|
||||
const folder = join(root, ".hyperframes");
|
||||
const lock = join(folder, "review.lock");
|
||||
await fs.mkdir(folder);
|
||||
await fs.writeFile(lock, "");
|
||||
await assert.rejects(
|
||||
withProjectShareLock(root, "review", () => assert.fail("legacy writer may still be alive"), {
|
||||
timeoutMs: 0,
|
||||
}),
|
||||
ShareBusyError,
|
||||
);
|
||||
assert.equal(await fs.readFile(lock, "utf8"), "");
|
||||
assert.deepEqual(await fs.readdir(folder), ["review.lock"]);
|
||||
});
|
||||
|
||||
test("a directory left empty during release does not block the next owner", async (t) => {
|
||||
const root = await project(t);
|
||||
const lock = join(root, ".hyperframes", "review-export.lock");
|
||||
await fs.mkdir(lock, { recursive: true });
|
||||
assert.equal(await withProjectShareLock(root, "export", async () => "recovered"), "recovered");
|
||||
await assert.rejects(fs.lstat(lock), { code: "ENOENT" });
|
||||
});
|
||||
|
||||
test("a delayed reaper cannot remove a replacement owner's directory", async (t) => {
|
||||
const root = await project(t);
|
||||
const lock = join(root, ".hyperframes", "review.lock");
|
||||
const stale = { token: randomUUID(), pid: await exitedPid(), hostname: hostname() };
|
||||
const live = { token: randomUUID(), pid: process.pid, hostname: hostname() };
|
||||
const staleMarker = await ownerDirectory(lock, stale);
|
||||
const replacement = join(root, "replacement");
|
||||
await ownerDirectory(replacement, live);
|
||||
|
||||
const readFile = fs.readFile;
|
||||
const read = t.mock.method(fs, "readFile", async (path, ...args) => {
|
||||
const contents = await readFile(path, ...args);
|
||||
if (path === staleMarker) {
|
||||
await fs.unlink(staleMarker);
|
||||
await fs.rmdir(lock);
|
||||
await fs.rename(replacement, lock);
|
||||
}
|
||||
return contents;
|
||||
});
|
||||
syncBuiltinESMExports();
|
||||
try {
|
||||
await assert.rejects(
|
||||
withProjectShareLock(root, "review", () => assert.fail("replacement owner holds the lock"), {
|
||||
timeoutMs: 100,
|
||||
}),
|
||||
ShareBusyError,
|
||||
);
|
||||
} finally {
|
||||
read.mock.restore();
|
||||
syncBuiltinESMExports();
|
||||
}
|
||||
assert.deepEqual(JSON.parse(await fs.readFile(join(lock, `${live.token}.json`), "utf8")), live);
|
||||
});
|
||||
|
||||
test("legacy files and fallback directories respect a foreign owner with a dead local PID", async (t) => {
|
||||
const root = await project(t);
|
||||
const lock = join(root, ".hyperframes", "review.lock");
|
||||
const owner = { token: randomUUID(), pid: await exitedPid(), hostname: hostname() + "-foreign" };
|
||||
await fs.mkdir(join(root, ".hyperframes"));
|
||||
await fs.writeFile(lock, JSON.stringify(owner));
|
||||
for (const marker of [lock, join(lock, `${owner.token}.json`)]) {
|
||||
if (marker !== lock) {
|
||||
await fs.unlink(lock);
|
||||
await ownerDirectory(lock, owner);
|
||||
}
|
||||
await assert.rejects(
|
||||
withProjectShareLock(root, "review", () => assert.fail("foreign owner holds the lock"), {
|
||||
timeoutMs: 100,
|
||||
}),
|
||||
ShareBusyError,
|
||||
);
|
||||
assert.deepEqual(JSON.parse(await fs.readFile(marker, "utf8")), owner);
|
||||
}
|
||||
});
|
||||
|
||||
test("Windows retries a transient fallback rename failure while the lock path is absent", async (t) => {
|
||||
const root = await project(t);
|
||||
const platform = Object.getOwnPropertyDescriptor(process, "platform");
|
||||
Object.defineProperty(process, "platform", { value: "win32" });
|
||||
t.after(() => Object.defineProperty(process, "platform", platform));
|
||||
const rename = fs.rename;
|
||||
const denied = Object.assign(new Error("open handle on the draft"), { code: "EPERM" });
|
||||
const attempt = t.mock.method(fs, "rename", async (from, to) => {
|
||||
if (attempt.mock.callCount() === 0) throw denied;
|
||||
await rename(from, to);
|
||||
});
|
||||
await assert.equal(
|
||||
await withProjectShareLock(root, "review", async () => "acquired"),
|
||||
"acquired",
|
||||
);
|
||||
assert.equal(attempt.mock.callCount(), 2);
|
||||
t.mock.method(fs, "rename", async () => {
|
||||
throw denied;
|
||||
});
|
||||
await assert.rejects(
|
||||
withProjectShareLock(root, "review", () => assert.fail("rename never succeeds"), {
|
||||
timeoutMs: 0,
|
||||
}),
|
||||
denied,
|
||||
);
|
||||
});
|
||||
@@ -48,8 +48,8 @@
|
||||
"unpacked": 825937
|
||||
},
|
||||
"@hyperframes/share": {
|
||||
"packed": 60820,
|
||||
"unpacked": 240537
|
||||
"packed": 64831,
|
||||
"unpacked": 257736
|
||||
},
|
||||
"@hyperframes/studio": {
|
||||
"packed": 5998328,
|
||||
|
||||
Reference in New Issue
Block a user