mirror of
https://github.com/heygen-com/hyperframes.git
synced 2026-11-17 15:24:19 +00:00
fix(studio-server): cancel background media adoption on close
This commit is contained in:
@@ -1,13 +1,14 @@
|
||||
import { createHash, randomUUID } from "node:crypto";
|
||||
import { constants, createReadStream, renameSync } from "node:fs";
|
||||
import { constants, createReadStream, createWriteStream, renameSync } from "node:fs";
|
||||
import { copyFile, mkdir, readdir, readFile, rename, rm, stat } from "node:fs/promises";
|
||||
import { pipeline } from "node:stream/promises";
|
||||
import { dirname, join } from "node:path";
|
||||
import { atomicTempPath } from "@hyperframes/core/atomic-file";
|
||||
|
||||
/** File contents stored once by sha256, text and binary alike. */
|
||||
export interface BlobStore {
|
||||
/** Copies the file in (a clone where the file system can) and returns the copy's hash. */
|
||||
put(absPath: string): Promise<string>;
|
||||
/** Archives the file and returns its hash; a signal makes copy and hash cancellable. */
|
||||
put(absPath: string, signal?: AbortSignal): Promise<string>;
|
||||
has(hash: string): boolean;
|
||||
read(hash: string): Promise<Buffer>;
|
||||
/** Writes the blob's bytes to `absPath` by clone-or-copy and rename, so a reader never sees half a file. */
|
||||
@@ -19,9 +20,9 @@ export interface BlobStore {
|
||||
|
||||
const BLOB_HASH = /^[0-9a-f]{64}$/;
|
||||
|
||||
async function hashFile(path: string): Promise<string> {
|
||||
async function hashFile(path: string, signal?: AbortSignal): Promise<string> {
|
||||
const hash = createHash("sha256");
|
||||
for await (const chunk of createReadStream(path)) hash.update(chunk);
|
||||
for await (const chunk of createReadStream(path, { signal })) hash.update(chunk);
|
||||
return hash.digest("hex");
|
||||
}
|
||||
|
||||
@@ -52,23 +53,30 @@ export async function openBlobStore(dir: string): Promise<BlobStore> {
|
||||
let total = [...sizes.values()].reduce((sum, size) => sum + size, 0);
|
||||
|
||||
return {
|
||||
async put(absPath) {
|
||||
async put(absPath, signal) {
|
||||
// Hash the copy, not the source, so a write racing the copy can never file bytes under the wrong hash.
|
||||
const temp = join(dir, `incoming-${randomUUID()}`);
|
||||
// Recreated if removed while open, so a missing folder is never mistaken for a deleted project file.
|
||||
await mkdir(dir, { recursive: true });
|
||||
await copyFile(absPath, temp, constants.COPYFILE_FICLONE);
|
||||
const hash = await hashFile(temp);
|
||||
if (sizes.has(hash)) {
|
||||
await rm(temp, { force: true });
|
||||
try {
|
||||
if (signal) {
|
||||
await pipeline(createReadStream(absPath), createWriteStream(temp), { signal });
|
||||
} else {
|
||||
await copyFile(absPath, temp, constants.COPYFILE_FICLONE);
|
||||
}
|
||||
const hash = await hashFile(temp, signal);
|
||||
signal?.throwIfAborted();
|
||||
if (sizes.has(hash)) return hash;
|
||||
await mkdir(dirname(pathOf(hash)), { recursive: true });
|
||||
const size = (await stat(temp)).size;
|
||||
signal?.throwIfAborted();
|
||||
await rename(temp, pathOf(hash));
|
||||
sizes.set(hash, size);
|
||||
total += size;
|
||||
return hash;
|
||||
} finally {
|
||||
await rm(temp, { force: true });
|
||||
}
|
||||
await mkdir(dirname(pathOf(hash)), { recursive: true });
|
||||
const size = (await stat(temp)).size;
|
||||
await rename(temp, pathOf(hash));
|
||||
sizes.set(hash, size);
|
||||
total += size;
|
||||
return hash;
|
||||
},
|
||||
has: (hash) => sizes.has(hash),
|
||||
read: async (hash) => readFile(pathOf(hash)),
|
||||
|
||||
@@ -8,7 +8,7 @@ export interface HistoryWho {
|
||||
name: string;
|
||||
}
|
||||
|
||||
/** A file's sha256 before and after; null is "did not exist", so a create or delete undoes like any edit. */
|
||||
/** A file's sha256 before and after; null is "did not exist" unless the entry names an unavailable original. */
|
||||
export interface HistoryFileChange {
|
||||
path: string;
|
||||
before: string | null;
|
||||
@@ -24,6 +24,8 @@ export interface HistoryEntry {
|
||||
startedAt: number;
|
||||
endedAt: number;
|
||||
files: HistoryFileChange[];
|
||||
/** These files existed before this entry, but their original bytes were never archived. */
|
||||
unavailableBefore?: string[];
|
||||
/** Set on an undo: the entry it reverted. Redo is undoing the undo. */
|
||||
undoes?: string;
|
||||
/** Set on a restore: the point (an entry id, or START) the files were made equal to. */
|
||||
|
||||
@@ -0,0 +1,298 @@
|
||||
// @vitest-environment node
|
||||
import {
|
||||
mkdirSync,
|
||||
mkdtempSync,
|
||||
writeFileSync,
|
||||
readdirSync,
|
||||
readFileSync,
|
||||
rmSync,
|
||||
statSync,
|
||||
} from "node:fs";
|
||||
import { createHash } from "node:crypto";
|
||||
import { HistoryBusyError } from "./ownerLock";
|
||||
import { tmpdir } from "node:os";
|
||||
import { join } from "node:path";
|
||||
import { afterEach, expect, it, vi } from "vitest";
|
||||
import { openProjectHistory } from "./projectHistory";
|
||||
import { START } from "./historyLog";
|
||||
|
||||
const archive = vi.hoisted(() => ({
|
||||
hold: false,
|
||||
entered: "",
|
||||
hash: false,
|
||||
failCleanup: false,
|
||||
partialPath: "",
|
||||
aborted: false,
|
||||
release: null as (() => void) | null,
|
||||
}));
|
||||
vi.mock("node:fs", async (importOriginal) => {
|
||||
const fs = await importOriginal<typeof import("node:fs")>();
|
||||
const { Readable, addAbortSignal } = await import("node:stream");
|
||||
return {
|
||||
...fs,
|
||||
createReadStream: (path: string, options?: Parameters<typeof fs.createReadStream>[1]) => {
|
||||
const heldPath = archive.hash
|
||||
? String(path).includes("incoming-")
|
||||
: String(path).endsWith(".mp4");
|
||||
if (!archive.hold || !heldPath) return fs.createReadStream(path, options);
|
||||
archive.entered = archive.hash ? "hash" : "copy";
|
||||
let sent = false;
|
||||
const stream = new Readable({
|
||||
read() {
|
||||
if (sent) return;
|
||||
sent = true;
|
||||
this.push(Buffer.alloc(65536, 7));
|
||||
},
|
||||
});
|
||||
archive.release = () => {
|
||||
stream.push(Buffer.alloc(2 * 1024 ** 2 - 65536, 7));
|
||||
stream.push(null);
|
||||
};
|
||||
if (typeof options === "object" && options?.signal) addAbortSignal(options.signal, stream);
|
||||
stream.on("close", () => {
|
||||
archive.aborted = true;
|
||||
});
|
||||
return stream;
|
||||
},
|
||||
};
|
||||
});
|
||||
vi.mock("node:fs/promises", async (importOriginal) => {
|
||||
const fs = await importOriginal<typeof import("node:fs/promises")>();
|
||||
return {
|
||||
...fs,
|
||||
rm: async (...args: Parameters<typeof fs.rm>) => {
|
||||
if (archive.failCleanup && String(args[0]) === archive.partialPath) {
|
||||
throw Object.assign(new Error("Cannot remove partial archive"), { code: "EPERM" });
|
||||
}
|
||||
return fs.rm(...args);
|
||||
},
|
||||
copyFile: async (...args: Parameters<typeof fs.copyFile>) => {
|
||||
if (archive.hold && !archive.hash && String(args[0]).endsWith(".mp4")) {
|
||||
await fs.writeFile(args[1], Buffer.alloc(65536, 7));
|
||||
await new Promise<void>((resolve) => {
|
||||
archive.release = resolve;
|
||||
});
|
||||
}
|
||||
return fs.copyFile(...args);
|
||||
},
|
||||
};
|
||||
});
|
||||
const roots: string[] = [];
|
||||
afterEach(() => {
|
||||
archive.hold = false;
|
||||
archive.hash = false;
|
||||
archive.entered = "";
|
||||
archive.failCleanup = false;
|
||||
archive.release = null;
|
||||
archive.aborted = false;
|
||||
for (const dir of roots.splice(0)) rmSync(dir, { recursive: true, force: true });
|
||||
});
|
||||
|
||||
async function heldProject(names = ["clip.mp4"], onError?: (error: Error) => void) {
|
||||
const root = mkdtempSync(join(tmpdir(), "hf-held-adoption-"));
|
||||
roots.push(root);
|
||||
const projectDir = join(root, "project");
|
||||
mkdirSync(projectDir);
|
||||
writeFileSync(join(projectDir, "index.html"), "before");
|
||||
for (const name of names) writeFileSync(join(projectDir, name), Buffer.alloc(2 * 1024 ** 2, 7));
|
||||
archive.hold = true;
|
||||
const historyRoot = join(root, "history");
|
||||
const history = await openProjectHistory({ projectDir, historyRoot, onError, quietMs: 30 });
|
||||
const home = join(historyRoot, history.projectId);
|
||||
return { root, projectDir, historyRoot, history, home, blobs: join(home, "blobs") };
|
||||
}
|
||||
|
||||
it("close cancels a held adoption, flushes text edits and resumes pending media on reopen", async () => {
|
||||
const onError = vi.fn();
|
||||
const { history, historyRoot, home, blobs, projectDir } = await heldProject(undefined, onError);
|
||||
try {
|
||||
await vi.waitFor(() => {
|
||||
const partial = readdirSync(blobs).find((name) => name.startsWith("incoming-"));
|
||||
expect(partial).toBeDefined();
|
||||
expect(statSync(join(blobs, partial!)).size).toBeGreaterThan(0);
|
||||
});
|
||||
writeFileSync(join(projectDir, "index.html"), "after");
|
||||
let closed = false;
|
||||
const closing = history.close().then(() => {
|
||||
closed = true;
|
||||
});
|
||||
try {
|
||||
await vi.waitFor(() => expect(closed).toBe(true));
|
||||
} finally {
|
||||
archive.release?.();
|
||||
await closing;
|
||||
}
|
||||
expect(archive.aborted).toBe(true);
|
||||
expect(JSON.parse(readFileSync(join(home, "adopting.json"), "utf8"))).toEqual(["clip.mp4"]);
|
||||
expect(onError).not.toHaveBeenCalled();
|
||||
const log = readFileSync(join(home, "log.jsonl"), "utf8")
|
||||
.trim()
|
||||
.split("\n")
|
||||
.map((line) => JSON.parse(line));
|
||||
const textEdit = log
|
||||
.filter((record) => record.type === "entry")
|
||||
.flatMap((record) => record.entry.files)
|
||||
.find((file) => file.path === "index.html");
|
||||
expect(textEdit).toMatchObject({
|
||||
before: createHash("sha256").update("before").digest("hex"),
|
||||
after: createHash("sha256").update("after").digest("hex"),
|
||||
});
|
||||
expect(readdirSync(blobs).some((name) => name.startsWith("incoming-"))).toBe(false);
|
||||
archive.hold = false;
|
||||
const reopened = await openProjectHistory({ projectDir, historyRoot, quietMs: 30 });
|
||||
try {
|
||||
await vi.waitFor(() => expect(reopened.peek(START)).toHaveProperty(["clip.mp4"]));
|
||||
expect(reopened.list().flatMap((entry) => entry.files.map((file) => file.path))).toContain(
|
||||
"index.html",
|
||||
);
|
||||
expect(reopened.peek(START)?.["clip.mp4"]).toMatch(/^[0-9a-f]{64}$/);
|
||||
} finally {
|
||||
await reopened.close();
|
||||
}
|
||||
} finally {
|
||||
archive.hold = false;
|
||||
await history.close();
|
||||
}
|
||||
}, 5000);
|
||||
|
||||
it.each(["copy", "hash"])(
|
||||
"cancelled %s leaves no partial blob before ownership release",
|
||||
async (phase) => {
|
||||
const { openBlobStore } = await import("./blobStore");
|
||||
const root = mkdtempSync(join(tmpdir(), "hf-cancel-blob-"));
|
||||
roots.push(root);
|
||||
const source = join(root, "clip.mp4");
|
||||
writeFileSync(source, Buffer.alloc(2 * 1024 ** 2, 7));
|
||||
const dir = join(root, "blobs");
|
||||
const store = await openBlobStore(dir);
|
||||
archive.hold = true;
|
||||
archive.hash = phase === "hash";
|
||||
const controller = new AbortController();
|
||||
const pending = store.put(source, controller.signal);
|
||||
let error: unknown;
|
||||
const settled = pending.catch((caught) => {
|
||||
error = caught;
|
||||
});
|
||||
await vi.waitFor(() =>
|
||||
expect(readdirSync(dir).some((name) => name.startsWith("incoming-"))).toBe(true),
|
||||
);
|
||||
await vi.waitFor(() => expect(archive.entered).toBe(phase));
|
||||
controller.abort();
|
||||
try {
|
||||
await vi.waitFor(() => expect(error).toMatchObject({ name: "AbortError" }));
|
||||
} finally {
|
||||
archive.release?.();
|
||||
await settled;
|
||||
}
|
||||
expect(readdirSync(dir)).toEqual([]);
|
||||
expect(store.bytes()).toBe(0);
|
||||
},
|
||||
);
|
||||
|
||||
it("failed cancellation cleanup flushes edits but refuses to release ownership", async () => {
|
||||
const { history, historyRoot, home, blobs, projectDir } = await heldProject();
|
||||
await vi.waitFor(() =>
|
||||
expect(readdirSync(blobs).some((name) => name.startsWith("incoming-"))).toBe(true),
|
||||
);
|
||||
writeFileSync(join(projectDir, "index.html"), "after");
|
||||
archive.partialPath = join(
|
||||
blobs,
|
||||
readdirSync(blobs).find((name) => name.startsWith("incoming-"))!,
|
||||
);
|
||||
archive.failCleanup = true;
|
||||
await expect(history.close()).rejects.toMatchObject({ code: "EPERM" });
|
||||
expect(readdirSync(blobs).some((name) => name.startsWith("incoming-"))).toBe(true);
|
||||
expect(readFileSync(join(home, "log.jsonl"), "utf8")).toContain(
|
||||
createHash("sha256").update("after").digest("hex"),
|
||||
);
|
||||
await expect(
|
||||
openProjectHistory({ projectDir, historyRoot, ownerWaitMs: 0 }),
|
||||
).rejects.toBeInstanceOf(HistoryBusyError);
|
||||
});
|
||||
|
||||
it.each(["replace", "delete", "missing-cache", "malformed-cache"])(
|
||||
"refuses unknown original media after %s while closed",
|
||||
async (change) => {
|
||||
const { root, history, historyRoot, home, projectDir } = await heldProject();
|
||||
const clip = join(projectDir, "clip.mp4");
|
||||
await vi.waitFor(() =>
|
||||
expect(readdirSync(join(home, "blobs")).some((name) => name.startsWith("incoming-"))).toBe(
|
||||
true,
|
||||
),
|
||||
);
|
||||
await history.close();
|
||||
if (change === "delete") rmSync(clip);
|
||||
else writeFileSync(clip, Buffer.alloc(2 * 1024 ** 2, 9));
|
||||
if (change === "missing-cache") rmSync(join(home, "stat.json"));
|
||||
if (change === "malformed-cache")
|
||||
writeFileSync(
|
||||
join(home, "stat.json"),
|
||||
JSON.stringify({ "clip.mp4": { hash: "invalid", stat: null } }),
|
||||
);
|
||||
archive.hold = false;
|
||||
const reopened = await openProjectHistory({ projectDir, historyRoot, quietMs: 30 });
|
||||
const who = { kind: "person" as const, name: "You" };
|
||||
try {
|
||||
await vi.waitFor(() =>
|
||||
expect(reopened.list().some((entry) => entry.unavailableBefore?.includes("clip.mp4"))).toBe(
|
||||
true,
|
||||
),
|
||||
);
|
||||
const entry = reopened.list().find((entry) => entry.unavailableBefore?.includes("clip.mp4"))!;
|
||||
await expect(reopened.step("back", who)).rejects.toThrow(
|
||||
"original media version is unavailable",
|
||||
);
|
||||
await expect(reopened.undo(entry.id, { who })).rejects.toThrow(
|
||||
"original media version is unavailable",
|
||||
);
|
||||
await expect(reopened.restore(START, who)).rejects.toThrow(
|
||||
"original media version is unavailable",
|
||||
);
|
||||
expect(() => reopened.peek(START)).toThrow("original media version is unavailable");
|
||||
const checkout = join(root, "checkout");
|
||||
mkdirSync(checkout);
|
||||
await expect(reopened.checkout(entry.id, "before", checkout)).rejects.toThrow(
|
||||
"original media version is unavailable",
|
||||
);
|
||||
expect(readdirSync(checkout)).toEqual([]);
|
||||
if (change === "delete") expect(readdirSync(projectDir)).not.toContain("clip.mp4");
|
||||
else expect(readFileSync(clip).equals(Buffer.alloc(2 * 1024 ** 2, 9))).toBe(true);
|
||||
} finally {
|
||||
await reopened.close();
|
||||
}
|
||||
const again = await openProjectHistory({ projectDir, historyRoot, quietMs: 30 });
|
||||
try {
|
||||
await expect(again.step("back", who)).rejects.toThrow(
|
||||
"original media version is unavailable",
|
||||
);
|
||||
} finally {
|
||||
await again.close();
|
||||
}
|
||||
},
|
||||
);
|
||||
|
||||
it("a later media replacement while serial adoption waits is never filed as the original baseline", async () => {
|
||||
const { history, projectDir } = await heldProject(["a.mp4", "b.mp4"]);
|
||||
try {
|
||||
await vi.waitFor(() => expect(archive.entered).toBe("copy"));
|
||||
writeFileSync(join(projectDir, "b.mp4"), Buffer.alloc(2 * 1024 ** 2, 9));
|
||||
archive.hold = false;
|
||||
archive.release!();
|
||||
await vi.waitFor(() =>
|
||||
expect(history.list().some((entry) => entry.unavailableBefore?.includes("b.mp4"))).toBe(true),
|
||||
);
|
||||
const entry = history.list().find((entry) => entry.unavailableBefore?.includes("b.mp4"))!;
|
||||
expect(entry.files.find((file) => file.path === "b.mp4")?.after).toBe(
|
||||
createHash("sha256")
|
||||
.update(Buffer.alloc(2 * 1024 ** 2, 9))
|
||||
.digest("hex"),
|
||||
);
|
||||
await expect(history.step("back", { kind: "person", name: "You" })).rejects.toThrow(
|
||||
"original media version is unavailable",
|
||||
);
|
||||
} finally {
|
||||
archive.hold = false;
|
||||
archive.release?.();
|
||||
await history.close();
|
||||
}
|
||||
});
|
||||
@@ -14,6 +14,7 @@ import {
|
||||
utimesSync,
|
||||
writeFileSync,
|
||||
} from "node:fs";
|
||||
import { once } from "node:events";
|
||||
import { spawnSync } from "node:child_process";
|
||||
import { tmpdir } from "node:os";
|
||||
import { basename, dirname, join, relative } from "node:path";
|
||||
@@ -29,6 +30,7 @@ const mediaCopy = vi.hoisted(() => ({
|
||||
held: null as Promise<void> | null,
|
||||
stored: null as (() => Promise<unknown>) | null,
|
||||
copies: [] as string[],
|
||||
entered: 0,
|
||||
fails: null as ((copy: number) => boolean) | null,
|
||||
}));
|
||||
vi.mock("./blobStore", async (importOriginal) => {
|
||||
@@ -39,13 +41,22 @@ vi.mock("./blobStore", async (importOriginal) => {
|
||||
const store = await real.openBlobStore(dir);
|
||||
return {
|
||||
...store,
|
||||
put: async (path: string) => {
|
||||
put: async (path: string, signal?: AbortSignal) => {
|
||||
if (path.endsWith(".mp4")) {
|
||||
await mediaCopy.held;
|
||||
mediaCopy.entered++;
|
||||
if (signal && mediaCopy.held) {
|
||||
signal.throwIfAborted();
|
||||
await Promise.race([
|
||||
mediaCopy.held,
|
||||
once(signal, "abort").then(() => {
|
||||
throw signal.reason;
|
||||
}),
|
||||
]);
|
||||
} else await mediaCopy.held;
|
||||
mediaCopy.copies.push(basename(path));
|
||||
if (mediaCopy.fails?.(mediaCopy.copies.length)) throw new Error("The disk is full.");
|
||||
}
|
||||
const hash = await store.put(path);
|
||||
const hash = await store.put(path, signal);
|
||||
if (path.endsWith(".mp4")) await mediaCopy.stored?.();
|
||||
return hash;
|
||||
},
|
||||
@@ -61,7 +72,7 @@ const cleanup: Array<() => unknown> = [];
|
||||
|
||||
afterEach(async () => {
|
||||
for (const step of cleanup.splice(0).reverse()) await step();
|
||||
Object.assign(mediaCopy, { held: null, stored: null, copies: [], fails: null });
|
||||
Object.assign(mediaCopy, { held: null, stored: null, copies: [], entered: 0, fails: null });
|
||||
});
|
||||
|
||||
const inside = (dir: string, path: string) => readFileSync(join(dir, path), "utf-8");
|
||||
@@ -372,7 +383,7 @@ describe("openProjectHistory", () => {
|
||||
expect(mediaCopy.copies).toEqual(["a.mp4"]);
|
||||
});
|
||||
|
||||
it("undoes a clip replaced while closed, though the history closed before its copy finished", async () => {
|
||||
it("undoes an archived clip replaced while closed", async () => {
|
||||
const clip = Buffer.alloc(2 * 1024 ** 2, 7);
|
||||
let release = () => {};
|
||||
mediaCopy.held = new Promise<void>((resolve) => (release = resolve));
|
||||
@@ -380,9 +391,9 @@ describe("openProjectHistory", () => {
|
||||
{ "index.html": "A", "clip.mp4": clip },
|
||||
{ quietMs: 30 },
|
||||
);
|
||||
const closed = history.close();
|
||||
release();
|
||||
await closed;
|
||||
await vi.waitFor(() => expect(history.peek(START)).toHaveProperty(["clip.mp4"]));
|
||||
await history.close();
|
||||
write("clip.mp4", Buffer.alloc(2 * 1024 ** 2, 9));
|
||||
const reopened = await open(projectDir, historyRoot, { quietMs: 30 });
|
||||
await reopened.flush();
|
||||
@@ -393,18 +404,88 @@ describe("openProjectHistory", () => {
|
||||
expect(readFileSync(join(projectDir, "clip.mp4")).equals(clip)).toBe(true);
|
||||
});
|
||||
|
||||
it("adopts media whose copy a process exit cut short, rather than log it as added", async () => {
|
||||
it("adopts interrupted legacy media with an explicit unavailable-original entry", async () => {
|
||||
const { history, write, projectDir, historyRoot } = await project({ "index.html": "A" });
|
||||
await history.close();
|
||||
// What an exit mid-copy leaves: the clip on disk, named as still being copied, not in the baseline.
|
||||
write("clip.mp4", Buffer.alloc(2 * 1024 ** 2, 7));
|
||||
writeFileSync(join(historyRoot, history.projectId, "adopting.json"), '["clip.mp4"]');
|
||||
const reopened = await open(projectDir, historyRoot, { quietMs: 30 });
|
||||
await vi.waitFor(() => expect(reopened.peek(START)).toHaveProperty(["clip.mp4"]), {
|
||||
await vi.waitFor(() => expect(reopened.list()[0]?.unavailableBefore).toEqual(["clip.mp4"]), {
|
||||
timeout: 10_000,
|
||||
});
|
||||
expect(reopened.peek(reopened.list()[0]!.id)).toHaveProperty(["clip.mp4"]);
|
||||
await reopened.flush();
|
||||
expect(reopened.list()).toEqual([]);
|
||||
expect(reopened.list()).toMatchObject([
|
||||
{ unavailableBefore: ["clip.mp4"], label: "Original media version is unavailable" },
|
||||
]);
|
||||
});
|
||||
|
||||
it("a saved API predecessor is not mistaken for the original media replaced while closed", async () => {
|
||||
mediaCopy.held = new Promise<void>(() => {});
|
||||
const { history, write, projectDir, historyRoot } = await project(
|
||||
{ "index.html": "A", "clip.mp4": Buffer.alloc(2 * 1024 ** 2, 7) },
|
||||
{ quietMs: 30 },
|
||||
);
|
||||
await vi.waitFor(() => expect(mediaCopy.entered).toBe(1));
|
||||
await history.close();
|
||||
const previous = Buffer.alloc(2 * 1024 ** 2, 9);
|
||||
write("clip.mp4", previous);
|
||||
let release = () => {};
|
||||
mediaCopy.held = new Promise<void>((resolve) => {
|
||||
release = resolve;
|
||||
});
|
||||
const reopened = await open(projectDir, historyRoot, { quietMs: 30 });
|
||||
await vi.waitFor(() => expect(mediaCopy.entered).toBe(2));
|
||||
const current = Buffer.alloc(2 * 1024 ** 2, 11);
|
||||
write("clip.mp4", current);
|
||||
recordFileWriteReceipt(join(projectDir, "clip.mp4"), {
|
||||
path: "clip.mp4",
|
||||
version: fileContentVersion(current),
|
||||
writeToken: "media-write",
|
||||
overwrote: previous,
|
||||
});
|
||||
await reopened.flush();
|
||||
release();
|
||||
await vi.waitFor(() =>
|
||||
expect(reopened.list().some((entry) => entry.unavailableBefore?.includes("clip.mp4"))).toBe(
|
||||
true,
|
||||
),
|
||||
);
|
||||
await reopened.flush();
|
||||
expect((await reopened.step("back", you)).ok).toBe(true);
|
||||
expect(readFileSync(join(projectDir, "clip.mp4")).equals(previous)).toBe(true);
|
||||
await expect(reopened.step("back", you)).rejects.toThrow(
|
||||
"original media version is unavailable",
|
||||
);
|
||||
await expect(reopened.restore(START, you)).rejects.toThrow(
|
||||
"original media version is unavailable",
|
||||
);
|
||||
});
|
||||
|
||||
it("close cancels adoption's prune-triggered recopy and preserves it for reopening", async () => {
|
||||
const clip = Buffer.alloc(2 * 1024 ** 2, 7);
|
||||
const { history, write, projectDir, historyRoot } = await project(
|
||||
{ "index.html": "A", "clip.mp4": clip },
|
||||
{ budgetBytes: 1, quietMs: 30 },
|
||||
);
|
||||
mediaCopy.stored = async () => {
|
||||
mediaCopy.stored = null;
|
||||
mediaCopy.held = new Promise<void>(() => {});
|
||||
await change(history, you, "Color", () => write("index.html", "B"));
|
||||
};
|
||||
await vi.waitFor(() => expect(mediaCopy.entered).toBe(2));
|
||||
await history.close();
|
||||
const home = join(historyRoot, history.projectId);
|
||||
expect(JSON.parse(readFileSync(join(home, "adopting.json"), "utf8"))).toEqual(["clip.mp4"]);
|
||||
expect(readdirSync(join(home, "blobs")).some((name) => name.startsWith("incoming-"))).toBe(
|
||||
false,
|
||||
);
|
||||
expect(readFileSync(join(home, "log.jsonl"), "utf8")).toContain("Color");
|
||||
mediaCopy.held = null;
|
||||
const reopened = await open(projectDir, historyRoot, { quietMs: 30 });
|
||||
await vi.waitFor(() => expect(reopened.peek(START)).toHaveProperty(["clip.mp4"]));
|
||||
expect(readFileSync(join(projectDir, "clip.mp4")).equals(clip)).toBe(true);
|
||||
});
|
||||
|
||||
it("keeps a media copy that a budget prune ran into before history recorded it", async () => {
|
||||
|
||||
@@ -180,6 +180,16 @@ interface Tracked {
|
||||
stat: string;
|
||||
}
|
||||
|
||||
type CachedFile = Tracked | { hash: null; stat: string | null };
|
||||
|
||||
function cachedFile(value: unknown): CachedFile | null {
|
||||
if (!value || typeof value !== "object") return null;
|
||||
const { hash, stat } = value as Record<string, unknown>;
|
||||
if (typeof hash === "string" && typeof stat === "string") return { hash, stat };
|
||||
if (hash === null && (typeof stat === "string" || stat === null)) return { hash, stat };
|
||||
return null;
|
||||
}
|
||||
|
||||
interface Group {
|
||||
id: string;
|
||||
who: HistoryWho;
|
||||
@@ -323,6 +333,8 @@ class Engine {
|
||||
/** A coalescing claim, open until another key, its idle timer, an operation, a window, or another write. */
|
||||
claimed: { group: Group; key: string; timer?: NodeJS.Timeout } | null = null;
|
||||
adopting = new Map<string, Promise<void>>();
|
||||
readonly adoptionController = new AbortController();
|
||||
adoptionStats = new Map<string, string | null>();
|
||||
/** Per path and hash an API write left, the hash of the bytes it replaced, until walked or written past. */
|
||||
overwritten = new Map<string, Map<string, string>>();
|
||||
stopHearing: (() => void) | undefined;
|
||||
@@ -417,15 +429,31 @@ class Engine {
|
||||
const cached = cache.get(path);
|
||||
this.tracked.set(path, { hash, stat: cached?.hash === hash ? cached.stat : "" });
|
||||
}
|
||||
const left = this.readAdopting();
|
||||
const unadopted = (file: { path: string }) =>
|
||||
left.has(file.path) && !this.tracked.has(file.path);
|
||||
this.adoptMediaInBackground(historyFiles(this.dir).filter(unadopted), this.now());
|
||||
await this.resumeAdoption(cache);
|
||||
// What changed while the project was closed is one outside entry, or the closed window's.
|
||||
this.reopenClosedWindow();
|
||||
await this.settleAll();
|
||||
}
|
||||
|
||||
async resumeAdoption(cache: Map<string, CachedFile>): Promise<void> {
|
||||
const left = this.readAdopting();
|
||||
for (const path of left) {
|
||||
const pending = cache.get(path);
|
||||
this.adoptionStats.set(path, pending?.hash === null ? pending.stat : null);
|
||||
}
|
||||
const unadopted = (file: { path: string }) =>
|
||||
left.has(file.path) && !this.tracked.has(file.path);
|
||||
const files = historyFiles(this.dir);
|
||||
const present = new Set(files.map((file) => file.path));
|
||||
for (const path of left) {
|
||||
if (!present.has(path) && !this.tracked.has(path)) {
|
||||
await this.recordUnavailableMedia(path, null);
|
||||
this.adoptionStats.delete(path);
|
||||
}
|
||||
}
|
||||
this.adoptMediaInBackground(files.filter(unadopted), this.now());
|
||||
}
|
||||
|
||||
hearWrites(): void {
|
||||
const realDir = realFilePath(this.dir);
|
||||
this.stopHearing = onFileOverwritten((absPath, version, bytes) => {
|
||||
@@ -510,8 +538,11 @@ class Engine {
|
||||
for (const file of files) {
|
||||
copied = copied.then(() => this.adopt(file, sweptAt));
|
||||
this.adopting.set(file.path, copied);
|
||||
if (!this.adoptionStats.has(file.path))
|
||||
this.adoptionStats.set(file.path, statKey(file, Infinity));
|
||||
}
|
||||
this.saveAdopting();
|
||||
this.saveStatCache();
|
||||
}
|
||||
|
||||
saveAdopting(): void {
|
||||
@@ -523,42 +554,79 @@ class Engine {
|
||||
|
||||
async adopt(file: ListedFile, sweptAt: number) {
|
||||
try {
|
||||
this.adoptionController.signal.throwIfAborted();
|
||||
if (this.whereFolder() !== "here") return;
|
||||
const stored = await this.storeIfPresent(file.path);
|
||||
const stored = await this.storeIfPresent(file.path, this.adoptionController.signal);
|
||||
await this.queue(async () => {
|
||||
// Still named for the next open until recorded, so a failed record is never logged as added.
|
||||
await this.recordAdopted(file, sweptAt, stored);
|
||||
this.adopting.delete(file.path);
|
||||
this.adoptionStats.delete(file.path);
|
||||
this.saveStatCache();
|
||||
this.saveAdopting();
|
||||
});
|
||||
} catch (error) {
|
||||
if (!(error instanceof HistoryClosedError)) this.options.onError?.(error);
|
||||
if (error instanceof HistoryClosedError) return;
|
||||
if (this.adoptionController.signal.aborted) {
|
||||
if ((error as Error).name === "AbortError") return;
|
||||
throw error;
|
||||
}
|
||||
this.options.onError?.(error);
|
||||
}
|
||||
}
|
||||
|
||||
async recordAdopted(file: ListedFile, sweptAt: number, stored: string | null) {
|
||||
if (stored === null || this.whereFolder() !== "here") return;
|
||||
this.adoptionController.signal.throwIfAborted();
|
||||
if (this.whereFolder() !== "here") return;
|
||||
if (stored === null) return this.recordUnavailableMedia(file.path, null);
|
||||
// A prune that ran while this copy waited for the queue may have taken its bytes.
|
||||
const hash = this.blobs.has(stored) ? stored : await this.storeIfPresent(file.path);
|
||||
if (hash === null) return;
|
||||
const hash = this.blobs.has(stored)
|
||||
? stored
|
||||
: await this.storeIfPresent(file.path, this.adoptionController.signal);
|
||||
this.adoptionController.signal.throwIfAborted();
|
||||
if (hash === null) return this.recordUnavailableMedia(file.path, null);
|
||||
await this.recordMediaBaseline(file, sweptAt, hash);
|
||||
}
|
||||
|
||||
async recordMediaBaseline(file: ListedFile, sweptAt: number, hash: string): Promise<void> {
|
||||
const change = { path: file.path, before: null, after: hash };
|
||||
const found = this.overwrittenBy(change, undefined, new Set());
|
||||
const start = found && this.blobs.has(found) ? found : hash;
|
||||
this.tracked.set(file.path, { hash, stat: statKey(file, sweptAt) });
|
||||
this.log.baseline.set(file.path, start);
|
||||
const originalStat = this.adoptionStats.get(file.path);
|
||||
const current = statSync(join(this.dir, file.path), { throwIfNoEntry: false });
|
||||
const unavailable =
|
||||
originalStat == null || !current || originalStat !== statKey(current, Infinity);
|
||||
if (unavailable) {
|
||||
await this.recordUnavailableMedia(file.path, start);
|
||||
} else {
|
||||
this.log.baseline.set(file.path, start);
|
||||
this.persistLog(baselineRecord(this.log));
|
||||
}
|
||||
if (start !== hash) addChange(this.outsideGroup(), file.path, start, hash);
|
||||
this.persistLog(baselineRecord(this.log));
|
||||
this.saveStatCache();
|
||||
}
|
||||
|
||||
async recordUnavailableMedia(path: string, after: string | null): Promise<void> {
|
||||
const group = this.newGroup(OUTSIDE, "Original media version is unavailable");
|
||||
group.changes.set(path, { path, before: null, after });
|
||||
await this.commit(group, { unavailableBefore: [path] });
|
||||
}
|
||||
|
||||
manifest(): Manifest {
|
||||
return new Map([...this.tracked].map(([path, file]) => [path, file.hash]));
|
||||
}
|
||||
|
||||
readStatCache(): Map<string, Tracked> {
|
||||
readStatCache(): Map<string, CachedFile> {
|
||||
try {
|
||||
const files = JSON.parse(readFileSync(join(this.home, "stat.json"), "utf-8"));
|
||||
return new Map(Object.entries(files as Record<string, Tracked>));
|
||||
const files: unknown = JSON.parse(readFileSync(join(this.home, "stat.json"), "utf-8"));
|
||||
const cache = new Map<string, CachedFile>();
|
||||
if (!files || typeof files !== "object") return cache;
|
||||
for (const [path, value] of Object.entries(files)) {
|
||||
const cached = cachedFile(value);
|
||||
if (cached) cache.set(path, cached);
|
||||
}
|
||||
return cache;
|
||||
} catch {
|
||||
return new Map();
|
||||
}
|
||||
@@ -575,7 +643,10 @@ class Engine {
|
||||
}
|
||||
|
||||
saveStatCache(): void {
|
||||
const files = JSON.stringify(Object.fromEntries(this.tracked));
|
||||
const pending = [...this.adoptionStats].map(
|
||||
([path, stat]) => [path, { hash: null, stat }] as const,
|
||||
);
|
||||
const files = JSON.stringify(Object.fromEntries([...pending, ...this.tracked]));
|
||||
mkdirSync(this.home, { recursive: true });
|
||||
replaceFileAtomically(join(this.home, "stat.json"), files, 0o644);
|
||||
}
|
||||
@@ -627,9 +698,9 @@ class Engine {
|
||||
}
|
||||
|
||||
/** The file's hash once copied in; null when it was removed before the copy (the next sweep records that). */
|
||||
async storeIfPresent(path: string): Promise<string | null> {
|
||||
async storeIfPresent(path: string, signal?: AbortSignal): Promise<string | null> {
|
||||
try {
|
||||
return await this.blobs.put(join(this.dir, path));
|
||||
return await this.blobs.put(join(this.dir, path), signal);
|
||||
} catch (error) {
|
||||
if ((error as NodeJS.ErrnoException).code === "ENOENT") return null;
|
||||
throw error;
|
||||
@@ -1070,6 +1141,7 @@ class Engine {
|
||||
|
||||
async undoNow(id: string, who: HistoryWho, mode?: UndoMode): Promise<HistoryResult> {
|
||||
const entry = this.entry(id);
|
||||
this.assertArchivedBefore([entry]);
|
||||
const changed = this.movedOn(entry);
|
||||
if (changed.length && !mode) return { ok: false, conflict: this.conflict(entry, changed) };
|
||||
if (mode === "back-to-before") {
|
||||
@@ -1129,7 +1201,22 @@ class Engine {
|
||||
return { files: [...paths], newer: newer.map((later) => later.id) };
|
||||
}
|
||||
|
||||
assertArchivedBefore(entries: readonly HistoryEntry[]): void {
|
||||
const paths = entries.flatMap((entry) => entry.unavailableBefore ?? []);
|
||||
if (paths.length)
|
||||
throw new Error(
|
||||
`Cannot restore ${paths.join(", ")}: the original media version is unavailable because its first archive did not complete before the file changed.`,
|
||||
);
|
||||
}
|
||||
|
||||
assertArchivedAt(point: string, side: HistoryEntrySide = "after"): void {
|
||||
const index = point === START ? -1 : this.log.entries.findIndex((entry) => entry.id === point);
|
||||
if (point !== START && index === -1) return;
|
||||
this.assertArchivedBefore(this.log.entries.slice(index + (side === "after" ? 1 : 0)));
|
||||
}
|
||||
|
||||
async restoreNow(point: string, who: HistoryWho, label: string): Promise<HistoryEntry | null> {
|
||||
this.assertArchivedAt(point);
|
||||
const files = manifestAt(this.log, point);
|
||||
if (!files) throw new Error("That point is no longer kept in this project's history.");
|
||||
const target = new Map<string, string | null>(
|
||||
@@ -1190,12 +1277,14 @@ class Engine {
|
||||
),
|
||||
peek: (point) => {
|
||||
this.assertOpen();
|
||||
this.assertArchivedAt(point);
|
||||
const files = manifestAt(this.log, point);
|
||||
return files && Object.fromEntries(files);
|
||||
},
|
||||
checkout: (entryId, side, emptyDir) =>
|
||||
this.queue(async () => {
|
||||
this.entry(entryId);
|
||||
this.assertArchivedAt(entryId, side);
|
||||
if (!existsSync(this.dir)) throw new Error(`The project folder is gone: ${this.dir}`);
|
||||
const dest = resolve(emptyDir);
|
||||
if (isWithin(this.dir, dest))
|
||||
@@ -1234,13 +1323,16 @@ class Engine {
|
||||
replacedAtPath: () => this.whereFolder() === "replaced",
|
||||
close: () =>
|
||||
(this.closing ??= (async () => {
|
||||
// A copy cut short would leave its media out of the baseline, so a change made while closed is lost.
|
||||
await Promise.all(this.adopting.values());
|
||||
// Stop resumable archives and clean their partial copies before releasing ownership.
|
||||
this.adoptionController.abort();
|
||||
const adoption = await Promise.allSettled(this.adopting.values());
|
||||
if (this.notedTimer) clearTimeout(this.notedTimer);
|
||||
this.stopHearing?.();
|
||||
const settled = this.queue(() => this.settleAll());
|
||||
this.closed = true;
|
||||
await settled;
|
||||
const failed = adoption.find((result) => result.status === "rejected");
|
||||
if (failed?.status === "rejected") throw failed.reason;
|
||||
})()),
|
||||
};
|
||||
}
|
||||
@@ -1290,7 +1382,7 @@ export async function openProjectHistory(options: ProjectHistoryOptions): Promis
|
||||
const api = engine.api();
|
||||
return {
|
||||
...api,
|
||||
close: () => api.close().finally(release),
|
||||
close: () => api.close().then(release),
|
||||
};
|
||||
} catch (error) {
|
||||
release();
|
||||
|
||||
Reference in New Issue
Block a user