fix(studio-server): cancel background media adoption on close

This commit is contained in:
Miguel Angel Simon Sierra
2026-10-07 03:46:45 -07:00
parent 1e711b087d
commit e3d65c5ff9
5 changed files with 528 additions and 47 deletions
+24 -16
View File
@@ -1,13 +1,14 @@
import { createHash, randomUUID } from "node:crypto";
import { constants, createReadStream, renameSync } from "node:fs";
import { constants, createReadStream, createWriteStream, renameSync } from "node:fs";
import { copyFile, mkdir, readdir, readFile, rename, rm, stat } from "node:fs/promises";
import { pipeline } from "node:stream/promises";
import { dirname, join } from "node:path";
import { atomicTempPath } from "@hyperframes/core/atomic-file";
/** File contents stored once by sha256, text and binary alike. */
export interface BlobStore {
/** Copies the file in (a clone where the file system can) and returns the copy's hash. */
put(absPath: string): Promise<string>;
/** Archives the file and returns its hash; a signal makes copy and hash cancellable. */
put(absPath: string, signal?: AbortSignal): Promise<string>;
has(hash: string): boolean;
read(hash: string): Promise<Buffer>;
/** Writes the blob's bytes to `absPath` by clone-or-copy and rename, so a reader never sees half a file. */
@@ -19,9 +20,9 @@ export interface BlobStore {
const BLOB_HASH = /^[0-9a-f]{64}$/;
async function hashFile(path: string): Promise<string> {
async function hashFile(path: string, signal?: AbortSignal): Promise<string> {
const hash = createHash("sha256");
for await (const chunk of createReadStream(path)) hash.update(chunk);
for await (const chunk of createReadStream(path, { signal })) hash.update(chunk);
return hash.digest("hex");
}
@@ -52,23 +53,30 @@ export async function openBlobStore(dir: string): Promise<BlobStore> {
let total = [...sizes.values()].reduce((sum, size) => sum + size, 0);
return {
async put(absPath) {
async put(absPath, signal) {
// Hash the copy, not the source, so a write racing the copy can never file bytes under the wrong hash.
const temp = join(dir, `incoming-${randomUUID()}`);
// Recreated if removed while open, so a missing folder is never mistaken for a deleted project file.
await mkdir(dir, { recursive: true });
await copyFile(absPath, temp, constants.COPYFILE_FICLONE);
const hash = await hashFile(temp);
if (sizes.has(hash)) {
await rm(temp, { force: true });
try {
if (signal) {
await pipeline(createReadStream(absPath), createWriteStream(temp), { signal });
} else {
await copyFile(absPath, temp, constants.COPYFILE_FICLONE);
}
const hash = await hashFile(temp, signal);
signal?.throwIfAborted();
if (sizes.has(hash)) return hash;
await mkdir(dirname(pathOf(hash)), { recursive: true });
const size = (await stat(temp)).size;
signal?.throwIfAborted();
await rename(temp, pathOf(hash));
sizes.set(hash, size);
total += size;
return hash;
} finally {
await rm(temp, { force: true });
}
await mkdir(dirname(pathOf(hash)), { recursive: true });
const size = (await stat(temp)).size;
await rename(temp, pathOf(hash));
sizes.set(hash, size);
total += size;
return hash;
},
has: (hash) => sizes.has(hash),
read: async (hash) => readFile(pathOf(hash)),
@@ -8,7 +8,7 @@ export interface HistoryWho {
name: string;
}
/** A file's sha256 before and after; null is "did not exist", so a create or delete undoes like any edit. */
/** A file's sha256 before and after; null is "did not exist" unless the entry names an unavailable original. */
export interface HistoryFileChange {
path: string;
before: string | null;
@@ -24,6 +24,8 @@ export interface HistoryEntry {
startedAt: number;
endedAt: number;
files: HistoryFileChange[];
/** These files existed before this entry, but their original bytes were never archived. */
unavailableBefore?: string[];
/** Set on an undo: the entry it reverted. Redo is undoing the undo. */
undoes?: string;
/** Set on a restore: the point (an entry id, or START) the files were made equal to. */
@@ -0,0 +1,298 @@
// @vitest-environment node
import {
mkdirSync,
mkdtempSync,
writeFileSync,
readdirSync,
readFileSync,
rmSync,
statSync,
} from "node:fs";
import { createHash } from "node:crypto";
import { HistoryBusyError } from "./ownerLock";
import { tmpdir } from "node:os";
import { join } from "node:path";
import { afterEach, expect, it, vi } from "vitest";
import { openProjectHistory } from "./projectHistory";
import { START } from "./historyLog";
const archive = vi.hoisted(() => ({
hold: false,
entered: "",
hash: false,
failCleanup: false,
partialPath: "",
aborted: false,
release: null as (() => void) | null,
}));
vi.mock("node:fs", async (importOriginal) => {
const fs = await importOriginal<typeof import("node:fs")>();
const { Readable, addAbortSignal } = await import("node:stream");
return {
...fs,
createReadStream: (path: string, options?: Parameters<typeof fs.createReadStream>[1]) => {
const heldPath = archive.hash
? String(path).includes("incoming-")
: String(path).endsWith(".mp4");
if (!archive.hold || !heldPath) return fs.createReadStream(path, options);
archive.entered = archive.hash ? "hash" : "copy";
let sent = false;
const stream = new Readable({
read() {
if (sent) return;
sent = true;
this.push(Buffer.alloc(65536, 7));
},
});
archive.release = () => {
stream.push(Buffer.alloc(2 * 1024 ** 2 - 65536, 7));
stream.push(null);
};
if (typeof options === "object" && options?.signal) addAbortSignal(options.signal, stream);
stream.on("close", () => {
archive.aborted = true;
});
return stream;
},
};
});
vi.mock("node:fs/promises", async (importOriginal) => {
const fs = await importOriginal<typeof import("node:fs/promises")>();
return {
...fs,
rm: async (...args: Parameters<typeof fs.rm>) => {
if (archive.failCleanup && String(args[0]) === archive.partialPath) {
throw Object.assign(new Error("Cannot remove partial archive"), { code: "EPERM" });
}
return fs.rm(...args);
},
copyFile: async (...args: Parameters<typeof fs.copyFile>) => {
if (archive.hold && !archive.hash && String(args[0]).endsWith(".mp4")) {
await fs.writeFile(args[1], Buffer.alloc(65536, 7));
await new Promise<void>((resolve) => {
archive.release = resolve;
});
}
return fs.copyFile(...args);
},
};
});
const roots: string[] = [];
afterEach(() => {
archive.hold = false;
archive.hash = false;
archive.entered = "";
archive.failCleanup = false;
archive.release = null;
archive.aborted = false;
for (const dir of roots.splice(0)) rmSync(dir, { recursive: true, force: true });
});
async function heldProject(names = ["clip.mp4"], onError?: (error: Error) => void) {
const root = mkdtempSync(join(tmpdir(), "hf-held-adoption-"));
roots.push(root);
const projectDir = join(root, "project");
mkdirSync(projectDir);
writeFileSync(join(projectDir, "index.html"), "before");
for (const name of names) writeFileSync(join(projectDir, name), Buffer.alloc(2 * 1024 ** 2, 7));
archive.hold = true;
const historyRoot = join(root, "history");
const history = await openProjectHistory({ projectDir, historyRoot, onError, quietMs: 30 });
const home = join(historyRoot, history.projectId);
return { root, projectDir, historyRoot, history, home, blobs: join(home, "blobs") };
}
it("close cancels a held adoption, flushes text edits and resumes pending media on reopen", async () => {
const onError = vi.fn();
const { history, historyRoot, home, blobs, projectDir } = await heldProject(undefined, onError);
try {
await vi.waitFor(() => {
const partial = readdirSync(blobs).find((name) => name.startsWith("incoming-"));
expect(partial).toBeDefined();
expect(statSync(join(blobs, partial!)).size).toBeGreaterThan(0);
});
writeFileSync(join(projectDir, "index.html"), "after");
let closed = false;
const closing = history.close().then(() => {
closed = true;
});
try {
await vi.waitFor(() => expect(closed).toBe(true));
} finally {
archive.release?.();
await closing;
}
expect(archive.aborted).toBe(true);
expect(JSON.parse(readFileSync(join(home, "adopting.json"), "utf8"))).toEqual(["clip.mp4"]);
expect(onError).not.toHaveBeenCalled();
const log = readFileSync(join(home, "log.jsonl"), "utf8")
.trim()
.split("\n")
.map((line) => JSON.parse(line));
const textEdit = log
.filter((record) => record.type === "entry")
.flatMap((record) => record.entry.files)
.find((file) => file.path === "index.html");
expect(textEdit).toMatchObject({
before: createHash("sha256").update("before").digest("hex"),
after: createHash("sha256").update("after").digest("hex"),
});
expect(readdirSync(blobs).some((name) => name.startsWith("incoming-"))).toBe(false);
archive.hold = false;
const reopened = await openProjectHistory({ projectDir, historyRoot, quietMs: 30 });
try {
await vi.waitFor(() => expect(reopened.peek(START)).toHaveProperty(["clip.mp4"]));
expect(reopened.list().flatMap((entry) => entry.files.map((file) => file.path))).toContain(
"index.html",
);
expect(reopened.peek(START)?.["clip.mp4"]).toMatch(/^[0-9a-f]{64}$/);
} finally {
await reopened.close();
}
} finally {
archive.hold = false;
await history.close();
}
}, 5000);
it.each(["copy", "hash"])(
"cancelled %s leaves no partial blob before ownership release",
async (phase) => {
const { openBlobStore } = await import("./blobStore");
const root = mkdtempSync(join(tmpdir(), "hf-cancel-blob-"));
roots.push(root);
const source = join(root, "clip.mp4");
writeFileSync(source, Buffer.alloc(2 * 1024 ** 2, 7));
const dir = join(root, "blobs");
const store = await openBlobStore(dir);
archive.hold = true;
archive.hash = phase === "hash";
const controller = new AbortController();
const pending = store.put(source, controller.signal);
let error: unknown;
const settled = pending.catch((caught) => {
error = caught;
});
await vi.waitFor(() =>
expect(readdirSync(dir).some((name) => name.startsWith("incoming-"))).toBe(true),
);
await vi.waitFor(() => expect(archive.entered).toBe(phase));
controller.abort();
try {
await vi.waitFor(() => expect(error).toMatchObject({ name: "AbortError" }));
} finally {
archive.release?.();
await settled;
}
expect(readdirSync(dir)).toEqual([]);
expect(store.bytes()).toBe(0);
},
);
it("failed cancellation cleanup flushes edits but refuses to release ownership", async () => {
const { history, historyRoot, home, blobs, projectDir } = await heldProject();
await vi.waitFor(() =>
expect(readdirSync(blobs).some((name) => name.startsWith("incoming-"))).toBe(true),
);
writeFileSync(join(projectDir, "index.html"), "after");
archive.partialPath = join(
blobs,
readdirSync(blobs).find((name) => name.startsWith("incoming-"))!,
);
archive.failCleanup = true;
await expect(history.close()).rejects.toMatchObject({ code: "EPERM" });
expect(readdirSync(blobs).some((name) => name.startsWith("incoming-"))).toBe(true);
expect(readFileSync(join(home, "log.jsonl"), "utf8")).toContain(
createHash("sha256").update("after").digest("hex"),
);
await expect(
openProjectHistory({ projectDir, historyRoot, ownerWaitMs: 0 }),
).rejects.toBeInstanceOf(HistoryBusyError);
});
it.each(["replace", "delete", "missing-cache", "malformed-cache"])(
"refuses unknown original media after %s while closed",
async (change) => {
const { root, history, historyRoot, home, projectDir } = await heldProject();
const clip = join(projectDir, "clip.mp4");
await vi.waitFor(() =>
expect(readdirSync(join(home, "blobs")).some((name) => name.startsWith("incoming-"))).toBe(
true,
),
);
await history.close();
if (change === "delete") rmSync(clip);
else writeFileSync(clip, Buffer.alloc(2 * 1024 ** 2, 9));
if (change === "missing-cache") rmSync(join(home, "stat.json"));
if (change === "malformed-cache")
writeFileSync(
join(home, "stat.json"),
JSON.stringify({ "clip.mp4": { hash: "invalid", stat: null } }),
);
archive.hold = false;
const reopened = await openProjectHistory({ projectDir, historyRoot, quietMs: 30 });
const who = { kind: "person" as const, name: "You" };
try {
await vi.waitFor(() =>
expect(reopened.list().some((entry) => entry.unavailableBefore?.includes("clip.mp4"))).toBe(
true,
),
);
const entry = reopened.list().find((entry) => entry.unavailableBefore?.includes("clip.mp4"))!;
await expect(reopened.step("back", who)).rejects.toThrow(
"original media version is unavailable",
);
await expect(reopened.undo(entry.id, { who })).rejects.toThrow(
"original media version is unavailable",
);
await expect(reopened.restore(START, who)).rejects.toThrow(
"original media version is unavailable",
);
expect(() => reopened.peek(START)).toThrow("original media version is unavailable");
const checkout = join(root, "checkout");
mkdirSync(checkout);
await expect(reopened.checkout(entry.id, "before", checkout)).rejects.toThrow(
"original media version is unavailable",
);
expect(readdirSync(checkout)).toEqual([]);
if (change === "delete") expect(readdirSync(projectDir)).not.toContain("clip.mp4");
else expect(readFileSync(clip).equals(Buffer.alloc(2 * 1024 ** 2, 9))).toBe(true);
} finally {
await reopened.close();
}
const again = await openProjectHistory({ projectDir, historyRoot, quietMs: 30 });
try {
await expect(again.step("back", who)).rejects.toThrow(
"original media version is unavailable",
);
} finally {
await again.close();
}
},
);
it("a later media replacement while serial adoption waits is never filed as the original baseline", async () => {
const { history, projectDir } = await heldProject(["a.mp4", "b.mp4"]);
try {
await vi.waitFor(() => expect(archive.entered).toBe("copy"));
writeFileSync(join(projectDir, "b.mp4"), Buffer.alloc(2 * 1024 ** 2, 9));
archive.hold = false;
archive.release!();
await vi.waitFor(() =>
expect(history.list().some((entry) => entry.unavailableBefore?.includes("b.mp4"))).toBe(true),
);
const entry = history.list().find((entry) => entry.unavailableBefore?.includes("b.mp4"))!;
expect(entry.files.find((file) => file.path === "b.mp4")?.after).toBe(
createHash("sha256")
.update(Buffer.alloc(2 * 1024 ** 2, 9))
.digest("hex"),
);
await expect(history.step("back", { kind: "person", name: "You" })).rejects.toThrow(
"original media version is unavailable",
);
} finally {
archive.hold = false;
archive.release?.();
await history.close();
}
});
@@ -14,6 +14,7 @@ import {
utimesSync,
writeFileSync,
} from "node:fs";
import { once } from "node:events";
import { spawnSync } from "node:child_process";
import { tmpdir } from "node:os";
import { basename, dirname, join, relative } from "node:path";
@@ -29,6 +30,7 @@ const mediaCopy = vi.hoisted(() => ({
held: null as Promise<void> | null,
stored: null as (() => Promise<unknown>) | null,
copies: [] as string[],
entered: 0,
fails: null as ((copy: number) => boolean) | null,
}));
vi.mock("./blobStore", async (importOriginal) => {
@@ -39,13 +41,22 @@ vi.mock("./blobStore", async (importOriginal) => {
const store = await real.openBlobStore(dir);
return {
...store,
put: async (path: string) => {
put: async (path: string, signal?: AbortSignal) => {
if (path.endsWith(".mp4")) {
await mediaCopy.held;
mediaCopy.entered++;
if (signal && mediaCopy.held) {
signal.throwIfAborted();
await Promise.race([
mediaCopy.held,
once(signal, "abort").then(() => {
throw signal.reason;
}),
]);
} else await mediaCopy.held;
mediaCopy.copies.push(basename(path));
if (mediaCopy.fails?.(mediaCopy.copies.length)) throw new Error("The disk is full.");
}
const hash = await store.put(path);
const hash = await store.put(path, signal);
if (path.endsWith(".mp4")) await mediaCopy.stored?.();
return hash;
},
@@ -61,7 +72,7 @@ const cleanup: Array<() => unknown> = [];
afterEach(async () => {
for (const step of cleanup.splice(0).reverse()) await step();
Object.assign(mediaCopy, { held: null, stored: null, copies: [], fails: null });
Object.assign(mediaCopy, { held: null, stored: null, copies: [], entered: 0, fails: null });
});
const inside = (dir: string, path: string) => readFileSync(join(dir, path), "utf-8");
@@ -372,7 +383,7 @@ describe("openProjectHistory", () => {
expect(mediaCopy.copies).toEqual(["a.mp4"]);
});
it("undoes a clip replaced while closed, though the history closed before its copy finished", async () => {
it("undoes an archived clip replaced while closed", async () => {
const clip = Buffer.alloc(2 * 1024 ** 2, 7);
let release = () => {};
mediaCopy.held = new Promise<void>((resolve) => (release = resolve));
@@ -380,9 +391,9 @@ describe("openProjectHistory", () => {
{ "index.html": "A", "clip.mp4": clip },
{ quietMs: 30 },
);
const closed = history.close();
release();
await closed;
await vi.waitFor(() => expect(history.peek(START)).toHaveProperty(["clip.mp4"]));
await history.close();
write("clip.mp4", Buffer.alloc(2 * 1024 ** 2, 9));
const reopened = await open(projectDir, historyRoot, { quietMs: 30 });
await reopened.flush();
@@ -393,18 +404,88 @@ describe("openProjectHistory", () => {
expect(readFileSync(join(projectDir, "clip.mp4")).equals(clip)).toBe(true);
});
it("adopts media whose copy a process exit cut short, rather than log it as added", async () => {
it("adopts interrupted legacy media with an explicit unavailable-original entry", async () => {
const { history, write, projectDir, historyRoot } = await project({ "index.html": "A" });
await history.close();
// What an exit mid-copy leaves: the clip on disk, named as still being copied, not in the baseline.
write("clip.mp4", Buffer.alloc(2 * 1024 ** 2, 7));
writeFileSync(join(historyRoot, history.projectId, "adopting.json"), '["clip.mp4"]');
const reopened = await open(projectDir, historyRoot, { quietMs: 30 });
await vi.waitFor(() => expect(reopened.peek(START)).toHaveProperty(["clip.mp4"]), {
await vi.waitFor(() => expect(reopened.list()[0]?.unavailableBefore).toEqual(["clip.mp4"]), {
timeout: 10_000,
});
expect(reopened.peek(reopened.list()[0]!.id)).toHaveProperty(["clip.mp4"]);
await reopened.flush();
expect(reopened.list()).toEqual([]);
expect(reopened.list()).toMatchObject([
{ unavailableBefore: ["clip.mp4"], label: "Original media version is unavailable" },
]);
});
it("a saved API predecessor is not mistaken for the original media replaced while closed", async () => {
mediaCopy.held = new Promise<void>(() => {});
const { history, write, projectDir, historyRoot } = await project(
{ "index.html": "A", "clip.mp4": Buffer.alloc(2 * 1024 ** 2, 7) },
{ quietMs: 30 },
);
await vi.waitFor(() => expect(mediaCopy.entered).toBe(1));
await history.close();
const previous = Buffer.alloc(2 * 1024 ** 2, 9);
write("clip.mp4", previous);
let release = () => {};
mediaCopy.held = new Promise<void>((resolve) => {
release = resolve;
});
const reopened = await open(projectDir, historyRoot, { quietMs: 30 });
await vi.waitFor(() => expect(mediaCopy.entered).toBe(2));
const current = Buffer.alloc(2 * 1024 ** 2, 11);
write("clip.mp4", current);
recordFileWriteReceipt(join(projectDir, "clip.mp4"), {
path: "clip.mp4",
version: fileContentVersion(current),
writeToken: "media-write",
overwrote: previous,
});
await reopened.flush();
release();
await vi.waitFor(() =>
expect(reopened.list().some((entry) => entry.unavailableBefore?.includes("clip.mp4"))).toBe(
true,
),
);
await reopened.flush();
expect((await reopened.step("back", you)).ok).toBe(true);
expect(readFileSync(join(projectDir, "clip.mp4")).equals(previous)).toBe(true);
await expect(reopened.step("back", you)).rejects.toThrow(
"original media version is unavailable",
);
await expect(reopened.restore(START, you)).rejects.toThrow(
"original media version is unavailable",
);
});
it("close cancels adoption's prune-triggered recopy and preserves it for reopening", async () => {
const clip = Buffer.alloc(2 * 1024 ** 2, 7);
const { history, write, projectDir, historyRoot } = await project(
{ "index.html": "A", "clip.mp4": clip },
{ budgetBytes: 1, quietMs: 30 },
);
mediaCopy.stored = async () => {
mediaCopy.stored = null;
mediaCopy.held = new Promise<void>(() => {});
await change(history, you, "Color", () => write("index.html", "B"));
};
await vi.waitFor(() => expect(mediaCopy.entered).toBe(2));
await history.close();
const home = join(historyRoot, history.projectId);
expect(JSON.parse(readFileSync(join(home, "adopting.json"), "utf8"))).toEqual(["clip.mp4"]);
expect(readdirSync(join(home, "blobs")).some((name) => name.startsWith("incoming-"))).toBe(
false,
);
expect(readFileSync(join(home, "log.jsonl"), "utf8")).toContain("Color");
mediaCopy.held = null;
const reopened = await open(projectDir, historyRoot, { quietMs: 30 });
await vi.waitFor(() => expect(reopened.peek(START)).toHaveProperty(["clip.mp4"]));
expect(readFileSync(join(projectDir, "clip.mp4")).equals(clip)).toBe(true);
});
it("keeps a media copy that a budget prune ran into before history recorded it", async () => {
@@ -180,6 +180,16 @@ interface Tracked {
stat: string;
}
type CachedFile = Tracked | { hash: null; stat: string | null };
function cachedFile(value: unknown): CachedFile | null {
if (!value || typeof value !== "object") return null;
const { hash, stat } = value as Record<string, unknown>;
if (typeof hash === "string" && typeof stat === "string") return { hash, stat };
if (hash === null && (typeof stat === "string" || stat === null)) return { hash, stat };
return null;
}
interface Group {
id: string;
who: HistoryWho;
@@ -323,6 +333,8 @@ class Engine {
/** A coalescing claim, open until another key, its idle timer, an operation, a window, or another write. */
claimed: { group: Group; key: string; timer?: NodeJS.Timeout } | null = null;
adopting = new Map<string, Promise<void>>();
readonly adoptionController = new AbortController();
adoptionStats = new Map<string, string | null>();
/** Per path and hash an API write left, the hash of the bytes it replaced, until walked or written past. */
overwritten = new Map<string, Map<string, string>>();
stopHearing: (() => void) | undefined;
@@ -417,15 +429,31 @@ class Engine {
const cached = cache.get(path);
this.tracked.set(path, { hash, stat: cached?.hash === hash ? cached.stat : "" });
}
const left = this.readAdopting();
const unadopted = (file: { path: string }) =>
left.has(file.path) && !this.tracked.has(file.path);
this.adoptMediaInBackground(historyFiles(this.dir).filter(unadopted), this.now());
await this.resumeAdoption(cache);
// What changed while the project was closed is one outside entry, or the closed window's.
this.reopenClosedWindow();
await this.settleAll();
}
async resumeAdoption(cache: Map<string, CachedFile>): Promise<void> {
const left = this.readAdopting();
for (const path of left) {
const pending = cache.get(path);
this.adoptionStats.set(path, pending?.hash === null ? pending.stat : null);
}
const unadopted = (file: { path: string }) =>
left.has(file.path) && !this.tracked.has(file.path);
const files = historyFiles(this.dir);
const present = new Set(files.map((file) => file.path));
for (const path of left) {
if (!present.has(path) && !this.tracked.has(path)) {
await this.recordUnavailableMedia(path, null);
this.adoptionStats.delete(path);
}
}
this.adoptMediaInBackground(files.filter(unadopted), this.now());
}
hearWrites(): void {
const realDir = realFilePath(this.dir);
this.stopHearing = onFileOverwritten((absPath, version, bytes) => {
@@ -510,8 +538,11 @@ class Engine {
for (const file of files) {
copied = copied.then(() => this.adopt(file, sweptAt));
this.adopting.set(file.path, copied);
if (!this.adoptionStats.has(file.path))
this.adoptionStats.set(file.path, statKey(file, Infinity));
}
this.saveAdopting();
this.saveStatCache();
}
saveAdopting(): void {
@@ -523,42 +554,79 @@ class Engine {
async adopt(file: ListedFile, sweptAt: number) {
try {
this.adoptionController.signal.throwIfAborted();
if (this.whereFolder() !== "here") return;
const stored = await this.storeIfPresent(file.path);
const stored = await this.storeIfPresent(file.path, this.adoptionController.signal);
await this.queue(async () => {
// Still named for the next open until recorded, so a failed record is never logged as added.
await this.recordAdopted(file, sweptAt, stored);
this.adopting.delete(file.path);
this.adoptionStats.delete(file.path);
this.saveStatCache();
this.saveAdopting();
});
} catch (error) {
if (!(error instanceof HistoryClosedError)) this.options.onError?.(error);
if (error instanceof HistoryClosedError) return;
if (this.adoptionController.signal.aborted) {
if ((error as Error).name === "AbortError") return;
throw error;
}
this.options.onError?.(error);
}
}
async recordAdopted(file: ListedFile, sweptAt: number, stored: string | null) {
if (stored === null || this.whereFolder() !== "here") return;
this.adoptionController.signal.throwIfAborted();
if (this.whereFolder() !== "here") return;
if (stored === null) return this.recordUnavailableMedia(file.path, null);
// A prune that ran while this copy waited for the queue may have taken its bytes.
const hash = this.blobs.has(stored) ? stored : await this.storeIfPresent(file.path);
if (hash === null) return;
const hash = this.blobs.has(stored)
? stored
: await this.storeIfPresent(file.path, this.adoptionController.signal);
this.adoptionController.signal.throwIfAborted();
if (hash === null) return this.recordUnavailableMedia(file.path, null);
await this.recordMediaBaseline(file, sweptAt, hash);
}
async recordMediaBaseline(file: ListedFile, sweptAt: number, hash: string): Promise<void> {
const change = { path: file.path, before: null, after: hash };
const found = this.overwrittenBy(change, undefined, new Set());
const start = found && this.blobs.has(found) ? found : hash;
this.tracked.set(file.path, { hash, stat: statKey(file, sweptAt) });
this.log.baseline.set(file.path, start);
const originalStat = this.adoptionStats.get(file.path);
const current = statSync(join(this.dir, file.path), { throwIfNoEntry: false });
const unavailable =
originalStat == null || !current || originalStat !== statKey(current, Infinity);
if (unavailable) {
await this.recordUnavailableMedia(file.path, start);
} else {
this.log.baseline.set(file.path, start);
this.persistLog(baselineRecord(this.log));
}
if (start !== hash) addChange(this.outsideGroup(), file.path, start, hash);
this.persistLog(baselineRecord(this.log));
this.saveStatCache();
}
async recordUnavailableMedia(path: string, after: string | null): Promise<void> {
const group = this.newGroup(OUTSIDE, "Original media version is unavailable");
group.changes.set(path, { path, before: null, after });
await this.commit(group, { unavailableBefore: [path] });
}
manifest(): Manifest {
return new Map([...this.tracked].map(([path, file]) => [path, file.hash]));
}
readStatCache(): Map<string, Tracked> {
readStatCache(): Map<string, CachedFile> {
try {
const files = JSON.parse(readFileSync(join(this.home, "stat.json"), "utf-8"));
return new Map(Object.entries(files as Record<string, Tracked>));
const files: unknown = JSON.parse(readFileSync(join(this.home, "stat.json"), "utf-8"));
const cache = new Map<string, CachedFile>();
if (!files || typeof files !== "object") return cache;
for (const [path, value] of Object.entries(files)) {
const cached = cachedFile(value);
if (cached) cache.set(path, cached);
}
return cache;
} catch {
return new Map();
}
@@ -575,7 +643,10 @@ class Engine {
}
saveStatCache(): void {
const files = JSON.stringify(Object.fromEntries(this.tracked));
const pending = [...this.adoptionStats].map(
([path, stat]) => [path, { hash: null, stat }] as const,
);
const files = JSON.stringify(Object.fromEntries([...pending, ...this.tracked]));
mkdirSync(this.home, { recursive: true });
replaceFileAtomically(join(this.home, "stat.json"), files, 0o644);
}
@@ -627,9 +698,9 @@ class Engine {
}
/** The file's hash once copied in; null when it was removed before the copy (the next sweep records that). */
async storeIfPresent(path: string): Promise<string | null> {
async storeIfPresent(path: string, signal?: AbortSignal): Promise<string | null> {
try {
return await this.blobs.put(join(this.dir, path));
return await this.blobs.put(join(this.dir, path), signal);
} catch (error) {
if ((error as NodeJS.ErrnoException).code === "ENOENT") return null;
throw error;
@@ -1070,6 +1141,7 @@ class Engine {
async undoNow(id: string, who: HistoryWho, mode?: UndoMode): Promise<HistoryResult> {
const entry = this.entry(id);
this.assertArchivedBefore([entry]);
const changed = this.movedOn(entry);
if (changed.length && !mode) return { ok: false, conflict: this.conflict(entry, changed) };
if (mode === "back-to-before") {
@@ -1129,7 +1201,22 @@ class Engine {
return { files: [...paths], newer: newer.map((later) => later.id) };
}
assertArchivedBefore(entries: readonly HistoryEntry[]): void {
const paths = entries.flatMap((entry) => entry.unavailableBefore ?? []);
if (paths.length)
throw new Error(
`Cannot restore ${paths.join(", ")}: the original media version is unavailable because its first archive did not complete before the file changed.`,
);
}
assertArchivedAt(point: string, side: HistoryEntrySide = "after"): void {
const index = point === START ? -1 : this.log.entries.findIndex((entry) => entry.id === point);
if (point !== START && index === -1) return;
this.assertArchivedBefore(this.log.entries.slice(index + (side === "after" ? 1 : 0)));
}
async restoreNow(point: string, who: HistoryWho, label: string): Promise<HistoryEntry | null> {
this.assertArchivedAt(point);
const files = manifestAt(this.log, point);
if (!files) throw new Error("That point is no longer kept in this project's history.");
const target = new Map<string, string | null>(
@@ -1190,12 +1277,14 @@ class Engine {
),
peek: (point) => {
this.assertOpen();
this.assertArchivedAt(point);
const files = manifestAt(this.log, point);
return files && Object.fromEntries(files);
},
checkout: (entryId, side, emptyDir) =>
this.queue(async () => {
this.entry(entryId);
this.assertArchivedAt(entryId, side);
if (!existsSync(this.dir)) throw new Error(`The project folder is gone: ${this.dir}`);
const dest = resolve(emptyDir);
if (isWithin(this.dir, dest))
@@ -1234,13 +1323,16 @@ class Engine {
replacedAtPath: () => this.whereFolder() === "replaced",
close: () =>
(this.closing ??= (async () => {
// A copy cut short would leave its media out of the baseline, so a change made while closed is lost.
await Promise.all(this.adopting.values());
// Stop resumable archives and clean their partial copies before releasing ownership.
this.adoptionController.abort();
const adoption = await Promise.allSettled(this.adopting.values());
if (this.notedTimer) clearTimeout(this.notedTimer);
this.stopHearing?.();
const settled = this.queue(() => this.settleAll());
this.closed = true;
await settled;
const failed = adoption.find((result) => result.status === "rejected");
if (failed?.status === "rejected") throw failed.reason;
})()),
};
}
@@ -1290,7 +1382,7 @@ export async function openProjectHistory(options: ProjectHistoryOptions): Promis
const api = engine.api();
return {
...api,
close: () => api.close().finally(release),
close: () => api.close().then(release),
};
} catch (error) {
release();