From e3d65c5ff9536d33ca6100376de78cb881912d21 Mon Sep 17 00:00:00 2001 From: Miguel Angel Simon Sierra Date: Wed, 7 Oct 2026 03:46:45 -0700 Subject: [PATCH] fix(studio-server): cancel background media adoption on close --- .../studio-server/src/history/blobStore.ts | 40 ++- .../studio-server/src/history/historyLog.ts | 4 +- .../src/history/projectHistory.close.test.ts | 298 ++++++++++++++++++ .../src/history/projectHistory.test.ts | 101 +++++- .../src/history/projectHistory.ts | 132 ++++++-- 5 files changed, 528 insertions(+), 47 deletions(-) create mode 100644 packages/studio-server/src/history/projectHistory.close.test.ts diff --git a/packages/studio-server/src/history/blobStore.ts b/packages/studio-server/src/history/blobStore.ts index a7e923697e..5bdbdc9217 100644 --- a/packages/studio-server/src/history/blobStore.ts +++ b/packages/studio-server/src/history/blobStore.ts @@ -1,13 +1,14 @@ import { createHash, randomUUID } from "node:crypto"; -import { constants, createReadStream, renameSync } from "node:fs"; +import { constants, createReadStream, createWriteStream, renameSync } from "node:fs"; import { copyFile, mkdir, readdir, readFile, rename, rm, stat } from "node:fs/promises"; +import { pipeline } from "node:stream/promises"; import { dirname, join } from "node:path"; import { atomicTempPath } from "@hyperframes/core/atomic-file"; /** File contents stored once by sha256, text and binary alike. */ export interface BlobStore { - /** Copies the file in (a clone where the file system can) and returns the copy's hash. */ - put(absPath: string): Promise; + /** Archives the file and returns its hash; a signal makes copy and hash cancellable. */ + put(absPath: string, signal?: AbortSignal): Promise; has(hash: string): boolean; read(hash: string): Promise; /** Writes the blob's bytes to `absPath` by clone-or-copy and rename, so a reader never sees half a file. */ @@ -19,9 +20,9 @@ export interface BlobStore { const BLOB_HASH = /^[0-9a-f]{64}$/; -async function hashFile(path: string): Promise { +async function hashFile(path: string, signal?: AbortSignal): Promise { const hash = createHash("sha256"); - for await (const chunk of createReadStream(path)) hash.update(chunk); + for await (const chunk of createReadStream(path, { signal })) hash.update(chunk); return hash.digest("hex"); } @@ -52,23 +53,30 @@ export async function openBlobStore(dir: string): Promise { let total = [...sizes.values()].reduce((sum, size) => sum + size, 0); return { - async put(absPath) { + async put(absPath, signal) { // Hash the copy, not the source, so a write racing the copy can never file bytes under the wrong hash. const temp = join(dir, `incoming-${randomUUID()}`); // Recreated if removed while open, so a missing folder is never mistaken for a deleted project file. await mkdir(dir, { recursive: true }); - await copyFile(absPath, temp, constants.COPYFILE_FICLONE); - const hash = await hashFile(temp); - if (sizes.has(hash)) { - await rm(temp, { force: true }); + try { + if (signal) { + await pipeline(createReadStream(absPath), createWriteStream(temp), { signal }); + } else { + await copyFile(absPath, temp, constants.COPYFILE_FICLONE); + } + const hash = await hashFile(temp, signal); + signal?.throwIfAborted(); + if (sizes.has(hash)) return hash; + await mkdir(dirname(pathOf(hash)), { recursive: true }); + const size = (await stat(temp)).size; + signal?.throwIfAborted(); + await rename(temp, pathOf(hash)); + sizes.set(hash, size); + total += size; return hash; + } finally { + await rm(temp, { force: true }); } - await mkdir(dirname(pathOf(hash)), { recursive: true }); - const size = (await stat(temp)).size; - await rename(temp, pathOf(hash)); - sizes.set(hash, size); - total += size; - return hash; }, has: (hash) => sizes.has(hash), read: async (hash) => readFile(pathOf(hash)), diff --git a/packages/studio-server/src/history/historyLog.ts b/packages/studio-server/src/history/historyLog.ts index ee5c3bf996..a15a29b73a 100644 --- a/packages/studio-server/src/history/historyLog.ts +++ b/packages/studio-server/src/history/historyLog.ts @@ -8,7 +8,7 @@ export interface HistoryWho { name: string; } -/** A file's sha256 before and after; null is "did not exist", so a create or delete undoes like any edit. */ +/** A file's sha256 before and after; null is "did not exist" unless the entry names an unavailable original. */ export interface HistoryFileChange { path: string; before: string | null; @@ -24,6 +24,8 @@ export interface HistoryEntry { startedAt: number; endedAt: number; files: HistoryFileChange[]; + /** These files existed before this entry, but their original bytes were never archived. */ + unavailableBefore?: string[]; /** Set on an undo: the entry it reverted. Redo is undoing the undo. */ undoes?: string; /** Set on a restore: the point (an entry id, or START) the files were made equal to. */ diff --git a/packages/studio-server/src/history/projectHistory.close.test.ts b/packages/studio-server/src/history/projectHistory.close.test.ts new file mode 100644 index 0000000000..3a059a5e1d --- /dev/null +++ b/packages/studio-server/src/history/projectHistory.close.test.ts @@ -0,0 +1,298 @@ +// @vitest-environment node +import { + mkdirSync, + mkdtempSync, + writeFileSync, + readdirSync, + readFileSync, + rmSync, + statSync, +} from "node:fs"; +import { createHash } from "node:crypto"; +import { HistoryBusyError } from "./ownerLock"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { afterEach, expect, it, vi } from "vitest"; +import { openProjectHistory } from "./projectHistory"; +import { START } from "./historyLog"; + +const archive = vi.hoisted(() => ({ + hold: false, + entered: "", + hash: false, + failCleanup: false, + partialPath: "", + aborted: false, + release: null as (() => void) | null, +})); +vi.mock("node:fs", async (importOriginal) => { + const fs = await importOriginal(); + const { Readable, addAbortSignal } = await import("node:stream"); + return { + ...fs, + createReadStream: (path: string, options?: Parameters[1]) => { + const heldPath = archive.hash + ? String(path).includes("incoming-") + : String(path).endsWith(".mp4"); + if (!archive.hold || !heldPath) return fs.createReadStream(path, options); + archive.entered = archive.hash ? "hash" : "copy"; + let sent = false; + const stream = new Readable({ + read() { + if (sent) return; + sent = true; + this.push(Buffer.alloc(65536, 7)); + }, + }); + archive.release = () => { + stream.push(Buffer.alloc(2 * 1024 ** 2 - 65536, 7)); + stream.push(null); + }; + if (typeof options === "object" && options?.signal) addAbortSignal(options.signal, stream); + stream.on("close", () => { + archive.aborted = true; + }); + return stream; + }, + }; +}); +vi.mock("node:fs/promises", async (importOriginal) => { + const fs = await importOriginal(); + return { + ...fs, + rm: async (...args: Parameters) => { + if (archive.failCleanup && String(args[0]) === archive.partialPath) { + throw Object.assign(new Error("Cannot remove partial archive"), { code: "EPERM" }); + } + return fs.rm(...args); + }, + copyFile: async (...args: Parameters) => { + if (archive.hold && !archive.hash && String(args[0]).endsWith(".mp4")) { + await fs.writeFile(args[1], Buffer.alloc(65536, 7)); + await new Promise((resolve) => { + archive.release = resolve; + }); + } + return fs.copyFile(...args); + }, + }; +}); +const roots: string[] = []; +afterEach(() => { + archive.hold = false; + archive.hash = false; + archive.entered = ""; + archive.failCleanup = false; + archive.release = null; + archive.aborted = false; + for (const dir of roots.splice(0)) rmSync(dir, { recursive: true, force: true }); +}); + +async function heldProject(names = ["clip.mp4"], onError?: (error: Error) => void) { + const root = mkdtempSync(join(tmpdir(), "hf-held-adoption-")); + roots.push(root); + const projectDir = join(root, "project"); + mkdirSync(projectDir); + writeFileSync(join(projectDir, "index.html"), "before"); + for (const name of names) writeFileSync(join(projectDir, name), Buffer.alloc(2 * 1024 ** 2, 7)); + archive.hold = true; + const historyRoot = join(root, "history"); + const history = await openProjectHistory({ projectDir, historyRoot, onError, quietMs: 30 }); + const home = join(historyRoot, history.projectId); + return { root, projectDir, historyRoot, history, home, blobs: join(home, "blobs") }; +} + +it("close cancels a held adoption, flushes text edits and resumes pending media on reopen", async () => { + const onError = vi.fn(); + const { history, historyRoot, home, blobs, projectDir } = await heldProject(undefined, onError); + try { + await vi.waitFor(() => { + const partial = readdirSync(blobs).find((name) => name.startsWith("incoming-")); + expect(partial).toBeDefined(); + expect(statSync(join(blobs, partial!)).size).toBeGreaterThan(0); + }); + writeFileSync(join(projectDir, "index.html"), "after"); + let closed = false; + const closing = history.close().then(() => { + closed = true; + }); + try { + await vi.waitFor(() => expect(closed).toBe(true)); + } finally { + archive.release?.(); + await closing; + } + expect(archive.aborted).toBe(true); + expect(JSON.parse(readFileSync(join(home, "adopting.json"), "utf8"))).toEqual(["clip.mp4"]); + expect(onError).not.toHaveBeenCalled(); + const log = readFileSync(join(home, "log.jsonl"), "utf8") + .trim() + .split("\n") + .map((line) => JSON.parse(line)); + const textEdit = log + .filter((record) => record.type === "entry") + .flatMap((record) => record.entry.files) + .find((file) => file.path === "index.html"); + expect(textEdit).toMatchObject({ + before: createHash("sha256").update("before").digest("hex"), + after: createHash("sha256").update("after").digest("hex"), + }); + expect(readdirSync(blobs).some((name) => name.startsWith("incoming-"))).toBe(false); + archive.hold = false; + const reopened = await openProjectHistory({ projectDir, historyRoot, quietMs: 30 }); + try { + await vi.waitFor(() => expect(reopened.peek(START)).toHaveProperty(["clip.mp4"])); + expect(reopened.list().flatMap((entry) => entry.files.map((file) => file.path))).toContain( + "index.html", + ); + expect(reopened.peek(START)?.["clip.mp4"]).toMatch(/^[0-9a-f]{64}$/); + } finally { + await reopened.close(); + } + } finally { + archive.hold = false; + await history.close(); + } +}, 5000); + +it.each(["copy", "hash"])( + "cancelled %s leaves no partial blob before ownership release", + async (phase) => { + const { openBlobStore } = await import("./blobStore"); + const root = mkdtempSync(join(tmpdir(), "hf-cancel-blob-")); + roots.push(root); + const source = join(root, "clip.mp4"); + writeFileSync(source, Buffer.alloc(2 * 1024 ** 2, 7)); + const dir = join(root, "blobs"); + const store = await openBlobStore(dir); + archive.hold = true; + archive.hash = phase === "hash"; + const controller = new AbortController(); + const pending = store.put(source, controller.signal); + let error: unknown; + const settled = pending.catch((caught) => { + error = caught; + }); + await vi.waitFor(() => + expect(readdirSync(dir).some((name) => name.startsWith("incoming-"))).toBe(true), + ); + await vi.waitFor(() => expect(archive.entered).toBe(phase)); + controller.abort(); + try { + await vi.waitFor(() => expect(error).toMatchObject({ name: "AbortError" })); + } finally { + archive.release?.(); + await settled; + } + expect(readdirSync(dir)).toEqual([]); + expect(store.bytes()).toBe(0); + }, +); + +it("failed cancellation cleanup flushes edits but refuses to release ownership", async () => { + const { history, historyRoot, home, blobs, projectDir } = await heldProject(); + await vi.waitFor(() => + expect(readdirSync(blobs).some((name) => name.startsWith("incoming-"))).toBe(true), + ); + writeFileSync(join(projectDir, "index.html"), "after"); + archive.partialPath = join( + blobs, + readdirSync(blobs).find((name) => name.startsWith("incoming-"))!, + ); + archive.failCleanup = true; + await expect(history.close()).rejects.toMatchObject({ code: "EPERM" }); + expect(readdirSync(blobs).some((name) => name.startsWith("incoming-"))).toBe(true); + expect(readFileSync(join(home, "log.jsonl"), "utf8")).toContain( + createHash("sha256").update("after").digest("hex"), + ); + await expect( + openProjectHistory({ projectDir, historyRoot, ownerWaitMs: 0 }), + ).rejects.toBeInstanceOf(HistoryBusyError); +}); + +it.each(["replace", "delete", "missing-cache", "malformed-cache"])( + "refuses unknown original media after %s while closed", + async (change) => { + const { root, history, historyRoot, home, projectDir } = await heldProject(); + const clip = join(projectDir, "clip.mp4"); + await vi.waitFor(() => + expect(readdirSync(join(home, "blobs")).some((name) => name.startsWith("incoming-"))).toBe( + true, + ), + ); + await history.close(); + if (change === "delete") rmSync(clip); + else writeFileSync(clip, Buffer.alloc(2 * 1024 ** 2, 9)); + if (change === "missing-cache") rmSync(join(home, "stat.json")); + if (change === "malformed-cache") + writeFileSync( + join(home, "stat.json"), + JSON.stringify({ "clip.mp4": { hash: "invalid", stat: null } }), + ); + archive.hold = false; + const reopened = await openProjectHistory({ projectDir, historyRoot, quietMs: 30 }); + const who = { kind: "person" as const, name: "You" }; + try { + await vi.waitFor(() => + expect(reopened.list().some((entry) => entry.unavailableBefore?.includes("clip.mp4"))).toBe( + true, + ), + ); + const entry = reopened.list().find((entry) => entry.unavailableBefore?.includes("clip.mp4"))!; + await expect(reopened.step("back", who)).rejects.toThrow( + "original media version is unavailable", + ); + await expect(reopened.undo(entry.id, { who })).rejects.toThrow( + "original media version is unavailable", + ); + await expect(reopened.restore(START, who)).rejects.toThrow( + "original media version is unavailable", + ); + expect(() => reopened.peek(START)).toThrow("original media version is unavailable"); + const checkout = join(root, "checkout"); + mkdirSync(checkout); + await expect(reopened.checkout(entry.id, "before", checkout)).rejects.toThrow( + "original media version is unavailable", + ); + expect(readdirSync(checkout)).toEqual([]); + if (change === "delete") expect(readdirSync(projectDir)).not.toContain("clip.mp4"); + else expect(readFileSync(clip).equals(Buffer.alloc(2 * 1024 ** 2, 9))).toBe(true); + } finally { + await reopened.close(); + } + const again = await openProjectHistory({ projectDir, historyRoot, quietMs: 30 }); + try { + await expect(again.step("back", who)).rejects.toThrow( + "original media version is unavailable", + ); + } finally { + await again.close(); + } + }, +); + +it("a later media replacement while serial adoption waits is never filed as the original baseline", async () => { + const { history, projectDir } = await heldProject(["a.mp4", "b.mp4"]); + try { + await vi.waitFor(() => expect(archive.entered).toBe("copy")); + writeFileSync(join(projectDir, "b.mp4"), Buffer.alloc(2 * 1024 ** 2, 9)); + archive.hold = false; + archive.release!(); + await vi.waitFor(() => + expect(history.list().some((entry) => entry.unavailableBefore?.includes("b.mp4"))).toBe(true), + ); + const entry = history.list().find((entry) => entry.unavailableBefore?.includes("b.mp4"))!; + expect(entry.files.find((file) => file.path === "b.mp4")?.after).toBe( + createHash("sha256") + .update(Buffer.alloc(2 * 1024 ** 2, 9)) + .digest("hex"), + ); + await expect(history.step("back", { kind: "person", name: "You" })).rejects.toThrow( + "original media version is unavailable", + ); + } finally { + archive.hold = false; + archive.release?.(); + await history.close(); + } +}); diff --git a/packages/studio-server/src/history/projectHistory.test.ts b/packages/studio-server/src/history/projectHistory.test.ts index 9f2eb29593..450d3d5755 100644 --- a/packages/studio-server/src/history/projectHistory.test.ts +++ b/packages/studio-server/src/history/projectHistory.test.ts @@ -14,6 +14,7 @@ import { utimesSync, writeFileSync, } from "node:fs"; +import { once } from "node:events"; import { spawnSync } from "node:child_process"; import { tmpdir } from "node:os"; import { basename, dirname, join, relative } from "node:path"; @@ -29,6 +30,7 @@ const mediaCopy = vi.hoisted(() => ({ held: null as Promise | null, stored: null as (() => Promise) | null, copies: [] as string[], + entered: 0, fails: null as ((copy: number) => boolean) | null, })); vi.mock("./blobStore", async (importOriginal) => { @@ -39,13 +41,22 @@ vi.mock("./blobStore", async (importOriginal) => { const store = await real.openBlobStore(dir); return { ...store, - put: async (path: string) => { + put: async (path: string, signal?: AbortSignal) => { if (path.endsWith(".mp4")) { - await mediaCopy.held; + mediaCopy.entered++; + if (signal && mediaCopy.held) { + signal.throwIfAborted(); + await Promise.race([ + mediaCopy.held, + once(signal, "abort").then(() => { + throw signal.reason; + }), + ]); + } else await mediaCopy.held; mediaCopy.copies.push(basename(path)); if (mediaCopy.fails?.(mediaCopy.copies.length)) throw new Error("The disk is full."); } - const hash = await store.put(path); + const hash = await store.put(path, signal); if (path.endsWith(".mp4")) await mediaCopy.stored?.(); return hash; }, @@ -61,7 +72,7 @@ const cleanup: Array<() => unknown> = []; afterEach(async () => { for (const step of cleanup.splice(0).reverse()) await step(); - Object.assign(mediaCopy, { held: null, stored: null, copies: [], fails: null }); + Object.assign(mediaCopy, { held: null, stored: null, copies: [], entered: 0, fails: null }); }); const inside = (dir: string, path: string) => readFileSync(join(dir, path), "utf-8"); @@ -372,7 +383,7 @@ describe("openProjectHistory", () => { expect(mediaCopy.copies).toEqual(["a.mp4"]); }); - it("undoes a clip replaced while closed, though the history closed before its copy finished", async () => { + it("undoes an archived clip replaced while closed", async () => { const clip = Buffer.alloc(2 * 1024 ** 2, 7); let release = () => {}; mediaCopy.held = new Promise((resolve) => (release = resolve)); @@ -380,9 +391,9 @@ describe("openProjectHistory", () => { { "index.html": "A", "clip.mp4": clip }, { quietMs: 30 }, ); - const closed = history.close(); release(); - await closed; + await vi.waitFor(() => expect(history.peek(START)).toHaveProperty(["clip.mp4"])); + await history.close(); write("clip.mp4", Buffer.alloc(2 * 1024 ** 2, 9)); const reopened = await open(projectDir, historyRoot, { quietMs: 30 }); await reopened.flush(); @@ -393,18 +404,88 @@ describe("openProjectHistory", () => { expect(readFileSync(join(projectDir, "clip.mp4")).equals(clip)).toBe(true); }); - it("adopts media whose copy a process exit cut short, rather than log it as added", async () => { + it("adopts interrupted legacy media with an explicit unavailable-original entry", async () => { const { history, write, projectDir, historyRoot } = await project({ "index.html": "A" }); await history.close(); // What an exit mid-copy leaves: the clip on disk, named as still being copied, not in the baseline. write("clip.mp4", Buffer.alloc(2 * 1024 ** 2, 7)); writeFileSync(join(historyRoot, history.projectId, "adopting.json"), '["clip.mp4"]'); const reopened = await open(projectDir, historyRoot, { quietMs: 30 }); - await vi.waitFor(() => expect(reopened.peek(START)).toHaveProperty(["clip.mp4"]), { + await vi.waitFor(() => expect(reopened.list()[0]?.unavailableBefore).toEqual(["clip.mp4"]), { timeout: 10_000, }); + expect(reopened.peek(reopened.list()[0]!.id)).toHaveProperty(["clip.mp4"]); await reopened.flush(); - expect(reopened.list()).toEqual([]); + expect(reopened.list()).toMatchObject([ + { unavailableBefore: ["clip.mp4"], label: "Original media version is unavailable" }, + ]); + }); + + it("a saved API predecessor is not mistaken for the original media replaced while closed", async () => { + mediaCopy.held = new Promise(() => {}); + const { history, write, projectDir, historyRoot } = await project( + { "index.html": "A", "clip.mp4": Buffer.alloc(2 * 1024 ** 2, 7) }, + { quietMs: 30 }, + ); + await vi.waitFor(() => expect(mediaCopy.entered).toBe(1)); + await history.close(); + const previous = Buffer.alloc(2 * 1024 ** 2, 9); + write("clip.mp4", previous); + let release = () => {}; + mediaCopy.held = new Promise((resolve) => { + release = resolve; + }); + const reopened = await open(projectDir, historyRoot, { quietMs: 30 }); + await vi.waitFor(() => expect(mediaCopy.entered).toBe(2)); + const current = Buffer.alloc(2 * 1024 ** 2, 11); + write("clip.mp4", current); + recordFileWriteReceipt(join(projectDir, "clip.mp4"), { + path: "clip.mp4", + version: fileContentVersion(current), + writeToken: "media-write", + overwrote: previous, + }); + await reopened.flush(); + release(); + await vi.waitFor(() => + expect(reopened.list().some((entry) => entry.unavailableBefore?.includes("clip.mp4"))).toBe( + true, + ), + ); + await reopened.flush(); + expect((await reopened.step("back", you)).ok).toBe(true); + expect(readFileSync(join(projectDir, "clip.mp4")).equals(previous)).toBe(true); + await expect(reopened.step("back", you)).rejects.toThrow( + "original media version is unavailable", + ); + await expect(reopened.restore(START, you)).rejects.toThrow( + "original media version is unavailable", + ); + }); + + it("close cancels adoption's prune-triggered recopy and preserves it for reopening", async () => { + const clip = Buffer.alloc(2 * 1024 ** 2, 7); + const { history, write, projectDir, historyRoot } = await project( + { "index.html": "A", "clip.mp4": clip }, + { budgetBytes: 1, quietMs: 30 }, + ); + mediaCopy.stored = async () => { + mediaCopy.stored = null; + mediaCopy.held = new Promise(() => {}); + await change(history, you, "Color", () => write("index.html", "B")); + }; + await vi.waitFor(() => expect(mediaCopy.entered).toBe(2)); + await history.close(); + const home = join(historyRoot, history.projectId); + expect(JSON.parse(readFileSync(join(home, "adopting.json"), "utf8"))).toEqual(["clip.mp4"]); + expect(readdirSync(join(home, "blobs")).some((name) => name.startsWith("incoming-"))).toBe( + false, + ); + expect(readFileSync(join(home, "log.jsonl"), "utf8")).toContain("Color"); + mediaCopy.held = null; + const reopened = await open(projectDir, historyRoot, { quietMs: 30 }); + await vi.waitFor(() => expect(reopened.peek(START)).toHaveProperty(["clip.mp4"])); + expect(readFileSync(join(projectDir, "clip.mp4")).equals(clip)).toBe(true); }); it("keeps a media copy that a budget prune ran into before history recorded it", async () => { diff --git a/packages/studio-server/src/history/projectHistory.ts b/packages/studio-server/src/history/projectHistory.ts index c894c7510e..efdd9ec877 100644 --- a/packages/studio-server/src/history/projectHistory.ts +++ b/packages/studio-server/src/history/projectHistory.ts @@ -180,6 +180,16 @@ interface Tracked { stat: string; } +type CachedFile = Tracked | { hash: null; stat: string | null }; + +function cachedFile(value: unknown): CachedFile | null { + if (!value || typeof value !== "object") return null; + const { hash, stat } = value as Record; + if (typeof hash === "string" && typeof stat === "string") return { hash, stat }; + if (hash === null && (typeof stat === "string" || stat === null)) return { hash, stat }; + return null; +} + interface Group { id: string; who: HistoryWho; @@ -323,6 +333,8 @@ class Engine { /** A coalescing claim, open until another key, its idle timer, an operation, a window, or another write. */ claimed: { group: Group; key: string; timer?: NodeJS.Timeout } | null = null; adopting = new Map>(); + readonly adoptionController = new AbortController(); + adoptionStats = new Map(); /** Per path and hash an API write left, the hash of the bytes it replaced, until walked or written past. */ overwritten = new Map>(); stopHearing: (() => void) | undefined; @@ -417,15 +429,31 @@ class Engine { const cached = cache.get(path); this.tracked.set(path, { hash, stat: cached?.hash === hash ? cached.stat : "" }); } - const left = this.readAdopting(); - const unadopted = (file: { path: string }) => - left.has(file.path) && !this.tracked.has(file.path); - this.adoptMediaInBackground(historyFiles(this.dir).filter(unadopted), this.now()); + await this.resumeAdoption(cache); // What changed while the project was closed is one outside entry, or the closed window's. this.reopenClosedWindow(); await this.settleAll(); } + async resumeAdoption(cache: Map): Promise { + const left = this.readAdopting(); + for (const path of left) { + const pending = cache.get(path); + this.adoptionStats.set(path, pending?.hash === null ? pending.stat : null); + } + const unadopted = (file: { path: string }) => + left.has(file.path) && !this.tracked.has(file.path); + const files = historyFiles(this.dir); + const present = new Set(files.map((file) => file.path)); + for (const path of left) { + if (!present.has(path) && !this.tracked.has(path)) { + await this.recordUnavailableMedia(path, null); + this.adoptionStats.delete(path); + } + } + this.adoptMediaInBackground(files.filter(unadopted), this.now()); + } + hearWrites(): void { const realDir = realFilePath(this.dir); this.stopHearing = onFileOverwritten((absPath, version, bytes) => { @@ -510,8 +538,11 @@ class Engine { for (const file of files) { copied = copied.then(() => this.adopt(file, sweptAt)); this.adopting.set(file.path, copied); + if (!this.adoptionStats.has(file.path)) + this.adoptionStats.set(file.path, statKey(file, Infinity)); } this.saveAdopting(); + this.saveStatCache(); } saveAdopting(): void { @@ -523,42 +554,79 @@ class Engine { async adopt(file: ListedFile, sweptAt: number) { try { + this.adoptionController.signal.throwIfAborted(); if (this.whereFolder() !== "here") return; - const stored = await this.storeIfPresent(file.path); + const stored = await this.storeIfPresent(file.path, this.adoptionController.signal); await this.queue(async () => { // Still named for the next open until recorded, so a failed record is never logged as added. await this.recordAdopted(file, sweptAt, stored); this.adopting.delete(file.path); + this.adoptionStats.delete(file.path); + this.saveStatCache(); this.saveAdopting(); }); } catch (error) { - if (!(error instanceof HistoryClosedError)) this.options.onError?.(error); + if (error instanceof HistoryClosedError) return; + if (this.adoptionController.signal.aborted) { + if ((error as Error).name === "AbortError") return; + throw error; + } + this.options.onError?.(error); } } async recordAdopted(file: ListedFile, sweptAt: number, stored: string | null) { - if (stored === null || this.whereFolder() !== "here") return; + this.adoptionController.signal.throwIfAborted(); + if (this.whereFolder() !== "here") return; + if (stored === null) return this.recordUnavailableMedia(file.path, null); // A prune that ran while this copy waited for the queue may have taken its bytes. - const hash = this.blobs.has(stored) ? stored : await this.storeIfPresent(file.path); - if (hash === null) return; + const hash = this.blobs.has(stored) + ? stored + : await this.storeIfPresent(file.path, this.adoptionController.signal); + this.adoptionController.signal.throwIfAborted(); + if (hash === null) return this.recordUnavailableMedia(file.path, null); + await this.recordMediaBaseline(file, sweptAt, hash); + } + + async recordMediaBaseline(file: ListedFile, sweptAt: number, hash: string): Promise { const change = { path: file.path, before: null, after: hash }; const found = this.overwrittenBy(change, undefined, new Set()); const start = found && this.blobs.has(found) ? found : hash; this.tracked.set(file.path, { hash, stat: statKey(file, sweptAt) }); - this.log.baseline.set(file.path, start); + const originalStat = this.adoptionStats.get(file.path); + const current = statSync(join(this.dir, file.path), { throwIfNoEntry: false }); + const unavailable = + originalStat == null || !current || originalStat !== statKey(current, Infinity); + if (unavailable) { + await this.recordUnavailableMedia(file.path, start); + } else { + this.log.baseline.set(file.path, start); + this.persistLog(baselineRecord(this.log)); + } if (start !== hash) addChange(this.outsideGroup(), file.path, start, hash); - this.persistLog(baselineRecord(this.log)); this.saveStatCache(); } + async recordUnavailableMedia(path: string, after: string | null): Promise { + const group = this.newGroup(OUTSIDE, "Original media version is unavailable"); + group.changes.set(path, { path, before: null, after }); + await this.commit(group, { unavailableBefore: [path] }); + } + manifest(): Manifest { return new Map([...this.tracked].map(([path, file]) => [path, file.hash])); } - readStatCache(): Map { + readStatCache(): Map { try { - const files = JSON.parse(readFileSync(join(this.home, "stat.json"), "utf-8")); - return new Map(Object.entries(files as Record)); + const files: unknown = JSON.parse(readFileSync(join(this.home, "stat.json"), "utf-8")); + const cache = new Map(); + if (!files || typeof files !== "object") return cache; + for (const [path, value] of Object.entries(files)) { + const cached = cachedFile(value); + if (cached) cache.set(path, cached); + } + return cache; } catch { return new Map(); } @@ -575,7 +643,10 @@ class Engine { } saveStatCache(): void { - const files = JSON.stringify(Object.fromEntries(this.tracked)); + const pending = [...this.adoptionStats].map( + ([path, stat]) => [path, { hash: null, stat }] as const, + ); + const files = JSON.stringify(Object.fromEntries([...pending, ...this.tracked])); mkdirSync(this.home, { recursive: true }); replaceFileAtomically(join(this.home, "stat.json"), files, 0o644); } @@ -627,9 +698,9 @@ class Engine { } /** The file's hash once copied in; null when it was removed before the copy (the next sweep records that). */ - async storeIfPresent(path: string): Promise { + async storeIfPresent(path: string, signal?: AbortSignal): Promise { try { - return await this.blobs.put(join(this.dir, path)); + return await this.blobs.put(join(this.dir, path), signal); } catch (error) { if ((error as NodeJS.ErrnoException).code === "ENOENT") return null; throw error; @@ -1070,6 +1141,7 @@ class Engine { async undoNow(id: string, who: HistoryWho, mode?: UndoMode): Promise { const entry = this.entry(id); + this.assertArchivedBefore([entry]); const changed = this.movedOn(entry); if (changed.length && !mode) return { ok: false, conflict: this.conflict(entry, changed) }; if (mode === "back-to-before") { @@ -1129,7 +1201,22 @@ class Engine { return { files: [...paths], newer: newer.map((later) => later.id) }; } + assertArchivedBefore(entries: readonly HistoryEntry[]): void { + const paths = entries.flatMap((entry) => entry.unavailableBefore ?? []); + if (paths.length) + throw new Error( + `Cannot restore ${paths.join(", ")}: the original media version is unavailable because its first archive did not complete before the file changed.`, + ); + } + + assertArchivedAt(point: string, side: HistoryEntrySide = "after"): void { + const index = point === START ? -1 : this.log.entries.findIndex((entry) => entry.id === point); + if (point !== START && index === -1) return; + this.assertArchivedBefore(this.log.entries.slice(index + (side === "after" ? 1 : 0))); + } + async restoreNow(point: string, who: HistoryWho, label: string): Promise { + this.assertArchivedAt(point); const files = manifestAt(this.log, point); if (!files) throw new Error("That point is no longer kept in this project's history."); const target = new Map( @@ -1190,12 +1277,14 @@ class Engine { ), peek: (point) => { this.assertOpen(); + this.assertArchivedAt(point); const files = manifestAt(this.log, point); return files && Object.fromEntries(files); }, checkout: (entryId, side, emptyDir) => this.queue(async () => { this.entry(entryId); + this.assertArchivedAt(entryId, side); if (!existsSync(this.dir)) throw new Error(`The project folder is gone: ${this.dir}`); const dest = resolve(emptyDir); if (isWithin(this.dir, dest)) @@ -1234,13 +1323,16 @@ class Engine { replacedAtPath: () => this.whereFolder() === "replaced", close: () => (this.closing ??= (async () => { - // A copy cut short would leave its media out of the baseline, so a change made while closed is lost. - await Promise.all(this.adopting.values()); + // Stop resumable archives and clean their partial copies before releasing ownership. + this.adoptionController.abort(); + const adoption = await Promise.allSettled(this.adopting.values()); if (this.notedTimer) clearTimeout(this.notedTimer); this.stopHearing?.(); const settled = this.queue(() => this.settleAll()); this.closed = true; await settled; + const failed = adoption.find((result) => result.status === "rejected"); + if (failed?.status === "rejected") throw failed.reason; })()), }; } @@ -1290,7 +1382,7 @@ export async function openProjectHistory(options: ProjectHistoryOptions): Promis const api = engine.api(); return { ...api, - close: () => api.close().finally(release), + close: () => api.close().then(release), }; } catch (error) { release();