diff --git a/packages/studio-server/src/history/__fixtures__/dieMidFirstCopy.ts b/packages/studio-server/src/history/__fixtures__/dieMidFirstCopy.ts new file mode 100644 index 0000000000..65b812ea80 --- /dev/null +++ b/packages/studio-server/src/history/__fixtures__/dieMidFirstCopy.ts @@ -0,0 +1,19 @@ +// A project's first history open, killed outright mid-copy with one copy half written. +import { writeFileSync } from "node:fs"; +import fsp from "node:fs/promises"; +import { syncBuiltinESMExports } from "node:module"; + +const [projectDir, historyRoot, dieAtWrite] = process.argv.slice(2); +const realWriteFile = fsp.writeFile; +let writes = 0; +fsp.writeFile = ((...args: Parameters) => { + if (String(args[0]).includes("incoming-") && ++writes === Number(dieAtWrite)) { + writeFileSync(String(args[0]), "half a co"); + process.kill(process.pid, "SIGKILL"); + } + return realWriteFile(...args); +}) as typeof realWriteFile; +syncBuiltinESMExports(); + +const { openProjectHistory } = await import("../projectHistory.js"); +await openProjectHistory({ projectDir: projectDir!, historyRoot: historyRoot! }); diff --git a/packages/studio-server/src/history/blobStore.test.ts b/packages/studio-server/src/history/blobStore.test.ts index 536f355314..07ced19953 100644 --- a/packages/studio-server/src/history/blobStore.test.ts +++ b/packages/studio-server/src/history/blobStore.test.ts @@ -107,14 +107,17 @@ describe("the history store on Windows, while another process briefly holds a fi const store = await openBlobStore(join(root, "history", "blobs")); let refused = 0; vi.mocked(fsp.rename).mockImplementation(async (from, to) => { - if (refused++ < 2) throw busy(); + if (refused < 2) { + refused += 1; + throw busy(); + } return realRename(from, to); }); const hash = await store.put(source); expect((await store.read(hash)).toString("utf8")).toBe("

kept

"); - expect(refused).toBe(3); + expect(refused).toBe(2); }); }); @@ -170,3 +173,20 @@ describe("a restore of a file the preview is serving", () => { expect(heard).toEqual([]); }); }); + +describe("the history store's count of what it holds", () => { + it("counts a file once when two copies of it are filed together", async () => { + const root = mkdtempSync(join(tmpdir(), "hf-blob-count-")); + try { + const small = join(root, "small.txt"); + const large = join(root, "large.txt"); + writeFileSync(small, "same bytes"); + writeFileSync(large, "x".repeat(200 * 1024)); + const store = await openBlobStore(join(root, "history", "blobs")); + await Promise.all([store.put(small), store.put(small), store.put(large), store.put(large)]); + expect(store.bytes()).toBe("same bytes".length + 200 * 1024); + } finally { + rmSync(root, { recursive: true, force: true }); + } + }); +}); diff --git a/packages/studio-server/src/history/blobStore.ts b/packages/studio-server/src/history/blobStore.ts index 9729b5a9ce..5aa1c16f19 100644 --- a/packages/studio-server/src/history/blobStore.ts +++ b/packages/studio-server/src/history/blobStore.ts @@ -1,6 +1,16 @@ import { createHash, randomUUID } from "node:crypto"; import { constants, createReadStream, createWriteStream } from "node:fs"; -import { copyFile, mkdir, open, readdir, readFile, rm, stat } from "node:fs/promises"; +import { + chmod, + copyFile, + mkdir, + open, + readdir, + readFile, + rm, + stat, + writeFile, +} from "node:fs/promises"; import { dirname, join } from "node:path"; import { pipeline } from "node:stream/promises"; import { atomicTempPath } from "@hyperframes/core/atomic-file"; @@ -13,7 +23,7 @@ import { stopServingAndClose } from "../helpers/servedFiles.js"; /** File contents stored once by sha256, text and binary alike. */ export interface BlobStore { - /** Copies the file in (a clone where the file system can) and returns the copy's hash. */ + /** Copies the file in (a clone where the file system can, for all but small files) and returns the copy's hash. */ put(absPath: string): Promise; reference(absPath: string): Promise; has(hash: string): boolean; @@ -29,6 +39,7 @@ export interface BlobStore { } const BLOB_HASH = /^[0-9a-f]{64}$/; +const READ_ONCE_AT_MOST_BYTES = 64 * 1024; export function blobPath(dir: string, hash: string): string { // Checked where the path is joined, so no caller can read or write outside the store. @@ -44,6 +55,16 @@ async function hashFile(path: string): Promise { return hash.digest("hex"); } +async function readIfSmall(path: string): Promise<{ bytes: Buffer; mode: number } | null> { + const handle = await open(path, "r"); + try { + const { size, mode } = await handle.stat(); + return size > READ_ONCE_AT_MOST_BYTES ? null : { bytes: await handle.readFile(), mode }; + } finally { + await handle.close(); + } +} + async function cloneOrStream(from: string, to: string): Promise { try { await copyFile(from, to, constants.COPYFILE_FICLONE_FORCE); @@ -144,8 +165,39 @@ export async function openBlobStore(dir: string): Promise { holds.delete(hash); }; + const file = (hash: string, size: number) => { + if (!sizes.has(hash)) total += size; + sizes.set(hash, size); + }; + const withFolderMadeIfMissing = (folder: string, write: () => Promise) => + write().catch(async (error: NodeJS.ErrnoException) => { + if (error.code !== "ENOENT") throw error; + await mkdir(folder, { recursive: true }); + await write(); + }); + // The bytes hashed are the bytes filed, so a write racing the read can never file bytes under the wrong hash. + const putBytes = async (bytes: Buffer, mode: number) => { + const hash = createHash("sha256").update(bytes).digest("hex"); + if (sizes.has(hash)) return hash; + const temp = join(dir, `incoming-${randomUUID()}`); + try { + await withFolderMadeIfMissing(dir, () => writeFile(temp, bytes, { mode })); + // A write applies the umask; the clone used for larger files keeps the mode exactly. + await chmod(temp, mode & 0o7777); + const to = blobPath(dir, hash); + await withFolderMadeIfMissing(dirname(to), () => renameWithRetryAsync(temp, to)); + } catch (error) { + await rm(temp, { force: true }); + throw error; + } + file(hash, bytes.length); + return hash; + }; + return { async put(absPath) { + const small = await readIfSmall(absPath); + if (small) return putBytes(small.bytes, small.mode); // Hash the copy, not the source, so a write racing the copy can never file bytes under the wrong hash. const temp = join(dir, `incoming-${randomUUID()}`); // Recreated if removed while open, so a missing folder is never mistaken for a deleted project file. @@ -162,8 +214,7 @@ export async function openBlobStore(dir: string): Promise { await mkdir(dirname(blobPath(dir, hash)), { recursive: true }); const size = (await stat(temp)).size; await renameWithRetryAsync(temp, blobPath(dir, hash)); - sizes.set(hash, size); - total += size; + file(hash, size); return hash; }, reference: (absPath) => hashFile(absPath), diff --git a/packages/studio-server/src/history/firstCopy.test.ts b/packages/studio-server/src/history/firstCopy.test.ts new file mode 100644 index 0000000000..89e11e166d --- /dev/null +++ b/packages/studio-server/src/history/firstCopy.test.ts @@ -0,0 +1,148 @@ +// @vitest-environment node +import { fork } from "node:child_process"; +import { createHash } from "node:crypto"; +import { + chmodSync, + existsSync, + mkdirSync, + mkdtempSync, + readdirSync, + readFileSync, + rmSync, + statSync, + writeFileSync, +} from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { afterEach, describe, expect, it, onTestFinished, vi } from "vitest"; + +vi.mock("node:fs/promises", async (importOriginal) => { + const actual = await importOriginal(); + return { ...actual, writeFile: vi.fn(actual.writeFile) }; +}); + +import * as fsp from "node:fs/promises"; +import { blobPath } from "./blobStore.js"; +import { START } from "./historyLog.js"; +import { openProjectHistory } from "./projectHistory.js"; + +const { writeFile: realWriteFile } = + await vi.importActual("node:fs/promises"); +const sha256 = (bytes: Buffer) => createHash("sha256").update(bytes).digest("hex"); +const temps: string[] = []; +const tempDir = (prefix: string) => { + const dir = mkdtempSync(join(tmpdir(), prefix)); + temps.push(dir); + return dir; +}; +const closes: (() => Promise)[] = []; + +afterEach(async () => { + vi.mocked(fsp.writeFile).mockImplementation(realWriteFile); + for (const close of closes.splice(0)) await close(); + for (const dir of temps.splice(0)) rmSync(dir, { recursive: true, force: true }); +}); + +function projectOf(count: number, tag: string): string { + const dir = tempDir("hf-first-copy-"); + mkdirSync(join(dir, "parts")); + writeFileSync(join(dir, "index.html"), `${tag}`); + for (let i = 0; i < count; i++) writeFileSync(join(dir, "parts", `p${i}.js`), `// ${tag} ${i}\n`); + return dir; +} +const homeIn = (historyRoot: string) => { + const homes = readdirSync(historyRoot, { withFileTypes: true }).filter((e) => e.isDirectory()); + return join(historyRoot, homes.find((e) => existsSync(join(historyRoot, e.name, "blobs")))!.name); +}; +async function open(projectDir: string, historyRoot: string) { + const history = await openProjectHistory({ projectDir, historyRoot }); + closes.push(() => history.close()); + return history; +} +/** Every file of the project is in the baseline, and its blob holds exactly the file's bytes. */ +function expectAllKept( + baseline: Record | null, + projectDir: string, + blobs: string, + paths: string[], +) { + expect(Object.keys(baseline ?? {}).sort()).toEqual([...paths].sort()); + for (const path of paths) { + const bytes = readFileSync(join(projectDir, path)); + expect(baseline![path]).toBe(sha256(bytes)); + expect(readFileSync(blobPath(blobs, sha256(bytes))).equals(bytes)).toBe(true); + } +} +const pathsOf = (count: number) => [ + "index.html", + ...Array.from({ length: count }, (_, i) => `parts/p${i}.js`), +]; + +describe("a project's first history copy", () => { + it("loses nothing when the process dies midway: the next open finishes it", async () => { + const projectDir = projectOf(120, "dies"); + const historyRoot = tempDir("hf-first-copy-root-"); + const before = readFileSync(join(projectDir, "parts", "p7.js")); + const died = await new Promise((done) => { + const fixture = join(import.meta.dirname, "__fixtures__", "dieMidFirstCopy.ts"); + fork(fixture, [projectDir, historyRoot, "40"], { execArgv: ["--import", "tsx"] }).once( + "exit", + (code, signal) => done(signal ?? code), + ); + }); + expect(died).toBe("SIGKILL"); + const home = homeIn(historyRoot); + expect(existsSync(join(home, "log.jsonl")), "the copy finished before the death").toBe(false); + const leftovers = () => + readdirSync(join(home, "blobs")).filter((name) => name.startsWith("incoming-")); + expect(leftovers().length, "the death left no half-written copy").toBeGreaterThan(0); + + const history = await open(projectDir, historyRoot); + + expectAllKept(history.peek(START), projectDir, join(home, "blobs"), pathsOf(120)); + expect(leftovers()).toEqual([]); + for (const shard of readdirSync(join(home, "blobs"))) + for (const hash of readdirSync(join(home, "blobs", shard))) + expect(sha256(readFileSync(join(home, "blobs", shard, hash)))).toBe(hash); + expect(readFileSync(join(projectDir, "parts", "p7.js")).equals(before)).toBe(true); + }); + + it("keeps the same contents and permissions, small files and large, under a strict umask", async () => { + const umask = process.umask(0o077); + onTestFinished(() => void process.umask(umask)); + const projectDir = projectOf(3, "kept"); + writeFileSync(join(projectDir, "large.js"), "x".repeat(200 * 1024)); + writeFileSync(join(projectDir, "same-a.js"), "same bytes"); + writeFileSync(join(projectDir, "same-b.js"), "same bytes"); + writeFileSync(join(projectDir, "run.sh"), "#!/bin/sh\n"); + chmodSync(join(projectDir, "run.sh"), 0o755); + const historyRoot = tempDir("hf-first-copy-root-"); + + const history = await open(projectDir, historyRoot); + + const blobs = join(homeIn(historyRoot), "blobs"); + const paths = [...pathsOf(3), "large.js", "same-a.js", "same-b.js", "run.sh"]; + expectAllKept(history.peek(START), projectDir, blobs, paths); + const script = readFileSync(join(projectDir, "run.sh")); + expect(statSync(blobPath(blobs, sha256(script))).mode & 0o111).toBe( + statSync(join(projectDir, "run.sh")).mode & 0o111, + ); + }); + + it("copies a few files at a time across every project opening together", async () => { + let writing = 0; + let most = 0; + vi.mocked(fsp.writeFile).mockImplementation(async (...args) => { + const counted = String(args[0]).includes("incoming-"); + if (counted) most = Math.max(most, ++writing); + await realWriteFile(...args); + if (counted) writing -= 1; + }); + const historyRoot = tempDir("hf-first-copy-root-"); + + await Promise.all(["a", "b", "c"].map((tag) => open(projectOf(30, tag), historyRoot))); + + expect(most).toBeGreaterThan(1); + expect(most).toBeLessThanOrEqual(4); + }); +}); diff --git a/packages/studio-server/src/history/projectHistory.ts b/packages/studio-server/src/history/projectHistory.ts index 87d8ea3e1f..c1c6ed33ff 100644 --- a/packages/studio-server/src/history/projectHistory.ts +++ b/packages/studio-server/src/history/projectHistory.ts @@ -212,6 +212,22 @@ const statKey = (file: { size: number; mtimeMs: number; ctimeMs: number }, swept ? "" : `${file.size}:${file.mtimeMs}:${file.ctimeMs}`; +const FIRST_COPY_SLOTS_IN_PROCESS = 4; +let firstCopiesRunning = 0; +const firstCopiesWaiting: (() => void)[] = []; + +async function inFirstCopySlot(copy: () => Promise): Promise { + if (firstCopiesRunning < FIRST_COPY_SLOTS_IN_PROCESS) firstCopiesRunning += 1; + else await new Promise((yours) => firstCopiesWaiting.push(yours)); + try { + return await copy(); + } finally { + const next = firstCopiesWaiting.shift(); + if (next) next(); + else firstCopiesRunning -= 1; + } +} + const MEDIA_LEDGER = ".media/manifest.jsonl"; const KEPT_HIDDEN_PATHS = new Set([...STUDIO_SIGNATURE_MANIFEST_PATHS, MEDIA_LEDGER]); @@ -548,11 +564,18 @@ class Engine { async firstOpen(): Promise { const sweptAt = this.now(); const files = historyFiles(this.dir); - for (const file of files.filter((file) => file.size <= COPY_LATER_ABOVE_BYTES)) { - const hash = await this.storeIfPresent(file.path); - if (this.whereFolder() !== "here") throw this.replaced(); - if (hash !== null) this.tracked.set(file.path, { hash, stat: statKey(file, sweptAt) }); - } + const copied = files.filter((file) => file.size <= COPY_LATER_ABOVE_BYTES); + const hashes = await Promise.allSettled( + copied.map((file) => inFirstCopySlot(() => this.storeIfPresent(file.path))), + ); + // Nothing names these copies until the log below is written, so one look at the end covers the whole copy. + if (this.whereFolder() !== "here") throw this.replaced(); + copied.forEach((file, i) => { + const result = hashes[i]!; + if (result.status === "rejected") throw result.reason; + if (result.value !== null) + this.tracked.set(file.path, { hash: result.value, stat: statKey(file, sweptAt) }); + }); this.log.baseline = this.manifest(); this.log.keepsLedger = true; this.persistLog();