perf(studio-server): a large project's first history copy takes a fraction of the time (#7415)

* perf(studio-server): a large project's first history copy takes a fraction of the time

* perf(studio-server): read a small file through one handle, with fewer comments

* fix(studio-server): small files keep their exact permissions in history under a strict umask
This commit is contained in:
Miguel Ángel
2026-11-08 22:25:44 +00:00
committed by GitHub
parent 327e20208c
commit 2927fe6155
5 changed files with 272 additions and 11 deletions
@@ -0,0 +1,19 @@
// A project's first history open, killed outright mid-copy with one copy half written.
import { writeFileSync } from "node:fs";
import fsp from "node:fs/promises";
import { syncBuiltinESMExports } from "node:module";
const [projectDir, historyRoot, dieAtWrite] = process.argv.slice(2);
const realWriteFile = fsp.writeFile;
let writes = 0;
fsp.writeFile = ((...args: Parameters<typeof realWriteFile>) => {
if (String(args[0]).includes("incoming-") && ++writes === Number(dieAtWrite)) {
writeFileSync(String(args[0]), "half a co");
process.kill(process.pid, "SIGKILL");
}
return realWriteFile(...args);
}) as typeof realWriteFile;
syncBuiltinESMExports();
const { openProjectHistory } = await import("../projectHistory.js");
await openProjectHistory({ projectDir: projectDir!, historyRoot: historyRoot! });
@@ -107,14 +107,17 @@ describe("the history store on Windows, while another process briefly holds a fi
const store = await openBlobStore(join(root, "history", "blobs"));
let refused = 0;
vi.mocked(fsp.rename).mockImplementation(async (from, to) => {
if (refused++ < 2) throw busy();
if (refused < 2) {
refused += 1;
throw busy();
}
return realRename(from, to);
});
const hash = await store.put(source);
expect((await store.read(hash)).toString("utf8")).toBe("<p>kept</p>");
expect(refused).toBe(3);
expect(refused).toBe(2);
});
});
@@ -170,3 +173,20 @@ describe("a restore of a file the preview is serving", () => {
expect(heard).toEqual([]);
});
});
describe("the history store's count of what it holds", () => {
it("counts a file once when two copies of it are filed together", async () => {
const root = mkdtempSync(join(tmpdir(), "hf-blob-count-"));
try {
const small = join(root, "small.txt");
const large = join(root, "large.txt");
writeFileSync(small, "same bytes");
writeFileSync(large, "x".repeat(200 * 1024));
const store = await openBlobStore(join(root, "history", "blobs"));
await Promise.all([store.put(small), store.put(small), store.put(large), store.put(large)]);
expect(store.bytes()).toBe("same bytes".length + 200 * 1024);
} finally {
rmSync(root, { recursive: true, force: true });
}
});
});
@@ -1,6 +1,16 @@
import { createHash, randomUUID } from "node:crypto";
import { constants, createReadStream, createWriteStream } from "node:fs";
import { copyFile, mkdir, open, readdir, readFile, rm, stat } from "node:fs/promises";
import {
chmod,
copyFile,
mkdir,
open,
readdir,
readFile,
rm,
stat,
writeFile,
} from "node:fs/promises";
import { dirname, join } from "node:path";
import { pipeline } from "node:stream/promises";
import { atomicTempPath } from "@hyperframes/core/atomic-file";
@@ -13,7 +23,7 @@ import { stopServingAndClose } from "../helpers/servedFiles.js";
/** File contents stored once by sha256, text and binary alike. */
export interface BlobStore {
/** Copies the file in (a clone where the file system can) and returns the copy's hash. */
/** Copies the file in (a clone where the file system can, for all but small files) and returns the copy's hash. */
put(absPath: string): Promise<string>;
reference(absPath: string): Promise<string>;
has(hash: string): boolean;
@@ -29,6 +39,7 @@ export interface BlobStore {
}
const BLOB_HASH = /^[0-9a-f]{64}$/;
const READ_ONCE_AT_MOST_BYTES = 64 * 1024;
export function blobPath(dir: string, hash: string): string {
// Checked where the path is joined, so no caller can read or write outside the store.
@@ -44,6 +55,16 @@ async function hashFile(path: string): Promise<string> {
return hash.digest("hex");
}
async function readIfSmall(path: string): Promise<{ bytes: Buffer; mode: number } | null> {
const handle = await open(path, "r");
try {
const { size, mode } = await handle.stat();
return size > READ_ONCE_AT_MOST_BYTES ? null : { bytes: await handle.readFile(), mode };
} finally {
await handle.close();
}
}
async function cloneOrStream(from: string, to: string): Promise<void> {
try {
await copyFile(from, to, constants.COPYFILE_FICLONE_FORCE);
@@ -144,8 +165,39 @@ export async function openBlobStore(dir: string): Promise<BlobStore> {
holds.delete(hash);
};
const file = (hash: string, size: number) => {
if (!sizes.has(hash)) total += size;
sizes.set(hash, size);
};
const withFolderMadeIfMissing = (folder: string, write: () => Promise<void>) =>
write().catch(async (error: NodeJS.ErrnoException) => {
if (error.code !== "ENOENT") throw error;
await mkdir(folder, { recursive: true });
await write();
});
// The bytes hashed are the bytes filed, so a write racing the read can never file bytes under the wrong hash.
const putBytes = async (bytes: Buffer, mode: number) => {
const hash = createHash("sha256").update(bytes).digest("hex");
if (sizes.has(hash)) return hash;
const temp = join(dir, `incoming-${randomUUID()}`);
try {
await withFolderMadeIfMissing(dir, () => writeFile(temp, bytes, { mode }));
// A write applies the umask; the clone used for larger files keeps the mode exactly.
await chmod(temp, mode & 0o7777);
const to = blobPath(dir, hash);
await withFolderMadeIfMissing(dirname(to), () => renameWithRetryAsync(temp, to));
} catch (error) {
await rm(temp, { force: true });
throw error;
}
file(hash, bytes.length);
return hash;
};
return {
async put(absPath) {
const small = await readIfSmall(absPath);
if (small) return putBytes(small.bytes, small.mode);
// Hash the copy, not the source, so a write racing the copy can never file bytes under the wrong hash.
const temp = join(dir, `incoming-${randomUUID()}`);
// Recreated if removed while open, so a missing folder is never mistaken for a deleted project file.
@@ -162,8 +214,7 @@ export async function openBlobStore(dir: string): Promise<BlobStore> {
await mkdir(dirname(blobPath(dir, hash)), { recursive: true });
const size = (await stat(temp)).size;
await renameWithRetryAsync(temp, blobPath(dir, hash));
sizes.set(hash, size);
total += size;
file(hash, size);
return hash;
},
reference: (absPath) => hashFile(absPath),
@@ -0,0 +1,148 @@
// @vitest-environment node
import { fork } from "node:child_process";
import { createHash } from "node:crypto";
import {
chmodSync,
existsSync,
mkdirSync,
mkdtempSync,
readdirSync,
readFileSync,
rmSync,
statSync,
writeFileSync,
} from "node:fs";
import { tmpdir } from "node:os";
import { join } from "node:path";
import { afterEach, describe, expect, it, onTestFinished, vi } from "vitest";
vi.mock("node:fs/promises", async (importOriginal) => {
const actual = await importOriginal<typeof import("node:fs/promises")>();
return { ...actual, writeFile: vi.fn(actual.writeFile) };
});
import * as fsp from "node:fs/promises";
import { blobPath } from "./blobStore.js";
import { START } from "./historyLog.js";
import { openProjectHistory } from "./projectHistory.js";
const { writeFile: realWriteFile } =
await vi.importActual<typeof import("node:fs/promises")>("node:fs/promises");
const sha256 = (bytes: Buffer) => createHash("sha256").update(bytes).digest("hex");
const temps: string[] = [];
const tempDir = (prefix: string) => {
const dir = mkdtempSync(join(tmpdir(), prefix));
temps.push(dir);
return dir;
};
const closes: (() => Promise<void>)[] = [];
afterEach(async () => {
vi.mocked(fsp.writeFile).mockImplementation(realWriteFile);
for (const close of closes.splice(0)) await close();
for (const dir of temps.splice(0)) rmSync(dir, { recursive: true, force: true });
});
function projectOf(count: number, tag: string): string {
const dir = tempDir("hf-first-copy-");
mkdirSync(join(dir, "parts"));
writeFileSync(join(dir, "index.html"), `<html>${tag}</html>`);
for (let i = 0; i < count; i++) writeFileSync(join(dir, "parts", `p${i}.js`), `// ${tag} ${i}\n`);
return dir;
}
const homeIn = (historyRoot: string) => {
const homes = readdirSync(historyRoot, { withFileTypes: true }).filter((e) => e.isDirectory());
return join(historyRoot, homes.find((e) => existsSync(join(historyRoot, e.name, "blobs")))!.name);
};
async function open(projectDir: string, historyRoot: string) {
const history = await openProjectHistory({ projectDir, historyRoot });
closes.push(() => history.close());
return history;
}
/** Every file of the project is in the baseline, and its blob holds exactly the file's bytes. */
function expectAllKept(
baseline: Record<string, string> | null,
projectDir: string,
blobs: string,
paths: string[],
) {
expect(Object.keys(baseline ?? {}).sort()).toEqual([...paths].sort());
for (const path of paths) {
const bytes = readFileSync(join(projectDir, path));
expect(baseline![path]).toBe(sha256(bytes));
expect(readFileSync(blobPath(blobs, sha256(bytes))).equals(bytes)).toBe(true);
}
}
const pathsOf = (count: number) => [
"index.html",
...Array.from({ length: count }, (_, i) => `parts/p${i}.js`),
];
describe("a project's first history copy", () => {
it("loses nothing when the process dies midway: the next open finishes it", async () => {
const projectDir = projectOf(120, "dies");
const historyRoot = tempDir("hf-first-copy-root-");
const before = readFileSync(join(projectDir, "parts", "p7.js"));
const died = await new Promise<NodeJS.Signals | number | null>((done) => {
const fixture = join(import.meta.dirname, "__fixtures__", "dieMidFirstCopy.ts");
fork(fixture, [projectDir, historyRoot, "40"], { execArgv: ["--import", "tsx"] }).once(
"exit",
(code, signal) => done(signal ?? code),
);
});
expect(died).toBe("SIGKILL");
const home = homeIn(historyRoot);
expect(existsSync(join(home, "log.jsonl")), "the copy finished before the death").toBe(false);
const leftovers = () =>
readdirSync(join(home, "blobs")).filter((name) => name.startsWith("incoming-"));
expect(leftovers().length, "the death left no half-written copy").toBeGreaterThan(0);
const history = await open(projectDir, historyRoot);
expectAllKept(history.peek(START), projectDir, join(home, "blobs"), pathsOf(120));
expect(leftovers()).toEqual([]);
for (const shard of readdirSync(join(home, "blobs")))
for (const hash of readdirSync(join(home, "blobs", shard)))
expect(sha256(readFileSync(join(home, "blobs", shard, hash)))).toBe(hash);
expect(readFileSync(join(projectDir, "parts", "p7.js")).equals(before)).toBe(true);
});
it("keeps the same contents and permissions, small files and large, under a strict umask", async () => {
const umask = process.umask(0o077);
onTestFinished(() => void process.umask(umask));
const projectDir = projectOf(3, "kept");
writeFileSync(join(projectDir, "large.js"), "x".repeat(200 * 1024));
writeFileSync(join(projectDir, "same-a.js"), "same bytes");
writeFileSync(join(projectDir, "same-b.js"), "same bytes");
writeFileSync(join(projectDir, "run.sh"), "#!/bin/sh\n");
chmodSync(join(projectDir, "run.sh"), 0o755);
const historyRoot = tempDir("hf-first-copy-root-");
const history = await open(projectDir, historyRoot);
const blobs = join(homeIn(historyRoot), "blobs");
const paths = [...pathsOf(3), "large.js", "same-a.js", "same-b.js", "run.sh"];
expectAllKept(history.peek(START), projectDir, blobs, paths);
const script = readFileSync(join(projectDir, "run.sh"));
expect(statSync(blobPath(blobs, sha256(script))).mode & 0o111).toBe(
statSync(join(projectDir, "run.sh")).mode & 0o111,
);
});
it("copies a few files at a time across every project opening together", async () => {
let writing = 0;
let most = 0;
vi.mocked(fsp.writeFile).mockImplementation(async (...args) => {
const counted = String(args[0]).includes("incoming-");
if (counted) most = Math.max(most, ++writing);
await realWriteFile(...args);
if (counted) writing -= 1;
});
const historyRoot = tempDir("hf-first-copy-root-");
await Promise.all(["a", "b", "c"].map((tag) => open(projectOf(30, tag), historyRoot)));
expect(most).toBeGreaterThan(1);
expect(most).toBeLessThanOrEqual(4);
});
});
@@ -212,6 +212,22 @@ const statKey = (file: { size: number; mtimeMs: number; ctimeMs: number }, swept
? ""
: `${file.size}:${file.mtimeMs}:${file.ctimeMs}`;
const FIRST_COPY_SLOTS_IN_PROCESS = 4;
let firstCopiesRunning = 0;
const firstCopiesWaiting: (() => void)[] = [];
async function inFirstCopySlot<T>(copy: () => Promise<T>): Promise<T> {
if (firstCopiesRunning < FIRST_COPY_SLOTS_IN_PROCESS) firstCopiesRunning += 1;
else await new Promise<void>((yours) => firstCopiesWaiting.push(yours));
try {
return await copy();
} finally {
const next = firstCopiesWaiting.shift();
if (next) next();
else firstCopiesRunning -= 1;
}
}
const MEDIA_LEDGER = ".media/manifest.jsonl";
const KEPT_HIDDEN_PATHS = new Set<string>([...STUDIO_SIGNATURE_MANIFEST_PATHS, MEDIA_LEDGER]);
@@ -548,11 +564,18 @@ class Engine {
async firstOpen(): Promise<void> {
const sweptAt = this.now();
const files = historyFiles(this.dir);
for (const file of files.filter((file) => file.size <= COPY_LATER_ABOVE_BYTES)) {
const hash = await this.storeIfPresent(file.path);
if (this.whereFolder() !== "here") throw this.replaced();
if (hash !== null) this.tracked.set(file.path, { hash, stat: statKey(file, sweptAt) });
}
const copied = files.filter((file) => file.size <= COPY_LATER_ABOVE_BYTES);
const hashes = await Promise.allSettled(
copied.map((file) => inFirstCopySlot(() => this.storeIfPresent(file.path))),
);
// Nothing names these copies until the log below is written, so one look at the end covers the whole copy.
if (this.whereFolder() !== "here") throw this.replaced();
copied.forEach((file, i) => {
const result = hashes[i]!;
if (result.status === "rejected") throw result.reason;
if (result.value !== null)
this.tracked.set(file.path, { hash: result.value, stat: statKey(file, sweptAt) });
});
this.log.baseline = this.manifest();
this.log.keepsLedger = true;
this.persistLog();