fix(studio-server): history restores stay inside the project and use only intact copies (#7902)

* fix(studio-server): history restores stay inside the project and use only intact copies

* fix(studio-server): a history path's trailing dots and spaces are trimmed by a scan, not a regex
This commit is contained in:
Michael Wang
2026-11-16 02:04:01 +00:00
committed by GitHub
parent 98208241e7
commit 478313ba0e
6 changed files with 400 additions and 35 deletions
@@ -5,6 +5,7 @@ import {
mkdtempSync,
readdirSync,
readFileSync,
realpathSync,
rmSync,
utimesSync,
writeFileSync,
@@ -25,7 +26,7 @@ vi.mock("node:fs/promises", async (importOriginal) => {
import * as fs from "node:fs";
import * as fsp from "node:fs/promises";
import { onRestoredInPlace, openBlobStore } from "./blobStore.js";
import { onRestoredInPlace, openBlobStore, readBlob } from "./blobStore.js";
import { serveWhileOpen } from "../helpers/servedFiles.js";
const {
@@ -69,7 +70,7 @@ describe("the history store on Windows, while another process briefly holds a fi
it("writes a file a holder keeps open in place, once the rename is still refused, and checks it", async () => {
const { target, store, hash, heard, stop } = await heldRestore();
try {
await store.writeTo(hash, target);
await store.writeTo(hash, root, "index.html");
} finally {
stop();
}
@@ -81,11 +82,11 @@ describe("the history store on Windows, while another process briefly holds a fi
it("fails a write in place that does not match the kept copy, and keeps the copy", async () => {
const { target, store, hash, heard, stop } = await heldRestore();
vi.mocked(fsp.copyFile).mockImplementation(async (from, to, mode) => {
if (to === target) return writeFileSync(target, "<p>torn</p>");
if (to === realpathSync(target)) return writeFileSync(target, "<p>torn</p>");
return realCopyFile(from, to, mode);
});
try {
await expect(store.writeTo(hash, target)).rejects.toThrow(/does not match/);
await expect(store.writeTo(hash, root, "index.html")).rejects.toThrow(/does not match/);
} finally {
stop();
}
@@ -103,7 +104,7 @@ describe("the history store on Windows, while another process briefly holds a fi
writeFileSync(target, "<p>after</p>");
fs.chmodSync(target, 0o444);
try {
await expect(store.writeTo(hash, target)).rejects.toMatchObject({
await expect(store.writeTo(hash, root, "index.html")).rejects.toMatchObject({
code: expect.stringMatching(/^(EACCES|EPERM)$/),
});
expect(readFileSync(target, "utf8")).toBe("<p>after</p>");
@@ -126,7 +127,7 @@ describe("the history store on Windows, while another process briefly holds a fi
return realRename(from, to);
});
await store.writeTo(hash, target);
await store.writeTo(hash, root, "index.html");
expect(readFileSync(target, "utf8")).toBe("<p>before</p>");
expect(refused).toBe(3);
@@ -176,7 +177,7 @@ describe("a restore of a file the preview is serving", () => {
return realRename(from, to);
});
await store.writeTo(hash, target);
await store.writeTo(hash, root, "clip.mp4");
expect(openAtReplace).toBe(false);
});
@@ -194,7 +195,9 @@ describe("a restore of a file the preview is serving", () => {
const stop = onRestoredInPlace((_path, outcome) => heard.push(outcome));
Object.defineProperty(process, "platform", { value: "linux", configurable: true });
try {
await expect(store.writeTo(hash, target)).rejects.toMatchObject({ code: "EPERM" });
await expect(store.writeTo(hash, root, "index.html")).rejects.toMatchObject({
code: "EPERM",
});
} finally {
Object.defineProperty(process, "platform", { value: platform, configurable: true });
stop();
@@ -294,3 +297,38 @@ describe("a file written to while the history reads it", () => {
expect(stored()).toEqual([]);
});
});
describe("a kept copy whose bytes no longer match its name", () => {
let root: string;
beforeEach(() => {
root = mkdtempSync(join(tmpdir(), "hf-blob-damaged-"));
});
afterEach(() => rmSync(root, { recursive: true, force: true }));
/** A filed copy of index.html, then changed in the store behind the history's back. */
async function damaged(size: number) {
const target = join(root, "index.html");
writeFileSync(target, "a".repeat(size));
const blobs = join(root, "history", "blobs");
const store = await openBlobStore(blobs);
const hash = (await store.put(target))!;
writeFileSync(join(blobs, hash.slice(0, 2), hash), "b".repeat(size));
writeFileSync(target, "current");
return { target, blobs, store, hash };
}
it.each([16, 256 * 1024])("is not written back into the project (%i bytes)", async (size) => {
const { target, store, hash } = await damaged(size);
await expect(store.writeTo(hash, root, "index.html")).rejects.toThrow(/damaged/);
expect(readFileSync(target, "utf8")).toBe("current");
expect(readdirSync(root).sort()).toEqual(["history", "index.html"]);
});
it("is not read back, through the store or the read-only view", async () => {
const { blobs, store, hash } = await damaged(16);
await expect(store.read(hash)).rejects.toThrow(/damaged/);
await expect(readBlob(blobs, hash)).rejects.toThrow(/damaged/);
});
});
+40 -15
View File
@@ -12,11 +12,12 @@ import {
stat,
writeFile,
} from "node:fs/promises";
import { dirname, join } from "node:path";
import { basename, dirname, join } from "node:path";
import { pipeline } from "node:stream/promises";
import { atomicTempPath, refuseReadOnly } from "@hyperframes/core/atomic-file";
import { isWindowsHold, renameWithRetryAsync } from "@hyperframes/core/rename-retry";
import { stopServingAndClose } from "../helpers/servedFiles.js";
import { assertLandedIn, assertStillWithin, folderWithin } from "./containedPath.js";
/** File contents stored once by sha256, text and binary alike. */
export interface BlobStore {
@@ -25,8 +26,8 @@ export interface BlobStore {
reference(absPath: string): Promise<string>;
has(hash: string): boolean;
read(hash: string): Promise<Buffer>;
/** Writes the blob's bytes to `absPath` by clone-or-copy and rename, so a reader never sees half a file. */
writeTo(hash: string, absPath: string, beforeReplace?: () => void): Promise<void>;
/** Writes the blob's bytes, checked against `hash`, to `path` in `root` by clone and rename; no link is followed. */
writeTo(hash: string, root: string, path: string, beforeReplace?: () => void): Promise<void>;
bytes(): number;
size(hash: string): number;
keep(path: string, absPath: string, remove: boolean): Promise<void>;
@@ -56,7 +57,18 @@ export function blobPath(dir: string, hash: string): string {
return join(dir, hash.slice(0, 2), hash);
}
export const readBlob = (dir: string, hash: string) => readFile(blobPath(dir, hash));
const sha256 = (bytes: Buffer) => createHash("sha256").update(bytes).digest("hex");
const damaged = () =>
new Error("History's copy of this version is damaged, so it was not used. Nothing was changed.");
async function readChecked(path: string, hash: string): Promise<Buffer> {
const bytes = await readFile(path);
if (sha256(bytes) !== hash) throw damaged();
return bytes;
}
export const readBlob = (dir: string, hash: string) => readChecked(blobPath(dir, hash), hash);
async function hashFile(path: string): Promise<string> {
const hash = createHash("sha256");
@@ -98,26 +110,32 @@ export function onRestoredInPlace(listener: InPlaceListener): () => void {
async function cloneOrCopy(
from: string,
to: string,
hash: string,
root: string,
path: string,
beforeReplace?: () => void,
): Promise<void> {
const folder = folderWithin(root, path, true)!;
const to = join(folder, basename(path));
const servedAs = join(root, path);
refuseReadOnly(to);
await mkdir(dirname(to), { recursive: true });
const temp = atomicTempPath(to);
try {
await copyFile(from, temp, constants.COPYFILE_FICLONE);
await stopServingAndClose(to);
await copyFile(from, temp, constants.COPYFILE_FICLONE | constants.COPYFILE_EXCL);
assertLandedIn(folder, temp);
if ((await hashFile(temp)) !== hash) throw damaged();
await stopServingAndClose(servedAs);
try {
await renameWithRetryAsync(temp, to, {
rename: async (from, to) => {
beforeReplace?.();
assertStillWithin(folder);
await rename(from, to);
},
});
} catch (error) {
if (!isWindowsHold(error)) throw error;
await writeInPlace(temp, to, hash);
await writeInPlace(temp, folder, to, servedAs, hash);
}
} finally {
await rm(temp, { force: true });
@@ -125,9 +143,16 @@ async function cloneOrCopy(
}
// Something outside the app still holds the file: write in place, sync, and fail on any mismatch with the blob.
async function writeInPlace(temp: string, to: string, hash: string): Promise<void> {
async function writeInPlace(
temp: string,
folder: string,
to: string,
servedAs: string,
hash: string,
): Promise<void> {
let outcome: InPlaceOutcome = "failed";
try {
assertLandedIn(folder, to);
await copyFile(temp, to);
const handle = await open(to, "r+");
try {
@@ -137,7 +162,7 @@ async function writeInPlace(temp: string, to: string, hash: string): Promise<voi
}
outcome = (await hashFile(to)) === hash ? "restored" : "mismatch";
} finally {
for (const listener of inPlaceListeners) listener(to, outcome);
for (const listener of inPlaceListeners) listener(servedAs, outcome);
}
if (outcome === "mismatch")
throw new Error("The file was written in place, but it does not match the copy history kept.");
@@ -195,7 +220,7 @@ export async function openBlobStore(dir: string): Promise<BlobStore> {
});
// The bytes hashed are the bytes filed, so a write racing the read can never file bytes under the wrong hash.
const putBytes = async (bytes: Buffer, mode: number) => {
const hash = createHash("sha256").update(bytes).digest("hex");
const hash = sha256(bytes);
if (sizes.has(hash)) return hash;
const temp = join(dir, `incoming-${randomUUID()}`);
try {
@@ -252,9 +277,9 @@ export async function openBlobStore(dir: string): Promise<BlobStore> {
},
reference: (absPath) => hashFile(absPath),
has: (hash) => sizes.has(hash) || asideSizes.has(hash),
read: (hash) => readFile(pathOf(hash)),
writeTo: async (hash, absPath, beforeReplace) =>
cloneOrCopy(pathOf(hash), absPath, hash, beforeReplace),
read: (hash) => readChecked(pathOf(hash), hash),
writeTo: async (hash, root, path, beforeReplace) =>
cloneOrCopy(pathOf(hash), hash, root, path, beforeReplace),
async release(held) {
for (const [hash, path] of held) {
holds.get(hash)?.delete(path);
@@ -0,0 +1,111 @@
import { lstatSync, mkdirSync, renameSync, rmdirSync, rmSync } from "node:fs";
import { basename, dirname, join } from "node:path";
import { atomicTempPath } from "@hyperframes/core/atomic-file";
import { realpath, realProjectRoot } from "@hyperframes/core/safe-path";
const DRIVE = /^[a-z]:/i;
/** The name as Windows reads it, without its trailing dots and spaces (a scan, so a long run of them stays cheap). */
function withoutTrailingDotsAndSpaces(name: string): string {
let end = name.length;
while (end > 0 && (name[end - 1] === "." || name[end - 1] === " ")) end -= 1;
return name.slice(0, end);
}
const isPlainName = (name: string) =>
withoutTrailingDotsAndSpaces(name) !== "" &&
!DRIVE.test(name) &&
!(process.platform === "win32" && name.includes(":"));
/** Relative, `/`-separated, and no `\`, drive, empty, `.` or `..` segment, as Windows reads it too (`.. ` is `..`). */
export function isRelativeProjectPath(path: string): boolean {
return !path.includes("\\") && path.split("/").every(isPlainName);
}
const inTheWay = (at: string) =>
new Error(`${at} is in the way; move or delete it, then try again.`);
const changedWhileWriting = (folder: string) =>
new Error(`${folder} changed while history was writing to it, so it stopped.`);
/** The real folder for `path` under `root`, walked folder by folder with no link; null when missing and not `make`. */
export function folderWithin(root: string, path: string, make: boolean): string | null {
if (!isRelativeProjectPath(path)) throw new Error(`${path} is not a path inside the project.`);
let folder = realProjectRoot(root);
const names = path.split("/").slice(0, -1);
for (const [depth, name] of names.entries()) {
const next = join(folder, name);
let made = false;
if (!lstatSync(next, { throwIfNoEntry: false })) {
if (!make) return null;
made = makeFolder(next);
}
const real = lstatSync(next, { throwIfNoEntry: false })?.isDirectory()
? realpath(next)
: undefined;
if (real === undefined || dirname(real) !== folder) {
if (made) rmdirSync(next);
throw inTheWay(names.slice(0, depth + 1).join("/"));
}
folder = real;
}
return folder;
}
function makeFolder(path: string): boolean {
try {
mkdirSync(path);
return true;
} catch (error) {
if ((error as NodeJS.ErrnoException).code !== "EEXIST") throw error;
return false;
}
}
export function assertStillWithin(folder: string): void {
let real: string | undefined;
try {
real = realpath(folder);
} catch {
real = undefined;
}
if (real !== folder) throw changedWhileWriting(folder);
}
export function assertLandedIn(folder: string, path: string): void {
assertStillWithin(folder);
if (dirname(path) !== folder || !lstatSync(path, { throwIfNoEntry: false })?.isFile())
throw changedWhileWriting(folder);
}
/** Renames the file aside in its own folder, checks it stayed in `folder`, then lets `keep` take it and removes it. */
export async function removeWithin(
folder: string,
path: string,
keep?: (aside: string) => Promise<void>,
): Promise<void> {
const at = join(folder, basename(path));
const standing = lstatSync(at, { throwIfNoEntry: false });
if (!standing) return;
if (standing.isDirectory()) throw inTheWay(path);
const aside = atomicTempPath(at);
assertStillWithin(folder);
try {
renameSync(at, aside);
} catch (error) {
if ((error as NodeJS.ErrnoException).code === "ENOENT") return;
throw error;
}
try {
assertLandedIn(folder, aside);
await keep?.(aside);
} catch (error) {
try {
renameSync(aside, at);
} catch {
// The error that brought it here says why.
}
throw error;
}
rmSync(aside, { force: true });
}
@@ -0,0 +1,175 @@
// @vitest-environment node
import {
appendFileSync,
mkdirSync,
mkdtempSync,
readdirSync,
readFileSync,
rmSync,
writeFileSync,
} from "node:fs";
import { tmpdir } from "node:os";
import { join } from "node:path";
import { afterEach, describe, expect, it, vi } from "vitest";
import { START, type HistoryWho } from "./historyLog";
import { openProjectHistory } from "./projectHistory";
// Counts every file-system call while armed, and runs `swap` just before call number `at`.
const { race, counted } = vi.hoisted(() => {
const race = { armed: false, calls: 0, at: 0, swap: null as null | (() => void) };
const before = () => {
if (!race.armed || !race.swap) return;
race.calls += 1;
if (race.calls !== race.at) return;
const swap = race.swap;
race.swap = null;
swap();
};
type Fn = (...args: unknown[]) => unknown;
const wrap = (fn: Fn): Fn => {
const wrapped: Fn = (...args) => {
before();
return fn(...args);
};
// realpathSync.native and the like.
for (const [key, inner] of Object.entries(fn))
if (typeof inner === "function") Object.assign(wrapped, { [key]: wrap(inner as Fn) });
return wrapped;
};
const counted = <T extends object>(module: T): T =>
Object.fromEntries(
Object.entries(module).map(([key, value]) => [
key,
typeof value === "function" && !/^[A-Z]/.test(key) ? wrap(value as Fn) : value,
]),
) as T;
return { race, counted };
});
vi.mock("node:fs", async (importOriginal) => {
const original = await importOriginal<typeof import("node:fs")>();
return { ...counted(original), default: counted(original) };
});
vi.mock("node:fs/promises", async (importOriginal) => {
const original = await importOriginal<typeof import("node:fs/promises")>();
return { ...counted(original), default: counted(original) };
});
const real = await vi.importActual<typeof import("node:fs")>("node:fs");
const you: HistoryWho = { kind: "person", name: "You" };
const cleanup: Array<() => unknown> = [];
afterEach(async () => {
race.armed = false;
race.swap = null;
for (const step of cleanup.splice(0).reverse()) await step();
});
/** Every file under `dir` and its text; a link is listed as where it points. */
function contents(dir: string): Record<string, string> {
const found: Record<string, string> = {};
for (const entry of real.readdirSync(dir, { withFileTypes: true, recursive: true })) {
const path = join(entry.parentPath, entry.name);
found[path] = entry.isSymbolicLink()
? `link to ${real.readlinkSync(path)}`
: entry.isFile()
? real.readFileSync(path, "utf-8")
: "folder";
}
return found;
}
/** A project whose `sub/` folder a restore rewrites, creates in and deletes from, and a folder outside it. */
async function restorable() {
const root = mkdtempSync(join(tmpdir(), "hf-history-contained-"));
cleanup.push(() => rmSync(root, { recursive: true, force: true }));
const projectDir = join(root, "project");
const sub = join(projectDir, "sub");
const outside = join(root, "outside");
mkdirSync(sub, { recursive: true });
mkdirSync(outside);
writeFileSync(join(projectDir, "index.html"), "i1");
for (const name of ["a", "b", "c"]) writeFileSync(join(sub, `${name}.html`), `${name}1`);
for (const name of ["a", "b", "d"])
writeFileSync(join(outside, `${name}.html`), `outside ${name}`);
const history = await openProjectHistory({ projectDir, historyRoot: join(root, "history") });
cleanup.push(() => history.close());
const window = await history.beginWindow(you, "Edit");
writeFileSync(join(projectDir, "index.html"), "i2");
writeFileSync(join(sub, "a.html"), "a2");
writeFileSync(join(sub, "b.html"), "b2");
rmSync(join(sub, "c.html"));
writeFileSync(join(sub, "d.html"), "d2");
await window.close();
const swap = () => {
real.renameSync(sub, join(projectDir, "sub-moved"));
real.symlinkSync(outside, sub, "junction");
};
return { history, projectDir, outside, swap };
}
describe("a restore while a folder in the project is swapped for a link", () => {
it("creates, replaces and deletes nothing outside the project, wherever in the restore the swap lands", async () => {
let calls = Infinity;
for (let at = 1; at <= calls; at += 1) {
const { history, outside, swap } = await restorable();
const before = contents(outside);
Object.assign(race, { armed: true, calls: 0, at, swap });
await history.restore(START, you).catch(() => undefined);
race.armed = false;
if (race.swap) calls = race.calls;
expect({ at, files: contents(outside) }).toEqual({ at, files: before });
for (const step of cleanup.splice(0).reverse()) await step();
}
expect(calls).toBeGreaterThan(20);
}, 120_000);
it("restores every file when nothing is swapped", async () => {
const { history, projectDir } = await restorable();
await history.restore(START, you);
expect(readdirSync(join(projectDir, "sub")).sort()).toEqual(["a.html", "b.html", "c.html"]);
expect(readFileSync(join(projectDir, "sub", "c.html"), "utf-8")).toBe("c1");
expect(readFileSync(join(projectDir, "index.html"), "utf-8")).toBe("i1");
});
});
describe("a history log with a path that could leave the project", () => {
it.each([
"sub\\..\\..\\outside\\a.html",
"/abs.html",
"C:/abs.html",
"sub/../../outside/a.html",
"sub//a.html",
"sub/./a.html",
".. /outside/a.html",
])("drops %s on open, and an undo of its entry writes nothing outside", async (forged) => {
const { history, projectDir, outside } = await restorable();
const [edit] = history.list();
const hash = edit!.files.find((file) => file.path === "sub/a.html")!.after!;
await history.close();
const logFile = join(projectDir, "..", "history", history.projectId, "log.jsonl");
const entry = {
id: "forged",
who: you,
label: "Forged",
startedAt: Date.now(),
endedAt: Date.now(),
files: [{ path: forged, before: hash, after: null }],
};
appendFileSync(logFile, `${JSON.stringify({ type: "entry", entry })}\n`);
const before = contents(outside);
const reopened = await openProjectHistory({
projectDir,
historyRoot: join(projectDir, "..", "history"),
});
cleanup.push(() => reopened.close());
const listed = reopened.list().find((item) => item.id === "forged");
expect(listed?.files ?? []).toEqual([]);
await reopened.undo("forged", { who: you }).catch(() => undefined);
expect(contents(outside)).toEqual(before);
expect(readdirSync(projectDir).sort()).toEqual([".hyperframes", "index.html", "sub"]);
});
});
@@ -48,9 +48,9 @@ vi.mock("./blobStore", async (importOriginal) => {
hooks.beforePut?.();
return store.put(path);
},
writeTo: async (hash: string, path: string, beforeReplace?: () => void) => {
writeTo: async (hash: string, root: string, path: string, beforeReplace?: () => void) => {
hooks.beforeWrite?.();
await store.writeTo(hash, path, beforeReplace);
await store.writeTo(hash, root, path, beforeReplace);
hooks.afterWrite?.();
},
};
@@ -12,7 +12,7 @@ import {
realpathSync,
rmSync,
} from "node:fs";
import { dirname, isAbsolute, join, relative, resolve, sep } from "node:path";
import { basename, dirname, isAbsolute, join, relative, resolve, sep } from "node:path";
import { replaceFileAtomically } from "@hyperframes/core/atomic-file";
import {
DELETED_VERSION,
@@ -31,6 +31,12 @@ import {
} from "../helpers/projectSignature.js";
import { isSettling, openBlobStore, readBlob, SETTLE_MS, type BlobStore } from "./blobStore.js";
import { isUncopied, uncopiedOnlyHashes } from "./uncopied.js";
import {
assertStillWithin,
folderWithin,
isRelativeProjectPath,
removeWithin,
} from "./containedPath.js";
import { pruneGoneProjectHistoriesDaily } from "./pruneHistories.js";
import {
ID_PATH,
@@ -242,8 +248,9 @@ async function inFirstCopySlot<T>(copy: () => Promise<T>): Promise<T> {
const MEDIA_LEDGER = ".media/manifest.jsonl";
const KEPT_HIDDEN_PATHS = new Set<string>([...STUDIO_SIGNATURE_MANIFEST_PATHS, MEDIA_LEDGER]);
/** A hidden name anywhere in a path (a tool's own record, .DS_Store) is nobody's work, except the kept ones above. */
/** A hidden name (a tool's own record, .DS_Store) is nobody's work, except the kept ones above; nor is a path out. */
function isHistoryPath(path: string): boolean {
if (!isRelativeProjectPath(path)) return false;
if (KEPT_HIDDEN_PATHS.has(path)) return true;
return !path.split("/").some((segment) => segment.startsWith("."));
}
@@ -306,6 +313,7 @@ async function removeEmptyFolders(dir: string): Promise<void> {
}
for (const entry of entries)
if (entry.isDirectory()) await removeEmptyFolders(join(dir, entry.name));
assertStillWithin(dirname(dir));
await rmdir(dir);
}
@@ -1264,16 +1272,23 @@ class Engine {
const version = hash === null ? DELETED_VERSION : hashVersion(hash);
recordFileWriteReceipt(absPath, { path, version, writeToken });
}
const standing = lstatSync(absPath, { throwIfNoEntry: false });
if (standing?.isFile() && isUncopied(path, standing.size))
await this.blobs.keep(path, absPath, hash === null);
const folder = folderWithin(this.dir, path, hash !== null);
if (!folder) return;
const at = join(folder, basename(path));
const standing = lstatSync(at, { throwIfNoEntry: false });
const uncopied = standing?.isFile() === true && isUncopied(path, standing.size);
if (hash === null) {
this.assertWritable();
rmSync(absPath, { force: true });
} else {
await removeEmptyFolders(absPath);
await this.blobs.writeTo(hash, absPath, () => this.assertWritable());
await removeWithin(
folder,
path,
uncopied ? (aside) => this.blobs.keep(path, aside, true) : undefined,
);
return;
}
if (uncopied) await this.blobs.keep(path, at, false);
await removeEmptyFolders(at);
await this.blobs.writeTo(hash, this.dir, path, () => this.assertWritable());
}
entry(id: string): HistoryEntry {
@@ -1449,7 +1464,8 @@ class Engine {
if (!this.blobs.has(hash))
throw new Error(`The version of ${path} from then is not kept.`);
try {
for (const [path, hash] of files) await this.blobs.writeTo(hash, join(dest, path));
await mkdir(dest, { recursive: true });
for (const [path, hash] of files) await this.blobs.writeTo(hash, dest, path);
} catch (error) {
for (const name of await readdir(dest).catch(() => []))
await rm(join(dest, name), { recursive: true, force: true });