mirror of
https://github.com/heygen-com/hyperframes.git
synced 2026-11-18 07:50:13 +00:00
fix(studio-server): history sweeps stop holding the host's event loop on large projects (#7491)
* fix(studio-server): project history lists files without holding the host's event loop * test(studio-server): the history walk test counts the hold up to the sweep's end * fix(studio-server): history rewrites video-state.json only when it changed * refactor(studio-server): names carry what the history walk comments said
This commit is contained in:
@@ -17,7 +17,7 @@ import {
|
||||
affectsProjectSignature,
|
||||
createProjectSignature,
|
||||
createProjectSignatureAsync,
|
||||
listProjectFiles,
|
||||
listProjectFilesAsync,
|
||||
} from "./projectSignature.js";
|
||||
|
||||
const temporaryProjects: string[] = [];
|
||||
@@ -72,15 +72,15 @@ describe("affectsProjectSignature", () => {
|
||||
});
|
||||
});
|
||||
|
||||
describe("listProjectFiles", () => {
|
||||
it("leaves out the temp file of a save in flight", () => {
|
||||
describe("listProjectFilesAsync", () => {
|
||||
it("leaves out the temp file of a save in flight", async () => {
|
||||
const project = mkdtempSync(resolve(tmpdir(), "hf-signature-"));
|
||||
temporaryProjects.push(project);
|
||||
writeFileSync(resolve(project, "index.html"), "<h1>Hello</h1>");
|
||||
writeFileSync(resolve(project, "index.html.hf0a1b2c.tmp"), "<h1>Bye</h1>");
|
||||
writeFileSync(resolve(project, "foo.12345678.tmp"), "mine");
|
||||
|
||||
expect(listProjectFiles(project).map((file) => file.path)).toEqual([
|
||||
expect((await listProjectFilesAsync(project)).map((file) => file.path)).toEqual([
|
||||
"foo.12345678.tmp",
|
||||
"index.html",
|
||||
]);
|
||||
|
||||
@@ -242,13 +242,16 @@ async function collectProjectFilesAsync(
|
||||
return collected;
|
||||
}
|
||||
|
||||
export type ListedProjectFile = { path: string; size: number; mtimeMs: number; ctimeMs: number };
|
||||
|
||||
/** The files a project is made of (the signature's set: source plus Studio's two manifests), paths with `/`. */
|
||||
export function listProjectFiles(
|
||||
projectDir: string,
|
||||
): Array<{ path: string; size: number; mtimeMs: number; ctimeMs: number }> {
|
||||
export async function listProjectFilesAsync(projectDir: string): Promise<ListedProjectFile[]> {
|
||||
const normalizedProjectDir = resolve(projectDir);
|
||||
return collectProjectFiles(normalizedProjectDir).map((entry) => ({
|
||||
path: relative(normalizedProjectDir, entry.file).split(sep).join("/"),
|
||||
const everyFileStartsWith = normalizedProjectDir.endsWith(sep)
|
||||
? normalizedProjectDir
|
||||
: normalizedProjectDir + sep;
|
||||
return (await collectProjectFilesAsync(normalizedProjectDir)).map((entry) => ({
|
||||
path: entry.file.slice(everyFileStartsWith.length).split(sep).join("/"),
|
||||
size: entry.size,
|
||||
mtimeMs: entry.mtimeMs,
|
||||
ctimeMs: entry.ctimeMs,
|
||||
|
||||
@@ -3,7 +3,7 @@ import { tmpdir } from "node:os";
|
||||
import { join } from "node:path";
|
||||
import { expect, it } from "vitest";
|
||||
import { registerRenderOutput, readRenderOutputs } from "./renderOutputs.js";
|
||||
import { listProjectFiles, affectsProjectSignature } from "./projectSignature.js";
|
||||
import { listProjectFilesAsync, affectsProjectSignature } from "./projectSignature.js";
|
||||
import { openProjectHistory } from "../history/projectHistory.js";
|
||||
import { beginArtifactCapture, recordRenderSuccess } from "../history/videoState.js";
|
||||
|
||||
@@ -66,7 +66,7 @@ it("registers concurrent custom outputs and directory renders without excluding
|
||||
writeFileSync(join(root, "hls", "chunk.ts"), "output");
|
||||
writeFileSync(join(root, "a.webm"), "output");
|
||||
writeFileSync(join(root, "b.webm"), "input");
|
||||
expect(listProjectFiles(root).map((file) => file.path)).toEqual(["b.webm"]);
|
||||
expect((await listProjectFilesAsync(root)).map((file) => file.path)).toEqual(["b.webm"]);
|
||||
await registerRenderOutput(root, join(root, "..", "external.mp4"));
|
||||
expect(readRenderOutputs(root)).toEqual(["a.webm", "hls"]);
|
||||
await expect(registerRenderOutput(root, root)).rejects.toThrow("project directory");
|
||||
@@ -75,7 +75,7 @@ it("registers concurrent custom outputs and directory renders without excluding
|
||||
}
|
||||
});
|
||||
|
||||
it("lists project files when the render output registry is unreadable", () => {
|
||||
it("lists project files when the render output registry is unreadable", async () => {
|
||||
const project = mkdtempSync(join(tmpdir(), "hf-render-registry-"));
|
||||
try {
|
||||
writeFileSync(join(project, "index.html"), "x");
|
||||
@@ -85,7 +85,7 @@ it("lists project files when the render output registry is unreadable", () => {
|
||||
mkdirSync(join(project, ".hyperframes"));
|
||||
writeFileSync(join(project, ".hyperframes", "render-outputs.json"), "{not json");
|
||||
expect(() => readRenderOutputs(project)).toThrow();
|
||||
expect(listProjectFiles(project).length).toBeGreaterThan(0);
|
||||
expect((await listProjectFilesAsync(project)).length).toBeGreaterThan(0);
|
||||
expect(affectsProjectSignature(project, join(project, "index.html"))).toBe(true);
|
||||
} finally {
|
||||
rmSync(project, { recursive: true, force: true });
|
||||
|
||||
@@ -26,7 +26,8 @@ import { realFilePath } from "../helpers/safePath.js";
|
||||
import {
|
||||
STUDIO_SIGNATURE_MANIFEST_PATHS,
|
||||
affectsProjectSignature,
|
||||
listProjectFiles,
|
||||
listProjectFilesAsync,
|
||||
type ListedProjectFile,
|
||||
} from "../helpers/projectSignature.js";
|
||||
import { isSettling, openBlobStore, readBlob, SETTLE_MS, type BlobStore } from "./blobStore.js";
|
||||
import { isUncopied, uncopiedOnlyHashes } from "./uncopied.js";
|
||||
@@ -263,9 +264,9 @@ function withoutHiddenPaths(log: HistoryLog): HistoryLog {
|
||||
return log;
|
||||
}
|
||||
|
||||
const historyFiles = (dir: string) =>
|
||||
listProjectFiles(dir).filter((file) => isHistoryPath(file.path));
|
||||
type ListedFile = ReturnType<typeof listProjectFiles>[number];
|
||||
const historyFiles = async (dir: string) =>
|
||||
(await listProjectFilesAsync(dir)).filter((file) => isHistoryPath(file.path));
|
||||
type ListedFile = ListedProjectFile;
|
||||
const COPY_LATER_ABOVE_BYTES = 1024 ** 2;
|
||||
|
||||
const sameWho = (a: HistoryWho, b: HistoryWho) => a.kind === b.kind && a.name === b.name;
|
||||
@@ -366,13 +367,29 @@ function keepTargetsOfItsFiles(entry: HistoryEntry): HistoryEntry {
|
||||
return entry;
|
||||
}
|
||||
|
||||
/** The tracked files; any change owes the project's video-state.json a rewrite. */
|
||||
class TrackedFiles extends Map<string, Tracked> {
|
||||
constructor(private readonly changed: () => void) {
|
||||
super();
|
||||
}
|
||||
override set(path: string, file: Tracked): this {
|
||||
this.changed();
|
||||
return super.set(path, file);
|
||||
}
|
||||
override delete(path: string): boolean {
|
||||
this.changed();
|
||||
return super.delete(path);
|
||||
}
|
||||
}
|
||||
|
||||
class Engine {
|
||||
readonly dir: string;
|
||||
readonly foldsCase: boolean;
|
||||
readonly folder: FolderIdentity & { dev: number };
|
||||
readonly home: string;
|
||||
log: HistoryLog = { baseline: new Map(), entries: [], pins: new Set() };
|
||||
tracked = new Map<string, Tracked>();
|
||||
owesVideoState = true;
|
||||
tracked = new TrackedFiles(() => (this.owesVideoState = true));
|
||||
windows: Group[] = [];
|
||||
outside: Group | null = null;
|
||||
/** A coalescing claim, open until another key, its idle timer, an operation, a window, or another write. */
|
||||
@@ -426,7 +443,8 @@ class Engine {
|
||||
try {
|
||||
return await task();
|
||||
} finally {
|
||||
if (this.whereFolder() === "here") {
|
||||
if (this.owesVideoState && this.whereFolder() === "here") {
|
||||
this.owesVideoState = false;
|
||||
await writeHistoryState(
|
||||
this.dir,
|
||||
this.projectId,
|
||||
@@ -434,7 +452,10 @@ class Engine {
|
||||
this.log.entries,
|
||||
this.manifest(),
|
||||
manifestAt(this.log, this.log.entries.at(-1)?.id ?? START) ?? new Map(),
|
||||
).catch((error) => this.options.onError?.(error));
|
||||
).catch((error) => {
|
||||
this.owesVideoState = true;
|
||||
this.options.onError?.(error);
|
||||
});
|
||||
}
|
||||
}
|
||||
});
|
||||
@@ -457,6 +478,7 @@ class Engine {
|
||||
}
|
||||
|
||||
persistLog(record?: LogRecord): void {
|
||||
this.owesVideoState = true;
|
||||
if (record) saveRecord(this.logFile, this.log, record);
|
||||
else writeLog(this.logFile, this.log);
|
||||
if (!existsSync(join(this.home, "project.json")))
|
||||
@@ -491,7 +513,7 @@ class Engine {
|
||||
const left = this.readAdopting();
|
||||
const unadopted = (file: { path: string }) =>
|
||||
left.has(file.path) && !this.tracked.has(file.path);
|
||||
this.adoptMediaInBackground(historyFiles(this.dir).filter(unadopted), this.now());
|
||||
this.adoptMediaInBackground((await historyFiles(this.dir)).filter(unadopted), this.now());
|
||||
// What changed while the project was closed is one outside entry, or the closed window's.
|
||||
this.reopenClosedWindow();
|
||||
await this.settleAll();
|
||||
@@ -554,7 +576,7 @@ class Engine {
|
||||
async takeInUnnamedLedger(): Promise<void> {
|
||||
const named = (path: string) =>
|
||||
this.log.entries.some((entry) => entry.files.some((file) => file.path === path));
|
||||
const ledger = historyFiles(this.dir).find((file) => file.path === MEDIA_LEDGER);
|
||||
const ledger = (await historyFiles(this.dir)).find((file) => file.path === MEDIA_LEDGER);
|
||||
if (ledger && !this.log.baseline.has(MEDIA_LEDGER) && !named(MEDIA_LEDGER)) {
|
||||
const copied = await this.storeIfPresent(MEDIA_LEDGER);
|
||||
if (typeof copied === "object") this.log.baseline.set(MEDIA_LEDGER, copied.hash);
|
||||
@@ -566,7 +588,7 @@ class Engine {
|
||||
|
||||
async firstOpen(): Promise<void> {
|
||||
const sweptAt = this.now();
|
||||
const files = historyFiles(this.dir);
|
||||
const files = await historyFiles(this.dir);
|
||||
const copied = files.filter((file) => file.size <= COPY_LATER_ABOVE_BYTES);
|
||||
const hashes = await Promise.allSettled(
|
||||
copied.map((file) => inFirstCopySlot(() => this.storeIfPresent(file.path))),
|
||||
@@ -690,7 +712,7 @@ class Engine {
|
||||
const sweptAt = this.now();
|
||||
const changedAt = (file: { mtimeMs: number; ctimeMs: number }) =>
|
||||
Math.min(sweptAt, Math.max(file.mtimeMs, file.ctimeMs));
|
||||
const seen = historyFiles(this.dir).filter((file) => !this.adopting.has(file.path));
|
||||
const seen = (await historyFiles(this.dir)).filter((file) => !this.adopting.has(file.path));
|
||||
const heard = new Map(this.overwritten);
|
||||
const present = new Set(seen.map((file) => file.path));
|
||||
const removed = [...this.tracked.keys()]
|
||||
|
||||
@@ -0,0 +1,98 @@
|
||||
// @vitest-environment node
|
||||
import { mkdirSync, mkdtempSync, rmSync, statSync, writeFileSync } from "node:fs";
|
||||
import { tmpdir } from "node:os";
|
||||
import { join } from "node:path";
|
||||
import { afterEach, expect, it, vi } from "vitest";
|
||||
import { openProjectHistory } from "./projectHistory";
|
||||
|
||||
// A slow disk, as a large project on a scanned drive is: each blocking stat or listing holds the thread this long.
|
||||
const slowDisk = vi.hoisted(() => ({ ms: 0 }));
|
||||
vi.mock("node:fs", async (importOriginal) => {
|
||||
const real = await importOriginal<typeof import("node:fs")>();
|
||||
const pause = new Int32Array(new SharedArrayBuffer(4));
|
||||
const slow =
|
||||
<A extends unknown[], R>(call: (...args: A) => R) =>
|
||||
(...args: A): R => {
|
||||
if (slowDisk.ms) Atomics.wait(pause, 0, 0, slowDisk.ms);
|
||||
return call(...args);
|
||||
};
|
||||
const lstatSync = slow(real.lstatSync);
|
||||
const readdirSync = slow(real.readdirSync);
|
||||
return { ...real, lstatSync, readdirSync, default: { ...real, lstatSync, readdirSync } };
|
||||
});
|
||||
|
||||
const cleanup: Array<() => unknown> = [];
|
||||
afterEach(async () => {
|
||||
slowDisk.ms = 0;
|
||||
for (const step of cleanup.splice(0).reverse()) await step();
|
||||
});
|
||||
|
||||
function tempDir(prefix: string): string {
|
||||
const dir = mkdtempSync(join(tmpdir(), prefix));
|
||||
cleanup.push(() => rmSync(dir, { recursive: true, force: true }));
|
||||
return dir;
|
||||
}
|
||||
|
||||
/** The longest stretch the event loop went without turning while `work` ran. */
|
||||
async function longestHold(work: () => Promise<unknown>): Promise<number> {
|
||||
let last = performance.now();
|
||||
let longest = 0;
|
||||
let running = true;
|
||||
const turn = () => {
|
||||
const now = performance.now();
|
||||
longest = Math.max(longest, now - last);
|
||||
last = now;
|
||||
if (running) setImmediate(turn);
|
||||
};
|
||||
setImmediate(turn);
|
||||
await work();
|
||||
running = false;
|
||||
// The stretch since the last turn counts too: a walk that never yields ends before a first turn.
|
||||
return Math.max(longest, performance.now() - last);
|
||||
}
|
||||
|
||||
// fails without the fix: the sweep listed the project with readdirSync and lstatSync, holding the loop for the walk.
|
||||
it("a sweep of a project on a slow disk lets the host keep answering while it lists the files", async () => {
|
||||
const projectDir = tempDir("hf-history-walk-");
|
||||
for (let folder = 0; folder < 20; folder++) {
|
||||
mkdirSync(join(projectDir, `scenes-${folder}`));
|
||||
for (let file = 0; file < 20; file++)
|
||||
writeFileSync(join(projectDir, `scenes-${folder}`, `clip-${file}.html`), `<p>${file}</p>`);
|
||||
}
|
||||
// A minute on, the files are settled, as in a project opened later: the sweep stats them and hashes none.
|
||||
const history = await openProjectHistory({
|
||||
projectDir,
|
||||
historyRoot: tempDir("hf-history-root-"),
|
||||
now: () => Date.now() + 60_000,
|
||||
});
|
||||
cleanup.push(() => history.close());
|
||||
|
||||
slowDisk.ms = 1;
|
||||
const longest = await longestHold(() => history.flush());
|
||||
|
||||
// A blocking walk of these 420 entries holds the loop at least 420 ms.
|
||||
expect(longest, `held the loop ${longest.toFixed(1)} ms`).toBeLessThan(200);
|
||||
});
|
||||
|
||||
// fails without the fix: every queued operation rebuilt and rewrote video-state.json, one record per project file.
|
||||
it("a sweep that finds nothing leaves the video state file alone, and a change rewrites it", async () => {
|
||||
const projectDir = tempDir("hf-history-state-");
|
||||
writeFileSync(join(projectDir, "index.html"), "<h1>Hello</h1>");
|
||||
const history = await openProjectHistory({
|
||||
projectDir,
|
||||
historyRoot: tempDir("hf-history-root-"),
|
||||
now: () => Date.now() + 60_000,
|
||||
});
|
||||
cleanup.push(() => history.close());
|
||||
await history.flush();
|
||||
// Each write replaces the file, so a rewrite shows as a new inode.
|
||||
const written = () => statSync(join(projectDir, ".hyperframes", "video-state.json")).ino;
|
||||
const before = written();
|
||||
|
||||
await history.flush();
|
||||
expect(written()).toBe(before);
|
||||
|
||||
writeFileSync(join(projectDir, "index.html"), "<h1>Bye</h1>");
|
||||
await history.flush();
|
||||
expect(written()).not.toBe(before);
|
||||
});
|
||||
@@ -3,7 +3,7 @@ import { createReadStream, existsSync, readFileSync, renameSync, statSync } from
|
||||
import { isAbsolute, join, resolve } from "node:path";
|
||||
import { replaceFileAtomically } from "@hyperframes/core/atomic-file";
|
||||
import { pinProjectRecord } from "../helpers/safePath.js";
|
||||
import { listProjectFiles } from "../helpers/projectSignature.js";
|
||||
import { listProjectFilesAsync } from "../helpers/projectSignature.js";
|
||||
import { readId } from "./historyId.js";
|
||||
import { takeHistoryOwnership } from "./ownerLock.js";
|
||||
import {
|
||||
@@ -198,7 +198,9 @@ function dropArtifactsOfGoneRevisions(state: VideoState): void {
|
||||
|
||||
async function fileHashes(dir: string): Promise<Record<string, string>> {
|
||||
const files: Record<string, string> = {};
|
||||
for (const { path } of listProjectFiles(dir).sort((a, b) => a.path.localeCompare(b.path))) {
|
||||
for (const { path } of (await listProjectFilesAsync(dir)).sort((a, b) =>
|
||||
a.path.localeCompare(b.path),
|
||||
)) {
|
||||
const hash = createHash("sha256");
|
||||
for await (const chunk of createReadStream(join(dir, path))) hash.update(chunk);
|
||||
files[path] = hash.digest("hex");
|
||||
|
||||
Reference in New Issue
Block a user