mirror of
https://github.com/heygen-com/hyperframes.git
synced 2026-10-30 09:16:55 +00:00
feat(cli): add hyperframes clean to find and remove what HyperFrames left on disk (#4701)
* feat(cli): add hyperframes clean to find and remove what HyperFrames left on disk `hyperframes clean` lists leftover render work dirs, debug renders, the history of projects that no longer exist, idle video proxies and extracted video frames, with sizes, and removes them. `--dry-run` removes nothing, `--json` is for agents, `--snapshots` also removes snapshot folders (otherwise listed as reclaimable). It never touches outputs, sources or anything a running render or preview still owns. * docs(cli): say that clean does not search a project's subfolders * fix(cli): clean test compares folder names on Windows and the skills manifest is regenerated * fix(studio-server): the proxy cache sweep removes only files the transcoder named It removed any idle .mp4 or .webm in .transcode-cache, so a user file placed there, or reached through a symlinked cache folder, could go. Proxies are always <sha256>.<ext> and their temp files .tmp-<uuid>-<proxy name>; nothing else is swept. * style(producer): keep a renderDirOwner doc comment within 120 columns
This commit is contained in:
@@ -517,6 +517,8 @@
|
||||
"packages/studio-server/src/routes/files.test.ts",
|
||||
"packages/studio-server/src/routes/render.test.ts",
|
||||
"packages/studio-server/src/routes/lint.test.ts",
|
||||
// extractionCache.test.ts: pre-existing parallel describe setup and publish-case arrange/assert blocks.
|
||||
"packages/engine/src/services/extractionCache.test.ts",
|
||||
"packages/studio-server/src/routes/preview.test.ts",
|
||||
"packages/studio-server/src/routes/projects.test.ts",
|
||||
"packages/studio-server/src/helpers/backupJournal.test.ts",
|
||||
|
||||
@@ -19,6 +19,7 @@ remains authoritative: run `npx hyperframes <command> --help`.
|
||||
| Render a video | `npx hyperframes render --output output.mp4` |
|
||||
| Publish a browser link | `npx hyperframes publish` |
|
||||
| Diagnose the machine | `npx hyperframes doctor` |
|
||||
| Free disk space HyperFrames used | `npx hyperframes clean` |
|
||||
|
||||
## Work with media
|
||||
|
||||
|
||||
@@ -1137,6 +1137,27 @@ In JSON mode, paths inside `detail` and `hint` are redacted — your home
|
||||
directory becomes the literal `$HOME`, so output is safe to paste into a bug
|
||||
report or an agent context.
|
||||
|
||||
### `clean`
|
||||
|
||||
Find and remove what HyperFrames left on disk.
|
||||
|
||||
```bash
|
||||
npx hyperframes clean --dry-run # list it, remove nothing
|
||||
npx hyperframes clean
|
||||
npx hyperframes clean --snapshots # also remove snapshots/ folders
|
||||
npx hyperframes clean --json
|
||||
```
|
||||
|
||||
It looks in the current project (not its subfolders) and every project the Studio history knows, and removes:
|
||||
|
||||
- frames and staging folders left by renders that were killed or crashed (a render that is still running keeps its own);
|
||||
- work folders of `--debug` renders nothing has written to for six hours;
|
||||
- the history of projects that no longer exist;
|
||||
- extracted video frames and video proxies nobody has used for an hour (they rebuild on demand).
|
||||
|
||||
`snapshots/` folders are listed as also reclaimable and removed only with `--snapshots`. Rendered outputs, source
|
||||
files, the browser download and models are never touched.
|
||||
|
||||
### `info` and `compositions`
|
||||
|
||||
`npx hyperframes info [dir]` prints project metadata: name, resolution,
|
||||
|
||||
@@ -172,6 +172,7 @@ const commandLoaders = {
|
||||
tts: () => import("./commands/tts.js").then((m) => m.default),
|
||||
docs: () => import("./commands/docs.js").then((m) => m.default),
|
||||
doctor: () => import("./commands/doctor.js").then((m) => m.default),
|
||||
clean: () => import("./commands/clean.js").then((m) => m.default),
|
||||
upgrade: () => import("./commands/upgrade.js").then((m) => m.default),
|
||||
skills: () => import("./commands/skills.js").then((m) => m.default),
|
||||
feedback: () => import("./commands/feedback.js").then((m) => m.default),
|
||||
|
||||
@@ -0,0 +1,206 @@
|
||||
import { type ChildProcess, spawn, spawnSync } from "node:child_process";
|
||||
import {
|
||||
chmodSync,
|
||||
existsSync,
|
||||
mkdirSync,
|
||||
mkdtempSync,
|
||||
readFileSync,
|
||||
readdirSync,
|
||||
realpathSync,
|
||||
rmSync,
|
||||
symlinkSync,
|
||||
utimesSync,
|
||||
writeFileSync,
|
||||
} from "node:fs";
|
||||
import { tmpdir } from "node:os";
|
||||
import { basename, join } from "node:path";
|
||||
import { runCommand } from "citty";
|
||||
import { afterEach, beforeEach, describe, expect, it, vi } from "vitest";
|
||||
import { createOwnedRenderDir } from "@hyperframes/producer";
|
||||
import { consumeCommandResult } from "../utils/commandResult.js";
|
||||
import cleanCommand, { cleanLeftovers } from "./clean.js";
|
||||
|
||||
const history = vi.hoisted(() => ({ unreadable: false }));
|
||||
vi.mock("@hyperframes/studio-server", async (importOriginal) => {
|
||||
const actual = await importOriginal<typeof import("@hyperframes/studio-server")>();
|
||||
return {
|
||||
...actual,
|
||||
listProjectHistories: (root: string) => {
|
||||
if (history.unreadable) throw new Error("history unreadable");
|
||||
return actual.listProjectHistories(root);
|
||||
},
|
||||
};
|
||||
});
|
||||
|
||||
const JOB = "0f8c2b1e-5d3a-4c7b-9e21-6a4f0d8b3c19";
|
||||
const SEVEN_HOURS_AGO = (Date.now() - 7 * 60 * 60 * 1000) / 1000;
|
||||
|
||||
describe("cleanLeftovers", () => {
|
||||
let scratch: string;
|
||||
let project: string;
|
||||
let renders: string;
|
||||
let liveRender: ChildProcess;
|
||||
// This machine's real owner stamp, so planted owners differ from it only in pid.
|
||||
let stamp: object;
|
||||
|
||||
/** A render temp dir holding one 1000-byte frame, optionally owned and optionally last written 7 h ago. */
|
||||
function plant(parent: string, name: string, owner?: number, stale = false): string {
|
||||
const dir = join(parent, name);
|
||||
mkdirSync(dir, { recursive: true });
|
||||
writeFileSync(join(dir, "frame_000001.jpg"), Buffer.alloc(1000));
|
||||
if (owner) writeFileSync(join(dir, "owner.json"), JSON.stringify({ ...stamp, pid: owner }));
|
||||
if (stale)
|
||||
for (const path of [join(dir, "frame_000001.jpg"), dir])
|
||||
utimesSync(path, SEVEN_HOURS_AGO, SEVEN_HOURS_AGO);
|
||||
return dir;
|
||||
}
|
||||
|
||||
const run = (overrides: { dryRun?: boolean; snapshots?: boolean } = {}) =>
|
||||
cleanLeftovers({
|
||||
projectDir: project,
|
||||
dryRun: overrides.dryRun ?? false,
|
||||
snapshots: overrides.snapshots ?? false,
|
||||
historyRoot: join(scratch, "history"),
|
||||
tempDir: join(scratch, "tmp"),
|
||||
debugDir: join(scratch, "debug"),
|
||||
});
|
||||
|
||||
beforeEach(() => {
|
||||
scratch = realpathSync(mkdtempSync(join(tmpdir(), "hf-clean-")));
|
||||
project = join(scratch, "project");
|
||||
renders = join(project, "renders");
|
||||
mkdirSync(renders, { recursive: true });
|
||||
mkdirSync(join(scratch, "tmp"));
|
||||
writeFileSync(join(project, "index.html"), '<div data-composition-id="main"></div>');
|
||||
const own = createOwnedRenderDir(join(scratch, "self-"));
|
||||
stamp = JSON.parse(readFileSync(join(own, "owner.json"), "utf-8"));
|
||||
rmSync(own, { recursive: true });
|
||||
vi.stubEnv("HYPERFRAMES_EXTRACT_CACHE_DIR", join(scratch, "extract-cache"));
|
||||
// Stands in for a render in progress: a running process that owns its work dir.
|
||||
liveRender = spawn(process.execPath, ["-e", "setInterval(() => {}, 1000)"], {
|
||||
stdio: "ignore",
|
||||
});
|
||||
});
|
||||
afterEach(() => {
|
||||
liveRender.kill("SIGKILL");
|
||||
vi.unstubAllEnvs();
|
||||
rmSync(scratch, { recursive: true, force: true });
|
||||
});
|
||||
|
||||
it("removes what dead renders left and keeps what a running render uses", async () => {
|
||||
const dead = spawnSync(process.execPath, ["-e", ""]).pid;
|
||||
const abandoned = plant(renders, `work-${JOB}-aB3dE9`, dead);
|
||||
const staging = plant(renders, ".final.hf-transaction-Zx81Qa", dead);
|
||||
const beforeOwners = plant(renders, `work-${JOB}-Oo0000`, undefined, true);
|
||||
const oldDebug = plant(join(scratch, "debug"), JOB, undefined, true);
|
||||
const windowsLeftover = plant(join(scratch, "tmp"), "hf-render-Qw12Er", dead);
|
||||
const live = plant(renders, `work-${JOB}-Ll0000`, liveRender.pid);
|
||||
const freshOwnerless = plant(renders, `work-${JOB}-Ff0000`);
|
||||
const userFolder = plant(renders, "work-in-progress", undefined, true);
|
||||
const debugNotes = plant(join(scratch, "debug"), "my-notes", undefined, true);
|
||||
writeFileSync(join(renders, "final.mp4"), "the render");
|
||||
plant(project, "snapshots");
|
||||
|
||||
const listed = await run({ dryRun: true });
|
||||
expect(listed.removed.map((item) => item.path).sort()).toEqual(
|
||||
[abandoned, staging, beforeOwners, oldDebug, windowsLeftover].sort(),
|
||||
);
|
||||
expect(listed.removed.every((item) => item.bytes >= 1000)).toBe(true);
|
||||
expect(existsSync(abandoned)).toBe(true);
|
||||
|
||||
await run();
|
||||
expect(readdirSync(renders).sort()).toEqual(
|
||||
["final.mp4", live, freshOwnerless, userFolder].map((path) => basename(path)).sort(),
|
||||
);
|
||||
expect(existsSync(oldDebug) || existsSync(windowsLeftover)).toBe(false);
|
||||
expect(existsSync(debugNotes)).toBe(true);
|
||||
});
|
||||
|
||||
it.skipIf(process.platform === "win32" || process.getuid?.() === 0)(
|
||||
"reports a leftover it cannot remove and still removes the rest",
|
||||
async () => {
|
||||
const dead = spawnSync(process.execPath, ["-e", ""]).pid;
|
||||
const stuck = plant(renders, `work-${JOB}-St0000`, dead);
|
||||
const locked = join(stuck, "locked");
|
||||
mkdirSync(locked);
|
||||
writeFileSync(join(locked, "frame.jpg"), "x");
|
||||
chmodSync(locked, 0o555);
|
||||
const other = plant(join(scratch, "tmp"), "hf-render-Ot0000", dead);
|
||||
try {
|
||||
const result = await run();
|
||||
expect(result.errors).toHaveLength(1);
|
||||
expect(result.removed.map((item) => item.path)).toEqual([other]);
|
||||
expect(existsSync(other)).toBe(false);
|
||||
} finally {
|
||||
chmodSync(locked, 0o755);
|
||||
}
|
||||
},
|
||||
);
|
||||
|
||||
it("leaves snapshots alone in a folder that is not a HyperFrames project", async () => {
|
||||
writeFileSync(join(project, "index.html"), "<title>my website</title>");
|
||||
const snapshots = plant(project, "snapshots");
|
||||
|
||||
const result = await run({ snapshots: true });
|
||||
expect(result.alsoReclaimable).toEqual([]);
|
||||
expect(existsSync(snapshots)).toBe(true);
|
||||
});
|
||||
|
||||
it.skipIf(process.platform === "win32" || process.getuid?.() === 0)(
|
||||
"still cleans when the history folder cannot be read",
|
||||
async () => {
|
||||
const history = join(scratch, "history");
|
||||
mkdirSync(history);
|
||||
chmodSync(history, 0o000);
|
||||
const dead = spawnSync(process.execPath, ["-e", ""]).pid;
|
||||
const leftover = plant(join(scratch, "tmp"), "hf-render-Hi0000", dead);
|
||||
try {
|
||||
const result = await run();
|
||||
expect(result.errors.length).toBeGreaterThan(0);
|
||||
expect(existsSync(leftover)).toBe(false);
|
||||
} finally {
|
||||
chmodSync(history, 0o755);
|
||||
}
|
||||
},
|
||||
);
|
||||
|
||||
it("lists a leftover once when the project folder is a link to the temp dir", async () => {
|
||||
const dead = spawnSync(process.execPath, ["-e", ""]).pid;
|
||||
const leftover = plant(join(scratch, "tmp"), "hf-render-Ln0000", dead);
|
||||
symlinkSync(join(scratch, "tmp"), join(scratch, "tmp-link"));
|
||||
|
||||
const result = await cleanLeftovers({
|
||||
projectDir: join(scratch, "tmp-link"),
|
||||
dryRun: true,
|
||||
snapshots: false,
|
||||
historyRoot: join(scratch, "history"),
|
||||
tempDir: join(scratch, "tmp"),
|
||||
debugDir: join(scratch, "debug"),
|
||||
});
|
||||
expect(result.removed.map((item) => item.path)).toEqual([leftover]);
|
||||
});
|
||||
|
||||
it("exits 1 when something could not be read or removed", async () => {
|
||||
history.unreadable = true;
|
||||
const log = vi.spyOn(console, "log").mockImplementation(() => {});
|
||||
try {
|
||||
await runCommand(cleanCommand, { rawArgs: [project, "--dry-run", "--json"] });
|
||||
expect(JSON.parse(String(log.mock.calls.at(-1)?.[0])).errors).toContain("history unreadable");
|
||||
expect(consumeCommandResult().exitCode).toBe(1);
|
||||
} finally {
|
||||
history.unreadable = false;
|
||||
log.mockRestore();
|
||||
}
|
||||
});
|
||||
|
||||
it("lists snapshots as reclaimable and removes them only when asked", async () => {
|
||||
const snapshots = plant(project, "snapshots");
|
||||
|
||||
const plain = await run();
|
||||
expect(plain.alsoReclaimable).toEqual([{ what: "Snapshots", path: snapshots, bytes: 1000 }]);
|
||||
expect(existsSync(snapshots)).toBe(true);
|
||||
|
||||
await run({ snapshots: true });
|
||||
expect(existsSync(snapshots)).toBe(false);
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,274 @@
|
||||
import { existsSync, readFileSync, realpathSync, rmSync } from "node:fs";
|
||||
import { tmpdir } from "node:os";
|
||||
import { join, resolve } from "node:path";
|
||||
import { defineCommand } from "citty";
|
||||
import {
|
||||
EXTRACT_CACHE_MIN_AGE_MS,
|
||||
directorySizeBytes,
|
||||
gcExtractionCache,
|
||||
resolveExtractCacheDir,
|
||||
} from "@hyperframes/engine";
|
||||
import {
|
||||
DEFAULT_HISTORY_ROOT,
|
||||
PROXY_CACHE_DIR_NAME,
|
||||
cleanupProxyCache,
|
||||
listProjectHistories,
|
||||
pruneGoneProjectHistories,
|
||||
} from "@hyperframes/studio-server";
|
||||
import type { Example } from "./_examples.js";
|
||||
import { c } from "../ui/colors.js";
|
||||
import { formatBytes } from "../ui/format.js";
|
||||
import { setCommandExitCode } from "../utils/commandResult.js";
|
||||
import { loadProducer } from "../utils/producer.js";
|
||||
import { redactHome } from "./doctor.js";
|
||||
import { withMeta } from "../utils/updateCheck.js";
|
||||
|
||||
export const examples: Example[] = [
|
||||
["See what HyperFrames left on disk, remove nothing", "hyperframes clean --dry-run"],
|
||||
["Remove it", "hyperframes clean"],
|
||||
["Also remove snapshot folders", "hyperframes clean --snapshots"],
|
||||
["For agents", "hyperframes clean --json"],
|
||||
];
|
||||
|
||||
/** A render dir with no owner record (made before owners were recorded) counts as abandoned after this long idle. */
|
||||
const OWNERLESS_IDLE_MS = 6 * 60 * 60 * 1000;
|
||||
/** A video proxy nobody has used for this long is regenerable; a Studio still using one keeps touching it. */
|
||||
const PROXY_IDLE_MS = 60 * 60 * 1000;
|
||||
|
||||
export interface Leftover {
|
||||
what: string;
|
||||
path: string;
|
||||
bytes: number;
|
||||
}
|
||||
|
||||
export interface CleanOptions {
|
||||
projectDir: string;
|
||||
dryRun: boolean;
|
||||
snapshots: boolean;
|
||||
/** Where to look; each defaults to the real location (tests point them at scratch dirs). */
|
||||
historyRoot?: string;
|
||||
tempDir?: string;
|
||||
debugDir?: string;
|
||||
}
|
||||
|
||||
export interface CleanResult {
|
||||
removed: Leftover[];
|
||||
alsoReclaimable: Leftover[];
|
||||
errors: string[];
|
||||
}
|
||||
|
||||
interface Sweep {
|
||||
dryRun: boolean;
|
||||
result: CleanResult;
|
||||
}
|
||||
|
||||
function recordError(sweep: Sweep, error: unknown): void {
|
||||
sweep.result.errors.push(error instanceof Error ? error.message : String(error));
|
||||
}
|
||||
|
||||
/** Runs one kind of sweep; a failure is reported and the rest of the clean goes on. */
|
||||
async function attempt(sweep: Sweep, step: () => unknown): Promise<void> {
|
||||
try {
|
||||
await step();
|
||||
} catch (error) {
|
||||
recordError(sweep, error);
|
||||
}
|
||||
}
|
||||
|
||||
/** A HyperFrames project: its index.html holds a composition root, not just any website. */
|
||||
function isProject(dir: string): boolean {
|
||||
try {
|
||||
return readFileSync(join(dir, "index.html"), "utf-8").includes("data-composition-id");
|
||||
} catch {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
function realDir(dir: string): string {
|
||||
try {
|
||||
return realpathSync(dir);
|
||||
} catch {
|
||||
return resolve(dir);
|
||||
}
|
||||
}
|
||||
|
||||
function knownProjectDirs(sweep: Sweep, historyRoot: string): string[] {
|
||||
try {
|
||||
return listProjectHistories(historyRoot).map((record) => record.projectDir);
|
||||
} catch (error) {
|
||||
recordError(sweep, error);
|
||||
return [];
|
||||
}
|
||||
}
|
||||
|
||||
function drop(sweep: Sweep, what: string, path: string): void {
|
||||
const bytes = directorySizeBytes(path);
|
||||
try {
|
||||
if (!sweep.dryRun)
|
||||
rmSync(path, { recursive: true, force: true, maxRetries: 3, retryDelay: 100 });
|
||||
sweep.result.removed.push({ what, path, bytes });
|
||||
} catch (error) {
|
||||
recordError(sweep, error);
|
||||
}
|
||||
}
|
||||
|
||||
function sweepProxies(sweep: Sweep, project: string): void {
|
||||
const proxies = join(project, PROXY_CACHE_DIR_NAME);
|
||||
const swept = cleanupProxyCache(proxies, {
|
||||
maxBytes: Number.POSITIVE_INFINITY,
|
||||
maxIdleMs: PROXY_IDLE_MS,
|
||||
minSweepIntervalMs: 0,
|
||||
dryRun: sweep.dryRun,
|
||||
});
|
||||
if (swept.removed.length === 0) return;
|
||||
sweep.result.removed.push({
|
||||
what: "Video proxies",
|
||||
path: proxies,
|
||||
bytes: swept.bytesBefore - swept.bytesAfter,
|
||||
});
|
||||
}
|
||||
|
||||
function sweepSnapshots(sweep: Sweep, project: string, remove: boolean): void {
|
||||
const snapshots = join(project, "snapshots");
|
||||
if (!existsSync(snapshots)) return;
|
||||
if (remove) drop(sweep, "Snapshots", snapshots);
|
||||
else
|
||||
sweep.result.alsoReclaimable.push({
|
||||
what: "Snapshots",
|
||||
path: snapshots,
|
||||
bytes: directorySizeBytes(snapshots),
|
||||
});
|
||||
}
|
||||
|
||||
async function sweepHistories(sweep: Sweep, historyRoot: string, tempDir: string): Promise<void> {
|
||||
const pruned = await pruneGoneProjectHistories(historyRoot, {
|
||||
dryRun: sweep.dryRun,
|
||||
onError: (error) => recordError(sweep, error),
|
||||
tempDir,
|
||||
});
|
||||
for (const history of pruned) {
|
||||
sweep.result.removed.push({
|
||||
what: "Project history",
|
||||
path: join(historyRoot, history.id),
|
||||
bytes: history.bytes,
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
function sweepExtractCache(sweep: Sweep): void {
|
||||
const extractCache = resolveExtractCacheDir().dir;
|
||||
if (!extractCache) return;
|
||||
const swept = gcExtractionCache(extractCache, {
|
||||
maxBytes: 0,
|
||||
minAgeMs: EXTRACT_CACHE_MIN_AGE_MS,
|
||||
dryRun: sweep.dryRun,
|
||||
});
|
||||
if (swept.evictedEntries + swept.agedPartialsRemoved === 0) return;
|
||||
sweep.result.removed.push({
|
||||
what: "Extracted video frames",
|
||||
path: extractCache,
|
||||
bytes: swept.evictedBytes,
|
||||
});
|
||||
}
|
||||
|
||||
/** Everything HyperFrames left behind that no running render or preview uses; removed unless `dryRun`. */
|
||||
export async function cleanLeftovers(options: CleanOptions): Promise<CleanResult> {
|
||||
const historyRoot = options.historyRoot ?? DEFAULT_HISTORY_ROOT;
|
||||
const tempDir = options.tempDir ?? tmpdir();
|
||||
const { RENDER_JOB_DIR, listAbandonedRenderDirs, resolveRenderDebugDir } = await loadProducer();
|
||||
const sweep: Sweep = {
|
||||
dryRun: options.dryRun,
|
||||
result: { removed: [], alsoReclaimable: [], errors: [] },
|
||||
};
|
||||
const dropAbandoned = (what: string, parent: string, names?: RegExp) => {
|
||||
for (const dir of listAbandonedRenderDirs(parent, {
|
||||
ownerlessIdleMs: OWNERLESS_IDLE_MS,
|
||||
names,
|
||||
}))
|
||||
drop(sweep, what, dir);
|
||||
};
|
||||
|
||||
const projects = new Set(
|
||||
[options.projectDir, ...knownProjectDirs(sweep, historyRoot)].map(realDir),
|
||||
);
|
||||
const renderParents = new Set([
|
||||
...[...projects].flatMap((project) => [project, join(project, "renders")]),
|
||||
realDir(tempDir),
|
||||
]);
|
||||
for (const parent of renderParents) dropAbandoned("Render leftovers", parent);
|
||||
dropAbandoned("Debug renders", options.debugDir ?? resolveRenderDebugDir(), RENDER_JOB_DIR);
|
||||
for (const project of projects) {
|
||||
if (!isProject(project)) continue;
|
||||
await attempt(sweep, () => sweepProxies(sweep, project));
|
||||
sweepSnapshots(sweep, project, options.snapshots);
|
||||
}
|
||||
await attempt(sweep, () => sweepHistories(sweep, historyRoot, tempDir));
|
||||
await attempt(sweep, () => sweepExtractCache(sweep));
|
||||
return sweep.result;
|
||||
}
|
||||
|
||||
function printTable(rows: Leftover[]): void {
|
||||
const whatWidth = Math.max(...rows.map((row) => row.what.length));
|
||||
for (const row of rows) {
|
||||
console.log(
|
||||
` ${row.what.padEnd(whatWidth)} ${formatBytes(row.bytes).padStart(9)} ${c.dim(row.path)}`,
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
const total = (rows: Leftover[]) => rows.reduce((sum, row) => sum + row.bytes, 0);
|
||||
const redactPaths = (rows: Leftover[]) =>
|
||||
rows.map((row) => ({ ...row, path: redactHome(row.path) }));
|
||||
|
||||
export default defineCommand({
|
||||
meta: { name: "clean", description: "Find and remove what HyperFrames left on disk" },
|
||||
args: {
|
||||
dir: {
|
||||
type: "positional",
|
||||
description: "Project directory (default: current); its subfolders are not searched",
|
||||
required: false,
|
||||
},
|
||||
"dry-run": {
|
||||
type: "boolean",
|
||||
description: "List what would be removed, remove nothing",
|
||||
default: false,
|
||||
},
|
||||
snapshots: { type: "boolean", description: "Also remove snapshots/ folders", default: false },
|
||||
json: { type: "boolean", description: "Output as JSON", default: false },
|
||||
},
|
||||
async run({ args }) {
|
||||
const dryRun = args["dry-run"];
|
||||
const result = await cleanLeftovers({
|
||||
projectDir: args.dir ?? process.cwd(),
|
||||
dryRun,
|
||||
snapshots: args.snapshots,
|
||||
});
|
||||
if (result.errors.length > 0) setCommandExitCode(1);
|
||||
if (args.json) {
|
||||
const report = withMeta({
|
||||
dryRun,
|
||||
removed: redactPaths(result.removed),
|
||||
alsoReclaimable: redactPaths(result.alsoReclaimable),
|
||||
errors: result.errors.map(redactHome),
|
||||
totalBytes: total(result.removed),
|
||||
});
|
||||
console.log(JSON.stringify(report, null, 2));
|
||||
return;
|
||||
}
|
||||
if (result.removed.length === 0) {
|
||||
console.log(`${c.success("◇")} Nothing to clean`);
|
||||
} else {
|
||||
console.log(
|
||||
`${c.success("◇")} ${dryRun ? "Would remove" : "Removed"} ${formatBytes(total(result.removed))}`,
|
||||
);
|
||||
printTable(result.removed);
|
||||
}
|
||||
if (result.alsoReclaimable.length > 0) {
|
||||
console.log(
|
||||
`\n Also reclaimable with --snapshots: ${formatBytes(total(result.alsoReclaimable))}`,
|
||||
);
|
||||
printTable(result.alsoReclaimable);
|
||||
}
|
||||
for (const error of result.errors) console.log(c.warn(` ${error}`));
|
||||
},
|
||||
});
|
||||
@@ -71,6 +71,7 @@ const GROUPS: Group[] = [
|
||||
],
|
||||
["browser", "Manage the Chrome browser used for rendering"],
|
||||
["doctor", "Check system dependencies and environment"],
|
||||
["clean", "Find and remove what HyperFrames left on disk"],
|
||||
["upgrade", "Check for updates and show upgrade instructions"],
|
||||
],
|
||||
},
|
||||
|
||||
@@ -232,7 +232,9 @@ export {
|
||||
type VideoFrameFormat,
|
||||
VIDEO_FRAME_FORMATS,
|
||||
isVideoFrameFormat,
|
||||
EXTRACT_CACHE_MIN_AGE_MS,
|
||||
} from "./services/videoFrameExtractor.js";
|
||||
export { directorySizeBytes, gcExtractionCache } from "./services/extractionCache.js";
|
||||
|
||||
export {
|
||||
resolveReferencedStart,
|
||||
|
||||
@@ -416,6 +416,19 @@ describe("gcExtractionCache", () => {
|
||||
expect(stats.evictedEntries).toBe(1);
|
||||
});
|
||||
|
||||
it("counts the same evictions in a dry run and removes nothing", () => {
|
||||
const old = makeEntry("old", 60, 120_000);
|
||||
const young = makeEntry("young", 60, 1_000);
|
||||
const options = { maxBytes: 0, minAgeMs: 60_000 };
|
||||
|
||||
const planned = gcExtractionCache(tmpRoot, { ...options, dryRun: true });
|
||||
expect(existsSync(old)).toBe(true);
|
||||
expect(gcExtractionCache(tmpRoot, options)).toEqual(planned);
|
||||
expect(planned.evictedEntries).toBe(1);
|
||||
expect(existsSync(old)).toBe(false);
|
||||
expect(existsSync(young)).toBe(true);
|
||||
});
|
||||
|
||||
it("evicts oldest complete entries first until under maxBytes while respecting minAge", () => {
|
||||
const oldest = makeEntry("oldest", 60, 120_000);
|
||||
const middle = makeEntry("middle", 60, 90_000);
|
||||
|
||||
@@ -336,7 +336,7 @@ function isPartialChild(name: string): boolean {
|
||||
return name.includes(".partial-");
|
||||
}
|
||||
|
||||
function directorySizeBytes(path: string): number {
|
||||
export function directorySizeBytes(path: string): number {
|
||||
try {
|
||||
const stat = lstatSync(path);
|
||||
if (!stat.isDirectory()) return stat.size;
|
||||
@@ -394,11 +394,12 @@ function collectGcEntry(
|
||||
now: number,
|
||||
minAgeMs: number,
|
||||
stats: GcStats,
|
||||
remove: (dir: string) => void,
|
||||
): GcEntry | null {
|
||||
try {
|
||||
const dirStat = statSync(dir);
|
||||
if (isPartialChild(name) && now - dirStat.mtimeMs >= minAgeMs) {
|
||||
removeDir(dir);
|
||||
remove(dir);
|
||||
stats.agedPartialsRemoved += 1;
|
||||
return null;
|
||||
}
|
||||
@@ -445,16 +446,21 @@ export function gcSweepDue(rootDir: string, maxAgeMs: number): boolean {
|
||||
}
|
||||
}
|
||||
|
||||
export function gcExtractionCache(
|
||||
rootDir: string,
|
||||
opts: { maxBytes: number; minAgeMs: number },
|
||||
): GcStats {
|
||||
const stats: GcStats = { evictedEntries: 0, evictedBytes: 0, agedPartialsRemoved: 0 };
|
||||
function markGcSweep(rootDir: string): void {
|
||||
try {
|
||||
writeFileSync(join(rootDir, GC_MARKER), "", "utf-8");
|
||||
} catch {
|
||||
// Unwritable root: the sweep below will no-op on the same root anyway.
|
||||
}
|
||||
}
|
||||
|
||||
export function gcExtractionCache(
|
||||
rootDir: string,
|
||||
opts: { maxBytes: number; minAgeMs: number; dryRun?: boolean },
|
||||
): GcStats {
|
||||
const stats: GcStats = { evictedEntries: 0, evictedBytes: 0, agedPartialsRemoved: 0 };
|
||||
const remove = opts.dryRun ? () => {} : removeDir;
|
||||
if (!opts.dryRun) markGcSweep(rootDir);
|
||||
try {
|
||||
const now = Date.now();
|
||||
const entries: GcEntry[] = [];
|
||||
@@ -466,6 +472,7 @@ export function gcExtractionCache(
|
||||
now,
|
||||
opts.minAgeMs,
|
||||
stats,
|
||||
remove,
|
||||
);
|
||||
if (entry) entries.push(entry);
|
||||
}
|
||||
@@ -477,7 +484,7 @@ export function gcExtractionCache(
|
||||
for (const entry of entries) {
|
||||
// ponytail: age-based liveness guard, not a lock; a render longer than minAge with a full cache could lose entries mid-read - acceptable, next render re-extracts.
|
||||
if (entry.ageMs < opts.minAgeMs) continue;
|
||||
removeDir(entry.dir);
|
||||
remove(entry.dir);
|
||||
stats.evictedEntries += 1;
|
||||
stats.evictedBytes += entry.size;
|
||||
totalBytes -= entry.size;
|
||||
|
||||
@@ -219,7 +219,7 @@ export interface ExtractionOptions {
|
||||
collectProbeFailures?: boolean;
|
||||
}
|
||||
|
||||
const EXTRACT_CACHE_MIN_AGE_MS = 60 * 60 * 1000;
|
||||
export const EXTRACT_CACHE_MIN_AGE_MS = 60 * 60 * 1000;
|
||||
const GC_STALENESS_MS = 24 * 60 * 60 * 1000;
|
||||
const SDR_TO_HDR_COLORSPACE_FILTER = "colorspace=all=bt2020:iall=bt709:range=tv";
|
||||
const HDR_TO_SDR_ZSCALE_TRANSFORM_KEY = "hdr2sdr-hable-srgb";
|
||||
@@ -2468,6 +2468,10 @@ export class FrameLookupTable {
|
||||
return frameIndex == null ? null : video.extracted.framePaths.get(frameIndex) || null;
|
||||
}
|
||||
|
||||
frameDirs(): string[] {
|
||||
return [...new Set(Array.from(this.videos.values(), (video) => video.extracted.outputDir))];
|
||||
}
|
||||
|
||||
private resetActiveState(): void {
|
||||
this.activeVideoIds.clear();
|
||||
this.startCursor = 0;
|
||||
|
||||
@@ -2,7 +2,7 @@
|
||||
import { describe, it, expect, beforeEach, afterEach, vi } from "vitest";
|
||||
import { mkdtempSync, rmSync, statSync, utimesSync, writeFileSync } from "node:fs";
|
||||
import { tmpdir } from "node:os";
|
||||
import { join } from "node:path";
|
||||
import { dirname, join } from "node:path";
|
||||
import { type Page } from "puppeteer-core";
|
||||
import { COMPLETE_SENTINEL } from "./extractionCache.js";
|
||||
|
||||
@@ -25,7 +25,7 @@ vi.mock("./screenshotService.js", () => ({
|
||||
}));
|
||||
|
||||
import { __testing, createVideoFrameInjector } from "./videoFrameInjector.js";
|
||||
import { type FrameLookupTable } from "./videoFrameExtractor.js";
|
||||
import { type ExtractedFrames, FrameLookupTable } from "./videoFrameExtractor.js";
|
||||
import { type BeforeCaptureHook } from "./frameCapture.js";
|
||||
import { DEFAULT_CONFIG } from "../config.js";
|
||||
|
||||
@@ -178,6 +178,7 @@ describe("createVideoFrameInjector cache hygiene against page-side skips", () =>
|
||||
// table is exercised exhaustively in videoFrameExtractor.test.ts.
|
||||
function fakeTable(payload: { videoId: string; framePath: string; frameIndex: number }) {
|
||||
return {
|
||||
frameDirs: () => [],
|
||||
getActiveFramePayloads: () =>
|
||||
new Map([
|
||||
[payload.videoId, { framePath: payload.framePath, frameIndex: payload.frameIndex }],
|
||||
@@ -338,6 +339,7 @@ describe("createVideoFrameInjector extraction-cache lease renewal", () => {
|
||||
|
||||
function makeHook(framePath: string): BeforeCaptureHook {
|
||||
const table = {
|
||||
frameDirs: () => [dirname(framePath)],
|
||||
getActiveFramePayloads: () => new Map([["v", { framePath, frameIndex: 0 }]]),
|
||||
} as unknown as FrameLookupTable;
|
||||
const hook = createVideoFrameInjector(table, { frameSrcResolver: inlineResolver });
|
||||
@@ -378,6 +380,18 @@ describe("createVideoFrameInjector extraction-cache lease renewal", () => {
|
||||
expect(sentinelMtimeMs()).toBeGreaterThan(before);
|
||||
});
|
||||
|
||||
it("renews the entry of a clip that is not on screen yet", async () => {
|
||||
const table = new FrameLookupTable();
|
||||
const lateClip = { videoId: "late", outputDir: cacheDir, framePaths: new Map() };
|
||||
table.addVideo(lateClip as unknown as ExtractedFrames, 3600, 3660, 0);
|
||||
const hook = createVideoFrameInjector(table, { frameSrcResolver: inlineResolver });
|
||||
const before = sentinelMtimeMs();
|
||||
|
||||
await hook?.(fakePage, 0);
|
||||
|
||||
expect(sentinelMtimeMs()).toBeGreaterThan(before);
|
||||
});
|
||||
|
||||
it("throttles renewal so repeated frames don't hammer the filesystem clock", async () => {
|
||||
const hook = makeHook(join(cacheDir, "frame_00001.jpg"));
|
||||
|
||||
|
||||
@@ -9,7 +9,6 @@
|
||||
|
||||
import { type Page } from "puppeteer-core";
|
||||
import { promises as fs } from "fs";
|
||||
import { dirname } from "node:path";
|
||||
import { type FrameLookupTable } from "./videoFrameExtractor.js";
|
||||
import { touchCacheDir } from "./extractionCache.js";
|
||||
import { injectVideoFramesBatch, syncVideoFrameVisibility } from "./screenshotService.js";
|
||||
@@ -191,13 +190,12 @@ export function createVideoFrameInjector(
|
||||
const lastCacheTouchByDir = new Map<string, number>();
|
||||
|
||||
/**
|
||||
* Renew this render's lease on the extraction-cache entry `framePath` lives
|
||||
* in. Called for every active video on every frame — including one whose
|
||||
* frame index hasn't moved, since a long-held-static frame needs its entry
|
||||
* kept alive just as much as a changing one — so it throttles per directory.
|
||||
* Renew this render's lease on an extraction-cache entry it reads from. Called
|
||||
* on every frame for every entry the render holds, including clips not on
|
||||
* screen yet (one that first shows an hour in still needs its frames), so it
|
||||
* throttles per directory.
|
||||
*/
|
||||
function renewCacheLease(framePath: string): void {
|
||||
const cacheDir = dirname(framePath);
|
||||
function renewCacheLease(cacheDir: string): void {
|
||||
const now = Date.now();
|
||||
const lastTouch = lastCacheTouchByDir.get(cacheDir);
|
||||
if (lastTouch !== undefined && now - lastTouch < CACHE_TOUCH_THROTTLE_MS) return;
|
||||
@@ -207,6 +205,7 @@ export function createVideoFrameInjector(
|
||||
|
||||
// fallow-ignore-next-line complexity
|
||||
return async (page: Page, time: number) => {
|
||||
for (const cacheDir of frameLookup.frameDirs()) renewCacheLease(cacheDir);
|
||||
const activePayloads = frameLookup.getActiveFramePayloads(time);
|
||||
|
||||
const updates: Array<{ videoId: string; dataUri: string; frameIndex: number }> = [];
|
||||
@@ -216,7 +215,6 @@ export function createVideoFrameInjector(
|
||||
[];
|
||||
for (const [videoId, payload] of activePayloads) {
|
||||
activeIds.add(videoId);
|
||||
renewCacheLease(payload.framePath);
|
||||
const lastFrameIndex = lastInjectedFrameByVideo.get(videoId);
|
||||
if (lastFrameIndex === payload.frameIndex) continue;
|
||||
pendingReads.push(
|
||||
|
||||
@@ -10,6 +10,7 @@
|
||||
export {
|
||||
createRenderJob,
|
||||
executeRenderJob,
|
||||
resolveRenderDebugDir,
|
||||
RenderCancelledError,
|
||||
RenderQualityError,
|
||||
applyRenderWarningPolicy,
|
||||
@@ -23,6 +24,11 @@ export {
|
||||
type RenderPerfSummary,
|
||||
type ProgressCallback,
|
||||
} from "./services/renderOrchestrator.js";
|
||||
export {
|
||||
RENDER_JOB_DIR,
|
||||
createOwnedRenderDir,
|
||||
listAbandonedRenderDirs,
|
||||
} from "./services/render/renderDirOwner.js";
|
||||
export {
|
||||
RENDER_REQUEST_VERSION,
|
||||
createRenderRequest,
|
||||
|
||||
@@ -1,5 +1,6 @@
|
||||
import {
|
||||
existsSync,
|
||||
lstatSync,
|
||||
mkdtempSync,
|
||||
readFileSync,
|
||||
readdirSync,
|
||||
@@ -16,6 +17,8 @@ export const TRANSACTION_BACKUP = "backup";
|
||||
// The names mkdtemp gives a render's temp dirs: work-<job uuid>-, hf-render- (Windows) and .<output>.hf-transaction-.
|
||||
const RENDER_TEMP_DIR =
|
||||
/^(work-[0-9a-f]{8}(-[0-9a-f]{4}){3}-[0-9a-f]{12}|hf-render|\..+\.hf-transaction)-[A-Za-z0-9]{6}$/;
|
||||
/** A `--debug` render's work dir, named by its job id. */
|
||||
export const RENDER_JOB_DIR = /^[0-9a-f]{8}(-[0-9a-f]{4}){3}-[0-9a-f]{12}$/;
|
||||
|
||||
/**
|
||||
* Where a pid means this process: host and boot, plus the Linux pid namespace. Containers and machines on a shared
|
||||
@@ -56,56 +59,88 @@ function alive(pid: number): boolean {
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* True only for an owner in this process's pid scope that has exited; no owner file, another scope or an unreadable
|
||||
* owner all mean "not provably abandoned".
|
||||
*/
|
||||
function ownerIsGone(dir: string): boolean {
|
||||
let owner: { pid?: unknown; host?: unknown; boot?: unknown; pidns?: unknown };
|
||||
type OwnerRecord = { pid?: unknown; host?: unknown; boot?: unknown; pidns?: unknown };
|
||||
|
||||
function inThisPidScope(owner: OwnerRecord): boolean {
|
||||
const scope = pidScope();
|
||||
return owner.host === scope.host && owner.boot === scope.boot && owner.pidns === scope.pidns;
|
||||
}
|
||||
|
||||
/** "gone" only for an owner in this pid scope that has exited; "none" when there is no owner file, else "unknown". */
|
||||
function ownerState(dir: string): "gone" | "live" | "unknown" | "none" {
|
||||
let owner: OwnerRecord;
|
||||
try {
|
||||
owner = JSON.parse(readFileSync(join(dir, OWNER_FILE), "utf-8"));
|
||||
} catch {
|
||||
return false;
|
||||
} catch (error) {
|
||||
return (error as NodeJS.ErrnoException).code === "ENOENT" ? "none" : "unknown";
|
||||
}
|
||||
const scope = pidScope();
|
||||
const { pid } = owner;
|
||||
return (
|
||||
owner.host === scope.host &&
|
||||
owner.boot === scope.boot &&
|
||||
owner.pidns === scope.pidns &&
|
||||
typeof pid === "number" &&
|
||||
Number.isInteger(pid) &&
|
||||
pid > 0 &&
|
||||
!alive(pid)
|
||||
if (!inThisPidScope(owner) || typeof pid !== "number" || !Number.isInteger(pid) || pid <= 0) {
|
||||
return "unknown";
|
||||
}
|
||||
return alive(pid) ? "live" : "gone";
|
||||
}
|
||||
|
||||
function newestWriteMs(path: string): number {
|
||||
const stat = lstatSync(path);
|
||||
if (!stat.isDirectory()) return stat.mtimeMs;
|
||||
return readdirSync(path).reduce(
|
||||
(newest, name) => Math.max(newest, newestWriteMs(join(path, name))),
|
||||
stat.mtimeMs,
|
||||
);
|
||||
}
|
||||
|
||||
/** A staging dir that still holds {@link TRANSACTION_BACKUP} (its render died mid-swap) is left alone. */
|
||||
function removeAbandonedRenderDirs(parent: string): void {
|
||||
function isAbandoned(dir: string, ownerlessIdleMs: number | undefined, now: number): boolean {
|
||||
try {
|
||||
if (!lstatSync(dir).isDirectory() || existsSync(join(dir, TRANSACTION_BACKUP))) return false;
|
||||
const owner = ownerState(dir);
|
||||
if (owner === "gone") return true;
|
||||
return (
|
||||
owner === "none" &&
|
||||
ownerlessIdleMs !== undefined &&
|
||||
now - newestWriteMs(dir) >= ownerlessIdleMs
|
||||
);
|
||||
} catch {
|
||||
// Vanished or unreadable mid-scan: not provably abandoned.
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
export interface AbandonedRenderDirOptions {
|
||||
/** Also count dirs with no owner record once nothing in them has been written for this long. */
|
||||
ownerlessIdleMs?: number;
|
||||
/** Names to consider instead of render temp dir names. */
|
||||
names?: RegExp;
|
||||
}
|
||||
|
||||
/** Render temp dirs under `parent` no running render owns; never a staging dir holding {@link TRANSACTION_BACKUP}. */
|
||||
export function listAbandonedRenderDirs(
|
||||
parent: string,
|
||||
options: AbandonedRenderDirOptions = {},
|
||||
): string[] {
|
||||
let names: string[];
|
||||
try {
|
||||
names = readdirSync(parent);
|
||||
} catch {
|
||||
return;
|
||||
}
|
||||
for (const name of names) {
|
||||
const dir = join(parent, name);
|
||||
if (
|
||||
!RENDER_TEMP_DIR.test(name) ||
|
||||
!ownerIsGone(dir) ||
|
||||
existsSync(join(dir, TRANSACTION_BACKUP))
|
||||
)
|
||||
continue;
|
||||
try {
|
||||
rmSync(dir, { recursive: true, force: true, maxRetries: 3, retryDelay: 100 });
|
||||
} catch {
|
||||
// Reclaiming a dead render's dir is best-effort; it must never fail the render that found it.
|
||||
}
|
||||
return [];
|
||||
}
|
||||
const now = Date.now();
|
||||
return names
|
||||
.filter((name) => (options.names ?? RENDER_TEMP_DIR).test(name))
|
||||
.map((name) => join(parent, name))
|
||||
.filter((dir) => isAbandoned(dir, options.ownerlessIdleMs, now));
|
||||
}
|
||||
|
||||
/** Creates a render's work dir after reclaiming dirs of renders that were killed outright (no cleanup ran). */
|
||||
export function createRenderWorkDir(prefix: string, outputDir: string): string {
|
||||
for (const dir of new Set([outputDir, dirname(prefix)])) removeAbandonedRenderDirs(dir);
|
||||
for (const parent of new Set([outputDir, dirname(prefix)])) {
|
||||
for (const dir of listAbandonedRenderDirs(parent)) {
|
||||
try {
|
||||
rmSync(dir, { recursive: true, force: true, maxRetries: 3, retryDelay: 100 });
|
||||
} catch {
|
||||
// Reclaiming a dead render's dir is best-effort; it must never fail the render that found it.
|
||||
}
|
||||
}
|
||||
}
|
||||
return createOwnedRenderDir(prefix);
|
||||
}
|
||||
|
||||
@@ -2825,6 +2825,14 @@ function deVerifyFallbackTelemetry(err: unknown): {
|
||||
};
|
||||
}
|
||||
|
||||
/** Where `--debug` renders keep their work dirs, one per job id. */
|
||||
export function resolveRenderDebugDir(): string {
|
||||
const producerRoot = process.env.PRODUCER_RENDERS_DIR
|
||||
? resolve(process.env.PRODUCER_RENDERS_DIR, "..")
|
||||
: resolve(dirname(fileURLToPath(import.meta.url)), "../..");
|
||||
return join(producerRoot, ".debug");
|
||||
}
|
||||
|
||||
/**
|
||||
* Render a `RenderJob` end-to-end: compile → probe → extract videos →
|
||||
* audio → capture → encode → assemble. The function body is a thin
|
||||
@@ -2846,11 +2854,7 @@ export async function executeRenderJob(
|
||||
// Ahead of the work dir / log file / execution context: a config the format
|
||||
// cannot honor must fail before anything is written to disk.
|
||||
validateHlsRenderConfig(job.config);
|
||||
const moduleDir = dirname(fileURLToPath(import.meta.url));
|
||||
const producerRoot = process.env.PRODUCER_RENDERS_DIR
|
||||
? resolve(process.env.PRODUCER_RENDERS_DIR, "..")
|
||||
: resolve(moduleDir, "../..");
|
||||
const debugDir = join(producerRoot, ".debug");
|
||||
const debugDir = resolveRenderDebugDir();
|
||||
const outputDir = dirname(outputPath);
|
||||
if (!existsSync(outputDir)) mkdirSync(outputDir, { recursive: true });
|
||||
const workDir = job.config.debug
|
||||
|
||||
@@ -1,6 +1,7 @@
|
||||
import { createHash, randomUUID } from "node:crypto";
|
||||
import { existsSync, mkdirSync, mkdtempSync, rmSync, utimesSync, writeFileSync } from "node:fs";
|
||||
import { tmpdir } from "node:os";
|
||||
import { join } from "node:path";
|
||||
import { basename, join } from "node:path";
|
||||
import { afterEach, describe, expect, it } from "vitest";
|
||||
import { cleanupProxyCache } from "./proxyCache.js";
|
||||
|
||||
@@ -14,6 +15,10 @@ function cacheDir(): string {
|
||||
return cache;
|
||||
}
|
||||
|
||||
/** A proxy path named the way the transcoder names one. */
|
||||
const proxy = (cache: string, label: string, extension = ".mp4") =>
|
||||
join(cache, `${createHash("sha256").update(label).digest("hex")}${extension}`);
|
||||
|
||||
function writeEntry(path: string, bytes: number, modifiedAt: number): void {
|
||||
writeFileSync(path, Buffer.alloc(bytes));
|
||||
const date = new Date(modifiedAt);
|
||||
@@ -28,9 +33,9 @@ describe("cleanupProxyCache", () => {
|
||||
it("removes idle and oldest entries until the cache is within its byte budget", () => {
|
||||
const cache = cacheDir();
|
||||
const now = 1_800_000_000_000;
|
||||
const expired = join(cache, "expired.mp4");
|
||||
const oldest = join(cache, "oldest.mp4");
|
||||
const newest = join(cache, "newest.mp4");
|
||||
const expired = proxy(cache, "expired");
|
||||
const oldest = proxy(cache, "oldest");
|
||||
const newest = proxy(cache, "newest");
|
||||
writeEntry(expired, 4, now - 31 * 24 * 60 * 60 * 1000);
|
||||
writeEntry(oldest, 6, now - 3_000);
|
||||
writeEntry(newest, 6, now - 1_000);
|
||||
@@ -49,11 +54,30 @@ describe("cleanupProxyCache", () => {
|
||||
expect(existsSync(newest)).toBe(true);
|
||||
});
|
||||
|
||||
it("reports the same removals in a dry run and removes nothing", () => {
|
||||
const cache = cacheDir();
|
||||
const now = 1_800_000_000_000;
|
||||
const idle = proxy(cache, "idle");
|
||||
const older = proxy(cache, "older");
|
||||
const recent = proxy(cache, "recent");
|
||||
writeEntry(idle, 4, now - 2 * 60 * 60 * 1000);
|
||||
writeEntry(older, 3, now - 10_000);
|
||||
writeEntry(recent, 6, now - 1_000);
|
||||
const options = { now, maxBytes: 7, maxIdleMs: 60 * 60 * 1000, minSweepIntervalMs: 0 };
|
||||
|
||||
const planned = cleanupProxyCache(cache, { ...options, dryRun: true });
|
||||
expect(existsSync(idle)).toBe(true);
|
||||
expect(cleanupProxyCache(cache, options)).toEqual(planned);
|
||||
expect(planned).toMatchObject({ removed: [idle, older], bytesBefore: 13, bytesAfter: 6 });
|
||||
expect(existsSync(idle)).toBe(false);
|
||||
expect(existsSync(recent)).toBe(true);
|
||||
});
|
||||
|
||||
it("counts and evicts WebM proxies alongside MP4 proxies", () => {
|
||||
const cache = cacheDir();
|
||||
const now = 1_800_000_000_000;
|
||||
const webm = join(cache, "alpha.webm");
|
||||
const mp4 = join(cache, "opaque.mp4");
|
||||
const webm = proxy(cache, "alpha", ".webm");
|
||||
const mp4 = proxy(cache, "opaque");
|
||||
writeEntry(webm, 6, now - 2_000);
|
||||
writeEntry(mp4, 6, now - 1_000);
|
||||
|
||||
@@ -72,8 +96,8 @@ describe("cleanupProxyCache", () => {
|
||||
it("preserves in-flight entries and removes stale temporary files", () => {
|
||||
const cache = cacheDir();
|
||||
const now = 1_800_000_000_000;
|
||||
const inFlight = join(cache, "active.mp4");
|
||||
const staleTemp = join(cache, ".tmp-crashed-active.mp4");
|
||||
const inFlight = proxy(cache, "active");
|
||||
const staleTemp = join(cache, `.tmp-${randomUUID()}-${basename(inFlight)}`);
|
||||
writeEntry(inFlight, 12, now - 40 * 24 * 60 * 60 * 1000);
|
||||
writeEntry(staleTemp, 3, now - 2 * 60 * 60 * 1000);
|
||||
|
||||
@@ -91,9 +115,27 @@ describe("cleanupProxyCache", () => {
|
||||
expect(existsSync(inFlight)).toBe(true);
|
||||
});
|
||||
|
||||
it("never removes a file the transcoder did not name, even from an idle cache over budget", () => {
|
||||
const cache = cacheDir();
|
||||
const now = 1_800_000_000_000;
|
||||
const userFiles = [join(cache, "holiday.mp4"), join(cache, ".tmp-notes.mp4")];
|
||||
for (const path of userFiles) writeEntry(path, 4, now - 40 * 24 * 60 * 60 * 1000);
|
||||
|
||||
const result = cleanupProxyCache(cache, {
|
||||
now,
|
||||
maxBytes: 0,
|
||||
maxIdleMs: 1,
|
||||
staleTempMs: 1,
|
||||
minSweepIntervalMs: 0,
|
||||
});
|
||||
|
||||
expect(result.removed).toEqual([]);
|
||||
for (const path of userFiles) expect(existsSync(path)).toBe(true);
|
||||
});
|
||||
|
||||
it("rate-limits repeated directory sweeps", () => {
|
||||
const cache = cacheDir();
|
||||
const path = join(cache, "entry.mp4");
|
||||
const path = proxy(cache, "entry");
|
||||
writeEntry(path, 2, 1_000);
|
||||
|
||||
const first = cleanupProxyCache(cache, {
|
||||
|
||||
@@ -8,6 +8,8 @@ const DEFAULT_MIN_SWEEP_INTERVAL_MS = 5 * 60 * 1000;
|
||||
const PROXY_EXTENSIONS: ReadonlySet<string> = new Set(
|
||||
Object.values(PROXY_VARIANT_CONFIG).map(({ extension }) => extension),
|
||||
);
|
||||
const TRANSCODER_PROXY_NAME = /^[0-9a-f]{64}\.\w+$/;
|
||||
const TRANSCODER_TEMP_NAME = /^\.tmp-[0-9a-f-]{36}-[0-9a-f]{64}\.\w+$/;
|
||||
|
||||
export interface ProxyCacheCleanupOptions {
|
||||
maxBytes?: number;
|
||||
@@ -16,6 +18,7 @@ export interface ProxyCacheCleanupOptions {
|
||||
minSweepIntervalMs?: number;
|
||||
protectedPaths?: ReadonlySet<string>;
|
||||
now?: number;
|
||||
dryRun?: boolean;
|
||||
}
|
||||
|
||||
export interface ProxyCacheCleanupResult {
|
||||
@@ -78,9 +81,12 @@ function readCacheInventory(
|
||||
modifiedAt: stat.mtimeMs,
|
||||
protected: protectedPaths.has(path),
|
||||
};
|
||||
if (dirent.name.startsWith(".tmp-")) {
|
||||
if (TRANSCODER_TEMP_NAME.test(dirent.name)) {
|
||||
if (now - stat.mtimeMs >= staleTempMs) staleTemps.push(entry);
|
||||
} else if (PROXY_EXTENSIONS.has(extname(dirent.name))) {
|
||||
} else if (
|
||||
TRANSCODER_PROXY_NAME.test(dirent.name) &&
|
||||
PROXY_EXTENSIONS.has(extname(dirent.name))
|
||||
) {
|
||||
entries.push(entry);
|
||||
}
|
||||
}
|
||||
@@ -97,21 +103,24 @@ function evictCacheEntries(
|
||||
now: number,
|
||||
maxIdleMs: number,
|
||||
maxBytes: number,
|
||||
dryRun: boolean,
|
||||
): Omit<ProxyCacheCleanupResult, "skipped"> {
|
||||
const bytesBefore = entries.reduce((total, entry) => total + entry.size, 0);
|
||||
let bytesAfter = bytesBefore;
|
||||
const removed: string[] = [];
|
||||
const remove = (entry: CacheEntry, countsTowardBudget: boolean): void => {
|
||||
unlinkSync(entry.path);
|
||||
if (!dryRun) unlinkSync(entry.path);
|
||||
removed.push(entry.path);
|
||||
if (countsTowardBudget) bytesAfter -= entry.size;
|
||||
};
|
||||
|
||||
for (const entry of staleTemps) remove(entry, false);
|
||||
const kept: CacheEntry[] = [];
|
||||
for (const entry of entries) {
|
||||
if (!entry.protected && now - entry.modifiedAt >= maxIdleMs) remove(entry, true);
|
||||
else kept.push(entry);
|
||||
}
|
||||
for (const entry of entries) {
|
||||
for (const entry of kept) {
|
||||
if (bytesAfter <= maxBytes) break;
|
||||
if (!entry.protected && existsSync(entry.path)) remove(entry, true);
|
||||
}
|
||||
@@ -145,7 +154,7 @@ export function cleanupProxyCache(
|
||||
const protectedPaths = options.protectedPaths ?? new Set<string>();
|
||||
const { entries, staleTemps } = readCacheInventory(cacheDir, protectedPaths, now, staleTempMs);
|
||||
return {
|
||||
...evictCacheEntries(entries, staleTemps, now, maxIdleMs, maxBytes),
|
||||
...evictCacheEntries(entries, staleTemps, now, maxIdleMs, maxBytes, options.dryRun ?? false),
|
||||
skipped: false,
|
||||
};
|
||||
}
|
||||
|
||||
@@ -28,7 +28,7 @@ import { mkdirWithinProject, realpath, realProjectRoot } from "./safePath.js";
|
||||
|
||||
export const PROXY_PARAMS_VERSION = "v5";
|
||||
|
||||
const CACHE_DIR_NAME = ".transcode-cache";
|
||||
export const CACHE_DIR_NAME = ".transcode-cache";
|
||||
|
||||
function boundedEnvInteger(name: string, fallback: number, min: number, max: number): number {
|
||||
const raw = process.env[name]?.trim();
|
||||
|
||||
@@ -3,6 +3,8 @@ export { createProjectSignature, affectsProjectSignature } from "./helpers/proje
|
||||
export { compositionsAffectedBy } from "./helpers/compositionInputs.js";
|
||||
export { affectsPreview } from "./helpers/previewReads.js";
|
||||
export * from "./history/index.js";
|
||||
export { cleanupProxyCache } from "./helpers/proxyCache.js";
|
||||
export { CACHE_DIR_NAME as PROXY_CACHE_DIR_NAME } from "./helpers/proxyTranscoder.js";
|
||||
export type {
|
||||
StudioApiAdapter,
|
||||
ResolvedProject,
|
||||
|
||||
@@ -30,7 +30,7 @@
|
||||
"files": 7
|
||||
},
|
||||
"hyperframes-cli": {
|
||||
"hash": "2617903119c32228",
|
||||
"hash": "c1b0b3d02ad38edc",
|
||||
"files": 11
|
||||
},
|
||||
"hyperframes-core": {
|
||||
|
||||
@@ -3,7 +3,7 @@ name: hyperframes-cli
|
||||
description: >
|
||||
Use the HyperFrames CLI development loop: init, add, catalog, capture, lint, check, snapshot,
|
||||
compare, grade-compare, preview, play, present, beats, keyframes, single or batch render, publish,
|
||||
cloud, cloudrun, feedback, lambda, doctor, browser, info, upgrade, skills, compositions, timeline, history, docs,
|
||||
cloud, cloudrun, feedback, lambda, doctor, browser, info, upgrade, skills, compositions, timeline, history, clean, docs,
|
||||
benchmark, telemetry, transcribe, auth, tts, and remove-background. Also use when diagnosing build
|
||||
or render failures. validate, inspect, and layout are deprecated aliases; use check. Covers local,
|
||||
HeyGen-hosted cloud, AWS Lambda, and Google Cloud Run rendering.
|
||||
@@ -100,6 +100,7 @@ Treat tiny unstyled content, canvas-sized icons, missing hero elements, or timel
|
||||
|
||||
- Non-TTY mode is automatic and scaffolds the centered blank. Pass `--example` only to start from a named example. Use `--non-interactive` to force flag-only mode on a TTY.
|
||||
- Use one `HYPERFRAMES_RUN_ID` for all commands in the same verification loop.
|
||||
- When disk is tight, run `npx hyperframes clean` (`--dry-run` to list first); it removes what dead renders left and idle caches that rebuild themselves, never outputs, sources or anything a running render uses. Write QC frames to a temp dir, not the project.
|
||||
- Use `--strict`, `--strict-all`, and `--strict-variables` when the corresponding warnings, variables, or CI conditions must gate the render.
|
||||
- JSON paths redact the home directory as `$HOME`; do not try to reverse the redaction.
|
||||
- When a hosted cloud project approaches or exceeds the 200 MB upload limit, use `cloud render --dry-run --json` and follow the `.hyperframesignore` investigation in `references/cloud.md`. Never ignore an asset merely because it is large.
|
||||
|
||||
Reference in New Issue
Block a user