fix(studio-server): a copy another process holds without progress ends in a named error (#6918)

* fix(studio-server): a copy another process holds without progress ends in a named error

* fix(studio-server): a long source's in-progress piece counts as the owner's progress
This commit is contained in:
Miguel Ángel
2026-11-01 09:47:43 +00:00
committed by GitHub
parent 8c22ddba0f
commit e5887959bb
3 changed files with 105 additions and 6 deletions
@@ -0,0 +1,62 @@
import { mkdtempSync, rmSync, utimesSync, writeFileSync } from "node:fs";
import { tmpdir } from "node:os";
import { join } from "node:path";
import { afterAll, describe, expect, it } from "vitest";
import { acquireCopyLock, CopyLockStalledError } from "./proxyLock.js";
const dir = mkdtempSync(join(tmpdir(), "hf-proxy-lock-"));
afterAll(() => rmSync(dir, { recursive: true, force: true }));
const hourAgo = new Date(Date.now() - 60 * 60 * 1000);
/** A lock held by a live process (this one) whose files last changed an hour ago. */
function staleLock(name: string): string {
const cachePath = join(dir, name);
const lock = join(dir, `.lock-${name}`);
writeFileSync(lock, String(process.pid));
utimesSync(lock, hourAgo, hourAgo);
return cachePath;
}
const within = <T>(work: Promise<T>, ms: number) =>
Promise.race([
work,
new Promise<never>((_, reject) =>
setTimeout(() => reject(new Error(`still waiting after ${ms} ms`)), ms),
),
]);
describe("acquireCopyLock", () => {
it("keeps waiting while the owner writes a piece of a long source", async () => {
const cachePath = staleLock("c.mp4");
writeFileSync(join(dir, ".tmp-1234-.part-c-0.mp4"), "x");
const abort = new AbortController();
const waiting = acquireCopyLock(cachePath, abort.signal, 60_000);
setTimeout(() => abort.abort(new Error("stopped")), 400);
await expect(within(waiting, 3_000)).rejects.toThrow("stopped");
});
it("gives up on a live owner that has made no progress for the stall window, naming the copy", async () => {
const cachePath = staleLock("a.mp4");
const waiting = acquireCopyLock(cachePath, new AbortController().signal, 300);
await expect(within(waiting, 3_000)).rejects.toThrow(CopyLockStalledError);
await expect(
within(acquireCopyLock(cachePath, new AbortController().signal, 300), 3_000),
).rejects.toThrow(/a\.mp4.*no progress/);
});
it("keeps waiting while the owner's temp file is still changing", async () => {
const cachePath = staleLock("b.mp4");
writeFileSync(join(dir, ".tmp-1234-b.mp4"), "x");
const abort = new AbortController();
const waiting = acquireCopyLock(cachePath, abort.signal, 60_000);
setTimeout(() => abort.abort(new Error("stopped")), 400);
await expect(within(waiting, 3_000)).rejects.toThrow("stopped");
});
});
@@ -2,12 +2,13 @@ import {
closeSync,
existsSync,
openSync,
readdirSync,
readFileSync,
rmSync,
statSync,
writeSync,
} from "node:fs";
import { basename, dirname, join } from "node:path";
import { basename, dirname, extname, join } from "node:path";
const POLL_MS = 500;
const UNREADABLE_LOCK_MS = 10_000;
@@ -16,6 +17,31 @@ function lockPathFor(cachePath: string): string {
return join(dirname(cachePath), `.lock-${basename(cachePath)}`);
}
export class CopyLockStalledError extends Error {
constructor(copy: string, stallMs: number) {
super(
`another process holds the copy of ${copy} and has made no progress for ${Math.round(stallMs / 1000)}s`,
);
this.name = "CopyLockStalledError";
}
}
function lastActivityMs(cachePath: string, lockPath: string): number {
const name = basename(cachePath);
const stem = basename(cachePath, extname(cachePath));
const mine = (file: string) =>
file === basename(lockPath) ||
(file.startsWith(".tmp-") && (file.endsWith(`-${name}`) || file.includes(`-.part-${stem}-`))) ||
file.startsWith(`.part-${stem}-`);
let newest = 0;
for (const file of readdirSync(dirname(cachePath))) {
if (!mine(file)) continue;
const changed = statSync(join(dirname(cachePath), file), { throwIfNoEntry: false })?.mtimeMs;
newest = Math.max(newest, changed ?? 0);
}
return newest;
}
function ownerIsAlive(lockPath: string): boolean {
try {
const pid = Number(readFileSync(lockPath, "utf8"));
@@ -47,10 +73,11 @@ function pause(ms: number, signal: AbortSignal): Promise<void> {
});
}
/** The right to make the copy at `cachePath`; null once another process finished it. Takes over a dead owner's lock. */
/** The right to make the copy at `cachePath`; null once another process finished it; throws on a stalled live owner. */
export async function acquireCopyLock(
cachePath: string,
signal: AbortSignal,
stallMs: number,
): Promise<(() => void) | null> {
const lockPath = lockPathFor(cachePath);
for (;;) {
@@ -66,7 +93,9 @@ export async function acquireCopyLock(
} catch (err) {
if ((err as NodeJS.ErrnoException).code !== "EEXIST") throw err;
}
if (ownerIsAlive(lockPath)) await pause(POLL_MS, signal);
else rmSync(lockPath, { force: true });
if (!ownerIsAlive(lockPath)) rmSync(lockPath, { force: true });
else if (Date.now() - lastActivityMs(cachePath, lockPath) > stallMs)
throw new CopyLockStalledError(basename(cachePath), stallMs);
else await pause(POLL_MS, signal);
}
}
@@ -26,7 +26,7 @@ import {
sourceSizeBytes,
writeCopyIndex,
} from "./proxyCacheKey.js";
import { acquireCopyLock } from "./proxyLock.js";
import { acquireCopyLock, CopyLockStalledError } from "./proxyLock.js";
import { PROXY_VARIANT_CONFIG, type ProxyVariant } from "./mediaCodecMap.js";
import { mkdirWithinProject, realpath, realProjectRoot } from "./safePath.js";
@@ -892,7 +892,15 @@ async function transcodeToCache(
writeCopyIndex(fastIndexPath(cachePath), sameContent);
return sameContent;
}
const release = await acquireCopyLock(cachePath, job.abandoned.signal);
const release = await acquireCopyLock(
cachePath,
job.abandoned.signal,
TRANSCODE_STALL_MS,
).catch((error: unknown) => {
throw error instanceof CopyLockStalledError
? new ProxyTranscodeError(error.message, null, "")
: error;
});
if (!release) return cachePath;
const tempPath = join(cacheDir, `.tmp-${randomUUID()}-${basename(cachePath)}`);
try {