From e5887959bb7c23c45ea56d44d9e6f0929de4eb3d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Miguel=20=C3=81ngel?= Date: Sun, 1 Nov 2026 09:47:43 +0000 Subject: [PATCH] fix(studio-server): a copy another process holds without progress ends in a named error (#6918) * fix(studio-server): a copy another process holds without progress ends in a named error * fix(studio-server): a long source's in-progress piece counts as the owner's progress --- .../src/helpers/proxyLock.test.ts | 62 +++++++++++++++++++ .../studio-server/src/helpers/proxyLock.ts | 37 +++++++++-- .../src/helpers/proxyTranscoder.ts | 12 +++- 3 files changed, 105 insertions(+), 6 deletions(-) create mode 100644 packages/studio-server/src/helpers/proxyLock.test.ts diff --git a/packages/studio-server/src/helpers/proxyLock.test.ts b/packages/studio-server/src/helpers/proxyLock.test.ts new file mode 100644 index 0000000000..ebb1a124cf --- /dev/null +++ b/packages/studio-server/src/helpers/proxyLock.test.ts @@ -0,0 +1,62 @@ +import { mkdtempSync, rmSync, utimesSync, writeFileSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { afterAll, describe, expect, it } from "vitest"; +import { acquireCopyLock, CopyLockStalledError } from "./proxyLock.js"; + +const dir = mkdtempSync(join(tmpdir(), "hf-proxy-lock-")); +afterAll(() => rmSync(dir, { recursive: true, force: true })); + +const hourAgo = new Date(Date.now() - 60 * 60 * 1000); + +/** A lock held by a live process (this one) whose files last changed an hour ago. */ +function staleLock(name: string): string { + const cachePath = join(dir, name); + const lock = join(dir, `.lock-${name}`); + writeFileSync(lock, String(process.pid)); + utimesSync(lock, hourAgo, hourAgo); + return cachePath; +} + +const within = (work: Promise, ms: number) => + Promise.race([ + work, + new Promise((_, reject) => + setTimeout(() => reject(new Error(`still waiting after ${ms} ms`)), ms), + ), + ]); + +describe("acquireCopyLock", () => { + it("keeps waiting while the owner writes a piece of a long source", async () => { + const cachePath = staleLock("c.mp4"); + writeFileSync(join(dir, ".tmp-1234-.part-c-0.mp4"), "x"); + const abort = new AbortController(); + + const waiting = acquireCopyLock(cachePath, abort.signal, 60_000); + setTimeout(() => abort.abort(new Error("stopped")), 400); + + await expect(within(waiting, 3_000)).rejects.toThrow("stopped"); + }); + + it("gives up on a live owner that has made no progress for the stall window, naming the copy", async () => { + const cachePath = staleLock("a.mp4"); + + const waiting = acquireCopyLock(cachePath, new AbortController().signal, 300); + + await expect(within(waiting, 3_000)).rejects.toThrow(CopyLockStalledError); + await expect( + within(acquireCopyLock(cachePath, new AbortController().signal, 300), 3_000), + ).rejects.toThrow(/a\.mp4.*no progress/); + }); + + it("keeps waiting while the owner's temp file is still changing", async () => { + const cachePath = staleLock("b.mp4"); + writeFileSync(join(dir, ".tmp-1234-b.mp4"), "x"); + const abort = new AbortController(); + + const waiting = acquireCopyLock(cachePath, abort.signal, 60_000); + setTimeout(() => abort.abort(new Error("stopped")), 400); + + await expect(within(waiting, 3_000)).rejects.toThrow("stopped"); + }); +}); diff --git a/packages/studio-server/src/helpers/proxyLock.ts b/packages/studio-server/src/helpers/proxyLock.ts index 4c154d697f..b357b1c3bd 100644 --- a/packages/studio-server/src/helpers/proxyLock.ts +++ b/packages/studio-server/src/helpers/proxyLock.ts @@ -2,12 +2,13 @@ import { closeSync, existsSync, openSync, + readdirSync, readFileSync, rmSync, statSync, writeSync, } from "node:fs"; -import { basename, dirname, join } from "node:path"; +import { basename, dirname, extname, join } from "node:path"; const POLL_MS = 500; const UNREADABLE_LOCK_MS = 10_000; @@ -16,6 +17,31 @@ function lockPathFor(cachePath: string): string { return join(dirname(cachePath), `.lock-${basename(cachePath)}`); } +export class CopyLockStalledError extends Error { + constructor(copy: string, stallMs: number) { + super( + `another process holds the copy of ${copy} and has made no progress for ${Math.round(stallMs / 1000)}s`, + ); + this.name = "CopyLockStalledError"; + } +} + +function lastActivityMs(cachePath: string, lockPath: string): number { + const name = basename(cachePath); + const stem = basename(cachePath, extname(cachePath)); + const mine = (file: string) => + file === basename(lockPath) || + (file.startsWith(".tmp-") && (file.endsWith(`-${name}`) || file.includes(`-.part-${stem}-`))) || + file.startsWith(`.part-${stem}-`); + let newest = 0; + for (const file of readdirSync(dirname(cachePath))) { + if (!mine(file)) continue; + const changed = statSync(join(dirname(cachePath), file), { throwIfNoEntry: false })?.mtimeMs; + newest = Math.max(newest, changed ?? 0); + } + return newest; +} + function ownerIsAlive(lockPath: string): boolean { try { const pid = Number(readFileSync(lockPath, "utf8")); @@ -47,10 +73,11 @@ function pause(ms: number, signal: AbortSignal): Promise { }); } -/** The right to make the copy at `cachePath`; null once another process finished it. Takes over a dead owner's lock. */ +/** The right to make the copy at `cachePath`; null once another process finished it; throws on a stalled live owner. */ export async function acquireCopyLock( cachePath: string, signal: AbortSignal, + stallMs: number, ): Promise<(() => void) | null> { const lockPath = lockPathFor(cachePath); for (;;) { @@ -66,7 +93,9 @@ export async function acquireCopyLock( } catch (err) { if ((err as NodeJS.ErrnoException).code !== "EEXIST") throw err; } - if (ownerIsAlive(lockPath)) await pause(POLL_MS, signal); - else rmSync(lockPath, { force: true }); + if (!ownerIsAlive(lockPath)) rmSync(lockPath, { force: true }); + else if (Date.now() - lastActivityMs(cachePath, lockPath) > stallMs) + throw new CopyLockStalledError(basename(cachePath), stallMs); + else await pause(POLL_MS, signal); } } diff --git a/packages/studio-server/src/helpers/proxyTranscoder.ts b/packages/studio-server/src/helpers/proxyTranscoder.ts index 89fc0229de..d6bf2d20a8 100644 --- a/packages/studio-server/src/helpers/proxyTranscoder.ts +++ b/packages/studio-server/src/helpers/proxyTranscoder.ts @@ -26,7 +26,7 @@ import { sourceSizeBytes, writeCopyIndex, } from "./proxyCacheKey.js"; -import { acquireCopyLock } from "./proxyLock.js"; +import { acquireCopyLock, CopyLockStalledError } from "./proxyLock.js"; import { PROXY_VARIANT_CONFIG, type ProxyVariant } from "./mediaCodecMap.js"; import { mkdirWithinProject, realpath, realProjectRoot } from "./safePath.js"; @@ -892,7 +892,15 @@ async function transcodeToCache( writeCopyIndex(fastIndexPath(cachePath), sameContent); return sameContent; } - const release = await acquireCopyLock(cachePath, job.abandoned.signal); + const release = await acquireCopyLock( + cachePath, + job.abandoned.signal, + TRANSCODE_STALL_MS, + ).catch((error: unknown) => { + throw error instanceof CopyLockStalledError + ? new ProxyTranscodeError(error.message, null, "") + : error; + }); if (!release) return cachePath; const tempPath = join(cacheDir, `.tmp-${randomUUID()}-${basename(cachePath)}`); try {