mirror of
https://github.com/heygen-com/hyperframes.git
synced 2026-11-17 15:24:19 +00:00
fix(studio-server): a copy another process holds without progress ends in a named error (#6918)
* fix(studio-server): a copy another process holds without progress ends in a named error * fix(studio-server): a long source's in-progress piece counts as the owner's progress
This commit is contained in:
@@ -0,0 +1,62 @@
|
||||
import { mkdtempSync, rmSync, utimesSync, writeFileSync } from "node:fs";
|
||||
import { tmpdir } from "node:os";
|
||||
import { join } from "node:path";
|
||||
import { afterAll, describe, expect, it } from "vitest";
|
||||
import { acquireCopyLock, CopyLockStalledError } from "./proxyLock.js";
|
||||
|
||||
const dir = mkdtempSync(join(tmpdir(), "hf-proxy-lock-"));
|
||||
afterAll(() => rmSync(dir, { recursive: true, force: true }));
|
||||
|
||||
const hourAgo = new Date(Date.now() - 60 * 60 * 1000);
|
||||
|
||||
/** A lock held by a live process (this one) whose files last changed an hour ago. */
|
||||
function staleLock(name: string): string {
|
||||
const cachePath = join(dir, name);
|
||||
const lock = join(dir, `.lock-${name}`);
|
||||
writeFileSync(lock, String(process.pid));
|
||||
utimesSync(lock, hourAgo, hourAgo);
|
||||
return cachePath;
|
||||
}
|
||||
|
||||
const within = <T>(work: Promise<T>, ms: number) =>
|
||||
Promise.race([
|
||||
work,
|
||||
new Promise<never>((_, reject) =>
|
||||
setTimeout(() => reject(new Error(`still waiting after ${ms} ms`)), ms),
|
||||
),
|
||||
]);
|
||||
|
||||
describe("acquireCopyLock", () => {
|
||||
it("keeps waiting while the owner writes a piece of a long source", async () => {
|
||||
const cachePath = staleLock("c.mp4");
|
||||
writeFileSync(join(dir, ".tmp-1234-.part-c-0.mp4"), "x");
|
||||
const abort = new AbortController();
|
||||
|
||||
const waiting = acquireCopyLock(cachePath, abort.signal, 60_000);
|
||||
setTimeout(() => abort.abort(new Error("stopped")), 400);
|
||||
|
||||
await expect(within(waiting, 3_000)).rejects.toThrow("stopped");
|
||||
});
|
||||
|
||||
it("gives up on a live owner that has made no progress for the stall window, naming the copy", async () => {
|
||||
const cachePath = staleLock("a.mp4");
|
||||
|
||||
const waiting = acquireCopyLock(cachePath, new AbortController().signal, 300);
|
||||
|
||||
await expect(within(waiting, 3_000)).rejects.toThrow(CopyLockStalledError);
|
||||
await expect(
|
||||
within(acquireCopyLock(cachePath, new AbortController().signal, 300), 3_000),
|
||||
).rejects.toThrow(/a\.mp4.*no progress/);
|
||||
});
|
||||
|
||||
it("keeps waiting while the owner's temp file is still changing", async () => {
|
||||
const cachePath = staleLock("b.mp4");
|
||||
writeFileSync(join(dir, ".tmp-1234-b.mp4"), "x");
|
||||
const abort = new AbortController();
|
||||
|
||||
const waiting = acquireCopyLock(cachePath, abort.signal, 60_000);
|
||||
setTimeout(() => abort.abort(new Error("stopped")), 400);
|
||||
|
||||
await expect(within(waiting, 3_000)).rejects.toThrow("stopped");
|
||||
});
|
||||
});
|
||||
@@ -2,12 +2,13 @@ import {
|
||||
closeSync,
|
||||
existsSync,
|
||||
openSync,
|
||||
readdirSync,
|
||||
readFileSync,
|
||||
rmSync,
|
||||
statSync,
|
||||
writeSync,
|
||||
} from "node:fs";
|
||||
import { basename, dirname, join } from "node:path";
|
||||
import { basename, dirname, extname, join } from "node:path";
|
||||
|
||||
const POLL_MS = 500;
|
||||
const UNREADABLE_LOCK_MS = 10_000;
|
||||
@@ -16,6 +17,31 @@ function lockPathFor(cachePath: string): string {
|
||||
return join(dirname(cachePath), `.lock-${basename(cachePath)}`);
|
||||
}
|
||||
|
||||
export class CopyLockStalledError extends Error {
|
||||
constructor(copy: string, stallMs: number) {
|
||||
super(
|
||||
`another process holds the copy of ${copy} and has made no progress for ${Math.round(stallMs / 1000)}s`,
|
||||
);
|
||||
this.name = "CopyLockStalledError";
|
||||
}
|
||||
}
|
||||
|
||||
function lastActivityMs(cachePath: string, lockPath: string): number {
|
||||
const name = basename(cachePath);
|
||||
const stem = basename(cachePath, extname(cachePath));
|
||||
const mine = (file: string) =>
|
||||
file === basename(lockPath) ||
|
||||
(file.startsWith(".tmp-") && (file.endsWith(`-${name}`) || file.includes(`-.part-${stem}-`))) ||
|
||||
file.startsWith(`.part-${stem}-`);
|
||||
let newest = 0;
|
||||
for (const file of readdirSync(dirname(cachePath))) {
|
||||
if (!mine(file)) continue;
|
||||
const changed = statSync(join(dirname(cachePath), file), { throwIfNoEntry: false })?.mtimeMs;
|
||||
newest = Math.max(newest, changed ?? 0);
|
||||
}
|
||||
return newest;
|
||||
}
|
||||
|
||||
function ownerIsAlive(lockPath: string): boolean {
|
||||
try {
|
||||
const pid = Number(readFileSync(lockPath, "utf8"));
|
||||
@@ -47,10 +73,11 @@ function pause(ms: number, signal: AbortSignal): Promise<void> {
|
||||
});
|
||||
}
|
||||
|
||||
/** The right to make the copy at `cachePath`; null once another process finished it. Takes over a dead owner's lock. */
|
||||
/** The right to make the copy at `cachePath`; null once another process finished it; throws on a stalled live owner. */
|
||||
export async function acquireCopyLock(
|
||||
cachePath: string,
|
||||
signal: AbortSignal,
|
||||
stallMs: number,
|
||||
): Promise<(() => void) | null> {
|
||||
const lockPath = lockPathFor(cachePath);
|
||||
for (;;) {
|
||||
@@ -66,7 +93,9 @@ export async function acquireCopyLock(
|
||||
} catch (err) {
|
||||
if ((err as NodeJS.ErrnoException).code !== "EEXIST") throw err;
|
||||
}
|
||||
if (ownerIsAlive(lockPath)) await pause(POLL_MS, signal);
|
||||
else rmSync(lockPath, { force: true });
|
||||
if (!ownerIsAlive(lockPath)) rmSync(lockPath, { force: true });
|
||||
else if (Date.now() - lastActivityMs(cachePath, lockPath) > stallMs)
|
||||
throw new CopyLockStalledError(basename(cachePath), stallMs);
|
||||
else await pause(POLL_MS, signal);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -26,7 +26,7 @@ import {
|
||||
sourceSizeBytes,
|
||||
writeCopyIndex,
|
||||
} from "./proxyCacheKey.js";
|
||||
import { acquireCopyLock } from "./proxyLock.js";
|
||||
import { acquireCopyLock, CopyLockStalledError } from "./proxyLock.js";
|
||||
import { PROXY_VARIANT_CONFIG, type ProxyVariant } from "./mediaCodecMap.js";
|
||||
import { mkdirWithinProject, realpath, realProjectRoot } from "./safePath.js";
|
||||
|
||||
@@ -892,7 +892,15 @@ async function transcodeToCache(
|
||||
writeCopyIndex(fastIndexPath(cachePath), sameContent);
|
||||
return sameContent;
|
||||
}
|
||||
const release = await acquireCopyLock(cachePath, job.abandoned.signal);
|
||||
const release = await acquireCopyLock(
|
||||
cachePath,
|
||||
job.abandoned.signal,
|
||||
TRANSCODE_STALL_MS,
|
||||
).catch((error: unknown) => {
|
||||
throw error instanceof CopyLockStalledError
|
||||
? new ProxyTranscodeError(error.message, null, "")
|
||||
: error;
|
||||
});
|
||||
if (!release) return cachePath;
|
||||
const tempPath = join(cacheDir, `.tmp-${randomUUID()}-${basename(cachePath)}`);
|
||||
try {
|
||||
|
||||
Reference in New Issue
Block a user