diff --git a/packages/studio-server/src/helpers/proxyLock.test.ts b/packages/studio-server/src/helpers/proxyLock.test.ts index 436bf15eda..b495ac65cd 100644 --- a/packages/studio-server/src/helpers/proxyLock.test.ts +++ b/packages/studio-server/src/helpers/proxyLock.test.ts @@ -79,6 +79,35 @@ describe("acquireCopyLock", () => { await expect(within(waiting, 3_000)).rejects.toThrow(CopyLockStalledError); }); + it("gives up within the stall window on a live owner whose lock and temp file are dated ahead of the clock", async () => { + const cachePath = staleLock("e.mp4"); + const nextYear = new Date(Date.now() + 365 * 24 * 60 * 60 * 1000); + for (const name of [".lock-e.mp4", ".tmp-1234-e.mp4"]) { + writeFileSync(join(dir, name), String(process.pid)); + utimesSync(join(dir, name), nextYear, nextYear); + } + + const waiting = acquireCopyLock(cachePath, () => false, new AbortController().signal, 300); + + await expect(within(waiting, 3_000)).rejects.toThrow(CopyLockStalledError); + }); + + it("keeps waiting for an owner still writing beside a temp file dated ahead of the clock", async () => { + const cachePath = staleLock("f.mp4"); + const nextYear = new Date(Date.now() + 365 * 24 * 60 * 60 * 1000); + writeFileSync(join(dir, ".tmp-9999-f.mp4"), "x"); + utimesSync(join(dir, ".tmp-9999-f.mp4"), nextYear, nextYear); + const writing = setInterval(() => writeFileSync(join(dir, ".tmp-1234-f.mp4"), "x"), 100); + const abort = new AbortController(); + + const waiting = acquireCopyLock(cachePath, () => false, abort.signal, 600); + setTimeout(() => abort.abort(new Error("stopped")), 1_500); + + await expect(within(waiting, 3_000).finally(() => clearInterval(writing))).rejects.toThrow( + "stopped", + ); + }); + it("keeps waiting while the owner's temp file is still changing", async () => { const cachePath = staleLock("b.mp4"); writeFileSync(join(dir, ".tmp-1234-b.mp4"), "x"); diff --git a/packages/studio-server/src/helpers/proxyLock.ts b/packages/studio-server/src/helpers/proxyLock.ts index 02238b16b6..cf48e19b7e 100644 --- a/packages/studio-server/src/helpers/proxyLock.ts +++ b/packages/studio-server/src/helpers/proxyLock.ts @@ -25,10 +25,20 @@ export class CopyLockStalledError extends Error { } } +type Sightings = Map; + +function changedNoLaterThanFirstSeen(sightings: Sightings, file: string, mtimeMs: number): number { + const earlier = sightings.get(file); + const seenAt = earlier?.mtimeMs === mtimeMs ? earlier.seenAt : Date.now(); + sightings.set(file, { mtimeMs, seenAt }); + return Math.min(mtimeMs, seenAt); +} + function lastActivityMs( cachePath: string, lockPath: string, unguessablePieceName: string | undefined, + sightings: Sightings, ): number { const name = basename(cachePath); const isPiece = (file: string) => @@ -43,7 +53,7 @@ function lastActivityMs( for (const file of readdirSync(dirname(cachePath))) { if (!mine(file)) continue; const changed = statSync(join(dirname(cachePath), file), { throwIfNoEntry: false })?.mtimeMs; - newest = Math.max(newest, changed ?? 0); + newest = Math.max(newest, changedNoLaterThanFirstSeen(sightings, file, changed ?? 0)); } return newest; } @@ -88,6 +98,7 @@ export async function acquireCopyLock( unguessablePieceName?: string, ): Promise<(() => void) | null> { const lockPath = lockPathFor(cachePath); + const sightings: Sightings = new Map(); for (;;) { signal.throwIfAborted(); if (copyMade(cachePath)) return null; @@ -102,7 +113,10 @@ export async function acquireCopyLock( if ((err as NodeJS.ErrnoException).code !== "EEXIST") throw err; } if (!ownerIsAlive(lockPath)) rmSync(lockPath, { force: true }); - else if (Date.now() - lastActivityMs(cachePath, lockPath, unguessablePieceName) > stallMs) + else if ( + Date.now() - lastActivityMs(cachePath, lockPath, unguessablePieceName, sightings) > + stallMs + ) throw new CopyLockStalledError(basename(cachePath), stallMs); else await pause(POLL_MS, signal); } diff --git a/packages/studio-server/src/helpers/proxyTranscoder.test.ts b/packages/studio-server/src/helpers/proxyTranscoder.test.ts index cd13eacd41..462bb286c0 100644 --- a/packages/studio-server/src/helpers/proxyTranscoder.test.ts +++ b/packages/studio-server/src/helpers/proxyTranscoder.test.ts @@ -81,7 +81,7 @@ function createFakeProc(): FakeProc { return proc; } -type SpawnOptions = { windowsHide?: boolean; signal?: AbortSignal }; +type SpawnOptions = { windowsHide?: boolean; signal?: AbortSignal; cwd?: string }; type SpawnCall = { command: string; args: string[]; options?: SpawnOptions; proc: FakeProc }; type SpawnImpl = (command: string, args: string[], options?: SpawnOptions) => FakeProc; @@ -283,12 +283,88 @@ describe("resolveProxy on a long source", () => { "-format_whitelist", "concat,mov,matroska", "-i", - expect.stringMatching(/pieces\.txt$/), + expect.stringMatching(/\.txt$/), "-c", "copy", ]); }); + it.skipIf(process.platform === "win32")( + "runs the join from the cache folder and names the list there by its bare name", + async () => { + const { spawn, calls } = createSpawnSpy(); + const { resolveProxy, getProxyCachePath } = await loadModule( + spawn, + FFMPEG_PATH, + false, + null, + undefined, + LONG_SECONDS, + ); + const projectDir = tmpProject(); + const sourcePath = join(projectDir, "long.mov"); + writeFileSync(sourcePath, "source-bytes"); + const result = resolveProxy(projectDir, sourcePath); + await answerAll(calls); + await result; + const joinCall = calls.at(-1)!; + expect(joinCall.options?.cwd).toBe(dirname(getProxyCachePath(projectDir, sourcePath))); + expect(joinCall.args[joinCall.args.indexOf("-i") + 1]).toMatch(/^[^/\\]+\.txt$/); + }, + ); + + it.skipIf(!realFfmpeg || !findFfBinary("ffprobe") || process.platform === "win32")( + "joins only its own pieces when a folder above the project has line breaks and list lines in its name", + async () => { + vi.resetModules(); + vi.doUnmock("./mediaMetadata.js"); + vi.doMock("@hyperframes/core/preview-proof", () => ({ previewToken: () => "a".repeat(64) })); + const { resolveProxy } = await import("./proxyTranscoder.js"); + const base = realpathSync(tmpProject()); + const clip = (path: string, colour: string, seconds: number) => + execFileSync(realFfmpeg!, [ + ...["-v", "error", "-f", "lavfi", "-i", `color=c=${colour}:s=32x32:r=2:d=${seconds}`], + ...["-c:v", "libx264", "-pix_fmt", "yuv420p", path], + ]); + clip(join(base, "a.mp4"), "blue", 2); + clip(join(base, "other.mp4"), "red", 2); + const projectDir = `${base}/a.mp4\nfile ${base}/other.mp4\n#/project`; + mkdirSync(projectDir, { recursive: true }); + const sourcePath = join(projectDir, "long.mp4"); + clip(sourcePath, "blue", LONG_SECONDS); + + const proxyPath = await resolveProxy(projectDir, sourcePath); + const seconds = execFileSync( + findFfBinary("ffprobe")!, + ["-v", "error", "-show_entries", "format=duration", "-of", "csv=p=0", "--", proxyPath], + { encoding: "utf8" }, + ); + expect(Number(seconds)).toBeCloseTo(LONG_SECONDS, 0); + // Red frames carry a high V (red difference) average; the source and a.mp4 are blue. + const redness = execFileSync( + realFfmpeg!, + [ + "-v", + "error", + "-i", + proxyPath, + "-vf", + "signalstats,metadata=print:file=-", + "-f", + "null", + "-", + ], + { encoding: "utf8" }, + ) + .split("\n") + .flatMap((line) => /VAVG=([\d.]+)/.exec(line)?.[1] ?? []) + .map(Number); + expect(redness.length).toBeGreaterThan(0); + expect(Math.max(...redness)).toBeLessThan(160); + }, + 60_000, + ); + it("encodes every piece again when the project shipped pieces at the names it can predict", async () => { const { spawn, calls } = createSpawnSpy(); const { resolveProxy, getProxyCachePath } = await loadModule( @@ -2147,6 +2223,45 @@ describe("resolveProxy", () => { await expect(result).resolves.toBe(cachePath); expect(existsSync(lockPath)).toBe(false); }); + + it("gives up on a lock dated ahead of the clock within the stall window, and frees its slot", async () => { + vi.useFakeTimers(); + process.env.HYPERFRAMES_PROXY_MAX_CONCURRENCY = "1"; + const { spawn, calls } = createSpawnSpy(); + const { resolveProxy, getProxyCachePath, TRANSCODE_STALL_MS } = await loadModule( + spawn, + FFMPEG_PATH, + ); + const projectDir = tmpProject(); + const lockedSource = join(projectDir, "locked.mov"); + const otherSource = join(projectDir, "other.mov"); + writeFileSync(lockedSource, "locked-bytes"); + writeFileSync(otherSource, "other-bytes"); + const cachePath = getProxyCachePath(projectDir, lockedSource); + mkdirSync(dirname(cachePath), { recursive: true }); + const nextYear = new Date(Date.now() + 365 * 24 * 60 * 60 * 1000); + for (const name of [`.lock-${basename(cachePath)}`, `.tmp-1234-${basename(cachePath)}`]) { + writeFileSync(join(dirname(cachePath), name), String(process.pid)); + utimesSync(join(dirname(cachePath), name), nextYear, nextYear); + } + + try { + const locked = resolveProxy(projectDir, lockedSource).catch((error: unknown) => error); + const other = resolveProxy(projectDir, otherSource); + await vi.advanceTimersByTimeAsync(0); + expect(calls).toHaveLength(0); + await vi.advanceTimersByTimeAsync(TRANSCODE_STALL_MS + 1_000); + + expect(await locked).toEqual( + expect.objectContaining({ message: expect.stringMatching(/no progress/) }), + ); + expect(calls).toHaveLength(1); + succeed(calls[0]!); + await expect(other).resolves.toBeTruthy(); + } finally { + vi.useRealTimers(); + } + }); }); it("throws a typed error when ffmpeg cannot be resolved, without spawning", async () => { diff --git a/packages/studio-server/src/helpers/proxyTranscoder.ts b/packages/studio-server/src/helpers/proxyTranscoder.ts index 36e948e281..e6d03e4c25 100644 --- a/packages/studio-server/src/helpers/proxyTranscoder.ts +++ b/packages/studio-server/src/helpers/proxyTranscoder.ts @@ -1,15 +1,7 @@ import { spawn } from "node:child_process"; import { createHmac, randomBytes, randomUUID } from "node:crypto"; -import { - existsSync, - mkdtempSync, - renameSync, - rmSync, - unlinkSync, - utimesSync, - writeFileSync, -} from "node:fs"; -import os, { availableParallelism, tmpdir, totalmem } from "node:os"; +import { existsSync, renameSync, rmSync, unlinkSync, utimesSync, writeFileSync } from "node:fs"; +import os, { availableParallelism, totalmem } from "node:os"; import { basename, dirname, extname, isAbsolute, join, relative, resolve, sep } from "node:path"; import { hdrToSdrToneMapFilter, type PreviewProxyBox } from "@hyperframes/core"; import { previewToken } from "@hyperframes/core/preview-proof"; @@ -101,6 +93,8 @@ function unguessablePiecePrefix(cacheDir: string, stem: string): string { } const CONCAT_TIMEOUT_MS = 2 * 60 * 1000; +// On POSIX a folder name can hold a line break that would start a new list line; on Windows the working folder is part of the DLL search, so the cwd stays default there. +const JOIN_FROM_CACHE_FOLDER = process.platform !== "win32"; const FAILURE_CACHE_TTL_MS = 60 * 1000; export const PROXY_PENDING_RETRY_AFTER_SECONDS = 2; const ENVIRONMENT_FAILURE_TTL_MS = 5 * PROXY_PENDING_RETRY_AFTER_SECONDS * 1000; @@ -405,8 +399,7 @@ function markCacheEntryUsed(cachePath: string): void { const now = new Date(); utimesSync(cachePath, now, now); } catch { - // A concurrent cleanup may have removed a stale entry after existsSync; - // the normal miss path below will recreate it on the next request. + // A concurrent cleanup may have removed the entry; the next request's miss path recreates it. } } @@ -662,18 +655,18 @@ async function encodeInChunks( ), ); if (failure !== undefined) throw failure; - const listDir = mkdtempSync(join(tmpdir(), "hf-proxy-join-")); + const listPath = join(dirname(tempPath), `${basename(tempPath, extension)}.txt`); try { - const listPath = join(listDir, "pieces.txt"); writeFileSync( listPath, pieces - .map((piece) => `file '${piece.replace(/\\/g, "/").replace(/'/g, "'\\''")}'\n`) + .map((piece) => (JOIN_FROM_CACHE_FOLDER ? basename(piece) : piece.replace(/\\/g, "/"))) + .map((piece) => `file '${piece.replace(/'/g, "'\\''")}'\n`) .join(""), ); await concat(listPath, tempPath); } finally { - rmSync(listDir, { recursive: true, force: true }); + rmSync(listPath, { force: true }); } for (const piece of pieces) if (existsSync(piece)) unlinkSync(piece); } @@ -833,8 +826,19 @@ async function runFfmpeg( chunks, outputPath, encode, - (listPath, target) => - spawnFfmpeg(ffmpegPath, concatArgs(listPath, target, variant), CONCAT_TIMEOUT_MS, signal), + (listPath, target) => { + // ffmpeg resolves relative list entries against the list path, which breaks on '#' or '?' in a folder name. + const cwd = JOIN_FROM_CACHE_FOLDER ? dirname(listPath) : undefined; + return spawnFfmpeg( + ffmpegPath, + concatArgs(cwd ? basename(listPath) : listPath, target, variant), + CONCAT_TIMEOUT_MS, + signal, + undefined, + false, + cwd, + ); + }, signal, gate, (doneSeconds) => onProgress({ doneSeconds, totalSeconds }), @@ -872,10 +876,12 @@ function spawnFfmpeg( signal: AbortSignal, onProgress?: (encodedSeconds: number) => void, background = false, + cwd?: string, ): Promise { return new Promise((resolvePromise, reject) => { signal.throwIfAborted(); const proc = spawn(ffmpegPath, args, { + cwd, stdio: ["ignore", "pipe", "pipe"], killSignal: "SIGKILL", signal, @@ -1006,8 +1012,7 @@ async function transcodeToCache( maintainProxyCache(cacheDir); return cachePath; } finally { - // No partial files: if anything above threw, remove whatever ffmpeg - // may have partially written under the temp name. + // No partial files: if anything above threw, remove what ffmpeg wrote under the temp name. if (existsSync(tempPath)) unlinkSync(tempPath); release(); }