From fadad3f12d61450eb53615dab50327f2dfde4fdd Mon Sep 17 00:00:00 2001 From: Michael Wang Date: Wed, 18 Nov 2026 19:34:09 +0000 Subject: [PATCH] fix(studio-server): proxy joins list pieces by name; a future-dated lock can't hold waiters (#8204) * fix(studio-server): proxy piece list names only the pieces, not their folder The chunked proxy join wrote each piece as an absolute path into a concat list in the system temp folder. A line break in a folder name above the project ended an entry early, so the rest of the path could be read as further list lines naming other local files. The list now sits in the cache folder beside the pieces and names each piece by its own generated file name, and ffmpeg runs the join from that folder. No project or folder name is written to the list, so folder names with line breaks, quotes, '#' or '?' all join only the copy's own pieces. The CLI publish bake uses the same join and is covered too. * fix(studio-server): a proxy copy lock dated ahead of the clock no longer holds waiters A copy waits for another process's lock while that owner's lock, temp and piece files keep changing. The newest modified time was taken as is, so a lock or temp file dated in the future looked like fresh progress forever, and the waiter held its transcode slot without end. Preview and the publish bake could then wait with nothing running. Each file's modified time now counts as no later than the moment the waiter first saw that time. A file dated ahead of the clock ages from then, so the waiter gives up within the stall window and frees its slot. An owner that keeps writing still sets a fresh time on each change and is waited for exactly as before, even beside a future-dated file. * fix(studio-server): keep the absolute piece list and default folder on Windows On Windows the working folder is part of the DLL search, and a long cache folder path may fail to start a process from it. Windows paths cannot hold a line break, so the piece join there lists absolute piece paths again and starts ffmpeg from the default folder. Elsewhere it still runs from the cache folder with bare piece names. The line-break test now injects an unquoted list line, so the old join really pulls in another file, and it checks that the copy holds none of that file's frames. A spawn test checks the join's folder and bare list name. --- .../src/helpers/proxyLock.test.ts | 29 +++++ .../studio-server/src/helpers/proxyLock.ts | 18 ++- .../src/helpers/proxyTranscoder.test.ts | 119 +++++++++++++++++- .../src/helpers/proxyTranscoder.ts | 45 ++++--- 4 files changed, 187 insertions(+), 24 deletions(-) diff --git a/packages/studio-server/src/helpers/proxyLock.test.ts b/packages/studio-server/src/helpers/proxyLock.test.ts index 436bf15eda..b495ac65cd 100644 --- a/packages/studio-server/src/helpers/proxyLock.test.ts +++ b/packages/studio-server/src/helpers/proxyLock.test.ts @@ -79,6 +79,35 @@ describe("acquireCopyLock", () => { await expect(within(waiting, 3_000)).rejects.toThrow(CopyLockStalledError); }); + it("gives up within the stall window on a live owner whose lock and temp file are dated ahead of the clock", async () => { + const cachePath = staleLock("e.mp4"); + const nextYear = new Date(Date.now() + 365 * 24 * 60 * 60 * 1000); + for (const name of [".lock-e.mp4", ".tmp-1234-e.mp4"]) { + writeFileSync(join(dir, name), String(process.pid)); + utimesSync(join(dir, name), nextYear, nextYear); + } + + const waiting = acquireCopyLock(cachePath, () => false, new AbortController().signal, 300); + + await expect(within(waiting, 3_000)).rejects.toThrow(CopyLockStalledError); + }); + + it("keeps waiting for an owner still writing beside a temp file dated ahead of the clock", async () => { + const cachePath = staleLock("f.mp4"); + const nextYear = new Date(Date.now() + 365 * 24 * 60 * 60 * 1000); + writeFileSync(join(dir, ".tmp-9999-f.mp4"), "x"); + utimesSync(join(dir, ".tmp-9999-f.mp4"), nextYear, nextYear); + const writing = setInterval(() => writeFileSync(join(dir, ".tmp-1234-f.mp4"), "x"), 100); + const abort = new AbortController(); + + const waiting = acquireCopyLock(cachePath, () => false, abort.signal, 600); + setTimeout(() => abort.abort(new Error("stopped")), 1_500); + + await expect(within(waiting, 3_000).finally(() => clearInterval(writing))).rejects.toThrow( + "stopped", + ); + }); + it("keeps waiting while the owner's temp file is still changing", async () => { const cachePath = staleLock("b.mp4"); writeFileSync(join(dir, ".tmp-1234-b.mp4"), "x"); diff --git a/packages/studio-server/src/helpers/proxyLock.ts b/packages/studio-server/src/helpers/proxyLock.ts index 02238b16b6..cf48e19b7e 100644 --- a/packages/studio-server/src/helpers/proxyLock.ts +++ b/packages/studio-server/src/helpers/proxyLock.ts @@ -25,10 +25,20 @@ export class CopyLockStalledError extends Error { } } +type Sightings = Map; + +function changedNoLaterThanFirstSeen(sightings: Sightings, file: string, mtimeMs: number): number { + const earlier = sightings.get(file); + const seenAt = earlier?.mtimeMs === mtimeMs ? earlier.seenAt : Date.now(); + sightings.set(file, { mtimeMs, seenAt }); + return Math.min(mtimeMs, seenAt); +} + function lastActivityMs( cachePath: string, lockPath: string, unguessablePieceName: string | undefined, + sightings: Sightings, ): number { const name = basename(cachePath); const isPiece = (file: string) => @@ -43,7 +53,7 @@ function lastActivityMs( for (const file of readdirSync(dirname(cachePath))) { if (!mine(file)) continue; const changed = statSync(join(dirname(cachePath), file), { throwIfNoEntry: false })?.mtimeMs; - newest = Math.max(newest, changed ?? 0); + newest = Math.max(newest, changedNoLaterThanFirstSeen(sightings, file, changed ?? 0)); } return newest; } @@ -88,6 +98,7 @@ export async function acquireCopyLock( unguessablePieceName?: string, ): Promise<(() => void) | null> { const lockPath = lockPathFor(cachePath); + const sightings: Sightings = new Map(); for (;;) { signal.throwIfAborted(); if (copyMade(cachePath)) return null; @@ -102,7 +113,10 @@ export async function acquireCopyLock( if ((err as NodeJS.ErrnoException).code !== "EEXIST") throw err; } if (!ownerIsAlive(lockPath)) rmSync(lockPath, { force: true }); - else if (Date.now() - lastActivityMs(cachePath, lockPath, unguessablePieceName) > stallMs) + else if ( + Date.now() - lastActivityMs(cachePath, lockPath, unguessablePieceName, sightings) > + stallMs + ) throw new CopyLockStalledError(basename(cachePath), stallMs); else await pause(POLL_MS, signal); } diff --git a/packages/studio-server/src/helpers/proxyTranscoder.test.ts b/packages/studio-server/src/helpers/proxyTranscoder.test.ts index cd13eacd41..462bb286c0 100644 --- a/packages/studio-server/src/helpers/proxyTranscoder.test.ts +++ b/packages/studio-server/src/helpers/proxyTranscoder.test.ts @@ -81,7 +81,7 @@ function createFakeProc(): FakeProc { return proc; } -type SpawnOptions = { windowsHide?: boolean; signal?: AbortSignal }; +type SpawnOptions = { windowsHide?: boolean; signal?: AbortSignal; cwd?: string }; type SpawnCall = { command: string; args: string[]; options?: SpawnOptions; proc: FakeProc }; type SpawnImpl = (command: string, args: string[], options?: SpawnOptions) => FakeProc; @@ -283,12 +283,88 @@ describe("resolveProxy on a long source", () => { "-format_whitelist", "concat,mov,matroska", "-i", - expect.stringMatching(/pieces\.txt$/), + expect.stringMatching(/\.txt$/), "-c", "copy", ]); }); + it.skipIf(process.platform === "win32")( + "runs the join from the cache folder and names the list there by its bare name", + async () => { + const { spawn, calls } = createSpawnSpy(); + const { resolveProxy, getProxyCachePath } = await loadModule( + spawn, + FFMPEG_PATH, + false, + null, + undefined, + LONG_SECONDS, + ); + const projectDir = tmpProject(); + const sourcePath = join(projectDir, "long.mov"); + writeFileSync(sourcePath, "source-bytes"); + const result = resolveProxy(projectDir, sourcePath); + await answerAll(calls); + await result; + const joinCall = calls.at(-1)!; + expect(joinCall.options?.cwd).toBe(dirname(getProxyCachePath(projectDir, sourcePath))); + expect(joinCall.args[joinCall.args.indexOf("-i") + 1]).toMatch(/^[^/\\]+\.txt$/); + }, + ); + + it.skipIf(!realFfmpeg || !findFfBinary("ffprobe") || process.platform === "win32")( + "joins only its own pieces when a folder above the project has line breaks and list lines in its name", + async () => { + vi.resetModules(); + vi.doUnmock("./mediaMetadata.js"); + vi.doMock("@hyperframes/core/preview-proof", () => ({ previewToken: () => "a".repeat(64) })); + const { resolveProxy } = await import("./proxyTranscoder.js"); + const base = realpathSync(tmpProject()); + const clip = (path: string, colour: string, seconds: number) => + execFileSync(realFfmpeg!, [ + ...["-v", "error", "-f", "lavfi", "-i", `color=c=${colour}:s=32x32:r=2:d=${seconds}`], + ...["-c:v", "libx264", "-pix_fmt", "yuv420p", path], + ]); + clip(join(base, "a.mp4"), "blue", 2); + clip(join(base, "other.mp4"), "red", 2); + const projectDir = `${base}/a.mp4\nfile ${base}/other.mp4\n#/project`; + mkdirSync(projectDir, { recursive: true }); + const sourcePath = join(projectDir, "long.mp4"); + clip(sourcePath, "blue", LONG_SECONDS); + + const proxyPath = await resolveProxy(projectDir, sourcePath); + const seconds = execFileSync( + findFfBinary("ffprobe")!, + ["-v", "error", "-show_entries", "format=duration", "-of", "csv=p=0", "--", proxyPath], + { encoding: "utf8" }, + ); + expect(Number(seconds)).toBeCloseTo(LONG_SECONDS, 0); + // Red frames carry a high V (red difference) average; the source and a.mp4 are blue. + const redness = execFileSync( + realFfmpeg!, + [ + "-v", + "error", + "-i", + proxyPath, + "-vf", + "signalstats,metadata=print:file=-", + "-f", + "null", + "-", + ], + { encoding: "utf8" }, + ) + .split("\n") + .flatMap((line) => /VAVG=([\d.]+)/.exec(line)?.[1] ?? []) + .map(Number); + expect(redness.length).toBeGreaterThan(0); + expect(Math.max(...redness)).toBeLessThan(160); + }, + 60_000, + ); + it("encodes every piece again when the project shipped pieces at the names it can predict", async () => { const { spawn, calls } = createSpawnSpy(); const { resolveProxy, getProxyCachePath } = await loadModule( @@ -2147,6 +2223,45 @@ describe("resolveProxy", () => { await expect(result).resolves.toBe(cachePath); expect(existsSync(lockPath)).toBe(false); }); + + it("gives up on a lock dated ahead of the clock within the stall window, and frees its slot", async () => { + vi.useFakeTimers(); + process.env.HYPERFRAMES_PROXY_MAX_CONCURRENCY = "1"; + const { spawn, calls } = createSpawnSpy(); + const { resolveProxy, getProxyCachePath, TRANSCODE_STALL_MS } = await loadModule( + spawn, + FFMPEG_PATH, + ); + const projectDir = tmpProject(); + const lockedSource = join(projectDir, "locked.mov"); + const otherSource = join(projectDir, "other.mov"); + writeFileSync(lockedSource, "locked-bytes"); + writeFileSync(otherSource, "other-bytes"); + const cachePath = getProxyCachePath(projectDir, lockedSource); + mkdirSync(dirname(cachePath), { recursive: true }); + const nextYear = new Date(Date.now() + 365 * 24 * 60 * 60 * 1000); + for (const name of [`.lock-${basename(cachePath)}`, `.tmp-1234-${basename(cachePath)}`]) { + writeFileSync(join(dirname(cachePath), name), String(process.pid)); + utimesSync(join(dirname(cachePath), name), nextYear, nextYear); + } + + try { + const locked = resolveProxy(projectDir, lockedSource).catch((error: unknown) => error); + const other = resolveProxy(projectDir, otherSource); + await vi.advanceTimersByTimeAsync(0); + expect(calls).toHaveLength(0); + await vi.advanceTimersByTimeAsync(TRANSCODE_STALL_MS + 1_000); + + expect(await locked).toEqual( + expect.objectContaining({ message: expect.stringMatching(/no progress/) }), + ); + expect(calls).toHaveLength(1); + succeed(calls[0]!); + await expect(other).resolves.toBeTruthy(); + } finally { + vi.useRealTimers(); + } + }); }); it("throws a typed error when ffmpeg cannot be resolved, without spawning", async () => { diff --git a/packages/studio-server/src/helpers/proxyTranscoder.ts b/packages/studio-server/src/helpers/proxyTranscoder.ts index 36e948e281..e6d03e4c25 100644 --- a/packages/studio-server/src/helpers/proxyTranscoder.ts +++ b/packages/studio-server/src/helpers/proxyTranscoder.ts @@ -1,15 +1,7 @@ import { spawn } from "node:child_process"; import { createHmac, randomBytes, randomUUID } from "node:crypto"; -import { - existsSync, - mkdtempSync, - renameSync, - rmSync, - unlinkSync, - utimesSync, - writeFileSync, -} from "node:fs"; -import os, { availableParallelism, tmpdir, totalmem } from "node:os"; +import { existsSync, renameSync, rmSync, unlinkSync, utimesSync, writeFileSync } from "node:fs"; +import os, { availableParallelism, totalmem } from "node:os"; import { basename, dirname, extname, isAbsolute, join, relative, resolve, sep } from "node:path"; import { hdrToSdrToneMapFilter, type PreviewProxyBox } from "@hyperframes/core"; import { previewToken } from "@hyperframes/core/preview-proof"; @@ -101,6 +93,8 @@ function unguessablePiecePrefix(cacheDir: string, stem: string): string { } const CONCAT_TIMEOUT_MS = 2 * 60 * 1000; +// On POSIX a folder name can hold a line break that would start a new list line; on Windows the working folder is part of the DLL search, so the cwd stays default there. +const JOIN_FROM_CACHE_FOLDER = process.platform !== "win32"; const FAILURE_CACHE_TTL_MS = 60 * 1000; export const PROXY_PENDING_RETRY_AFTER_SECONDS = 2; const ENVIRONMENT_FAILURE_TTL_MS = 5 * PROXY_PENDING_RETRY_AFTER_SECONDS * 1000; @@ -405,8 +399,7 @@ function markCacheEntryUsed(cachePath: string): void { const now = new Date(); utimesSync(cachePath, now, now); } catch { - // A concurrent cleanup may have removed a stale entry after existsSync; - // the normal miss path below will recreate it on the next request. + // A concurrent cleanup may have removed the entry; the next request's miss path recreates it. } } @@ -662,18 +655,18 @@ async function encodeInChunks( ), ); if (failure !== undefined) throw failure; - const listDir = mkdtempSync(join(tmpdir(), "hf-proxy-join-")); + const listPath = join(dirname(tempPath), `${basename(tempPath, extension)}.txt`); try { - const listPath = join(listDir, "pieces.txt"); writeFileSync( listPath, pieces - .map((piece) => `file '${piece.replace(/\\/g, "/").replace(/'/g, "'\\''")}'\n`) + .map((piece) => (JOIN_FROM_CACHE_FOLDER ? basename(piece) : piece.replace(/\\/g, "/"))) + .map((piece) => `file '${piece.replace(/'/g, "'\\''")}'\n`) .join(""), ); await concat(listPath, tempPath); } finally { - rmSync(listDir, { recursive: true, force: true }); + rmSync(listPath, { force: true }); } for (const piece of pieces) if (existsSync(piece)) unlinkSync(piece); } @@ -833,8 +826,19 @@ async function runFfmpeg( chunks, outputPath, encode, - (listPath, target) => - spawnFfmpeg(ffmpegPath, concatArgs(listPath, target, variant), CONCAT_TIMEOUT_MS, signal), + (listPath, target) => { + // ffmpeg resolves relative list entries against the list path, which breaks on '#' or '?' in a folder name. + const cwd = JOIN_FROM_CACHE_FOLDER ? dirname(listPath) : undefined; + return spawnFfmpeg( + ffmpegPath, + concatArgs(cwd ? basename(listPath) : listPath, target, variant), + CONCAT_TIMEOUT_MS, + signal, + undefined, + false, + cwd, + ); + }, signal, gate, (doneSeconds) => onProgress({ doneSeconds, totalSeconds }), @@ -872,10 +876,12 @@ function spawnFfmpeg( signal: AbortSignal, onProgress?: (encodedSeconds: number) => void, background = false, + cwd?: string, ): Promise { return new Promise((resolvePromise, reject) => { signal.throwIfAborted(); const proc = spawn(ffmpegPath, args, { + cwd, stdio: ["ignore", "pipe", "pipe"], killSignal: "SIGKILL", signal, @@ -1006,8 +1012,7 @@ async function transcodeToCache( maintainProxyCache(cacheDir); return cachePath; } finally { - // No partial files: if anything above threw, remove whatever ffmpeg - // may have partially written under the temp name. + // No partial files: if anything above threw, remove what ffmpeg wrote under the temp name. if (existsSync(tempPath)) unlinkSync(tempPath); release(); }