From a4139e9ead02bea455feb3409c0076d2de050896 Mon Sep 17 00:00:00 2001 From: Michael Wang Date: Mon, 16 Nov 2026 02:10:35 +0000 Subject: [PATCH] fix(engine): give each superset extraction a folder of its own (#7901) * fix(engine): give each superset extraction a folder of its own A superset group extracted into a folder named from the group index and the process id, and deleted and recreated it by that name in the cache root that every render shares. Two renders in one process could share the folder and remove each other's frames. Partial folders are now created with mkdtemp, so each extraction writes only into a folder it made. A cache entry is published only when its frame count is the one the extraction wrote. * fix(engine): make the partial folders with createStagingDir; raise the engine size budget --- .../src/services/extractionCache.test.ts | 65 ++++++++-- .../engine/src/services/extractionCache.ts | 53 +++++--- ...eoFrameExtractor.supersetIsolation.test.ts | 121 ++++++++++++++++++ .../src/services/videoFrameExtractor.ts | 48 +++---- scripts/package-size-budgets.json | 2 +- 5 files changed, 236 insertions(+), 53 deletions(-) create mode 100644 packages/engine/src/services/videoFrameExtractor.supersetIsolation.test.ts diff --git a/packages/engine/src/services/extractionCache.test.ts b/packages/engine/src/services/extractionCache.test.ts index 2c261daa8d..19ab42f2ad 100644 --- a/packages/engine/src/services/extractionCache.test.ts +++ b/packages/engine/src/services/extractionCache.test.ts @@ -6,13 +6,14 @@ import { mkdirSync, mkdtempSync, readdirSync, + readFileSync, rmSync, statSync, symlinkSync, utimesSync, writeFileSync, } from "node:fs"; -import { join } from "node:path"; +import { basename, join } from "node:path"; import { tmpdir } from "node:os"; import { @@ -25,7 +26,8 @@ import { gcExtractionCache, lookupCacheEntry, markCacheEntryComplete, - partialCacheEntryDir, + createPartialCacheEntryDir, + createPartialDir, privateCacheRoot, publishCacheEntry, readContentSha256, @@ -64,8 +66,7 @@ function removeCacheRoot(tmpRoot: string): void { /** Create and populate a partial dir for `entry` with one frame file. */ function seedPartialDir(entry: { dir: string; keyHash: string }, frameContent: string): string { - const partialDir = partialCacheEntryDir(entry); - mkdirSync(partialDir, { recursive: true }); + const partialDir = createPartialCacheEntryDir(entry); writeFileSync(join(partialDir, "frame_00001.jpg"), frameContent, "utf-8"); return partialDir; } @@ -387,7 +388,7 @@ describe("publishCacheEntry", () => { const cacheEntry = entry(); const partialDir = seedPartialDir(cacheEntry, "frame"); - const result = publishCacheEntry(cacheEntry, partialDir); + const result = publishCacheEntry(cacheEntry, partialDir, 1); expect(result).toEqual({ dir: cacheEntry.dir, published: true }); expect(existsSync(partialDir)).toBe(false); @@ -403,7 +404,7 @@ describe("publishCacheEntry", () => { const partialDir = seedPartialDir(cacheEntry, "loser"); - const result = publishCacheEntry(cacheEntry, partialDir); + const result = publishCacheEntry(cacheEntry, partialDir, 1); expect(result).toEqual({ dir: cacheEntry.dir, published: true }); expect(existsSync(partialDir)).toBe(false); @@ -418,7 +419,7 @@ describe("publishCacheEntry", () => { const partialDir = seedPartialDir(cacheEntry, "fresh"); - const result = publishCacheEntry(cacheEntry, partialDir); + const result = publishCacheEntry(cacheEntry, partialDir, 1); expect(result).toEqual({ dir: cacheEntry.dir, published: true }); expect(existsSync(partialDir)).toBe(false); @@ -433,13 +434,61 @@ describe("publishCacheEntry", () => { expect(lookupCacheEntry(tmpRoot, keyFor(sourceFile)).hit).toBe(false); const partialDir = seedPartialDir(cacheEntry, "fresh"); - const result = publishCacheEntry(cacheEntry, partialDir); + const result = publishCacheEntry(cacheEntry, partialDir, 1); expect(result).toEqual({ dir: cacheEntry.dir, published: true }); expect(existsSync(partialDir)).toBe(false); expect(lookupCacheEntry(tmpRoot, keyFor(sourceFile)).hit).toBe(true); expect(statSync(join(cacheEntry.dir, "frame_00001.jpg")).size).toBe("fresh".length); }); + + it.each([ + ["fewer", 2], + ["more", 0], + ])("does not publish a partial directory holding %s frames than its writer made", (_, made) => { + const cacheEntry = entry(); + const partialDir = seedPartialDir(cacheEntry, "frame"); + + const result = publishCacheEntry(cacheEntry, partialDir, made); + + expect(result).toEqual({ dir: partialDir, published: false }); + expect(existsSync(cacheEntry.dir)).toBe(false); + expect(existsSync(join(partialDir, COMPLETE_SENTINEL))).toBe(false); + }); +}); + +describe("createPartialDir", () => { + let tmpRoot: string; + + beforeEach(() => { + ({ tmpRoot } = makeCacheRoot()); + }); + + afterEach(() => { + removeCacheRoot(tmpRoot); + }); + + it("creates a new empty directory per call under a name the aged-partial sweep matches", () => { + const first = createPartialDir(tmpRoot, "__superset-0"); + const second = createPartialDir(tmpRoot, "__superset-0"); + + expect(first).not.toBe(second); + for (const dir of [first, second]) { + expect(readdirSync(dir)).toEqual([]); + expect(basename(dir)).toMatch( + new RegExp(`^__superset-0\\.partial-${process.pid}-[A-Za-z0-9]{6}$`), + ); + } + }); + + it("leaves another writer's directory of the same group in place", () => { + const other = createPartialDir(tmpRoot, "__superset-0"); + writeFileSync(join(other, "frame_00001.jpg"), "other", "utf-8"); + + createPartialDir(tmpRoot, "__superset-0"); + + expect(readFileSync(join(other, "frame_00001.jpg"), "utf-8")).toBe("other"); + }); }); describe("gcExtractionCache", () => { diff --git a/packages/engine/src/services/extractionCache.ts b/packages/engine/src/services/extractionCache.ts index 06da678371..37fdd698b3 100644 --- a/packages/engine/src/services/extractionCache.ts +++ b/packages/engine/src/services/extractionCache.ts @@ -16,8 +16,9 @@ * - Cache entries live under `//` so * `ls` output and tracing logs stay short. Truncation to 16 hex chars * leaves 64 bits of entropy — collision risk at cache scale is negligible. - * - Frames are extracted into a unique `.partial--/` dir. - * Once all frames are written, the partial dir receives the `.hf-complete` + * - Frames are extracted into a `.partial--/` dir that + * `mkdtemp` creates for that one writer. Once all frames are written and + * their count is the writer's own, the partial dir receives the `.hf-complete` * sentinel and is atomically renamed to the final key dir. Concurrent * same-key writers may duplicate ffmpeg work, but readers only ever serve * complete entries. @@ -32,7 +33,7 @@ * become inert and can be gc'd by the caller. */ -import { createHash, randomUUID } from "node:crypto"; +import { createHash } from "node:crypto"; import { chmodSync, createReadStream, @@ -45,8 +46,9 @@ import { utimesSync, writeFileSync, } from "node:fs"; +import { createStagingDir } from "@hyperframes/core/atomic-file"; import { renameWithRetry } from "@hyperframes/core/rename-retry"; -import { join } from "node:path"; +import { basename, dirname, join } from "node:path"; import { defaultExtractCacheDir } from "../config.js"; import type { VideoMetadata } from "../utils/ffprobe.js"; import { FRAME_FILENAME_PREFIX, framePathsFromDirectory } from "./extractedFrameIndex.js"; @@ -234,12 +236,23 @@ export function ensureCacheEntryDir(entry: CacheEntry): void { mkdirSync(entry.dir, { recursive: true }); } +/** A new render-owned directory to populate a cache entry in before the atomic publish rename. */ +export function createPartialCacheEntryDir(entry: CacheEntry): string { + return createPartialDir(dirname(entry.dir), basename(entry.dir)); +} + /** - * Unique render-owned directory used to populate a cache entry before the - * atomic publish rename. + * A new, empty `.partial--` directory under `parent`, named so the GC's aged-partial + * sweep removes it if its writer dies. `mkdtemp` creates it atomically under a name no other writer + * can predict or reuse, so only its creator writes to it. */ -export function partialCacheEntryDir(entry: CacheEntry): string { - return `${entry.dir}.partial-${process.pid}-${randomUUID().slice(0, 8)}`; +export function createPartialDir(parent: string, name: string): string { + mkdirSync(parent, { recursive: true }); + return createStagingDir(parent, `${name}.partial-${process.pid}-`); +} + +function frameFileCount(dir: string): number { + return readdirSync(dir).filter((file) => file.startsWith(FRAME_FILENAME_PREFIX)).length; } function isTargetExistsRenameError(err: unknown): boolean { @@ -247,14 +260,6 @@ function isTargetExistsRenameError(err: unknown): boolean { return code === "EEXIST" || code === "ENOTEMPTY" || code === "EPERM"; } -/** - * Publish an extracted partial directory as the final cache entry. - * - * Same-filesystem directory rename is atomic: readers either see no entry or - * a complete sentineled entry. When another writer wins the race, the caller - * should rehydrate from the final dir. If publish cannot complete safely, the - * partial remains render-owned and must be cleaned up by the render cleanup. - */ /** * If a concurrent writer's completed entry is visible, discard our partial * and serve theirs. Identical keys produce identical frames, so adopting the @@ -266,8 +271,22 @@ function adoptPublishedWinner(entry: CacheEntry, partialDir: string): CachePubli return { dir: entry.dir, published: true }; } -export function publishCacheEntry(entry: CacheEntry, partialDir: string): CachePublishResult { +/** + * Publish an extracted partial directory as the final cache entry, only when it + * holds exactly the `expectedFrames` its writer put there. + * + * Same-filesystem directory rename is atomic: readers either see no entry or + * a complete sentineled entry. When another writer wins the race, the caller + * should rehydrate from the final dir. If publish cannot complete safely, the + * partial remains render-owned and must be cleaned up by the render cleanup. + */ +export function publishCacheEntry( + entry: CacheEntry, + partialDir: string, + expectedFrames: number, +): CachePublishResult { try { + if (frameFileCount(partialDir) !== expectedFrames) return { dir: partialDir, published: false }; writeFileSync(join(partialDir, COMPLETE_SENTINEL), "", "utf-8"); } catch { return { dir: partialDir, published: false }; diff --git a/packages/engine/src/services/videoFrameExtractor.supersetIsolation.test.ts b/packages/engine/src/services/videoFrameExtractor.supersetIsolation.test.ts new file mode 100644 index 0000000000..486c784936 --- /dev/null +++ b/packages/engine/src/services/videoFrameExtractor.supersetIsolation.test.ts @@ -0,0 +1,121 @@ +import { spawnSync } from "node:child_process"; +import { existsSync, mkdtempSync, readdirSync, readFileSync, rmSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { afterAll, describe, expect, it, vi } from "vitest"; +import type { VideoElement } from "../types.js"; + +// Enough extraction slots that one render waiting inside ffmpeg never starves the other. +vi.mock("os", async (importOriginal) => ({ + ...(await importOriginal()), + availableParallelism: () => 16, +})); + +// Lets a test hold one source's superset ffmpeg until it says so. +const held = vi.hoisted(() => ({ + source: "", + reached: () => {}, + release: Promise.resolve(), +})); + +vi.mock("../utils/runFfmpeg.js", async (importOriginal) => { + const real = await importOriginal(); + return { + ...real, + runFfmpeg: vi.fn(async (args: string[], opts?: Parameters[1]) => { + if (args.includes(held.source) && args.some((arg) => arg.includes("__superset-"))) { + held.reached(); + await held.release; + } + return real.runFfmpeg(args, opts); + }), + }; +}); + +const { extractAllVideoFrames, extractVideoFramesRange } = await import("./videoFrameExtractor.js"); +const { runFfmpeg } = await import("../utils/runFfmpeg.js"); + +const HAS_FFMPEG = spawnSync("ffmpeg", ["-version"]).status === 0; + +describe.skipIf(!HAS_FFMPEG)("superset extraction in concurrent renders", () => { + const dir = mkdtempSync(join(tmpdir(), "hf-superset-isolation-")); + afterAll(() => rmSync(dir, { recursive: true, force: true })); + + async function synth(name: string, lavfi: string): Promise { + const src = join(dir, name); + const made = await runFfmpeg([ + ...["-y", "-v", "error", "-f", "lavfi", "-i", lavfi], + ...["-c:v", "libx264", "-preset", "ultrafast", "-pix_fmt", "yuv420p", src], + ]); + expect(made.success, made.stderr).toBe(true); + return src; + } + + // Two overlapping, frame-aligned trims of one source: a render extracts them as one superset group. + const overlappingTrims = (prefix: string, src: string): VideoElement[] => + [0, 2].map((mediaStart, i) => ({ + ...{ id: `${prefix}-${i}`, src, start: 0, end: 4, mediaStart }, + ...{ loop: false, hasAudio: false }, + })); + + const render = (videos: VideoElement[], out: string, cacheDir: string) => + extractAllVideoFrames(videos, dir, { fps: 30, outputDir: join(dir, out) }, undefined, { + extractCacheDir: cacheDir, + }); + + const supersetFolders = (root: string): string[] => + readdirSync(root).filter((name) => name.startsWith("__superset-")); + + async function expectOwnFrames( + result: Awaited>, + videos: VideoElement[], + ): Promise { + expect(result.errors).toEqual([]); + for (const video of videos) { + const extracted = result.extracted.find((item) => item.videoId === video.id); + expect(extracted?.totalFrames).toBe(120); + const direct = await extractVideoFramesRange( + video.src, + `direct-${video.id}`, + video.mediaStart, + 4, + { fps: 30, outputDir: join(dir, "direct"), format: "jpg" }, + ); + for (const frame of [0, 60, 119]) { + const own = readFileSync(direct.framePaths.get(frame)!); + const got = readFileSync(extracted!.framePaths.get(frame)!); + expect(got.equals(own), `${video.id} frame ${frame}`).toBe(true); + } + } + } + + it("keeps each render's frames and folder its own while another runs in the same process", async () => { + const cacheDir = mkdtempSync(join(dir, "cache-")); + const first = overlappingTrims("first", await synth("first.mp4", "testsrc2=d=10:s=320x180")); + const second = overlappingTrims( + "second", + await synth("second.mp4", "mandelbrot=s=320x180,trim=duration=10"), + ); + + let release = () => {}; + const reached = new Promise((resolve) => { + held.source = first[0]!.src; + held.reached = resolve; + held.release = new Promise((done) => (release = done)); + }); + const firstRender = render(first, "out-first", cacheDir); + try { + await reached; + const firstFolders = supersetFolders(cacheDir); + expect(firstFolders).toHaveLength(1); + + // The second render runs start to finish, cleanup included, while the first waits mid-extraction. + await expectOwnFrames(await render(second, "out-second", cacheDir), second); + expect(existsSync(join(cacheDir, firstFolders[0]!))).toBe(true); + } finally { + release(); + } + await expectOwnFrames(await firstRender, first); + expect(supersetFolders(cacheDir)).toEqual([]); + }, 120_000); +}); diff --git a/packages/engine/src/services/videoFrameExtractor.ts b/packages/engine/src/services/videoFrameExtractor.ts index 1758fe1a87..3bbc1b7414 100644 --- a/packages/engine/src/services/videoFrameExtractor.ts +++ b/packages/engine/src/services/videoFrameExtractor.ts @@ -93,10 +93,11 @@ import { unwrapTemplate } from "../utils/htmlTemplate.js"; import { mediaIdPathSegment } from "../utils/mediaPathSegment.js"; import { FRAME_FILENAME_PREFIX, + createPartialCacheEntryDir, + createPartialDir, gcExtractionCache, gcSweepDue, lookupCacheEntry, - partialCacheEntryDir, publishCacheEntry, readContentSha256, readKeyStat, @@ -2184,8 +2185,6 @@ function sliceSupersetMember( configuredFps: FpsInput, ): ExtractedFrames { const { work } = member.miss; - rmSync(outputDir, { recursive: true, force: true }); - mkdirSync(outputDir, { recursive: true }); // Sample-time correctness: member frame k uses superset frame // offset_i + k, so its source time is @@ -2204,7 +2203,11 @@ function sliceSupersetMember( linkOrCopyFrame(sourceFrame, join(outputDir, frameFileName(i + 1, work.format))); } - return extractedFramesFromDirectory(work, outputDir, work.videoPath, fps); + const sliced = extractedFramesFromDirectory(work, outputDir, work.videoPath, fps); + if (sliced.totalFrames !== frameCount) { + throw new Error(`superset slice holds ${sliced.totalFrames} frames, not ${frameCount}`); + } + return sliced; } export async function extractAllVideoFrames( @@ -2647,9 +2650,7 @@ export async function extractAllVideoFrames( return attempted.result; } - const partialDir = partialCacheEntryDir(cacheTarget.entry); - rmSync(partialDir, { recursive: true, force: true }); - mkdirSync(partialDir, { recursive: true }); + let partialDir = createPartialCacheEntryDir(cacheTarget.entry); const attempted = await runVideoExtractionWithRetry( () => extractVideoFramesRange( @@ -2668,12 +2669,12 @@ export async function extractAllVideoFrames( onRetry: () => { recordTransientRetries(1); rmSync(partialDir, { recursive: true, force: true }); - mkdirSync(partialDir, { recursive: true }); + partialDir = createPartialCacheEntryDir(cacheTarget.entry); }, }, ); const result = attempted.result; - const published = publishCacheEntry(cacheTarget.entry, partialDir); + const published = publishCacheEntry(cacheTarget.entry, partialDir, result.totalFrames); if (!published.published) { breakdown.cachePublishFailures += 1; return { ...result, ownedByLookup: false }; @@ -2791,18 +2792,15 @@ export async function extractAllVideoFrames( const { miss } = member; const { work, cacheTarget } = miss; if (!cacheTarget) { - return sliceSupersetMember( - member, - superset, - join(options.outputDir, mediaIdPathSegment(work.video.id)), - fps, - configuredFps, - ); + const outputDir = join(options.outputDir, mediaIdPathSegment(work.video.id)); + rmSync(outputDir, { recursive: true, force: true }); + mkdirSync(outputDir, { recursive: true }); + return sliceSupersetMember(member, superset, outputDir, fps, configuredFps); } - const partialDir = partialCacheEntryDir(cacheTarget.entry); + const partialDir = createPartialCacheEntryDir(cacheTarget.entry); const sliced = sliceSupersetMember(member, superset, partialDir, fps, configuredFps); - const published = publishCacheEntry(cacheTarget.entry, partialDir); + const published = publishCacheEntry(cacheTarget.entry, partialDir, sliced.totalFrames); if (!published.published) { breakdown.cachePublishFailures += 1; return { ...sliced, ownedByLookup: false }; @@ -2819,15 +2817,11 @@ export async function extractAllVideoFrames( // members link into partial dirs under cacheRootDir, which is commonly a // different mount than the render's outputDir — extracting the superset // next to the cache keeps linkSync viable there (the EXDEV copyFileSync - // fallback would silently multiply disk usage per member). The - // `.partial-` name puts crashed leftovers under the GC's aged-partial - // sweep. - const tempDir = cacheRootDir - ? join(cacheRootDir, `${group.groupId}.partial-${process.pid}`) - : join(options.outputDir, group.groupId); - + // fallback would silently multiply disk usage per member). Renders share the + // cache root and group ids, so each gets a new folder of its own. + let tempDir: string | undefined; try { - rmSync(tempDir, { recursive: true, force: true }); + tempDir = createPartialDir(cacheRootDir ?? options.outputDir, group.groupId); // A long union can fail even when each shorter // member range succeeds. Do not retry the optimization itself; preserve // the established grouped→direct fallback and apply bounded retries only @@ -2871,7 +2865,7 @@ export async function extractAllVideoFrames( ); return fallback; } finally { - rmSync(tempDir, { recursive: true, force: true }); + if (tempDir) rmSync(tempDir, { recursive: true, force: true }); } } diff --git a/scripts/package-size-budgets.json b/scripts/package-size-budgets.json index defdc32846..21e629d6f1 100644 --- a/scripts/package-size-budgets.json +++ b/scripts/package-size-budgets.json @@ -12,7 +12,7 @@ "unpacked": 4374000 }, "@hyperframes/engine": { - "packed": 500000, + "packed": 510000, "unpacked": 1742862 }, "@hyperframes/gcp-cloud-run": {