diff --git a/packages/engine/src/services/extractionCache.test.ts b/packages/engine/src/services/extractionCache.test.ts index 2c261daa8d..19ab42f2ad 100644 --- a/packages/engine/src/services/extractionCache.test.ts +++ b/packages/engine/src/services/extractionCache.test.ts @@ -6,13 +6,14 @@ import { mkdirSync, mkdtempSync, readdirSync, + readFileSync, rmSync, statSync, symlinkSync, utimesSync, writeFileSync, } from "node:fs"; -import { join } from "node:path"; +import { basename, join } from "node:path"; import { tmpdir } from "node:os"; import { @@ -25,7 +26,8 @@ import { gcExtractionCache, lookupCacheEntry, markCacheEntryComplete, - partialCacheEntryDir, + createPartialCacheEntryDir, + createPartialDir, privateCacheRoot, publishCacheEntry, readContentSha256, @@ -64,8 +66,7 @@ function removeCacheRoot(tmpRoot: string): void { /** Create and populate a partial dir for `entry` with one frame file. */ function seedPartialDir(entry: { dir: string; keyHash: string }, frameContent: string): string { - const partialDir = partialCacheEntryDir(entry); - mkdirSync(partialDir, { recursive: true }); + const partialDir = createPartialCacheEntryDir(entry); writeFileSync(join(partialDir, "frame_00001.jpg"), frameContent, "utf-8"); return partialDir; } @@ -387,7 +388,7 @@ describe("publishCacheEntry", () => { const cacheEntry = entry(); const partialDir = seedPartialDir(cacheEntry, "frame"); - const result = publishCacheEntry(cacheEntry, partialDir); + const result = publishCacheEntry(cacheEntry, partialDir, 1); expect(result).toEqual({ dir: cacheEntry.dir, published: true }); expect(existsSync(partialDir)).toBe(false); @@ -403,7 +404,7 @@ describe("publishCacheEntry", () => { const partialDir = seedPartialDir(cacheEntry, "loser"); - const result = publishCacheEntry(cacheEntry, partialDir); + const result = publishCacheEntry(cacheEntry, partialDir, 1); expect(result).toEqual({ dir: cacheEntry.dir, published: true }); expect(existsSync(partialDir)).toBe(false); @@ -418,7 +419,7 @@ describe("publishCacheEntry", () => { const partialDir = seedPartialDir(cacheEntry, "fresh"); - const result = publishCacheEntry(cacheEntry, partialDir); + const result = publishCacheEntry(cacheEntry, partialDir, 1); expect(result).toEqual({ dir: cacheEntry.dir, published: true }); expect(existsSync(partialDir)).toBe(false); @@ -433,13 +434,61 @@ describe("publishCacheEntry", () => { expect(lookupCacheEntry(tmpRoot, keyFor(sourceFile)).hit).toBe(false); const partialDir = seedPartialDir(cacheEntry, "fresh"); - const result = publishCacheEntry(cacheEntry, partialDir); + const result = publishCacheEntry(cacheEntry, partialDir, 1); expect(result).toEqual({ dir: cacheEntry.dir, published: true }); expect(existsSync(partialDir)).toBe(false); expect(lookupCacheEntry(tmpRoot, keyFor(sourceFile)).hit).toBe(true); expect(statSync(join(cacheEntry.dir, "frame_00001.jpg")).size).toBe("fresh".length); }); + + it.each([ + ["fewer", 2], + ["more", 0], + ])("does not publish a partial directory holding %s frames than its writer made", (_, made) => { + const cacheEntry = entry(); + const partialDir = seedPartialDir(cacheEntry, "frame"); + + const result = publishCacheEntry(cacheEntry, partialDir, made); + + expect(result).toEqual({ dir: partialDir, published: false }); + expect(existsSync(cacheEntry.dir)).toBe(false); + expect(existsSync(join(partialDir, COMPLETE_SENTINEL))).toBe(false); + }); +}); + +describe("createPartialDir", () => { + let tmpRoot: string; + + beforeEach(() => { + ({ tmpRoot } = makeCacheRoot()); + }); + + afterEach(() => { + removeCacheRoot(tmpRoot); + }); + + it("creates a new empty directory per call under a name the aged-partial sweep matches", () => { + const first = createPartialDir(tmpRoot, "__superset-0"); + const second = createPartialDir(tmpRoot, "__superset-0"); + + expect(first).not.toBe(second); + for (const dir of [first, second]) { + expect(readdirSync(dir)).toEqual([]); + expect(basename(dir)).toMatch( + new RegExp(`^__superset-0\\.partial-${process.pid}-[A-Za-z0-9]{6}$`), + ); + } + }); + + it("leaves another writer's directory of the same group in place", () => { + const other = createPartialDir(tmpRoot, "__superset-0"); + writeFileSync(join(other, "frame_00001.jpg"), "other", "utf-8"); + + createPartialDir(tmpRoot, "__superset-0"); + + expect(readFileSync(join(other, "frame_00001.jpg"), "utf-8")).toBe("other"); + }); }); describe("gcExtractionCache", () => { diff --git a/packages/engine/src/services/extractionCache.ts b/packages/engine/src/services/extractionCache.ts index 06da678371..37fdd698b3 100644 --- a/packages/engine/src/services/extractionCache.ts +++ b/packages/engine/src/services/extractionCache.ts @@ -16,8 +16,9 @@ * - Cache entries live under `//` so * `ls` output and tracing logs stay short. Truncation to 16 hex chars * leaves 64 bits of entropy — collision risk at cache scale is negligible. - * - Frames are extracted into a unique `.partial--/` dir. - * Once all frames are written, the partial dir receives the `.hf-complete` + * - Frames are extracted into a `.partial--/` dir that + * `mkdtemp` creates for that one writer. Once all frames are written and + * their count is the writer's own, the partial dir receives the `.hf-complete` * sentinel and is atomically renamed to the final key dir. Concurrent * same-key writers may duplicate ffmpeg work, but readers only ever serve * complete entries. @@ -32,7 +33,7 @@ * become inert and can be gc'd by the caller. */ -import { createHash, randomUUID } from "node:crypto"; +import { createHash } from "node:crypto"; import { chmodSync, createReadStream, @@ -45,8 +46,9 @@ import { utimesSync, writeFileSync, } from "node:fs"; +import { createStagingDir } from "@hyperframes/core/atomic-file"; import { renameWithRetry } from "@hyperframes/core/rename-retry"; -import { join } from "node:path"; +import { basename, dirname, join } from "node:path"; import { defaultExtractCacheDir } from "../config.js"; import type { VideoMetadata } from "../utils/ffprobe.js"; import { FRAME_FILENAME_PREFIX, framePathsFromDirectory } from "./extractedFrameIndex.js"; @@ -234,12 +236,23 @@ export function ensureCacheEntryDir(entry: CacheEntry): void { mkdirSync(entry.dir, { recursive: true }); } +/** A new render-owned directory to populate a cache entry in before the atomic publish rename. */ +export function createPartialCacheEntryDir(entry: CacheEntry): string { + return createPartialDir(dirname(entry.dir), basename(entry.dir)); +} + /** - * Unique render-owned directory used to populate a cache entry before the - * atomic publish rename. + * A new, empty `.partial--` directory under `parent`, named so the GC's aged-partial + * sweep removes it if its writer dies. `mkdtemp` creates it atomically under a name no other writer + * can predict or reuse, so only its creator writes to it. */ -export function partialCacheEntryDir(entry: CacheEntry): string { - return `${entry.dir}.partial-${process.pid}-${randomUUID().slice(0, 8)}`; +export function createPartialDir(parent: string, name: string): string { + mkdirSync(parent, { recursive: true }); + return createStagingDir(parent, `${name}.partial-${process.pid}-`); +} + +function frameFileCount(dir: string): number { + return readdirSync(dir).filter((file) => file.startsWith(FRAME_FILENAME_PREFIX)).length; } function isTargetExistsRenameError(err: unknown): boolean { @@ -247,14 +260,6 @@ function isTargetExistsRenameError(err: unknown): boolean { return code === "EEXIST" || code === "ENOTEMPTY" || code === "EPERM"; } -/** - * Publish an extracted partial directory as the final cache entry. - * - * Same-filesystem directory rename is atomic: readers either see no entry or - * a complete sentineled entry. When another writer wins the race, the caller - * should rehydrate from the final dir. If publish cannot complete safely, the - * partial remains render-owned and must be cleaned up by the render cleanup. - */ /** * If a concurrent writer's completed entry is visible, discard our partial * and serve theirs. Identical keys produce identical frames, so adopting the @@ -266,8 +271,22 @@ function adoptPublishedWinner(entry: CacheEntry, partialDir: string): CachePubli return { dir: entry.dir, published: true }; } -export function publishCacheEntry(entry: CacheEntry, partialDir: string): CachePublishResult { +/** + * Publish an extracted partial directory as the final cache entry, only when it + * holds exactly the `expectedFrames` its writer put there. + * + * Same-filesystem directory rename is atomic: readers either see no entry or + * a complete sentineled entry. When another writer wins the race, the caller + * should rehydrate from the final dir. If publish cannot complete safely, the + * partial remains render-owned and must be cleaned up by the render cleanup. + */ +export function publishCacheEntry( + entry: CacheEntry, + partialDir: string, + expectedFrames: number, +): CachePublishResult { try { + if (frameFileCount(partialDir) !== expectedFrames) return { dir: partialDir, published: false }; writeFileSync(join(partialDir, COMPLETE_SENTINEL), "", "utf-8"); } catch { return { dir: partialDir, published: false }; diff --git a/packages/engine/src/services/videoFrameExtractor.supersetIsolation.test.ts b/packages/engine/src/services/videoFrameExtractor.supersetIsolation.test.ts new file mode 100644 index 0000000000..486c784936 --- /dev/null +++ b/packages/engine/src/services/videoFrameExtractor.supersetIsolation.test.ts @@ -0,0 +1,121 @@ +import { spawnSync } from "node:child_process"; +import { existsSync, mkdtempSync, readdirSync, readFileSync, rmSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { afterAll, describe, expect, it, vi } from "vitest"; +import type { VideoElement } from "../types.js"; + +// Enough extraction slots that one render waiting inside ffmpeg never starves the other. +vi.mock("os", async (importOriginal) => ({ + ...(await importOriginal()), + availableParallelism: () => 16, +})); + +// Lets a test hold one source's superset ffmpeg until it says so. +const held = vi.hoisted(() => ({ + source: "", + reached: () => {}, + release: Promise.resolve(), +})); + +vi.mock("../utils/runFfmpeg.js", async (importOriginal) => { + const real = await importOriginal(); + return { + ...real, + runFfmpeg: vi.fn(async (args: string[], opts?: Parameters[1]) => { + if (args.includes(held.source) && args.some((arg) => arg.includes("__superset-"))) { + held.reached(); + await held.release; + } + return real.runFfmpeg(args, opts); + }), + }; +}); + +const { extractAllVideoFrames, extractVideoFramesRange } = await import("./videoFrameExtractor.js"); +const { runFfmpeg } = await import("../utils/runFfmpeg.js"); + +const HAS_FFMPEG = spawnSync("ffmpeg", ["-version"]).status === 0; + +describe.skipIf(!HAS_FFMPEG)("superset extraction in concurrent renders", () => { + const dir = mkdtempSync(join(tmpdir(), "hf-superset-isolation-")); + afterAll(() => rmSync(dir, { recursive: true, force: true })); + + async function synth(name: string, lavfi: string): Promise { + const src = join(dir, name); + const made = await runFfmpeg([ + ...["-y", "-v", "error", "-f", "lavfi", "-i", lavfi], + ...["-c:v", "libx264", "-preset", "ultrafast", "-pix_fmt", "yuv420p", src], + ]); + expect(made.success, made.stderr).toBe(true); + return src; + } + + // Two overlapping, frame-aligned trims of one source: a render extracts them as one superset group. + const overlappingTrims = (prefix: string, src: string): VideoElement[] => + [0, 2].map((mediaStart, i) => ({ + ...{ id: `${prefix}-${i}`, src, start: 0, end: 4, mediaStart }, + ...{ loop: false, hasAudio: false }, + })); + + const render = (videos: VideoElement[], out: string, cacheDir: string) => + extractAllVideoFrames(videos, dir, { fps: 30, outputDir: join(dir, out) }, undefined, { + extractCacheDir: cacheDir, + }); + + const supersetFolders = (root: string): string[] => + readdirSync(root).filter((name) => name.startsWith("__superset-")); + + async function expectOwnFrames( + result: Awaited>, + videos: VideoElement[], + ): Promise { + expect(result.errors).toEqual([]); + for (const video of videos) { + const extracted = result.extracted.find((item) => item.videoId === video.id); + expect(extracted?.totalFrames).toBe(120); + const direct = await extractVideoFramesRange( + video.src, + `direct-${video.id}`, + video.mediaStart, + 4, + { fps: 30, outputDir: join(dir, "direct"), format: "jpg" }, + ); + for (const frame of [0, 60, 119]) { + const own = readFileSync(direct.framePaths.get(frame)!); + const got = readFileSync(extracted!.framePaths.get(frame)!); + expect(got.equals(own), `${video.id} frame ${frame}`).toBe(true); + } + } + } + + it("keeps each render's frames and folder its own while another runs in the same process", async () => { + const cacheDir = mkdtempSync(join(dir, "cache-")); + const first = overlappingTrims("first", await synth("first.mp4", "testsrc2=d=10:s=320x180")); + const second = overlappingTrims( + "second", + await synth("second.mp4", "mandelbrot=s=320x180,trim=duration=10"), + ); + + let release = () => {}; + const reached = new Promise((resolve) => { + held.source = first[0]!.src; + held.reached = resolve; + held.release = new Promise((done) => (release = done)); + }); + const firstRender = render(first, "out-first", cacheDir); + try { + await reached; + const firstFolders = supersetFolders(cacheDir); + expect(firstFolders).toHaveLength(1); + + // The second render runs start to finish, cleanup included, while the first waits mid-extraction. + await expectOwnFrames(await render(second, "out-second", cacheDir), second); + expect(existsSync(join(cacheDir, firstFolders[0]!))).toBe(true); + } finally { + release(); + } + await expectOwnFrames(await firstRender, first); + expect(supersetFolders(cacheDir)).toEqual([]); + }, 120_000); +}); diff --git a/packages/engine/src/services/videoFrameExtractor.ts b/packages/engine/src/services/videoFrameExtractor.ts index 1758fe1a87..3bbc1b7414 100644 --- a/packages/engine/src/services/videoFrameExtractor.ts +++ b/packages/engine/src/services/videoFrameExtractor.ts @@ -93,10 +93,11 @@ import { unwrapTemplate } from "../utils/htmlTemplate.js"; import { mediaIdPathSegment } from "../utils/mediaPathSegment.js"; import { FRAME_FILENAME_PREFIX, + createPartialCacheEntryDir, + createPartialDir, gcExtractionCache, gcSweepDue, lookupCacheEntry, - partialCacheEntryDir, publishCacheEntry, readContentSha256, readKeyStat, @@ -2184,8 +2185,6 @@ function sliceSupersetMember( configuredFps: FpsInput, ): ExtractedFrames { const { work } = member.miss; - rmSync(outputDir, { recursive: true, force: true }); - mkdirSync(outputDir, { recursive: true }); // Sample-time correctness: member frame k uses superset frame // offset_i + k, so its source time is @@ -2204,7 +2203,11 @@ function sliceSupersetMember( linkOrCopyFrame(sourceFrame, join(outputDir, frameFileName(i + 1, work.format))); } - return extractedFramesFromDirectory(work, outputDir, work.videoPath, fps); + const sliced = extractedFramesFromDirectory(work, outputDir, work.videoPath, fps); + if (sliced.totalFrames !== frameCount) { + throw new Error(`superset slice holds ${sliced.totalFrames} frames, not ${frameCount}`); + } + return sliced; } export async function extractAllVideoFrames( @@ -2647,9 +2650,7 @@ export async function extractAllVideoFrames( return attempted.result; } - const partialDir = partialCacheEntryDir(cacheTarget.entry); - rmSync(partialDir, { recursive: true, force: true }); - mkdirSync(partialDir, { recursive: true }); + let partialDir = createPartialCacheEntryDir(cacheTarget.entry); const attempted = await runVideoExtractionWithRetry( () => extractVideoFramesRange( @@ -2668,12 +2669,12 @@ export async function extractAllVideoFrames( onRetry: () => { recordTransientRetries(1); rmSync(partialDir, { recursive: true, force: true }); - mkdirSync(partialDir, { recursive: true }); + partialDir = createPartialCacheEntryDir(cacheTarget.entry); }, }, ); const result = attempted.result; - const published = publishCacheEntry(cacheTarget.entry, partialDir); + const published = publishCacheEntry(cacheTarget.entry, partialDir, result.totalFrames); if (!published.published) { breakdown.cachePublishFailures += 1; return { ...result, ownedByLookup: false }; @@ -2791,18 +2792,15 @@ export async function extractAllVideoFrames( const { miss } = member; const { work, cacheTarget } = miss; if (!cacheTarget) { - return sliceSupersetMember( - member, - superset, - join(options.outputDir, mediaIdPathSegment(work.video.id)), - fps, - configuredFps, - ); + const outputDir = join(options.outputDir, mediaIdPathSegment(work.video.id)); + rmSync(outputDir, { recursive: true, force: true }); + mkdirSync(outputDir, { recursive: true }); + return sliceSupersetMember(member, superset, outputDir, fps, configuredFps); } - const partialDir = partialCacheEntryDir(cacheTarget.entry); + const partialDir = createPartialCacheEntryDir(cacheTarget.entry); const sliced = sliceSupersetMember(member, superset, partialDir, fps, configuredFps); - const published = publishCacheEntry(cacheTarget.entry, partialDir); + const published = publishCacheEntry(cacheTarget.entry, partialDir, sliced.totalFrames); if (!published.published) { breakdown.cachePublishFailures += 1; return { ...sliced, ownedByLookup: false }; @@ -2819,15 +2817,11 @@ export async function extractAllVideoFrames( // members link into partial dirs under cacheRootDir, which is commonly a // different mount than the render's outputDir — extracting the superset // next to the cache keeps linkSync viable there (the EXDEV copyFileSync - // fallback would silently multiply disk usage per member). The - // `.partial-` name puts crashed leftovers under the GC's aged-partial - // sweep. - const tempDir = cacheRootDir - ? join(cacheRootDir, `${group.groupId}.partial-${process.pid}`) - : join(options.outputDir, group.groupId); - + // fallback would silently multiply disk usage per member). Renders share the + // cache root and group ids, so each gets a new folder of its own. + let tempDir: string | undefined; try { - rmSync(tempDir, { recursive: true, force: true }); + tempDir = createPartialDir(cacheRootDir ?? options.outputDir, group.groupId); // A long union can fail even when each shorter // member range succeeds. Do not retry the optimization itself; preserve // the established grouped→direct fallback and apply bounded retries only @@ -2871,7 +2865,7 @@ export async function extractAllVideoFrames( ); return fallback; } finally { - rmSync(tempDir, { recursive: true, force: true }); + if (tempDir) rmSync(tempDir, { recursive: true, force: true }); } } diff --git a/scripts/package-size-budgets.json b/scripts/package-size-budgets.json index defdc32846..21e629d6f1 100644 --- a/scripts/package-size-budgets.json +++ b/scripts/package-size-budgets.json @@ -12,7 +12,7 @@ "unpacked": 4374000 }, "@hyperframes/engine": { - "packed": 500000, + "packed": 510000, "unpacked": 1742862 }, "@hyperframes/gcp-cloud-run": {