perf(engine): extraction cache on by default with atomic publish and LRU gc (#1901)

* perf(engine): extraction cache on by default with atomic publish and LRU gc

Warm re-renders now skip source-video frame extraction entirely
(video_extract 400ms -> 13ms on a 4-video composition; outputs are
pixel-identical, PSNR inf). What made default-on safe:

- Atomic entry publish: frames extract into a unique .partial-<pid>-<uuid>
  dir, the completion sentinel is written there, and the dir is renamed
  into the final key atomically. Concurrent renders sharing a cache can
  duplicate work but can never serve a torn entry (previously documented
  as single-writer only).
- Size-capped LRU gc: best-effort sweep after extraction evicts
  oldest-used entries past a 2 GiB default budget
  (HYPERFRAMES_EXTRACT_CACHE_MAX_MB) and clears crashed writers'
  partials. Entries younger than 60 min are never evicted so live
  renders keep their frames.
- Default cache dir: <tmpdir>/hyperframes-extract-cache-<uid>. Opt out
  with HYPERFRAMES_EXTRACT_CACHE_DIR=off (or none/false/0); a
  non-writable dir degrades to uncached with a single warning instead
  of failing the render.

* fix(engine): harden extraction cache publish and surface cache ops signals

Review hardening for the default-on extraction cache:

- Bypass the cache for HDR-converted intermediates: the key snapshot
  describes the original source, so publishing converted frames under
  it would poison later plain-SDR renders of the same trim. (The
  follow-up transform-keyed change re-enables caching for these.)
- publishCacheEntry TOCTOU: adopt a concurrent writer's completed
  entry both before removing an apparently-stale dir and after a
  failed retry rename, so a winner's publish is never destroyed or
  reported as a failure.
- Observability for the failure paths: cachePublishFailures,
  cacheGcEvictions, cacheGcBytesFreed, and cacheAgedPartialsCleared on
  ExtractionPhaseBreakdown; gcExtractionCache now returns sweep stats.

* fix(engine): sweep superseded cache generations in gc

After a SCHEMA_PREFIX bump, old-generation entries (hfcache-v2-*)
no longer matched the sweep's prefix filter and would orphan their
disk forever. The gc now matches any hfcache-v* generation; superseded
entries never receive sentinel touches, so the LRU evicts them first.

packages/engine/src/config.test.ts

@@ -1,4 +1,6 @@
 import { describe, it, expect, beforeEach, afterEach } from "vitest";
+import { join } from "node:path";
+import { tmpdir } from "node:os";
 import { resolveConfig, DEFAULT_CONFIG, scaleProtocolTimeoutForComposition } from "./config.js";
 import { isLowMemorySystem } from "./services/systemMemory.js";
 
@@ -6,10 +8,15 @@ describe("resolveConfig", () => {
   const savedEnv = new Map<string, string | undefined>();
 
   function setEnv(key: string, value: string) {
-    savedEnv.set(key, process.env[key]);
+    if (!savedEnv.has(key)) savedEnv.set(key, process.env[key]);
     process.env[key] = value;
   }
 
+  function unsetEnv(key: string) {
+    if (!savedEnv.has(key)) savedEnv.set(key, process.env[key]);
+    delete process.env[key];
+  }
+
   beforeEach(() => {
     savedEnv.clear();
   });
@@ -182,6 +189,38 @@ describe("resolveConfig", () => {
     });
   });
 
+  describe("extraction cache env", () => {
+    it("defaults the extract cache directory to tmpdir plus uid when env is unset", () => {
+      unsetEnv("HYPERFRAMES_EXTRACT_CACHE_DIR");
+
+      const config = resolveConfig();
+
+      expect(config.extractCacheDir).toBe(
+        join(tmpdir(), `hyperframes-extract-cache-${process.getuid?.() ?? "u"}`),
+      );
+    });
+
+    it("disables the extract cache when env is an opt-out token", () => {
+      for (const value of ["off", "none", "false", "0", " OFF "]) {
+        setEnv("HYPERFRAMES_EXTRACT_CACHE_DIR", value);
+
+        expect(resolveConfig().extractCacheDir).toBeUndefined();
+      }
+    });
+
+    it("uses an explicit extract cache path from env", () => {
+      setEnv("HYPERFRAMES_EXTRACT_CACHE_DIR", "/tmp/custom-hf-cache");
+
+      expect(resolveConfig().extractCacheDir).toBe("/tmp/custom-hf-cache");
+    });
+
+    it("converts HYPERFRAMES_EXTRACT_CACHE_MAX_MB to bytes", () => {
+      setEnv("HYPERFRAMES_EXTRACT_CACHE_MAX_MB", "512");
+
+      expect(resolveConfig().extractCacheMaxBytes).toBe(512 * 1024 ** 2);
+    });
+  });
+
   describe("lowMemoryMode", () => {
     it("forces on for truthy PRODUCER_LOW_MEMORY_MODE values", () => {
       setEnv("PRODUCER_LOW_MEMORY_MODE", "true");

packages/engine/src/config.ts

@@ -6,6 +6,8 @@
  * fallbacks for backward compatibility during migration.
  */
 
+import { tmpdir } from "node:os";
+import { join } from "node:path";
 import {
   getSystemTotalMb,
   isLowMemorySystem,
@@ -178,16 +180,19 @@ export interface EngineConfig {
   /**
    * Directory where the content-addressed extraction cache persists frame
    * bundles keyed on (path, mtime, size, mediaStart, duration, fps, format).
-   * Undefined disables caching — extraction runs into the render's workDir
-   * and cleanup removes it when the render ends, preserving the pre-cache
-   * behaviour.
+   * Defaults on under the OS temp directory:
+   * `<tmpdir>/hyperframes-extract-cache-<uid>`.
    *
-   * **Single-writer.** The cache is not safe for concurrent renders pointing
-   * at the same directory. A `.hf-complete` sentinel prevents another render
-   * from serving an entry that hasn't finished extracting, but individual
-   * frame files are written non-atomically — a second render reading during
-   * the write window can observe a truncated frame. Give each concurrent
-   * render pipeline its own `extractCacheDir`, or gate with an external mutex.
+   * New entries publish atomically: frames are extracted into a unique
+   * partial directory, the `.hf-complete` sentinel is written there, and the
+   * partial directory is renamed into the final key directory. Concurrent
+   * renders against the same cache are safe; at worst, two renders duplicate
+   * ffmpeg work and one rehydrates from the winner.
+   *
+   * Set `HYPERFRAMES_EXTRACT_CACHE_DIR` to a path to override the default, or
+   * to `off`, `none`, `false`, or `0` to disable caching for the process.
+   * When disabled, extraction runs into the render's workDir and cleanup
+   * removes it when the render ends, preserving the pre-cache behaviour.
    *
    * **Network filesystems.** `mtime` resolution on NFS/SMB mounts can be
    * coarser than expected (seconds rather than nanoseconds), which may
@@ -197,6 +202,15 @@ export interface EngineConfig {
    * Env fallback: `HYPERFRAMES_EXTRACT_CACHE_DIR`.
    */
   extractCacheDir?: string;
+  /**
+   * Soft disk budget for `extractCacheDir`, in bytes. The renderer runs a
+   * best-effort LRU sweep after extraction and evicts oldest sentineled
+   * entries until the cache is under this cap, while protecting young entries
+   * that may belong to live renders.
+   *
+   * Env fallback: `HYPERFRAMES_EXTRACT_CACHE_MAX_MB` (megabytes).
+   */
+  extractCacheMaxBytes: number;
 
   // ── Debug ────────────────────────────────────────────────────────────
   debug: boolean;
@@ -249,6 +263,8 @@ export const DEFAULT_CONFIG: EngineConfig = {
 
   verifyRuntime: true,
 
+  extractCacheMaxBytes: 2 * 1024 ** 3,
+
   debug: false,
 };
 
@@ -353,6 +369,23 @@ export function resolveConfig(overrides?: Partial<EngineConfig>): EngineConfig {
     const raw = env("HF_STATIC_DEDUP")?.trim().toLowerCase();
     return !(raw === "false" || raw === "off" || raw === "0");
   };
+  const resolveExtractCacheDir = (): string | undefined => {
+    const raw = env("HYPERFRAMES_EXTRACT_CACHE_DIR");
+    if (raw === undefined) {
+      return join(tmpdir(), `hyperframes-extract-cache-${process.getuid?.() ?? "u"}`);
+    }
+    const trimmed = raw.trim();
+    const normalized = trimmed.toLowerCase();
+    if (
+      normalized === "off" ||
+      normalized === "none" ||
+      normalized === "false" ||
+      normalized === "0"
+    ) {
+      return undefined;
+    }
+    return raw;
+  };
 
   // Env-var layer (backward compat)
   const fromEnv: Partial<EngineConfig> = {
@@ -446,7 +479,10 @@ export function resolveConfig(overrides?: Partial<EngineConfig>): EngineConfig {
     verifyRuntime: env("PRODUCER_VERIFY_HYPERFRAME_RUNTIME") !== "false",
     runtimeManifestPath: env("PRODUCER_HYPERFRAME_MANIFEST_PATH"),
 
-    extractCacheDir: env("HYPERFRAMES_EXTRACT_CACHE_DIR"),
+    extractCacheDir: resolveExtractCacheDir(),
+    extractCacheMaxBytes:
+      envNum("HYPERFRAMES_EXTRACT_CACHE_MAX_MB", DEFAULT_CONFIG.extractCacheMaxBytes / 1024 ** 2) *
+      1024 ** 2,
   };
 
   // Remove undefined values so they don't override defaults

packages/engine/src/services/extractionCache.test.ts

@@ -1,5 +1,14 @@
 import { afterEach, beforeEach, describe, expect, it } from "vitest";
-import { existsSync, mkdirSync, mkdtempSync, rmSync, writeFileSync } from "node:fs";
+import {
+  existsSync,
+  mkdirSync,
+  mkdtempSync,
+  readdirSync,
+  rmSync,
+  statSync,
+  utimesSync,
+  writeFileSync,
+} from "node:fs";
 import { join } from "node:path";
 import { tmpdir } from "node:os";
 
@@ -10,8 +19,11 @@ import {
   cacheEntryDirName,
   computeCacheKey,
   ensureCacheEntryDir,
+  gcExtractionCache,
   lookupCacheEntry,
   markCacheEntryComplete,
+  partialCacheEntryDir,
+  publishCacheEntry,
   readKeyStat,
   type CacheKeyInput,
 } from "./extractionCache.js";
@@ -31,6 +43,25 @@ const keyFor = (videoPath: string, overrides: Partial<CacheKeyInput> = {}): Cach
   };
 };
 
+function makeCacheRoot(): { tmpRoot: string; sourceFile: string } {
+  const tmpRoot = mkdtempSync(join(tmpdir(), "hf-extract-cache-test-"));
+  const sourceFile = join(tmpRoot, "clip.mp4");
+  writeFileSync(sourceFile, "fake-video-bytes", "utf-8");
+  return { tmpRoot, sourceFile };
+}
+
+function removeCacheRoot(tmpRoot: string): void {
+  if (existsSync(tmpRoot)) rmSync(tmpRoot, { recursive: true, force: true });
+}
+
+/** Create and populate a partial dir for `entry` with one frame file. */
+function seedPartialDir(entry: { dir: string; keyHash: string }, frameContent: string): string {
+  const partialDir = partialCacheEntryDir(entry);
+  mkdirSync(partialDir, { recursive: true });
+  writeFileSync(join(partialDir, "frame_00001.jpg"), frameContent, "utf-8");
+  return partialDir;
+}
+
 describe("extractionCache constants", () => {
   it("exposes the v2 schema prefix", () => {
     expect(SCHEMA_PREFIX).toBe("hfcache-v3-");
@@ -50,13 +81,11 @@ describe("computeCacheKey", () => {
   let sourceFile: string;
 
   beforeEach(() => {
-    tmpRoot = mkdtempSync(join(tmpdir(), "hf-extract-cache-test-"));
-    sourceFile = join(tmpRoot, "clip.mp4");
-    writeFileSync(sourceFile, "fake-video-bytes", "utf-8");
+    ({ tmpRoot, sourceFile } = makeCacheRoot());
   });
 
   afterEach(() => {
-    if (existsSync(tmpRoot)) rmSync(tmpRoot, { recursive: true, force: true });
+    removeCacheRoot(tmpRoot);
   });
 
   const base = (videoPath: string): CacheKeyInput => keyFor(videoPath);
@@ -148,13 +177,11 @@ describe("lookupCacheEntry / markCacheEntryComplete", () => {
   let sourceFile: string;
 
   beforeEach(() => {
-    tmpRoot = mkdtempSync(join(tmpdir(), "hf-extract-cache-test-"));
-    sourceFile = join(tmpRoot, "clip.mp4");
-    writeFileSync(sourceFile, "fake-video-bytes", "utf-8");
+    ({ tmpRoot, sourceFile } = makeCacheRoot());
   });
 
   afterEach(() => {
-    if (existsSync(tmpRoot)) rmSync(tmpRoot, { recursive: true, force: true });
+    removeCacheRoot(tmpRoot);
   });
 
   const base = (videoPath: string): CacheKeyInput => keyFor(videoPath);
@@ -197,3 +224,144 @@ describe("lookupCacheEntry / markCacheEntryComplete", () => {
     expect(a.entry.dir).toBe(b.entry.dir);
   });
 });
+
+describe("publishCacheEntry", () => {
+  let tmpRoot: string;
+  let sourceFile: string;
+
+  beforeEach(() => {
+    ({ tmpRoot, sourceFile } = makeCacheRoot());
+  });
+
+  afterEach(() => {
+    removeCacheRoot(tmpRoot);
+  });
+
+  function entry() {
+    return lookupCacheEntry(tmpRoot, keyFor(sourceFile)).entry;
+  }
+
+  it("publishes a partial directory atomically with the complete sentinel inside", () => {
+    const cacheEntry = entry();
+    const partialDir = seedPartialDir(cacheEntry, "frame");
+
+    const result = publishCacheEntry(cacheEntry, partialDir);
+
+    expect(result).toEqual({ dir: cacheEntry.dir, published: true });
+    expect(existsSync(partialDir)).toBe(false);
+    expect(existsSync(join(cacheEntry.dir, "frame_00001.jpg"))).toBe(true);
+    expect(existsSync(join(cacheEntry.dir, COMPLETE_SENTINEL))).toBe(true);
+  });
+
+  it("serves a complete winner when another writer publishes the same entry first", () => {
+    const cacheEntry = entry();
+    mkdirSync(cacheEntry.dir, { recursive: true });
+    writeFileSync(join(cacheEntry.dir, "frame_00001.jpg"), "winner", "utf-8");
+    markCacheEntryComplete(cacheEntry);
+
+    const partialDir = seedPartialDir(cacheEntry, "loser");
+
+    const result = publishCacheEntry(cacheEntry, partialDir);
+
+    expect(result).toEqual({ dir: cacheEntry.dir, published: true });
+    expect(existsSync(partialDir)).toBe(false);
+    expect(existsSync(join(cacheEntry.dir, COMPLETE_SENTINEL))).toBe(true);
+    expect(statSync(join(cacheEntry.dir, "frame_00001.jpg")).size).toBe("winner".length);
+  });
+
+  it("replaces a stale unsentineled final directory and retries publish once", () => {
+    const cacheEntry = entry();
+    mkdirSync(cacheEntry.dir, { recursive: true });
+    writeFileSync(join(cacheEntry.dir, "frame_00001.jpg"), "stale", "utf-8");
+
+    const partialDir = seedPartialDir(cacheEntry, "fresh");
+
+    const result = publishCacheEntry(cacheEntry, partialDir);
+
+    expect(result).toEqual({ dir: cacheEntry.dir, published: true });
+    expect(existsSync(partialDir)).toBe(false);
+    expect(statSync(join(cacheEntry.dir, "frame_00001.jpg")).size).toBe("fresh".length);
+    expect(existsSync(join(cacheEntry.dir, COMPLETE_SENTINEL))).toBe(true);
+  });
+});
+
+describe("gcExtractionCache", () => {
+  let tmpRoot: string;
+
+  beforeEach(() => {
+    tmpRoot = mkdtempSync(join(tmpdir(), "hf-extract-cache-gc-test-"));
+  });
+
+  afterEach(() => {
+    if (existsSync(tmpRoot)) rmSync(tmpRoot, { recursive: true, force: true });
+  });
+
+  function makeEntry(name: string, bytes: number, ageMs: number): string {
+    const dir = join(tmpRoot, `${SCHEMA_PREFIX}${name}`);
+    mkdirSync(dir, { recursive: true });
+    writeFileSync(join(dir, "frame_00001.jpg"), "x".repeat(bytes), "utf-8");
+    markCacheEntryComplete({ dir, keyHash: name.padEnd(64, "0") });
+    const when = new Date(Date.now() - ageMs);
+    utimesSync(join(dir, COMPLETE_SENTINEL), when, when);
+    return dir;
+  }
+
+  it("evicts superseded-generation entries (hfcache-v2-*) under the size cap", () => {
+    const oldGen = join(tmpRoot, "hfcache-v2-0123456789abcdef");
+    mkdirSync(oldGen, { recursive: true });
+    writeFileSync(join(oldGen, "frame_00001.jpg"), "x".repeat(2048), "utf-8");
+    writeFileSync(join(oldGen, ".hf-complete"), "", "utf-8");
+    const aged = new Date(Date.now() - 2 * 60 * 60 * 1000);
+    utimesSync(join(oldGen, ".hf-complete"), aged, aged);
+    utimesSync(oldGen, aged, aged);
+
+    const stats = gcExtractionCache(tmpRoot, { maxBytes: 1024, minAgeMs: 60 * 60 * 1000 });
+
+    expect(existsSync(oldGen)).toBe(false);
+    expect(stats.evictedEntries).toBe(1);
+  });
+
+  it("evicts oldest complete entries first until under maxBytes while respecting minAge", () => {
+    const oldest = makeEntry("oldest", 60, 120_000);
+    const middle = makeEntry("middle", 60, 90_000);
+    const young = makeEntry("young", 60, 1_000);
+
+    gcExtractionCache(tmpRoot, { maxBytes: 100, minAgeMs: 60_000 });
+
+    expect(existsSync(oldest)).toBe(false);
+    expect(existsSync(middle)).toBe(false);
+    expect(existsSync(young)).toBe(true);
+  });
+
+  it("removes aged partial directories", () => {
+    const agedPartial = join(tmpRoot, `${SCHEMA_PREFIX}abc.partial-1234-deadbeef`);
+    const freshPartial = join(tmpRoot, `${SCHEMA_PREFIX}def.partial-1234-feedface`);
+    mkdirSync(agedPartial, { recursive: true });
+    mkdirSync(freshPartial, { recursive: true });
+    const old = new Date(Date.now() - 120_000);
+    utimesSync(agedPartial, old, old);
+
+    gcExtractionCache(tmpRoot, { maxBytes: 1_000_000, minAgeMs: 60_000 });
+
+    expect(existsSync(agedPartial)).toBe(false);
+    expect(existsSync(freshPartial)).toBe(true);
+  });
+
+  it("ignores non-cache-prefix directories under the same root", () => {
+    const animatedGif = join(tmpRoot, "animated-gif");
+    mkdirSync(animatedGif, { recursive: true });
+    writeFileSync(join(animatedGif, "frame.png"), "keep", "utf-8");
+    makeEntry("old", 200, 120_000);
+
+    gcExtractionCache(tmpRoot, { maxBytes: 1, minAgeMs: 60_000 });
+
+    expect(existsSync(animatedGif)).toBe(true);
+    expect(readdirSync(animatedGif)).toEqual(["frame.png"]);
+  });
+
+  it("never throws when the cache root is missing", () => {
+    expect(() =>
+      gcExtractionCache(join(tmpRoot, "missing"), { maxBytes: 1, minAgeMs: 60_000 }),
+    ).not.toThrow();
+  });
+});

packages/engine/src/services/extractionCache.ts

@@ -15,10 +15,14 @@
  * - Cache entries live under `<rootDir>/<SCHEMA_PREFIX><key[0..16]>/` so
  *   `ls` output and tracing logs stay short. Truncation to 16 hex chars
  *   leaves 64 bits of entropy — collision risk at cache scale is negligible.
- * - A completed entry is marked by writing the `.hf-complete` sentinel file
- *   after all frames are on disk. A dir without the sentinel is treated as
- *   absent (stale/abandoned) and re-extracted into a fresh key (the old dir
- *   is left for external gc — the cache owns keys, not deletion policy).
+ * - Frames are extracted into a unique `<entry>.partial-<pid>-<uuid>/` dir.
+ *   Once all frames are written, the partial dir receives the `.hf-complete`
+ *   sentinel and is atomically renamed to the final key dir. Concurrent
+ *   same-key writers may duplicate ffmpeg work, but readers only ever serve
+ *   complete entries.
+ * - The sentinel mtime is touched on hits and used as the cache's LRU clock.
+ *   `gcExtractionCache` evicts by that mtime and also clears old partial dirs
+ *   left behind by crashed writers.
  *
  * ### Versioning
  *
@@ -27,9 +31,18 @@
  * become inert and can be gc'd by the caller.
  */
 
-import { createHash } from "node:crypto";
-import { mkdirSync, readdirSync, statSync, writeFileSync } from "node:fs";
-import { existsSync } from "node:fs";
+import { createHash, randomUUID } from "node:crypto";
+import {
+  existsSync,
+  lstatSync,
+  mkdirSync,
+  readdirSync,
+  renameSync,
+  rmSync,
+  statSync,
+  utimesSync,
+  writeFileSync,
+} from "node:fs";
 import { join } from "node:path";
 import type { VideoMetadata } from "../utils/ffprobe.js";
 
@@ -82,12 +95,17 @@ export interface CacheEntry {
 
 export interface CacheLookup {
   /** Cache entry information — always returned even on a miss so the caller
-   *  can extract directly into `dir` then call `markCacheEntryComplete`. */
+   *  can derive a partial dir and publish it after extraction. */
   entry: CacheEntry;
   /** True when the entry exists AND carries the completion sentinel. */
   hit: boolean;
 }
 
+export interface CachePublishResult {
+  dir: string;
+  published: boolean;
+}
+
 /**
  * Read `(mtimeMs, size)` for a path. Returns `null` if the file is missing —
  * callers should skip the cache path for that entry so the extractor surfaces
@@ -134,9 +152,9 @@ export function cacheEntryDirName(keyHash: string): string {
 
 /**
  * Look up a cache entry by key input. Returns the resolved entry path plus a
- * `hit` flag. On miss, callers should extract frames into `entry.dir`
- * (after calling `ensureCacheEntryDir`) and then call `markCacheEntryComplete`
- * once the extraction succeeds.
+ * `hit` flag. On miss, callers should extract frames into a
+ * `partialCacheEntryDir(entry)` directory and publish it with
+ * `publishCacheEntry` once extraction succeeds.
  */
 export function lookupCacheEntry(rootDir: string, input: CacheKeyInput): CacheLookup {
   const keyHash = computeCacheKey(input);
@@ -154,20 +172,249 @@ export function ensureCacheEntryDir(entry: CacheEntry): void {
 }
 
 /**
- * Write the completion sentinel so subsequent lookups treat this entry as a
- * hit. Must be called only after every frame has been written.
+ * Unique render-owned directory used to populate a cache entry before the
+ * atomic publish rename.
+ */
+export function partialCacheEntryDir(entry: CacheEntry): string {
+  return `${entry.dir}.partial-${process.pid}-${randomUUID().slice(0, 8)}`;
+}
+
+function isTargetExistsRenameError(err: unknown): boolean {
+  const code = (err as NodeJS.ErrnoException).code;
+  return code === "EEXIST" || code === "ENOTEMPTY" || code === "EPERM";
+}
+
+/**
+ * Publish an extracted partial directory as the final cache entry.
  *
- * Concurrency: lookup→populate→mark is non-atomic. Two concurrent renders of
- * the same key may both miss, both extract into the same dir, and the later
- * writer's frames win. The result is correct (identical inputs yield identical
- * frames) but wasteful. Acceptable for a single-process render pipeline;
- * anyone running concurrent renders against a shared cache root should front
- * it with an external lock.
+ * Same-filesystem directory rename is atomic: readers either see no entry or
+ * a complete sentineled entry. When another writer wins the race, the caller
+ * should rehydrate from the final dir. If publish cannot complete safely, the
+ * partial remains render-owned and must be cleaned up by the render cleanup.
+ */
+/**
+ * If a concurrent writer's completed entry is visible, discard our partial
+ * and serve theirs. Identical keys produce identical frames, so adopting the
+ * winner is always correct. Returns null when no winner is present.
+ */
+function adoptPublishedWinner(entry: CacheEntry, partialDir: string): CachePublishResult | null {
+  if (!existsSync(join(entry.dir, COMPLETE_SENTINEL))) return null;
+  removeDir(partialDir);
+  return { dir: entry.dir, published: true };
+}
+
+export function publishCacheEntry(entry: CacheEntry, partialDir: string): CachePublishResult {
+  try {
+    writeFileSync(join(partialDir, COMPLETE_SENTINEL), "", "utf-8");
+  } catch {
+    return { dir: partialDir, published: false };
+  }
+
+  try {
+    renameSync(partialDir, entry.dir);
+    return { dir: entry.dir, published: true };
+  } catch (err) {
+    if (!isTargetExistsRenameError(err)) return { dir: partialDir, published: false };
+  }
+
+  const winner = adoptPublishedWinner(entry, partialDir);
+  if (winner) return winner;
+
+  try {
+    rmSync(entry.dir, { recursive: true, force: true });
+  } catch {
+    return { dir: partialDir, published: false };
+  }
+
+  try {
+    renameSync(partialDir, entry.dir);
+    return { dir: entry.dir, published: true };
+  } catch {
+    // TOCTOU: a concurrent writer can publish between the winner check, the
+    // rm above, and this retry. Re-run the adopt check so a winner that
+    // landed inside that window is served rather than reported as a failure.
+    return adoptPublishedWinner(entry, partialDir) ?? { dir: partialDir, published: false };
+  }
+}
+
+/**
+ * Update the LRU clock for a complete cache entry. Misses and filesystem
+ * races are harmless: the caller can still use the entry it already found.
+ */
+export function touchCacheEntry(entry: CacheEntry): void {
+  try {
+    const now = new Date();
+    utimesSync(join(entry.dir, COMPLETE_SENTINEL), now, now);
+  } catch {
+    // Best effort LRU touch.
+  }
+}
+
+/**
+ * Write the completion sentinel so subsequent lookups treat this entry as a
+ * hit. Must be called only after every frame has been written. The extractor
+ * now publishes new entries via `publishCacheEntry`; this helper remains
+ * exported for tests and legacy callers that materialize entries directly.
+ *
+ * Concurrency: direct mark is non-atomic and should not be used for shared
+ * writer paths. `publishCacheEntry` writes the sentinel inside a partial dir
+ * and atomically renames it into place, so concurrent writers duplicate work
+ * but never serve torn frames.
  */
 export function markCacheEntryComplete(entry: CacheEntry): void {
   writeFileSync(join(entry.dir, COMPLETE_SENTINEL), "", "utf-8");
 }
 
+/** Any generation of this cache's entries ("hfcache-v*"), current or superseded. */
+const CACHE_GENERATION_PREFIX = "hfcache-v";
+
+function isCacheLikeChild(name: string): boolean {
+  // Match every schema generation, not just SCHEMA_PREFIX: after a schema
+  // bump, superseded-version entries would otherwise be invisible to the
+  // sweep and orphan their disk forever. Old-generation entries never get
+  // sentinel touches, so the LRU evicts them first.
+  return name.startsWith(CACHE_GENERATION_PREFIX) || name.includes(".partial-");
+}
+
+function isPartialChild(name: string): boolean {
+  return name.includes(".partial-");
+}
+
+function directorySizeBytes(path: string): number {
+  try {
+    const stat = lstatSync(path);
+    if (!stat.isDirectory()) return stat.size;
+  } catch {
+    return 0;
+  }
+
+  let total = 0;
+  let children: string[];
+  try {
+    children = readdirSync(path);
+  } catch {
+    return 0;
+  }
+
+  for (const child of children) {
+    const childPath = join(path, child);
+    try {
+      const stat = lstatSync(childPath);
+      if (stat.isDirectory()) {
+        total += directorySizeBytes(childPath);
+      } else {
+        total += stat.size;
+      }
+    } catch {
+      // Ignore entries deleted or made unreadable during the sweep.
+    }
+  }
+  return total;
+}
+
+function removeDir(path: string): void {
+  try {
+    rmSync(path, { recursive: true, force: true });
+  } catch {
+    // Cache GC is opportunistic; one bad entry must not abort the sweep.
+  }
+}
+
+interface GcEntry {
+  dir: string;
+  size: number;
+  lastUseMs: number;
+  ageMs: number;
+}
+
+/**
+ * Stat one cache-looking child for the GC sweep. Aged partial dirs (crashed
+ * writers) are removed immediately and yield `null`; entries that disappear
+ * or fail to stat mid-sweep also yield `null`.
+ */
+function collectGcEntry(
+  dir: string,
+  name: string,
+  now: number,
+  minAgeMs: number,
+  stats: GcStats,
+): GcEntry | null {
+  try {
+    const dirStat = statSync(dir);
+    if (isPartialChild(name) && now - dirStat.mtimeMs >= minAgeMs) {
+      removeDir(dir);
+      stats.agedPartialsRemoved += 1;
+      return null;
+    }
+
+    let lastUseMs = dirStat.mtimeMs;
+    try {
+      lastUseMs = statSync(join(dir, COMPLETE_SENTINEL)).mtimeMs;
+    } catch {
+      // Unsentineled entries use directory mtime as a stale-entry clock.
+    }
+
+    return { dir, size: directorySizeBytes(dir), lastUseMs, ageMs: now - lastUseMs };
+  } catch {
+    return null;
+  }
+}
+
+export interface GcStats {
+  /** Complete entries evicted by the LRU size sweep. */
+  evictedEntries: number;
+  /** Bytes reclaimed by evicted entries. */
+  evictedBytes: number;
+  /** Aged `.partial-*` dirs (crashed writers) removed. */
+  agedPartialsRemoved: number;
+}
+
+/**
+ * Opportunistic size-capped LRU cleanup for extracted video frames.
+ *
+ * Scans only direct cache-looking children and never throws. The age guard is
+ * a liveness heuristic, not a lock. Returns counts so the caller can surface
+ * eviction pressure in render observability.
+ */
+export function gcExtractionCache(
+  rootDir: string,
+  opts: { maxBytes: number; minAgeMs: number },
+): GcStats {
+  const stats: GcStats = { evictedEntries: 0, evictedBytes: 0, agedPartialsRemoved: 0 };
+  try {
+    const now = Date.now();
+    const entries: GcEntry[] = [];
+    for (const child of readdirSync(rootDir, { withFileTypes: true })) {
+      if (!child.isDirectory() || !isCacheLikeChild(child.name)) continue;
+      const entry = collectGcEntry(
+        join(rootDir, child.name),
+        child.name,
+        now,
+        opts.minAgeMs,
+        stats,
+      );
+      if (entry) entries.push(entry);
+    }
+
+    let totalBytes = entries.reduce((sum, e) => sum + e.size, 0);
+    if (totalBytes <= opts.maxBytes) return stats;
+
+    entries.sort((a, b) => a.lastUseMs - b.lastUseMs);
+    for (const entry of entries) {
+      // ponytail: age-based liveness guard, not a lock; a render longer than minAge with a full cache could lose entries mid-read - acceptable, next render re-extracts.
+      if (entry.ageMs < opts.minAgeMs) continue;
+      removeDir(entry.dir);
+      stats.evictedEntries += 1;
+      stats.evictedBytes += entry.size;
+      totalBytes -= entry.size;
+      if (totalBytes <= opts.maxBytes) break;
+    }
+  } catch {
+    // Missing root or unreadable cache: no cleanup this sweep.
+  }
+  return stats;
+}
+
 /**
  * Rebuild the in-memory frame index for a cached entry. Called on cache hits
  * so the extractor's caller receives the same `ExtractedFrames` shape it

packages/engine/src/services/videoFrameExtractor.test.ts

@@ -1,5 +1,14 @@
-import { afterAll, beforeAll, describe, expect, it } from "vitest";
-import { existsSync, mkdirSync, mkdtempSync, readdirSync, rmSync, writeFileSync } from "node:fs";
+import { afterAll, beforeAll, describe, expect, it, vi } from "vitest";
+import {
+  existsSync,
+  mkdirSync,
+  mkdtempSync,
+  readdirSync,
+  rmSync,
+  statSync,
+  utimesSync,
+  writeFileSync,
+} from "node:fs";
 import { join } from "node:path";
 import { tmpdir } from "node:os";
 import { spawnSync } from "node:child_process";
@@ -16,9 +25,11 @@ import {
   analyzeClipMediaFit,
   type VideoElement,
   type ExtractedFrames,
+  type ExtractionResult,
 } from "./videoFrameExtractor.js";
 import { extractVideoMetadata, type VideoMetadata } from "../utils/ffprobe.js";
 import { runFfmpeg } from "../utils/runFfmpeg.js";
+import { COMPLETE_SENTINEL, SCHEMA_PREFIX } from "./extractionCache.js";
 
 // ffmpeg is not preinstalled on GitHub's ubuntu-24.04 runners. The producer
 // regression test at packages/producer/tests/vfr-screen-recording/ runs inside
@@ -812,12 +823,11 @@ describe.skipIf(!HAS_FFMPEG)("extractAllVideoFrames on a VFR source", () => {
     expect(result.phaseBreakdown.vfrPreflightMs).toBeGreaterThanOrEqual(0);
   }, 60_000);
 
-  it("reuses extracted frames on a warm cache hit", async () => {
-    const CACHE_DIR = mkdtempSync(join(tmpdir(), "hf-extract-cache-test-"));
-    const SRC = join(FIXTURE_DIR, "cache-src.mp4");
-
-    // Synthesize a clean CFR SDR clip — keeps VFR preflight count at zero so
-    // the cache key is stable across the two runs.
+  // Shared fixture helpers for the cache tests below. All synthesize clean
+  // CFR SDR clips — keeps VFR preflight count at zero so cache keys are
+  // stable across runs within a test.
+  async function synthCfrClip(name: string, durationSeconds: number): Promise<string> {
+    const src = join(FIXTURE_DIR, name);
     const synth = await runFfmpeg([
       "-y",
       "-hide_banner",
@@ -826,51 +836,49 @@ describe.skipIf(!HAS_FFMPEG)("extractAllVideoFrames on a VFR source", () => {
       "-f",
       "lavfi",
       "-i",
-      "testsrc2=s=320x180:d=2:rate=30",
+      `testsrc2=s=320x180:d=${durationSeconds}:rate=30`,
       "-c:v",
       "libx264",
       "-preset",
       "ultrafast",
       "-pix_fmt",
       "yuv420p",
-      SRC,
+      src,
     ]);
     if (!synth.success) {
-      throw new Error(`Cache fixture synthesis failed: ${synth.stderr.slice(-400)}`);
+      throw new Error(`Fixture synthesis failed (${name}): ${synth.stderr.slice(-400)}`);
     }
+    return src;
+  }
 
-    const video: VideoElement = {
-      id: "cv1",
-      src: SRC,
-      start: 0,
-      end: 2,
-      mediaStart: 0,
-      loop: false,
-      hasAudio: false,
-    };
+  function cfrClipElement(id: string, src: string, endSeconds: number): VideoElement {
+    return { id, src, start: 0, end: endSeconds, mediaStart: 0, loop: false, hasAudio: false };
+  }
 
-    const outDirA = join(FIXTURE_DIR, "out-cache-miss");
-    mkdirSync(outDirA, { recursive: true });
-    const miss = await extractAllVideoFrames(
-      [video],
-      FIXTURE_DIR,
-      { fps: 30, outputDir: outDirA },
-      undefined,
-      { extractCacheDir: CACHE_DIR },
-    );
+  async function extractWithCache(
+    video: VideoElement,
+    outName: string,
+    cacheDir: string,
+    fps = 30,
+  ): Promise<ExtractionResult> {
+    const outputDir = join(FIXTURE_DIR, outName);
+    mkdirSync(outputDir, { recursive: true });
+    return extractAllVideoFrames([video], FIXTURE_DIR, { fps, outputDir }, undefined, {
+      extractCacheDir: cacheDir,
+    });
+  }
+
+  it("reuses extracted frames on a warm cache hit", async () => {
+    const CACHE_DIR = mkdtempSync(join(tmpdir(), "hf-extract-cache-test-"));
+    const SRC = await synthCfrClip("cache-src.mp4", 2);
+    const video = cfrClipElement("cv1", SRC, 2);
+
+    const miss = await extractWithCache(video, "out-cache-miss", CACHE_DIR);
     expect(miss.errors).toEqual([]);
     expect(miss.phaseBreakdown.cacheHits).toBe(0);
     expect(miss.phaseBreakdown.cacheMisses).toBe(1);
 
-    const outDirB = join(FIXTURE_DIR, "out-cache-hit");
-    mkdirSync(outDirB, { recursive: true });
-    const hit = await extractAllVideoFrames(
-      [video],
-      FIXTURE_DIR,
-      { fps: 30, outputDir: outDirB },
-      undefined,
-      { extractCacheDir: CACHE_DIR },
-    );
+    const hit = await extractWithCache(video, "out-cache-hit", CACHE_DIR);
     expect(hit.errors).toEqual([]);
     expect(hit.phaseBreakdown.cacheHits).toBe(1);
     expect(hit.phaseBreakdown.cacheMisses).toBe(0);
@@ -884,61 +892,65 @@ describe.skipIf(!HAS_FFMPEG)("extractAllVideoFrames on a VFR source", () => {
     rmSync(CACHE_DIR, { recursive: true, force: true });
   }, 60_000);
 
+  it("updates the cache sentinel mtime on a hit", async () => {
+    const CACHE_DIR = mkdtempSync(join(tmpdir(), "hf-extract-cache-touch-test-"));
+    const SRC = await synthCfrClip("cache-touch-src.mp4", 1);
+    const video = cfrClipElement("touch", SRC, 1);
+
+    const miss = await extractWithCache(video, "out-cache-touch-miss", CACHE_DIR);
+    expect(miss.errors).toEqual([]);
+    expect(miss.phaseBreakdown.cacheMisses).toBe(1);
+
+    const cacheEntryNames = readdirSync(CACHE_DIR).filter((name) => name.startsWith(SCHEMA_PREFIX));
+    expect(cacheEntryNames).toHaveLength(1);
+    const sentinel = join(CACHE_DIR, cacheEntryNames[0]!, COMPLETE_SENTINEL);
+    const old = new Date(Date.now() - 120_000);
+    utimesSync(sentinel, old, old);
+    const before = statSync(sentinel).mtimeMs;
+
+    const hit = await extractWithCache(video, "out-cache-touch-hit", CACHE_DIR);
+
+    expect(hit.errors).toEqual([]);
+    expect(hit.phaseBreakdown.cacheHits).toBe(1);
+    expect(statSync(sentinel).mtimeMs).toBeGreaterThan(before);
+
+    rmSync(CACHE_DIR, { recursive: true, force: true });
+  }, 60_000);
+
+  it("disables caching for this render when the cache dir is not writable", async () => {
+    const CACHE_FILE = join(FIXTURE_DIR, "cache-dir-is-a-file");
+    writeFileSync(CACHE_FILE, "not a directory", "utf-8");
+    const SRC = await synthCfrClip("cache-disabled-src.mp4", 1);
+
+    const stderr = vi.spyOn(process.stderr, "write").mockImplementation(() => true);
+    try {
+      const result = await extractWithCache(
+        cfrClipElement("uncached", SRC, 1),
+        "out-cache-disabled",
+        CACHE_FILE,
+      );
+
+      expect(result.errors).toEqual([]);
+      expect(result.extracted).toHaveLength(1);
+      expect(result.phaseBreakdown.cacheHits).toBe(0);
+      expect(result.phaseBreakdown.cacheMisses).toBe(0);
+      expect(stderr).toHaveBeenCalledTimes(1);
+      expect(String(stderr.mock.calls[0]?.[0])).toContain("extraction cache dir");
+      expect(String(stderr.mock.calls[0]?.[0])).toContain("caching disabled for this render");
+    } finally {
+      stderr.mockRestore();
+    }
+  }, 60_000);
+
   it("invalidates the cache when fps changes", async () => {
     const CACHE_DIR = mkdtempSync(join(tmpdir(), "hf-extract-cache-test-"));
-    const SRC = join(FIXTURE_DIR, "cache-fps-src.mp4");
+    const SRC = await synthCfrClip("cache-fps-src.mp4", 1);
+    const video = cfrClipElement("cv2", SRC, 1);
 
-    const synth = await runFfmpeg([
-      "-y",
-      "-hide_banner",
-      "-loglevel",
-      "error",
-      "-f",
-      "lavfi",
-      "-i",
-      "testsrc2=s=320x180:d=1:rate=30",
-      "-c:v",
-      "libx264",
-      "-preset",
-      "ultrafast",
-      "-pix_fmt",
-      "yuv420p",
-      SRC,
-    ]);
-    if (!synth.success) {
-      throw new Error(`Cache-fps fixture synthesis failed: ${synth.stderr.slice(-400)}`);
-    }
-
-    const video: VideoElement = {
-      id: "cv2",
-      src: SRC,
-      start: 0,
-      end: 1,
-      mediaStart: 0,
-      loop: false,
-      hasAudio: false,
-    };
-
-    const outA = join(FIXTURE_DIR, "out-cache-fps-30");
-    mkdirSync(outA, { recursive: true });
-    const first = await extractAllVideoFrames(
-      [video],
-      FIXTURE_DIR,
-      { fps: 30, outputDir: outA },
-      undefined,
-      { extractCacheDir: CACHE_DIR },
-    );
+    const first = await extractWithCache(video, "out-cache-fps-30", CACHE_DIR);
     expect(first.phaseBreakdown.cacheMisses).toBe(1);
 
-    const outB = join(FIXTURE_DIR, "out-cache-fps-60");
-    mkdirSync(outB, { recursive: true });
-    const second = await extractAllVideoFrames(
-      [video],
-      FIXTURE_DIR,
-      { fps: 60, outputDir: outB },
-      undefined,
-      { extractCacheDir: CACHE_DIR },
-    );
+    const second = await extractWithCache(video, "out-cache-fps-60", CACHE_DIR, 60);
     expect(second.phaseBreakdown.cacheMisses).toBe(1);
     expect(second.phaseBreakdown.cacheHits).toBe(0);
 

packages/engine/src/services/videoFrameExtractor.ts

@@ -25,11 +25,13 @@ import { DEFAULT_CONFIG, type EngineConfig } from "../config.js";
 import { unwrapTemplate } from "../utils/htmlTemplate.js";
 import {
   FRAME_FILENAME_PREFIX,
-  ensureCacheEntryDir,
+  gcExtractionCache,
   lookupCacheEntry,
-  markCacheEntryComplete,
+  partialCacheEntryDir,
+  publishCacheEntry,
   readKeyStat,
   rehydrateCacheEntry,
+  touchCacheEntry,
   type CacheFrameFormat,
 } from "./extractionCache.js";
 
@@ -82,6 +84,8 @@ export interface ExtractionOptions {
   format?: VideoFrameFormat;
 }
 
+const EXTRACT_CACHE_MIN_AGE_MS = 60 * 60 * 1000;
+
 /**
  * Per-phase timings and counters emitted by `extractAllVideoFrames`.
  *
@@ -106,6 +110,16 @@ export interface ExtractionOptions {
  */
 export interface ExtractionPhaseBreakdown {
   resolveMs: number;
+  /** Publishes that could not land atomically — the render still succeeded
+   *  from the partial dir, but future renders re-extract. A rising rate is
+   *  the first signal that warm renders are silently going cold. */
+  cachePublishFailures: number;
+  /** Entries evicted by the post-extraction LRU sweep. */
+  cacheGcEvictions: number;
+  /** Bytes reclaimed by the LRU sweep. */
+  cacheGcBytesFreed: number;
+  /** Aged .partial-* dirs (crashed writers) removed by the sweep. */
+  cacheAgedPartialsCleared: number;
   hdrProbeMs: number;
   hdrPreflightMs: number;
   hdrPreflightCount: number;
@@ -529,7 +543,9 @@ export async function extractAllVideoFrames(
   baseDir: string,
   options: ExtractionOptions,
   signal?: AbortSignal,
-  config?: Partial<Pick<EngineConfig, "ffmpegProcessTimeout" | "extractCacheDir">>,
+  config?: Partial<
+    Pick<EngineConfig, "ffmpegProcessTimeout" | "extractCacheDir" | "extractCacheMaxBytes">
+  >,
   compiledDir?: string,
 ): Promise<ExtractionResult> {
   const startTime = Date.now();
@@ -538,6 +554,10 @@ export async function extractAllVideoFrames(
   let totalFramesExtracted = 0;
   const breakdown: ExtractionPhaseBreakdown = {
     resolveMs: 0,
+    cachePublishFailures: 0,
+    cacheGcEvictions: 0,
+    cacheGcBytesFreed: 0,
+    cacheAgedPartialsCleared: 0,
     hdrProbeMs: 0,
     hdrPreflightMs: 0,
     hdrPreflightCount: 0,
@@ -691,6 +711,13 @@ export async function extractAllVideoFrames(
             config,
           );
           entry.videoPath = convertedPath;
+          // The converted intermediate carries BT.2020-mapped pixels but the
+          // cache key snapshot above still describes the ORIGINAL source.
+          // Publishing converted frames under that key would poison later
+          // plain-SDR renders of the same trim, so bypass the cache for
+          // converted entries. (The follow-up transform-keyed cache change
+          // re-enables caching for these with a discriminated key.)
+          cacheKeyInputs[i] = null;
           // Segment-scoped re-encode starts the new file at t=0, so downstream
           // extraction must seek from 0, not the original mediaStart. Shallow-copy
           // to avoid mutating the caller's VideoElement.
@@ -739,7 +766,18 @@ export async function extractAllVideoFrames(
   breakdown.vfrPreflightMs = Date.now() - vfrPreflightStart;
 
   const phase3Start = Date.now();
-  const cacheRootDir = config?.extractCacheDir;
+  const configuredCacheRootDir = config?.extractCacheDir;
+  let cacheRootDir: string | undefined;
+  if (configuredCacheRootDir) {
+    try {
+      mkdirSync(configuredCacheRootDir, { recursive: true });
+      cacheRootDir = configuredCacheRootDir;
+    } catch {
+      process.stderr.write(
+        `[hyperframes:render] WARNING: extraction cache dir ${configuredCacheRootDir} is not writable; caching disabled for this render\n`,
+      );
+    }
+  }
 
   async function tryCachedExtract(
     video: VideoElement,
@@ -770,6 +808,7 @@ export async function extractAllVideoFrames(
 
     if (lookup.hit) {
       breakdown.cacheHits += 1;
+      touchCacheEntry(lookup.entry);
       const rehydrated = rehydrateCacheEntry(lookup.entry, {
         videoId: video.id,
         srcPath: keyInput.videoPath,
@@ -781,7 +820,8 @@ export async function extractAllVideoFrames(
     }
 
     breakdown.cacheMisses += 1;
-    ensureCacheEntryDir(lookup.entry);
+    const partialDir = partialCacheEntryDir(lookup.entry);
+    mkdirSync(partialDir, { recursive: true });
     const result = await extractVideoFramesRange(
       videoPath,
       video.id,
@@ -790,12 +830,22 @@ export async function extractAllVideoFrames(
       { ...options, format: cacheFormat },
       signal,
       config,
-      lookup.entry.dir,
+      partialDir,
     );
-    // Mark complete only AFTER frames are on disk — a crash mid-extract
-    // leaves the entry un-sentineled so the next lookup re-extracts over it.
-    markCacheEntryComplete(lookup.entry);
-    return { ...result, ownedByLookup: true };
+    const published = publishCacheEntry(lookup.entry, partialDir);
+    if (!published.published) {
+      breakdown.cachePublishFailures += 1;
+      return { ...result, ownedByLookup: false };
+    }
+
+    const rehydrated = rehydrateCacheEntry(lookup.entry, {
+      videoId: video.id,
+      srcPath: keyInput.videoPath,
+      fps: options.fps,
+      format: cacheFormat,
+      metadata,
+    });
+    return { ...rehydrated, ownedByLookup: true };
   }
 
   function extractionError(videoId: string, err: unknown): { videoId: string; error: string } {
@@ -926,6 +976,16 @@ export async function extractAllVideoFrames(
     }
   }
 
+  if (cacheRootDir) {
+    const gcStats = gcExtractionCache(cacheRootDir, {
+      maxBytes: config?.extractCacheMaxBytes ?? DEFAULT_CONFIG.extractCacheMaxBytes,
+      minAgeMs: EXTRACT_CACHE_MIN_AGE_MS,
+    });
+    breakdown.cacheGcEvictions = gcStats.evictedEntries;
+    breakdown.cacheGcBytesFreed = gcStats.evictedBytes;
+    breakdown.cacheAgedPartialsCleared = gcStats.agedPartialsRemoved;
+  }
+
   return {
     success: errors.length === 0,
     extracted,