feat(media): alpha-capable authoring proxies (#2598)

* feat(media): alpha-capable authoring proxies

Alpha sources were refused a proxy before the codec map ever asked whether the
browser could decode them, so a ProRes 4444 alpha file (which no browser
previews at all) rendered black forever, while an alpha WebM (which previews
fine) was already covered by the browser-safe check on the next line. The alpha
veto earned nothing and cost the one case that needed help.

Alpha is now a target-codec choice rather than a veto: alpha sources transcode
to VP9 + yuva420p in WebM, everything else keeps the existing H.264/MP4 path
byte for byte. Only files no browser can preview are proxied, which is the rule
the runtime already followed everywhere else.

WebM cannot carry AAC, so the VP9 path uses Opus and drops the MP4-only
faststart flag. PROXY_PARAMS_VERSION moves to v3 so clients stop serving the
previously cached proxies.

Safari does not decode VP9 alpha and still shows black for alpha sources, as it
does today: this is better on Chromium and Firefox and no worse anywhere.

* fix(media): infer proxy variant for rescue

* fix(media): preserve alpha proxy hardening after restack

packages/studio-server/src/helpers/mediaCodecMap.test.ts

@@ -8,6 +8,8 @@ import {
   decideMediaProxyEligibility,
   createMediaCodecProbeCache,
   probeAssetCodec,
+  proxyVariantFor,
+  resolveProxyVariantRequest,
   scanProjectMediaCodecMap,
 } from "./mediaCodecMap.js";
 
@@ -171,7 +173,7 @@ describe("probeAssetCodec", () => {
 });
 
 describe("decideMediaProxyEligibility", () => {
-  it("allows only hostile opaque video through the H.264 proxy path", () => {
+  it("allows browser-hostile video with or without alpha", () => {
     expect(
       decideMediaProxyEligibility({
         codecName: "hevc",
@@ -180,9 +182,6 @@ describe("decideMediaProxyEligibility", () => {
         hasAlpha: false,
       }),
     ).toEqual({ eligible: true });
-  });
-
-  it("rejects alpha and browser-safe sources before transcoding", () => {
     expect(
       decideMediaProxyEligibility({
         codecName: "prores",
@@ -190,7 +189,18 @@ describe("decideMediaProxyEligibility", () => {
         representativeMime: null,
         hasAlpha: true,
       }),
-    ).toEqual({ eligible: false, reason: "alpha_source" });
+    ).toEqual({ eligible: true });
+  });
+
+  it("rejects browser-safe sources even when they carry alpha", () => {
+    expect(
+      decideMediaProxyEligibility({
+        codecName: "vp8",
+        browserHostile: false,
+        representativeMime: "video/webm",
+        hasAlpha: true,
+      }),
+    ).toEqual({ eligible: false, reason: "browser_safe_codec" });
     expect(
       decideMediaProxyEligibility({
         codecName: "h264",
@@ -204,6 +214,43 @@ describe("decideMediaProxyEligibility", () => {
       reason: "unknown_codec",
     });
   });
+
+  it("does not recycle an alpha VP9 source into the same proxy codec", () => {
+    expect(
+      decideMediaProxyEligibility({
+        codecName: "vp9",
+        browserHostile: true,
+        representativeMime: 'video/webm; codecs="vp09.00.10.08"',
+        hasAlpha: true,
+      }),
+    ).toEqual({ eligible: false, reason: "proxy_target_codec" });
+  });
+});
+
+describe("proxyVariantFor", () => {
+  it("uses VP9 for alpha and H.264 otherwise", () => {
+    const facts = {
+      codecName: "prores",
+      browserHostile: true,
+      representativeMime: null,
+      hasAlpha: true,
+    };
+    expect(proxyVariantFor(facts)).toBe("vp9");
+    expect(proxyVariantFor({ ...facts, hasAlpha: false })).toBe("h264");
+  });
+});
+
+describe("resolveProxyVariantRequest", () => {
+  it("infers the alpha-aware variant for an unlisted runtime rescue", () => {
+    const facts = {
+      codecName: "prores",
+      browserHostile: true,
+      representativeMime: null,
+      hasAlpha: true,
+    };
+    expect(resolveProxyVariantRequest("auto", facts)).toBe("vp9");
+    expect(resolveProxyVariantRequest("h264", facts)).toBeNull();
+  });
 });
 
 describe("scanProjectMediaCodecMap", () => {

packages/studio-server/src/helpers/mediaCodecMap.ts

@@ -22,9 +22,8 @@ export interface AssetCodecFacts {
    * when no representative mime exists (ProRes: browsers never decode it, so
    * the runtime always proxies rather than probing `canPlayType`). */
   representativeMime: string | null;
-  /** Source carries an alpha channel (ffprobe pix_fmt). Alpha sources are
-   * never proxied — an H.264 proxy would destroy the transparency (e.g.
-   * ProRes 4444 alpha). */
+  /** Source carries an alpha channel (ffprobe pix_fmt). Alpha sources use a
+   * VP9/WebM proxy so their transparency is preserved. */
   hasAlpha: boolean;
 }
 
@@ -48,7 +47,41 @@ export const BROWSER_HOSTILE_CODECS: Record<string, string | null> = {
   vp9: 'video/webm; codecs="vp09.00.10.08"',
 };
 
-export type MediaProxyIneligibilityReason = "alpha_source" | "browser_safe_codec" | "unknown_codec";
+export type ProxyVariant = "h264" | "vp9";
+export type ProxyVariantRequest = ProxyVariant | "auto";
+
+export const PROXY_VARIANT_CONFIG: Record<
+  ProxyVariant,
+  { extension: ".mp4" | ".webm"; contentType: "video/mp4" | "video/webm" }
+> = {
+  h264: { extension: ".mp4", contentType: "video/mp4" },
+  vp9: { extension: ".webm", contentType: "video/webm" },
+};
+
+export function isProxyVariant(value: string): value is ProxyVariant {
+  return Object.hasOwn(PROXY_VARIANT_CONFIG, value);
+}
+
+export function isProxyVariantRequest(value: string): value is ProxyVariantRequest {
+  return value === "auto" || isProxyVariant(value);
+}
+
+export function proxyVariantFor(facts: AssetCodecFacts): ProxyVariant {
+  return facts.hasAlpha ? "vp9" : "h264";
+}
+
+export function resolveProxyVariantRequest(
+  request: ProxyVariantRequest,
+  facts: AssetCodecFacts,
+): ProxyVariant | null {
+  const expected = proxyVariantFor(facts);
+  return request === "auto" || request === expected ? expected : null;
+}
+
+export type MediaProxyIneligibilityReason =
+  | "browser_safe_codec"
+  | "proxy_target_codec"
+  | "unknown_codec";
 
 export type MediaProxyEligibility =
   | { eligible: true }
@@ -57,8 +90,10 @@ export type MediaProxyEligibility =
 /** Single policy gate shared by proactive scans and on-demand proxy routes. */
 export function decideMediaProxyEligibility(facts: AssetCodecFacts | null): MediaProxyEligibility {
   if (!facts) return { eligible: false, reason: "unknown_codec" };
-  if (facts.hasAlpha) return { eligible: false, reason: "alpha_source" };
   if (!facts.browserHostile) return { eligible: false, reason: "browser_safe_codec" };
+  if (facts.hasAlpha && facts.codecName === "vp9") {
+    return { eligible: false, reason: "proxy_target_codec" };
+  }
   return { eligible: true };
 }
 

packages/studio-server/src/helpers/mediaProxyPreview.ts

@@ -2,6 +2,7 @@ import { resolve } from "node:path";
 import type { StudioApiAdapter } from "../types.js";
 import {
   createMediaCodecProbeCache,
+  proxyVariantFor,
   scanProjectMediaCodecMap,
   type HtmlSourceLike,
   type MediaCodecMap,
@@ -72,8 +73,7 @@ function injectScriptTagIntoHead(html: string, scriptTag: string): string {
  * responses). No second concurrency limiter here — the transcoder's own
  * global bound throttles both pre-warm and element-triggered calls.
  * Pre-warm failures are swallowed; an actual `?hf-proxy=` request surfaces
- * them as a 502. Alpha-bearing entries are never pre-warmed: the runtime
- * never proxies them (transparency would be destroyed).
+ * them as a 502. Alpha-bearing entries pre-warm their VP9/WebM variant.
  *
  * The single shared implementation for every auto-proxy surface — the studio
  * preview route (via `injectMediaCodecMap` below) and the CLI's composition /
@@ -100,13 +100,15 @@ export async function injectMediaCodecMapIntoHtml(
   }
   if (Object.keys(map).length === 0) return html;
   for (const [rootRelativePathname, facts] of Object.entries(map)) {
-    if (!facts.browserHostile || facts.hasAlpha) continue;
-    resolveProxy(projectDir, resolve(projectDir, rootRelativePathname.replace(/^\/+/, ""))).catch(
-      () => {
-        // Swallowed: the pre-warm is best-effort. A real `?hf-proxy=` request
-        // for this asset re-attempts the transcode and reports failure (502).
-      },
-    );
+    if (!facts.browserHostile) continue;
+    resolveProxy(
+      projectDir,
+      resolve(projectDir, rootRelativePathname.replace(/^\/+/, "")),
+      proxyVariantFor(facts),
+    ).catch(() => {
+      // Swallowed: the pre-warm is best-effort. A real `?hf-proxy=` request
+      // for this asset re-attempts the transcode and reports failure (502).
+    });
   }
   // <-escape prevents a src path containing "</script>" from breaking out of
   // the injected tag, mirroring injectPreviewVariables in routes/preview.ts.

packages/studio-server/src/helpers/proxyCache.test.ts

@@ -49,6 +49,26 @@ describe("cleanupProxyCache", () => {
     expect(existsSync(newest)).toBe(true);
   });
 
+  it("counts and evicts WebM proxies alongside MP4 proxies", () => {
+    const cache = cacheDir();
+    const now = 1_800_000_000_000;
+    const webm = join(cache, "alpha.webm");
+    const mp4 = join(cache, "opaque.mp4");
+    writeEntry(webm, 6, now - 2_000);
+    writeEntry(mp4, 6, now - 1_000);
+
+    const result = cleanupProxyCache(cache, {
+      now,
+      maxBytes: 6,
+      maxIdleMs: 10_000,
+      minSweepIntervalMs: 0,
+    });
+
+    expect(result.bytesBefore).toBe(12);
+    expect(result.removed).toEqual([webm]);
+    expect(result.bytesAfter).toBe(6);
+  });
+
   it("preserves in-flight entries and removes stale temporary files", () => {
     const cache = cacheDir();
     const now = 1_800_000_000_000;

packages/studio-server/src/helpers/proxyCache.ts

@@ -1,9 +1,13 @@
 import { existsSync, readdirSync, statSync, unlinkSync } from "node:fs";
-import { join } from "node:path";
+import { extname, join } from "node:path";
+import { PROXY_VARIANT_CONFIG } from "./mediaCodecMap.js";
 
 const DEFAULT_MAX_BYTES = 10 * 1024 * 1024 * 1024;
 const DEFAULT_STALE_TEMP_MS = 60 * 60 * 1000;
 const DEFAULT_MIN_SWEEP_INTERVAL_MS = 5 * 60 * 1000;
+const PROXY_EXTENSIONS: ReadonlySet<string> = new Set(
+  Object.values(PROXY_VARIANT_CONFIG).map(({ extension }) => extension),
+);
 
 export interface ProxyCacheCleanupOptions {
   maxBytes?: number;
@@ -76,7 +80,7 @@ function readCacheInventory(
     };
     if (dirent.name.startsWith(".tmp-")) {
       if (now - stat.mtimeMs >= staleTempMs) staleTemps.push(entry);
-    } else if (dirent.name.endsWith(".mp4")) {
+    } else if (PROXY_EXTENSIONS.has(extname(dirent.name))) {
       entries.push(entry);
     }
   }

packages/studio-server/src/helpers/proxyTranscoder.test.ts

@@ -157,6 +157,41 @@ describe("resolveProxy", () => {
     expect(cacheDirEntries).toEqual([expectedCachePath.split("/").at(-1)]);
   });
 
+  it("uses VP9 alpha-safe args and a distinct WebM cache path", async () => {
+    const { spawn, calls } = createSpawnSpy();
+    const { resolveProxy, getProxyCachePath } = await loadModule(spawn, FFMPEG_PATH);
+    const projectDir = tmpProject();
+    const sourcePath = join(projectDir, "alpha.mov");
+    writeFileSync(sourcePath, "source-bytes");
+
+    const h264Path = getProxyCachePath(projectDir, sourcePath, "h264");
+    const vp9Path = getProxyCachePath(projectDir, sourcePath, "vp9");
+    const resultPromise = resolveProxy(projectDir, sourcePath, "vp9");
+    await flush();
+
+    expect(vp9Path).not.toBe(h264Path);
+    expect(vp9Path).toMatch(/\.webm$/);
+    expect(h264Path).toMatch(/\.mp4$/);
+    const args = calls[0]!.args;
+    expect(args).toContain("libvpx-vp9");
+    expect(args).toContain("yuva420p");
+    expect(args).toContain("libopus");
+    expect(args[args.indexOf("-b:v") + 1]).toBe("0");
+    expect(args[args.indexOf("-crf") + 1]).toBe("23");
+    expect(args[args.indexOf("-deadline") + 1]).toBe("good");
+    expect(args[args.indexOf("-auto-alt-ref") + 1]).toBe("0");
+    expect(args[args.indexOf("-metadata:s:v:0") + 1]).toBe("alpha_mode=1");
+    expect(args[args.indexOf("-ac") + 1]).toBe("2");
+    expect(args).toContain("-row-mt");
+    expect(args).toContain("-cpu-used");
+    expect(args).not.toContain("-movflags");
+    expect(args).not.toContain("+faststart");
+    expect(args[args.indexOf("-vf") + 1]).toContain("format=yuva420p");
+
+    succeed(calls[0]!, "fake-vp9-bytes");
+    await expect(resultPromise).resolves.toBe(vp9Path);
+  });
+
   it("returns without spawning on a cache hit", async () => {
     const { spawn, calls } = createSpawnSpy();
     const { resolveProxy, getProxyCachePath } = await loadModule(spawn, FFMPEG_PATH);
@@ -201,6 +236,24 @@ describe("resolveProxy", () => {
     await result;
   });
 
+  it("preserves alpha on HDR-tagged VP9 proxies by bypassing the opaque tonemap chain", async () => {
+    const { spawn, calls } = createSpawnSpy();
+    const { resolveProxy } = await loadModule(spawn, FFMPEG_PATH, true);
+    const projectDir = tmpProject();
+    const sourcePath = join(projectDir, "hdr-alpha.mov");
+    writeFileSync(sourcePath, "source-bytes");
+
+    const result = resolveProxy(projectDir, sourcePath, "vp9");
+    await flush();
+
+    expect(calls).toHaveLength(1);
+    const filter = calls[0]!.args[calls[0]!.args.indexOf("-vf") + 1];
+    expect(filter).toContain("format=yuva420p");
+    expect(filter).not.toContain("tonemap=");
+    succeed(calls[0]!, "fake-vp9-alpha-bytes");
+    await result;
+  });
+
   it("rejects HDR proxying with a typed actionable error when ffmpeg lacks zscale", async () => {
     const { spawn, calls } = createSpawnSpy();
     const { resolveProxy, FfmpegMissingFilterError } = await loadModule(spawn, FFMPEG_PATH, true);

packages/studio-server/src/helpers/proxyTranscoder.ts

@@ -13,11 +13,12 @@ import { basename, dirname, isAbsolute, join, relative, sep } from "node:path";
 import { findFfBinary } from "@hyperframes/parsers/ff-binaries";
 import { probeMediaMetadata } from "./mediaMetadata.js";
 import { cleanupProxyCache } from "./proxyCache.js";
+import { PROXY_VARIANT_CONFIG, type ProxyVariant } from "./mediaCodecMap.js";
 
 /**
  * Transcodes browser-hostile local video sources (HEVC, ProRes, ...) into a
- * cached, seekable H.264 authoring proxy. Consumed by the preview/play/static
- * project routes (U3/U4) to serve a `?hf-proxy=h264` request; never used on
+ * cached, seekable authoring proxy. Consumed by the preview/play/static
+ * project routes (U3/U4) to serve a `?hf-proxy=` request; never used on
  * the render path (render always sees the original file).
  *
  * IMPORTANT — request-lifecycle detachment: nothing here accepts or wires an
@@ -31,7 +32,7 @@ import { cleanupProxyCache } from "./proxyCache.js";
  * entry still lands for the next request.
  */
 
-export const PROXY_PARAMS_VERSION = "v2";
+export const PROXY_PARAMS_VERSION = "v3";
 
 const CACHE_DIR_NAME = ".transcode-cache";
 
@@ -158,16 +159,22 @@ function canonicalizeProxySource(
   };
 }
 
-function buildProxyCacheKey(source: CanonicalProxySource): string {
+function buildProxyCacheKey(source: CanonicalProxySource, variant: ProxyVariant): string {
   const stat = statSync(source.sourcePath);
   return createHash("sha256")
-    .update(`${source.relativePath}\0${stat.mtimeMs}\0${stat.size}\0${PROXY_PARAMS_VERSION}`)
+    .update(
+      `${source.relativePath}\0${stat.mtimeMs}\0${stat.size}\0${PROXY_PARAMS_VERSION}\0${variant}`,
+    )
     .digest("hex");
 }
 
-function getCanonicalProxyCachePath(source: CanonicalProxySource): string {
-  const key = buildProxyCacheKey(source);
-  return join(source.projectDir, CACHE_DIR_NAME, `${key}.mp4`);
+function getCanonicalProxyCachePath(source: CanonicalProxySource, variant: ProxyVariant): string {
+  const key = buildProxyCacheKey(source, variant);
+  return join(
+    source.projectDir,
+    CACHE_DIR_NAME,
+    `${key}${PROXY_VARIANT_CONFIG[variant].extension}`,
+  );
 }
 
 /**
@@ -175,8 +182,15 @@ function getCanonicalProxyCachePath(source: CanonicalProxySource): string {
  * transcoding anything. Route handlers use this to check cache state (e.g.
  * for ETag/If-None-Match) before deciding whether to await a transcode.
  */
-export function getProxyCachePath(projectDir: string, absoluteSourcePath: string): string {
-  return getCanonicalProxyCachePath(canonicalizeProxySource(projectDir, absoluteSourcePath));
+export function getProxyCachePath(
+  projectDir: string,
+  absoluteSourcePath: string,
+  variant: ProxyVariant = "h264",
+): string {
+  return getCanonicalProxyCachePath(
+    canonicalizeProxySource(projectDir, absoluteSourcePath),
+    variant,
+  );
 }
 
 // --- global concurrency limiter -------------------------------------------
@@ -290,31 +304,35 @@ export function clearFailedTranscodesForTest(): void {
   failedTranscodes.clear();
 }
 
-async function runFfmpeg(sourcePath: string, outputPath: string): Promise<void> {
+async function runFfmpeg(
+  sourcePath: string,
+  outputPath: string,
+  variant: ProxyVariant,
+): Promise<void> {
   const metadata = await probeMediaMetadata(sourcePath);
   const ffmpegPath = findFfBinary("ffmpeg", { configuredMustExist: true });
   if (!ffmpegPath) {
     throw new FfmpegUnavailableError();
   }
-  if (metadata.color.isHdr) await ensureHdrFilters(ffmpegPath);
+  // The HDR tonemap filters discard alpha. VP9 is the alpha-preserving proxy
+  // variant, so retain its source color values instead of making it opaque.
+  if (metadata.color.isHdr && variant !== "vp9") await ensureHdrFilters(ffmpegPath);
   const evenScale = "scale=trunc(iw/2)*2:trunc(ih/2)*2";
-  const videoFilter = metadata.color.isHdr
-    ? [
-        "zscale=t=linear:npl=100",
-        "tonemap=hable:desat=0",
-        "zscale=p=bt709:t=bt709:m=bt709:r=tv",
-        evenScale,
-        "format=yuv420p",
-      ].join(",")
-    : [evenScale, "format=yuv420p"].join(",");
+  const pixelFormat = variant === "vp9" ? "yuva420p" : "yuv420p";
+  const videoFilter =
+    metadata.color.isHdr && variant !== "vp9"
+      ? [
+          "zscale=t=linear:npl=100",
+          "tonemap=hable:desat=0",
+          "zscale=p=bt709:t=bt709:m=bt709:r=tv",
+          evenScale,
+          `format=${pixelFormat}`,
+        ].join(",")
+      : [evenScale, `format=${pixelFormat}`].join(",");
 
   return new Promise((resolvePromise, reject) => {
-    const args = [
-      "-y",
-      "-i",
-      sourcePath,
-      "-vf",
-      videoFilter,
+    const commonArgs = ["-y", "-i", sourcePath, "-vf", videoFilter];
+    const h264Args = [
       "-c:v",
       "libx264",
       "-profile:v",
@@ -335,8 +353,38 @@ async function runFfmpeg(sourcePath: string, outputPath: string): Promise<void>
       "aac",
       "-movflags",
       "+faststart",
-      outputPath,
     ];
+    const vp9Args = [
+      "-c:v",
+      "libvpx-vp9",
+      "-b:v",
+      "0",
+      "-crf",
+      "23",
+      "-deadline",
+      "good",
+      "-pix_fmt",
+      "yuva420p",
+      "-colorspace",
+      "bt709",
+      "-color_primaries",
+      "bt709",
+      "-color_trc",
+      "bt709",
+      "-row-mt",
+      "1",
+      "-cpu-used",
+      "4",
+      "-auto-alt-ref",
+      "0",
+      "-metadata:s:v:0",
+      "alpha_mode=1",
+      "-ac",
+      "2",
+      "-c:a",
+      "libopus",
+    ];
+    const args = [...commonArgs, ...(variant === "vp9" ? vp9Args : h264Args), outputPath];
 
     // Hard ceiling so a hung ffmpeg can never permanently occupy one of the
     // global transcode slots: the child is killed and the slot released via
@@ -372,7 +420,11 @@ async function runFfmpeg(sourcePath: string, outputPath: string): Promise<void>
   });
 }
 
-async function transcodeToCache(absoluteSourcePath: string, cachePath: string): Promise<string> {
+async function transcodeToCache(
+  absoluteSourcePath: string,
+  cachePath: string,
+  variant: ProxyVariant,
+): Promise<string> {
   await acquireSlot();
   try {
     // Another caller may have finished (or a pre-warm beat us) while queued.
@@ -382,7 +434,7 @@ async function transcodeToCache(absoluteSourcePath: string, cachePath: string):
     mkdirSync(cacheDir, { recursive: true });
     const tempPath = join(cacheDir, `.tmp-${randomUUID()}-${basename(cachePath)}`);
     try {
-      await runFfmpeg(absoluteSourcePath, tempPath);
+      await runFfmpeg(absoluteSourcePath, tempPath, variant);
       renameSync(tempPath, cachePath);
       maintainProxyCache(cacheDir);
       return cachePath;
@@ -397,7 +449,7 @@ async function transcodeToCache(absoluteSourcePath: string, cachePath: string):
 }
 
 /**
- * Resolves the cached H.264 proxy for `absoluteSourcePath`, transcoding it at
+ * Resolves the cached proxy variant for `absoluteSourcePath`, transcoding it at
  * most once per cache key. Concurrent calls for the same key (including a
  * pre-warm call racing an element-triggered one) share one ffmpeg child and
  * one promise; calls for different keys queue through the global concurrency
@@ -407,9 +459,10 @@ async function transcodeToCache(absoluteSourcePath: string, cachePath: string):
 export async function resolveProxy(
   projectDir: string,
   absoluteSourcePath: string,
+  variant: ProxyVariant = "h264",
 ): Promise<string> {
   const source = canonicalizeProxySource(projectDir, absoluteSourcePath);
-  const cachePath = getCanonicalProxyCachePath(source);
+  const cachePath = getCanonicalProxyCachePath(source, variant);
   if (existsSync(cachePath)) {
     markCacheEntryUsed(cachePath);
     maintainProxyCache(dirname(cachePath));
@@ -425,7 +478,7 @@ export async function resolveProxy(
   const existing = inFlight.get(cachePath);
   if (existing) return existing;
 
-  const promise = transcodeToCache(source.sourcePath, cachePath)
+  const promise = transcodeToCache(source.sourcePath, cachePath, variant)
     .catch((err: unknown) => {
       if (
         err instanceof ProxyTranscodeError &&