From 39bc3749b4f2d5fe4528c19fb5ce5c6615a59dd1 Mon Sep 17 00:00:00 2001
From: James <james.russo@heygen.com>
Date: Mon, 4 May 2026 19:25:49 -0700
Subject: [PATCH] fix(engine): default to codec-based alpha capability instead
 of relying on tags
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Tag-based alpha detection (alpha_mode / ALPHA_MODE / pix_fmt yuva*) is
fundamentally brittle. Failure modes seen in the wild:
- case-sensitivity across ffmpeg versions (alpha_mode vs ALPHA_MODE)
- older muxers that omit the sidecar tag entirely
- mp4-as-webm rewraps that drop the tag
- ffprobe reporting yuv420p for VP9-with-alpha because the alpha plane
  lives in a Matroska BlockAdditional sidecar, not the main pix_fmt

Each of those silently strips alpha at extraction time. The bug doesn't
surface until the rendered output is missing layers — frustrating to debug,
silent in stdout. The previous case-insensitive fix patched one of the
failure modes; this commit removes the class.

The robust alternative is codec-based: any bitstream that CAN carry alpha
(VP9, VP8, ProRes 4444) gets the alpha-aware decoder and PNG output by
default, regardless of what the tag says. The cost is a small file-size
increase on opaque VP9/VP8 sources (cached PNGs vs JPGs); the benefit is
no class of silent alpha loss from tag misdetection.

- Adds codecMayHaveAlpha() + decoderForCodec() helpers and exports them.
- Updates extractVideoFramesRange to force libvpx-vp9 / libvpx for VP9 / VP8
  unconditionally (was: only when metadata.hasAlpha).
- Updates resolveFrameFormat to default to PNG for any alpha-capable codec
  (was: only when metadata.hasAlpha).
- +4 unit tests covering the codec table.

Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
---
 .../src/services/videoFrameExtractor.test.ts  | 37 +++++++++++++++++++
 .../src/services/videoFrameExtractor.ts       | 37 +++++++++++++++++--
 2 files changed, 71 insertions(+), 3 deletions(-)

diff --git a/packages/engine/src/services/videoFrameExtractor.test.ts b/packages/engine/src/services/videoFrameExtractor.test.ts
index 84635f12b..e83ca1254 100644
--- a/packages/engine/src/services/videoFrameExtractor.test.ts
+++ b/packages/engine/src/services/videoFrameExtractor.test.ts
@@ -10,6 +10,8 @@ import {
   extractAllVideoFrames,
   createFrameLookupTable,
   resolveProjectRelativeSrc,
+  codecMayHaveAlpha,
+  decoderForCodec,
   type VideoElement,
   type ExtractedFrames,
 } from "./videoFrameExtractor.js";
@@ -33,6 +35,41 @@ const HAS_FFMPEG = spawnSync("ffmpeg", ["-version"]).status === 0;
 // <video>'s first decoded frame for the whole clip duration. The resolver
 // now mirrors browser semantics by stripping leading `..` segments as a
 // fallback when the literal join doesn't exist.
+// Codec-based alpha defaulting is the deeper fix for the
+// `alpha_mode`-vs-`ALPHA_MODE` tag-detection bug (see ffprobe.test.ts). The
+// extractor uses these helpers to decide:
+//   1. whether to force the alpha-aware decoder (libvpx-vp9)
+//   2. whether to default the cached frame format to PNG (with alpha) vs JPG
+// The "default to capable" trade is small file-size growth on opaque VP9
+// content for correctness on alpha-having content even when the sidecar tag
+// is missing or muxed with the wrong case.
+describe("codec alpha capability", () => {
+  it("flags VP9, VP8, and ProRes as alpha-capable", () => {
+    expect(codecMayHaveAlpha("vp9")).toBe(true);
+    expect(codecMayHaveAlpha("VP9")).toBe(true);
+    expect(codecMayHaveAlpha("vp8")).toBe(true);
+    expect(codecMayHaveAlpha("prores")).toBe(true);
+  });
+
+  it("does not flag h264 / h265 / mpeg4 (no alpha in their bitstreams)", () => {
+    expect(codecMayHaveAlpha("h264")).toBe(false);
+    expect(codecMayHaveAlpha("h265")).toBe(false);
+    expect(codecMayHaveAlpha("hevc")).toBe(false);
+    expect(codecMayHaveAlpha("mpeg4")).toBe(false);
+  });
+
+  it("treats undefined / empty input as non-alpha", () => {
+    expect(codecMayHaveAlpha(undefined)).toBe(false);
+    expect(codecMayHaveAlpha("")).toBe(false);
+  });
+
+  it("returns the alpha-aware decoder name for VP9 and VP8", () => {
+    expect(decoderForCodec("vp9")).toBe("libvpx-vp9");
+    expect(decoderForCodec("VP9")).toBe("libvpx-vp9");
+    expect(decoderForCodec("vp8")).toBe("libvpx");
+  });
+});
+
 describe("resolveProjectRelativeSrc — sub-composition path clamping", () => {
   let tmp: string;
 
diff --git a/packages/engine/src/services/videoFrameExtractor.ts b/packages/engine/src/services/videoFrameExtractor.ts
index 9992d9966..33d3173da 100644
--- a/packages/engine/src/services/videoFrameExtractor.ts
+++ b/packages/engine/src/services/videoFrameExtractor.ts
@@ -230,8 +230,17 @@ export async function extractVideoFramesRange(
   if (isHdr && isMacOS) {
     args.push("-hwaccel", "videotoolbox");
   }
-  if (metadata.hasAlpha && metadata.videoCodec === "vp9") {
-    args.push("-c:v", "libvpx-vp9");
+  // Always force the alpha-aware decoder on codecs that can carry alpha. The
+  // alternative — gating on `metadata.hasAlpha` — relies on tag detection that
+  // has at least three known failure modes: case-sensitivity across ffmpeg
+  // versions (`alpha_mode` vs `ALPHA_MODE`), missing tags from older muxers,
+  // and mp4-as-webm rewraps that drop the sidecar. A wrong negative there
+  // silently strips alpha during decode and the bug doesn't surface until
+  // the rendered video is missing layers. Codec-based default has no such
+  // ambiguity: libvpx-vp9 reads the alpha sidecar when present and decodes
+  // normally when it isn't.
+  if (codecMayHaveAlpha(metadata.videoCodec)) {
+    args.push("-c:v", decoderForCodec(metadata.videoCodec));
   }
   args.push("-ss", String(startTime), "-i", videoPath, "-t", String(duration));
 
@@ -398,9 +407,31 @@ function resolveSegmentDuration(
   return sourceRemaining > 0 ? sourceRemaining : metadata.durationSeconds;
 }
 
+/**
+ * Codecs whose bitstream is allowed to carry an alpha channel. Default the
+ * extraction path to PNG output for these regardless of `metadata.hasAlpha`
+ * so a missed sidecar tag doesn't silently strip transparency. Opaque content
+ * encoded in one of these codecs pays a small file-size cost on the cached
+ * frames but stays correct on the rare case where alpha IS present and the
+ * tag was missed.
+ */
+const ALPHA_CAPABLE_CODECS = new Set(["vp9", "vp8", "prores"]);
+
+export function codecMayHaveAlpha(codec: string | undefined): boolean {
+  return ALPHA_CAPABLE_CODECS.has((codec ?? "").toLowerCase());
+}
+
+export function decoderForCodec(codec: string | undefined): string {
+  const c = (codec ?? "").toLowerCase();
+  if (c === "vp9") return "libvpx-vp9";
+  if (c === "vp8") return "libvpx";
+  return c;
+}
+
 function resolveFrameFormat(metadata: VideoMetadata, requested?: "jpg" | "png"): CacheFrameFormat {
   if (requested) return requested;
-  return metadata.hasAlpha ? "png" : "jpg";
+  if (metadata.hasAlpha || codecMayHaveAlpha(metadata.videoCodec)) return "png";
+  return "jpg";
 }
 
 /**