fix(engine): warn when a video input declares alpha but decodes opaque (#3771)

* fix(engine): warn when a video input declares alpha but decodes opaque

- Problem: a video input whose alpha_mode=1 tag outlives its alpha plane (a remux drops the BlockAdditional sidecar while keeping the tag) composites as a solid rectangle, and nothing tells the user their file, not the renderer, is the problem (#3220 / #3226).
- Fix: during extraction preflight, when an input declares alpha (hasAlpha) and its codec can carry alpha, sample the decoded alpha plane (up to 3 frames at 8x8 rgba, bounded 30s, ~768-byte ceiling). If uniformly opaque, emit a non-blocking stderr warning naming the file with the re-export remedy. Inconclusive probes stay silent; opaque full-frame backgrounds are legitimate and never fail the render.
- Verification: 8 new unit tests (byte logic + message); engine suite 1483/1487 with 4 pre-existing failures on clean main (missing hdr-regression PNG fixture in this clone); typecheck/lint/format clean; real-fixture checks: lying-tag WebM warns, genuine-alpha WebM stays silent, both agree with ffmpeg alphaextract.

Signed-off-by: Santhi Prakash <b.santhiprakash@gmail.com>

* fix(engine): probe input alpha with the decoder that matches its codec

* style(engine): format the alpha probe test

* fix(engine): keep the alpha probe from failing or racing a render

---------

Signed-off-by: Santhi Prakash <b.santhiprakash@gmail.com>
Co-authored-by: Miguel Ángel <miguel.sierra@heygen.com>

packages/engine/src/services/videoFrameExtractor.ts

@@ -41,6 +41,7 @@ import {
   extractMediaMetadata,
   type VideoMetadata,
 } from "../utils/ffprobe.js";
+import { inputAlphaOpaqueWarning, probeInputAlphaPlane } from "../utils/alphaPlaneProbe.js";
 import {
   analyzeCompositionHdr,
   isHdrColorSpace as isHdrColorSpaceUtil,
@@ -1844,6 +1845,23 @@ export async function extractAllVideoFrames(
   const sdrToHdrTransfers: Array<HdrTransfer | undefined> = resolvedVideos.map(() => undefined);
   breakdown.hdrProbeMs = Date.now() - phase2ProbeStart;
 
+  // Warning only: an opaque video used as a full-frame background is legitimate.
+  const alphaWarnedSrcs = new Set<string>();
+  if (resolvedVideos.length > 0) {
+    await Promise.all(
+      resolvedVideos.map(async ({ video, videoPath }, index) => {
+        if (signal?.aborted) return;
+        const metadata = videoMetadata[index];
+        if (!metadata?.hasAlpha || !codecMayHaveAlpha(metadata.videoCodec)) return;
+        if (alphaWarnedSrcs.has(video.src)) return;
+        alphaWarnedSrcs.add(video.src);
+        const decoder = decoderForCodec(metadata.videoCodec);
+        if ((await probeInputAlphaPlane(videoPath, decoder, signal)) !== true) return;
+        process.stderr.write(inputAlphaOpaqueWarning(video.src));
+      }),
+    );
+  }
+
   const hdrPreflightStart = Date.now();
   const hdrInfo = analyzeCompositionHdr(videoColorSpaces);
   // Track entries the HDR preflight validated as non-extractable so they can

packages/engine/src/utils/alphaPlaneProbe.test.ts

@@ -0,0 +1,115 @@
+import { spawnSync } from "node:child_process";
+import { mkdtempSync, rmSync } from "node:fs";
+import { tmpdir } from "node:os";
+import { join } from "node:path";
+import { afterAll, beforeAll, describe, expect, it } from "vitest";
+import { decoderForCodec } from "../services/videoFrameExtractor.js";
+import {
+  inputAlphaOpaqueWarning,
+  probeInputAlphaPlane,
+  sampledRgbaAlphaIsFullyOpaque,
+} from "./alphaPlaneProbe.js";
+import { getFfmpegBinary } from "./ffmpegBinaries.js";
+
+const BYTES_PER_FRAME = 8 * 8 * 4; // 256 — one 8x8 rgba frame
+
+/** Build a raw rgba sample where the alpha byte of every pixel is `alpha`. */
+function sample(alpha: number, frameCount: number): Buffer {
+  const buf = Buffer.alloc(BYTES_PER_FRAME * frameCount, 0);
+  for (let i = 3; i < buf.length; i += 4) {
+    buf[i] = alpha;
+  }
+  return buf;
+}
+
+describe("sampledRgbaAlphaIsFullyOpaque", () => {
+  it("returns true when every alpha byte in a single frame is 255", () => {
+    expect(sampledRgbaAlphaIsFullyOpaque(sample(255, 1))).toBe(true);
+  });
+
+  it("returns true across a multi-frame sample (2 and 3 frames)", () => {
+    expect(sampledRgbaAlphaIsFullyOpaque(sample(255, 2))).toBe(true);
+    expect(sampledRgbaAlphaIsFullyOpaque(sample(255, 3))).toBe(true);
+  });
+
+  it("returns false when any pixel shows full transparency", () => {
+    const buf = sample(255, 1);
+    buf[3] = 0; // first pixel's alpha
+    expect(sampledRgbaAlphaIsFullyOpaque(buf)).toBe(false);
+  });
+
+  it("returns false when any pixel shows partial transparency", () => {
+    const buf = sample(255, 2);
+    buf[3 + 4 * 10] = 254; // partial alpha on the 11th pixel
+    expect(sampledRgbaAlphaIsFullyOpaque(buf)).toBe(false);
+  });
+
+  it("returns undefined for an empty sample (inconclusive, not a warning)", () => {
+    expect(sampledRgbaAlphaIsFullyOpaque(Buffer.alloc(0))).toBeUndefined();
+  });
+
+  it("returns undefined for a byte count that is not a whole-frame multiple", () => {
+    expect(sampledRgbaAlphaIsFullyOpaque(Buffer.alloc(BYTES_PER_FRAME - 1))).toBeUndefined();
+  });
+
+  it("returns undefined for an oversized sample beyond the 3-frame ceiling", () => {
+    expect(sampledRgbaAlphaIsFullyOpaque(Buffer.alloc(BYTES_PER_FRAME * 4))).toBeUndefined();
+  });
+});
+
+describe("inputAlphaOpaqueWarning", () => {
+  it("names the offending file and carries the re-export remedy", () => {
+    const line = inputAlphaOpaqueWarning("avatar.webm");
+    expect(line).toContain('src="avatar.webm"');
+    expect(line).toContain("declares an alpha channel");
+    expect(line).toContain("first frames decode fully opaque");
+    expect(line).toContain("alpha pixel format");
+    expect(line).not.toContain("yuva420p");
+    expect(line.endsWith("\n")).toBe(true);
+  });
+});
+
+const HAS_FFMPEG = spawnSync(getFfmpegBinary(), ["-version"], { encoding: "utf-8" }).status === 0;
+
+describe.skipIf(!HAS_FFMPEG)("probeInputAlphaPlane on real files", () => {
+  let dir: string;
+  beforeAll(() => {
+    dir = mkdtempSync(join(tmpdir(), "hf-alpha-probe-test-"));
+  });
+  afterAll(() => {
+    rmSync(dir, { recursive: true, force: true });
+  });
+
+  const encode = (name: string, color: string, codecArgs: string[]): string => {
+    const out = join(dir, name);
+    const args = [
+      "-v",
+      "error",
+      "-f",
+      "lavfi",
+      "-i",
+      `color=c=${color}:s=32x32:d=0.2,format=rgba`,
+      ...codecArgs,
+    ];
+    const res = spawnSync(getFfmpegBinary(), [...args, "-y", out]);
+    expect(res.status).toBe(0);
+    return out;
+  };
+  const vp9 = ["-c:v", "libvpx-vp9", "-pix_fmt", "yuva420p"];
+  const prores = ["-c:v", "prores_ks", "-profile:v", "4", "-pix_fmt", "yuva444p10le"];
+
+  it("reports a VP9 input whose alpha is all opaque", async () => {
+    const file = encode("opaque.webm", "red", vp9);
+    expect(await probeInputAlphaPlane(file, decoderForCodec("vp9"))).toBe(true);
+  });
+
+  it("does not flag a VP9 input that is really transparent", async () => {
+    const file = encode("clear.webm", "red@0.0", vp9);
+    expect(await probeInputAlphaPlane(file, decoderForCodec("vp9"))).toBe(false);
+  });
+
+  it("probes ProRes 4444 with its own decoder, not the VP9 one", async () => {
+    const file = encode("opaque.mov", "red", prores);
+    expect(await probeInputAlphaPlane(file, decoderForCodec("prores"))).toBe(true);
+  });
+});

packages/engine/src/utils/alphaPlaneProbe.ts

@@ -0,0 +1,74 @@
+import { mkdtempSync, readFileSync, rmSync } from "node:fs";
+import { tmpdir } from "node:os";
+import { join } from "node:path";
+import { runFfmpeg } from "./runFfmpeg.js";
+
+// Up to 3 frames at 8x8 rgba; a 1- or 2-frame clip is still a whole sample.
+const BYTES_PER_SAMPLE_FRAME = 8 * 8 * 4;
+const MAX_SAMPLE_BYTES = BYTES_PER_SAMPLE_FRAME * 3;
+
+/** true: every sampled alpha byte is 255; false: some transparency; undefined: malformed sample. */
+export function sampledRgbaAlphaIsFullyOpaque(buf: Buffer): boolean | undefined {
+  if (
+    buf.length === 0 ||
+    buf.length > MAX_SAMPLE_BYTES ||
+    buf.length % BYTES_PER_SAMPLE_FRAME !== 0
+  ) {
+    return undefined;
+  }
+  for (let i = 3; i < buf.length; i += 4) {
+    if (buf[i] !== 255) return false;
+  }
+  return true;
+}
+
+/**
+ * Whether a video input that declares alpha decodes fully opaque (a remux can keep `alpha_mode=1`
+ * while dropping the alpha sidecar). `decoder` must keep alpha. Any failure returns undefined.
+ */
+export async function probeInputAlphaPlane(
+  videoPath: string,
+  decoder: string,
+  signal?: AbortSignal,
+): Promise<boolean | undefined> {
+  let probeDir: string | undefined;
+  try {
+    probeDir = mkdtempSync(join(tmpdir(), "hf-alpha-probe-"));
+    const samplePath = join(probeDir, "alpha.raw");
+    const result = await runFfmpeg(
+      [
+        "-v",
+        "error",
+        "-c:v",
+        decoder,
+        "-i",
+        videoPath,
+        "-frames:v",
+        "3",
+        "-vf",
+        "scale=8:8",
+        "-pix_fmt",
+        "rgba",
+        "-f",
+        "rawvideo",
+        samplePath,
+      ],
+      { timeout: 30_000, signal },
+    );
+    if (!result.success) return undefined;
+    return sampledRgbaAlphaIsFullyOpaque(readFileSync(samplePath));
+  } catch {
+    return undefined;
+  } finally {
+    if (probeDir) rmSync(probeDir, { recursive: true, force: true });
+  }
+}
+
+export function inputAlphaOpaqueWarning(src: string): string {
+  return (
+    `[hyperframes:render] WARNING: video src="${src}" declares an alpha channel ` +
+    "but its first frames decode fully opaque. If it should be transparent, re-export " +
+    "it with an alpha pixel format and avoid remuxing afterward, which can drop the " +
+    "alpha while keeping the tag.\n"
+  );
+}