Merge 0b3f1866ed into 0e558d5916

packages/engine/src/services/videoFrameExtractor.ts

@@ -28,6 +28,7 @@ import {
   extractMediaMetadata,
   type VideoMetadata,
 } from "../utils/ffprobe.js";
+import { inputAlphaOpaqueWarning, probeInputAlphaPlane } from "../utils/alphaPlaneProbe.js";
 import {
   analyzeCompositionHdr,
   isHdrColorSpace as isHdrColorSpaceUtil,
@@ -1616,6 +1617,31 @@ export async function extractAllVideoFrames(
   const sdrToHdrTransfers: Array<HdrTransfer | undefined> = resolvedVideos.map(() => undefined);
   breakdown.hdrProbeMs = Date.now() - phase2ProbeStart;
 
+  // Phase 2a: warn when a video input declares alpha but its decoded alpha
+  // plane is uniformly opaque. `alpha_mode=1` is container metadata that can
+  // outlive the alpha it describes — a remux can drop the BlockAdditional
+  // sidecar while keeping the tag — so a file can promise transparency it no
+  // longer contains and then composite as a solid rectangle with no way for
+  // the user to know their file, not the renderer, is the problem
+  // (heygen-com/hyperframes#3220 / #3226). Warning only, never an error: an
+  // opaque video used as a full-frame background is legitimate, and an
+  // inconclusive probe stays silent.
+  const alphaWarnedSrcs = new Set<string>();
+  if (resolvedVideos.length > 0) {
+    await Promise.all(
+      resolvedVideos.map(async ({ video, videoPath }, index) => {
+        if (signal?.aborted) return;
+        const metadata = videoMetadata[index];
+        if (!metadata?.hasAlpha || !codecMayHaveAlpha(metadata.videoCodec)) return;
+        if (alphaWarnedSrcs.has(video.src)) return;
+        const opaque = await probeInputAlphaPlane(videoPath);
+        if (opaque !== true) return;
+        alphaWarnedSrcs.add(video.src);
+        process.stderr.write(inputAlphaOpaqueWarning(video.src));
+      }),
+    );
+  }
+
   const hdrPreflightStart = Date.now();
   const hdrInfo = analyzeCompositionHdr(videoColorSpaces);
   // Track entries the HDR preflight validated as non-extractable so they can

packages/engine/src/utils/alphaPlaneProbe.test.ts

@@ -0,0 +1,60 @@
+import { describe, expect, it } from "vitest";
+import { inputAlphaOpaqueWarning, sampledRgbaAlphaIsFullyOpaque } from "./alphaPlaneProbe.js";
+
+const BYTES_PER_FRAME = 8 * 8 * 4; // 256 — one 8x8 rgba frame
+
+/** Build a raw rgba sample where the alpha byte of every pixel is `alpha`. */
+function sample(alpha: number, frameCount: number): Buffer {
+  const buf = Buffer.alloc(BYTES_PER_FRAME * frameCount, 0);
+  for (let i = 3; i < buf.length; i += 4) {
+    buf[i] = alpha;
+  }
+  return buf;
+}
+
+describe("sampledRgbaAlphaIsFullyOpaque", () => {
+  it("returns true when every alpha byte in a single frame is 255", () => {
+    expect(sampledRgbaAlphaIsFullyOpaque(sample(255, 1))).toBe(true);
+  });
+
+  it("returns true across a multi-frame sample (2 and 3 frames)", () => {
+    expect(sampledRgbaAlphaIsFullyOpaque(sample(255, 2))).toBe(true);
+    expect(sampledRgbaAlphaIsFullyOpaque(sample(255, 3))).toBe(true);
+  });
+
+  it("returns false when any pixel shows full transparency", () => {
+    const buf = sample(255, 1);
+    buf[3] = 0; // first pixel's alpha
+    expect(sampledRgbaAlphaIsFullyOpaque(buf)).toBe(false);
+  });
+
+  it("returns false when any pixel shows partial transparency", () => {
+    const buf = sample(255, 2);
+    buf[3 + 4 * 10] = 254; // partial alpha on the 11th pixel
+    expect(sampledRgbaAlphaIsFullyOpaque(buf)).toBe(false);
+  });
+
+  it("returns undefined for an empty sample (inconclusive, not a warning)", () => {
+    expect(sampledRgbaAlphaIsFullyOpaque(Buffer.alloc(0))).toBeUndefined();
+  });
+
+  it("returns undefined for a byte count that is not a whole-frame multiple", () => {
+    expect(sampledRgbaAlphaIsFullyOpaque(Buffer.alloc(BYTES_PER_FRAME - 1))).toBeUndefined();
+  });
+
+  it("returns undefined for an oversized sample beyond the 3-frame ceiling", () => {
+    expect(sampledRgbaAlphaIsFullyOpaque(Buffer.alloc(BYTES_PER_FRAME * 4))).toBeUndefined();
+  });
+});
+
+describe("inputAlphaOpaqueWarning", () => {
+  it("names the offending file and carries the re-export remedy", () => {
+    const line = inputAlphaOpaqueWarning("avatar.webm");
+    expect(line).toContain('src="avatar.webm"');
+    expect(line).toContain("declares an alpha channel");
+    expect(line).toContain("decodes fully opaque");
+    expect(line).toContain("yuva420p");
+    expect(line).toContain("alpha sidecar");
+    expect(line.endsWith("\n")).toBe(true);
+  });
+});

packages/engine/src/utils/alphaPlaneProbe.ts

@@ -0,0 +1,130 @@
+/**
+ * Alpha-plane probe for video INPUTS.
+ *
+ * `alpha_mode=1` (or an alpha-capable `pix_fmt`) is metadata that can outlive
+ * the alpha it describes: a remux can keep the tag while dropping the
+ * BlockAdditional sidecar that carries the VP9 alpha plane, so a file can
+ * keep promising transparency it no longer contains. Such an input renders
+ * as a solid opaque rectangle over whatever is beneath it, and nothing tells
+ * the user their file — not the renderer — is the problem.
+ *
+ * This module answers one question about a video input: it declares alpha,
+ * but does the decoded alpha plane come out uniformly opaque? The answer is
+ * surfaced as a warning only, never an error — an opaque video used as a
+ * full-frame background is perfectly legitimate, so this can never fail a
+ * render.
+ *
+ * The sibling CLI-side check (`packages/cli/src/utils/webmAlphaCheck.ts`)
+ * verifies the same property on the render OUTPUT after a WebM encode; this
+ * probe is the input-side counterpart, implemented in the engine so it runs
+ * for every render path (local CLI, Docker, cloud) at the point where all
+ * input metadata is already in hand.
+ */
+
+import { mkdtempSync, readFileSync, rmSync } from "node:fs";
+import { tmpdir } from "node:os";
+import { join } from "node:path";
+import { runFfmpeg } from "./runFfmpeg.js";
+
+/**
+ * Bytes per sampled frame at 8x8 rgba: 8 * 8 * 4 = 256. `-frames:v 3` samples
+ * AT MOST 3 frames — a legitimate 1-frame WebM (a still) yields 256 bytes and
+ * a 2-frame yields 512, both valid opaque samples that must be evaluated.
+ * Mirrors the byte accounting in the CLI's `webmAlphaCheck.sampledAlphaIsFullyOpaque`.
+ */
+const BYTES_PER_SAMPLE_FRAME = 8 * 8 * 4;
+const MAX_SAMPLE_BYTES = BYTES_PER_SAMPLE_FRAME * 3;
+
+/**
+ * Decode a raw rgba sample buffer and decide whether the alpha plane is
+ * uniformly opaque. Pure — exported so tests exercise the shipped byte logic
+ * directly without spawning ffmpeg.
+ *
+ * Returns:
+ *  - `true`  — every alpha byte across all sampled frames is 255;
+ *  - `false` — any pixel shows partial or full transparency;
+ *  - `undefined` — the sample is malformed (empty, oversized, or not a
+ *    positive whole-frame byte count). An inconclusive probe is not a warning
+ *    trigger.
+ */
+export function sampledRgbaAlphaIsFullyOpaque(buf: Buffer): boolean | undefined {
+  if (
+    buf.length === 0 ||
+    buf.length > MAX_SAMPLE_BYTES ||
+    buf.length % BYTES_PER_SAMPLE_FRAME !== 0
+  ) {
+    return undefined;
+  }
+  for (let i = 3; i < buf.length; i += 4) {
+    if (buf[i] !== 255) return false;
+  }
+  return true;
+}
+
+/**
+ * Sample a video input's alpha plane and report whether it decodes fully
+ * opaque. Runs ffmpeg with the libvpx-vp9 input decoder forced (the default
+ * decoder silently discards VP9 alpha), sampling up to 3 frames at 8x8 rgba —
+ * the same command the CLI's post-render WebM check uses. The decision to
+ * sample a few frames rather than the first frame only is deliberate: it
+ * catches a clip that is opaque at the head and transparent later, at a fixed
+ * ~768-byte ceiling.
+ *
+ * Best-effort and non-blocking: any failure (missing binary, decode error,
+ * malformed output) returns `undefined` — a diagnostic can never take down a
+ * render.
+ */
+export async function probeInputAlphaPlane(videoPath: string): Promise<boolean | undefined> {
+  const probeDir = mkdtempSync(join(tmpdir(), "hf-alpha-probe-"));
+  const samplePath = join(probeDir, "alpha.raw");
+  try {
+    const result = await runFfmpeg(
+      [
+        "-v",
+        "error",
+        "-c:v",
+        "libvpx-vp9",
+        "-i",
+        videoPath,
+        "-frames:v",
+        "3",
+        "-vf",
+        "scale=8:8",
+        "-pix_fmt",
+        "rgba",
+        "-f",
+        "rawvideo",
+        samplePath,
+      ],
+      { timeout: 30_000 },
+    );
+    if (!result.success) return undefined;
+    let buf: Buffer;
+    try {
+      buf = readFileSync(samplePath);
+    } catch {
+      return undefined;
+    }
+    return sampledRgbaAlphaIsFullyOpaque(buf);
+  } catch {
+    return undefined;
+  } finally {
+    rmSync(probeDir, { recursive: true, force: true });
+  }
+}
+
+/**
+ * The stderr warning line written when a video input declares alpha but its
+ * decoded alpha plane is uniformly opaque. Matches the existing
+ * `[hyperframes:render] WARNING:` convention in videoFrameExtractor (missing
+ * src, unwritable cache dir) so the same surface carries the same shape.
+ * Pure — exported for tests.
+ */
+export function inputAlphaOpaqueWarning(src: string): string {
+  return (
+    `[hyperframes:render] WARNING: video src="${src}" declares an alpha channel ` +
+    "but decodes fully opaque. Transparency will not composite. If it should " +
+    "be transparent, re-export with `-pix_fmt yuva420p` and avoid remuxing " +
+    "afterward, which can drop the alpha sidecar while keeping the tag.\n"
+  );
+}