From cad7dbed0dc29d53ec3a4bbb2b0bea2fa3c1495e Mon Sep 17 00:00:00 2001 From: Santhi Prakash <38608178+santhiprakash@users.noreply.github.com> Date: Fri, 2 Oct 2026 17:29:16 +0530 Subject: [PATCH] fix(engine): warn when a video input declares alpha but decodes opaque (#3771) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * fix(engine): warn when a video input declares alpha but decodes opaque - Problem: a video input whose alpha_mode=1 tag outlives its alpha plane (a remux drops the BlockAdditional sidecar while keeping the tag) composites as a solid rectangle, and nothing tells the user their file, not the renderer, is the problem (#3220 / #3226). - Fix: during extraction preflight, when an input declares alpha (hasAlpha) and its codec can carry alpha, sample the decoded alpha plane (up to 3 frames at 8x8 rgba, bounded 30s, ~768-byte ceiling). If uniformly opaque, emit a non-blocking stderr warning naming the file with the re-export remedy. Inconclusive probes stay silent; opaque full-frame backgrounds are legitimate and never fail the render. - Verification: 8 new unit tests (byte logic + message); engine suite 1483/1487 with 4 pre-existing failures on clean main (missing hdr-regression PNG fixture in this clone); typecheck/lint/format clean; real-fixture checks: lying-tag WebM warns, genuine-alpha WebM stays silent, both agree with ffmpeg alphaextract. Signed-off-by: Santhi Prakash * fix(engine): probe input alpha with the decoder that matches its codec * style(engine): format the alpha probe test * fix(engine): keep the alpha probe from failing or racing a render --------- Signed-off-by: Santhi Prakash Co-authored-by: Miguel Ángel --- .../src/services/videoFrameExtractor.ts | 18 +++ .../engine/src/utils/alphaPlaneProbe.test.ts | 115 ++++++++++++++++++ packages/engine/src/utils/alphaPlaneProbe.ts | 74 +++++++++++ 3 files changed, 207 insertions(+) create mode 100644 packages/engine/src/utils/alphaPlaneProbe.test.ts create mode 100644 packages/engine/src/utils/alphaPlaneProbe.ts diff --git a/packages/engine/src/services/videoFrameExtractor.ts b/packages/engine/src/services/videoFrameExtractor.ts index 3b8879f87d..d5de062553 100644 --- a/packages/engine/src/services/videoFrameExtractor.ts +++ b/packages/engine/src/services/videoFrameExtractor.ts @@ -41,6 +41,7 @@ import { extractMediaMetadata, type VideoMetadata, } from "../utils/ffprobe.js"; +import { inputAlphaOpaqueWarning, probeInputAlphaPlane } from "../utils/alphaPlaneProbe.js"; import { analyzeCompositionHdr, isHdrColorSpace as isHdrColorSpaceUtil, @@ -1844,6 +1845,23 @@ export async function extractAllVideoFrames( const sdrToHdrTransfers: Array = resolvedVideos.map(() => undefined); breakdown.hdrProbeMs = Date.now() - phase2ProbeStart; + // Warning only: an opaque video used as a full-frame background is legitimate. + const alphaWarnedSrcs = new Set(); + if (resolvedVideos.length > 0) { + await Promise.all( + resolvedVideos.map(async ({ video, videoPath }, index) => { + if (signal?.aborted) return; + const metadata = videoMetadata[index]; + if (!metadata?.hasAlpha || !codecMayHaveAlpha(metadata.videoCodec)) return; + if (alphaWarnedSrcs.has(video.src)) return; + alphaWarnedSrcs.add(video.src); + const decoder = decoderForCodec(metadata.videoCodec); + if ((await probeInputAlphaPlane(videoPath, decoder, signal)) !== true) return; + process.stderr.write(inputAlphaOpaqueWarning(video.src)); + }), + ); + } + const hdrPreflightStart = Date.now(); const hdrInfo = analyzeCompositionHdr(videoColorSpaces); // Track entries the HDR preflight validated as non-extractable so they can diff --git a/packages/engine/src/utils/alphaPlaneProbe.test.ts b/packages/engine/src/utils/alphaPlaneProbe.test.ts new file mode 100644 index 0000000000..85d8ff117e --- /dev/null +++ b/packages/engine/src/utils/alphaPlaneProbe.test.ts @@ -0,0 +1,115 @@ +import { spawnSync } from "node:child_process"; +import { mkdtempSync, rmSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { afterAll, beforeAll, describe, expect, it } from "vitest"; +import { decoderForCodec } from "../services/videoFrameExtractor.js"; +import { + inputAlphaOpaqueWarning, + probeInputAlphaPlane, + sampledRgbaAlphaIsFullyOpaque, +} from "./alphaPlaneProbe.js"; +import { getFfmpegBinary } from "./ffmpegBinaries.js"; + +const BYTES_PER_FRAME = 8 * 8 * 4; // 256 — one 8x8 rgba frame + +/** Build a raw rgba sample where the alpha byte of every pixel is `alpha`. */ +function sample(alpha: number, frameCount: number): Buffer { + const buf = Buffer.alloc(BYTES_PER_FRAME * frameCount, 0); + for (let i = 3; i < buf.length; i += 4) { + buf[i] = alpha; + } + return buf; +} + +describe("sampledRgbaAlphaIsFullyOpaque", () => { + it("returns true when every alpha byte in a single frame is 255", () => { + expect(sampledRgbaAlphaIsFullyOpaque(sample(255, 1))).toBe(true); + }); + + it("returns true across a multi-frame sample (2 and 3 frames)", () => { + expect(sampledRgbaAlphaIsFullyOpaque(sample(255, 2))).toBe(true); + expect(sampledRgbaAlphaIsFullyOpaque(sample(255, 3))).toBe(true); + }); + + it("returns false when any pixel shows full transparency", () => { + const buf = sample(255, 1); + buf[3] = 0; // first pixel's alpha + expect(sampledRgbaAlphaIsFullyOpaque(buf)).toBe(false); + }); + + it("returns false when any pixel shows partial transparency", () => { + const buf = sample(255, 2); + buf[3 + 4 * 10] = 254; // partial alpha on the 11th pixel + expect(sampledRgbaAlphaIsFullyOpaque(buf)).toBe(false); + }); + + it("returns undefined for an empty sample (inconclusive, not a warning)", () => { + expect(sampledRgbaAlphaIsFullyOpaque(Buffer.alloc(0))).toBeUndefined(); + }); + + it("returns undefined for a byte count that is not a whole-frame multiple", () => { + expect(sampledRgbaAlphaIsFullyOpaque(Buffer.alloc(BYTES_PER_FRAME - 1))).toBeUndefined(); + }); + + it("returns undefined for an oversized sample beyond the 3-frame ceiling", () => { + expect(sampledRgbaAlphaIsFullyOpaque(Buffer.alloc(BYTES_PER_FRAME * 4))).toBeUndefined(); + }); +}); + +describe("inputAlphaOpaqueWarning", () => { + it("names the offending file and carries the re-export remedy", () => { + const line = inputAlphaOpaqueWarning("avatar.webm"); + expect(line).toContain('src="avatar.webm"'); + expect(line).toContain("declares an alpha channel"); + expect(line).toContain("first frames decode fully opaque"); + expect(line).toContain("alpha pixel format"); + expect(line).not.toContain("yuva420p"); + expect(line.endsWith("\n")).toBe(true); + }); +}); + +const HAS_FFMPEG = spawnSync(getFfmpegBinary(), ["-version"], { encoding: "utf-8" }).status === 0; + +describe.skipIf(!HAS_FFMPEG)("probeInputAlphaPlane on real files", () => { + let dir: string; + beforeAll(() => { + dir = mkdtempSync(join(tmpdir(), "hf-alpha-probe-test-")); + }); + afterAll(() => { + rmSync(dir, { recursive: true, force: true }); + }); + + const encode = (name: string, color: string, codecArgs: string[]): string => { + const out = join(dir, name); + const args = [ + "-v", + "error", + "-f", + "lavfi", + "-i", + `color=c=${color}:s=32x32:d=0.2,format=rgba`, + ...codecArgs, + ]; + const res = spawnSync(getFfmpegBinary(), [...args, "-y", out]); + expect(res.status).toBe(0); + return out; + }; + const vp9 = ["-c:v", "libvpx-vp9", "-pix_fmt", "yuva420p"]; + const prores = ["-c:v", "prores_ks", "-profile:v", "4", "-pix_fmt", "yuva444p10le"]; + + it("reports a VP9 input whose alpha is all opaque", async () => { + const file = encode("opaque.webm", "red", vp9); + expect(await probeInputAlphaPlane(file, decoderForCodec("vp9"))).toBe(true); + }); + + it("does not flag a VP9 input that is really transparent", async () => { + const file = encode("clear.webm", "red@0.0", vp9); + expect(await probeInputAlphaPlane(file, decoderForCodec("vp9"))).toBe(false); + }); + + it("probes ProRes 4444 with its own decoder, not the VP9 one", async () => { + const file = encode("opaque.mov", "red", prores); + expect(await probeInputAlphaPlane(file, decoderForCodec("prores"))).toBe(true); + }); +}); diff --git a/packages/engine/src/utils/alphaPlaneProbe.ts b/packages/engine/src/utils/alphaPlaneProbe.ts new file mode 100644 index 0000000000..b963ca1156 --- /dev/null +++ b/packages/engine/src/utils/alphaPlaneProbe.ts @@ -0,0 +1,74 @@ +import { mkdtempSync, readFileSync, rmSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { runFfmpeg } from "./runFfmpeg.js"; + +// Up to 3 frames at 8x8 rgba; a 1- or 2-frame clip is still a whole sample. +const BYTES_PER_SAMPLE_FRAME = 8 * 8 * 4; +const MAX_SAMPLE_BYTES = BYTES_PER_SAMPLE_FRAME * 3; + +/** true: every sampled alpha byte is 255; false: some transparency; undefined: malformed sample. */ +export function sampledRgbaAlphaIsFullyOpaque(buf: Buffer): boolean | undefined { + if ( + buf.length === 0 || + buf.length > MAX_SAMPLE_BYTES || + buf.length % BYTES_PER_SAMPLE_FRAME !== 0 + ) { + return undefined; + } + for (let i = 3; i < buf.length; i += 4) { + if (buf[i] !== 255) return false; + } + return true; +} + +/** + * Whether a video input that declares alpha decodes fully opaque (a remux can keep `alpha_mode=1` + * while dropping the alpha sidecar). `decoder` must keep alpha. Any failure returns undefined. + */ +export async function probeInputAlphaPlane( + videoPath: string, + decoder: string, + signal?: AbortSignal, +): Promise { + let probeDir: string | undefined; + try { + probeDir = mkdtempSync(join(tmpdir(), "hf-alpha-probe-")); + const samplePath = join(probeDir, "alpha.raw"); + const result = await runFfmpeg( + [ + "-v", + "error", + "-c:v", + decoder, + "-i", + videoPath, + "-frames:v", + "3", + "-vf", + "scale=8:8", + "-pix_fmt", + "rgba", + "-f", + "rawvideo", + samplePath, + ], + { timeout: 30_000, signal }, + ); + if (!result.success) return undefined; + return sampledRgbaAlphaIsFullyOpaque(readFileSync(samplePath)); + } catch { + return undefined; + } finally { + if (probeDir) rmSync(probeDir, { recursive: true, force: true }); + } +} + +export function inputAlphaOpaqueWarning(src: string): string { + return ( + `[hyperframes:render] WARNING: video src="${src}" declares an alpha channel ` + + "but its first frames decode fully opaque. If it should be transparent, re-export " + + "it with an alpha pixel format and avoid remuxing afterward, which can drop the " + + "alpha while keeping the tag.\n" + ); +}