From 6cfb05e38bcb0ad02fa848500b462f0193a639b1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Miguel=20=C3=81ngel?= Date: Wed, 29 Jul 2026 20:05:42 +0200 Subject: [PATCH] fix(render): preserve transparency in GIF output (#2327) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ## What - Treat GIF as an alpha-capable output format and capture its frames as RGBA PNGs. - Encode transparent GIFs with explicit FFmpeg palette semantics: `reserve_transparent=1` and `alpha_threshold=128`. - Keep page-side shader compositing enabled for GIF while the resulting composite is captured through the RGBA disk-frame path. - Extend the real render harness to verify decoded GIF alpha and compare a GIF shader-transition frame against the existing MP4 golden. - Preserve the existing opaque encoder contract: `needsAlpha=false` continues to use JPEG frames without alpha-only palette filters. ## Why Direct `--format gif` renders silently flattened transparent compositions when frames are captured as JPEG, because the palette encoder receives no alpha plane to preserve. GIF also needs page-side shader compositing. A blanket `needsAlpha` exclusion disabled that path after enabling RGBA capture, while the layered compositor intentionally excludes GIF. That left shader GIFs on the DOM fallback and produced hard cuts instead of the authored WebGL blend. ## How - Centralize output alpha detection in `outputNeedsAlpha`, shared by in-process and distributed planning. - Select PNG or JPEG GIF frame input from the resolved alpha requirement. - Make palette transparency flags explicit and conditional so the legacy opaque path retains its existing arguments. - Add an explicit output-format capability for page-side shader compositing: MP4 keeps its opaque streaming path, GIF uses RGBA PNG disk frames, and WebM/MOV/PNG sequence retain their existing paths. - Add `data-no-timeline` to the static transparency fixture so the artifact regression does not wait for a timeline it intentionally does not register. ## Test plan - [x] RED on base: direct GIF decoded with an opaque corner instead of alpha 0. - [x] RED on the previous PR head: the real GIF shader-transition frame scored 11.05 dB against the existing golden because neither shader compositor was active. - [x] `bun test packages/producer/src/services/render/renderFormat.test.ts packages/producer/src/services/render/stages/encodeStage.test.ts packages/producer/src/services/render/capturePlan.test.ts` — 23 passed. - [x] `bun run --filter @hyperframes/producer typecheck` - [x] `bun run --filter @hyperframes/producer build` - [x] `bun run --filter @hyperframes/producer test:transparency` — WebM, GIF, and PNG sequence alpha assertions passed; GIF shader control/transition frames scored 28.11/26.57 dB against the golden. - [x] `bun run --cwd packages/producer tsx src/regression-harness.ts page-side-shader-compositor-render-compat --sequential` — all 100 visual checkpoints passed, stream parity passed, and audio correlation was 1.000. - [x] Changed-file oxlint, oxfmt check, pre-commit checks, and `git diff --check`. --- .../producer/src/services/distributed/plan.ts | 6 +- .../src/services/render/renderFormat.test.ts | 28 ++++ .../src/services/render/renderFormat.ts | 9 ++ .../render/stages/encodeStage.test.ts | 60 ++++++++ .../src/services/render/stages/encodeStage.ts | 6 +- .../services/render/stages/gifEncodeArgs.ts | 7 +- .../src/services/renderOrchestrator.ts | 35 ++--- packages/producer/src/transparency-test.ts | 137 +++++++++++++++++- .../tests/transparency-regression/meta.json | 2 +- .../transparency-regression/src/index.html | 1 + 10 files changed, 260 insertions(+), 31 deletions(-) create mode 100644 packages/producer/src/services/render/renderFormat.test.ts create mode 100644 packages/producer/src/services/render/renderFormat.ts diff --git a/packages/producer/src/services/distributed/plan.ts b/packages/producer/src/services/distributed/plan.ts index 63121e289..ef989c367 100644 --- a/packages/producer/src/services/distributed/plan.ts +++ b/packages/producer/src/services/distributed/plan.ts @@ -66,6 +66,7 @@ import { validateNoSystemFonts, } from "../render/planValidation.js"; import { snapshotRuntimeEnv } from "../render/runtimeEnvSnapshot.js"; +import { outputNeedsAlpha } from "../render/renderFormat.js"; import { buildSyntheticRenderJob, buildPlanVideosJson, @@ -888,7 +889,7 @@ export async function plan( // move the contents over once the staged work completes. const finalCompiledDir = join(planDir, "compiled"); - // webm + mov + png-sequence carry alpha — flip force-screenshot so + // Alpha-capable distributed formats flip force-screenshot so // compileStage takes the alpha-aware capture path (BeginFrame doesn't // preserve alpha on Linux headless-shell). Must match the in-process // renderer's needsAlpha logic in `renderOrchestrator.ts` so chunked @@ -897,8 +898,7 @@ export async function plan( // into the planDir and every chunk worker captures opaque RGB — the // libvpx-vp9 alpha sub-stream then encodes either uniform alpha or // gets downgraded by the encoder, producing un-keyable webm output. - const needsAlpha = - config.format === "png-sequence" || config.format === "mov" || config.format === "webm"; + const needsAlpha = outputNeedsAlpha(config.format); // ── Compile ── const compileResult = await runCompileStage({ diff --git a/packages/producer/src/services/render/renderFormat.test.ts b/packages/producer/src/services/render/renderFormat.test.ts new file mode 100644 index 000000000..1435d42cf --- /dev/null +++ b/packages/producer/src/services/render/renderFormat.test.ts @@ -0,0 +1,28 @@ +import { describe, expect, it } from "bun:test"; +import { outputNeedsAlpha, outputSupportsPageSideShaderCompositing } from "./renderFormat.js"; + +describe("outputNeedsAlpha", () => { + it("uses alpha-aware capture for transparent-capable formats", () => { + expect(outputNeedsAlpha("gif")).toBe(true); + expect(outputNeedsAlpha("webm")).toBe(true); + expect(outputNeedsAlpha("mov")).toBe(true); + expect(outputNeedsAlpha("png-sequence")).toBe(true); + }); + + it("preserves opaque capture for mp4", () => { + expect(outputNeedsAlpha("mp4")).toBe(false); + }); +}); + +describe("outputSupportsPageSideShaderCompositing", () => { + it("supports opaque MP4 capture and RGBA GIF disk frames", () => { + expect(outputSupportsPageSideShaderCompositing("mp4")).toBe(true); + expect(outputSupportsPageSideShaderCompositing("gif")).toBe(true); + }); + + it("keeps the alpha video and PNG sequence formats on their existing paths", () => { + expect(outputSupportsPageSideShaderCompositing("webm")).toBe(false); + expect(outputSupportsPageSideShaderCompositing("mov")).toBe(false); + expect(outputSupportsPageSideShaderCompositing("png-sequence")).toBe(false); + }); +}); diff --git a/packages/producer/src/services/render/renderFormat.ts b/packages/producer/src/services/render/renderFormat.ts new file mode 100644 index 000000000..fd654e738 --- /dev/null +++ b/packages/producer/src/services/render/renderFormat.ts @@ -0,0 +1,9 @@ +export type RenderOutputFormat = "mp4" | "webm" | "mov" | "png-sequence" | "gif"; + +export function outputNeedsAlpha(format: RenderOutputFormat): boolean { + return format !== "mp4"; +} + +export function outputSupportsPageSideShaderCompositing(format: RenderOutputFormat): boolean { + return format === "mp4" || format === "gif"; +} diff --git a/packages/producer/src/services/render/stages/encodeStage.test.ts b/packages/producer/src/services/render/stages/encodeStage.test.ts index 58df97ae6..537fc401e 100644 --- a/packages/producer/src/services/render/stages/encodeStage.test.ts +++ b/packages/producer/src/services/render/stages/encodeStage.test.ts @@ -120,6 +120,7 @@ describe("gif encode args", () => { outputPath: "/tmp/hf/demo.gif", fps: { num: 15, den: 1 }, loop: 0, + preserveAlpha: false, }; it("builds the palettegen pass with diff statistics", () => { @@ -151,6 +152,21 @@ describe("gif encode args", () => { "/tmp/hf/demo.gif", ]); }); + + it("reserves transparency and applies the GIF alpha threshold for RGBA frames", () => { + const transparentInput = { + ...input, + framePattern: "frame_%06d.png", + preserveAlpha: true, + }; + + expect(buildGifPalettegenArgs(transparentInput)).toContain( + "fps=15,palettegen=stats_mode=diff:reserve_transparent=1", + ); + expect(buildGifPaletteuseArgs(transparentInput)).toContain( + "fps=15 [x]; [x][1:v] paletteuse=dither=sierra2_4a:alpha_threshold=128", + ); + }); }); describe("runEncodeStage config plumbing", () => { @@ -221,4 +237,48 @@ describe("runEncodeStage config plumbing", () => { resolvedEngineConfig.ffmpegEncodeTimeout, ); }); + + it("encodes alpha GIFs from PNG frames with explicit transparency filters", async () => { + const { runEncodeStage } = await import("./encodeStage.js"); + const paths = createFramesDir("png"); + + await runEncodeStage( + makeInput({ + framesDir: paths.framesDir, + outputPath: join(paths.root, "out.gif"), + videoOnlyPath: join(paths.root, "video-only.mp4"), + isGif: true, + needsAlpha: true, + }), + ); + + expect(runFfmpegMock).toHaveBeenCalledTimes(2); + expect(runFfmpegMock.mock.calls[0]?.[0]).toContain(join(paths.framesDir, "frame_%06d.png")); + expect(runFfmpegMock.mock.calls[0]?.[0]).toContain( + "fps=30,palettegen=stats_mode=diff:reserve_transparent=1", + ); + expect(runFfmpegMock.mock.calls[1]?.[0]).toContain(join(paths.framesDir, "frame_%06d.png")); + expect(runFfmpegMock.mock.calls[1]?.[0]).toContain( + "fps=30 [x]; [x][1:v] paletteuse=dither=sierra2_4a:alpha_threshold=128", + ); + }); + + it("keeps opaque GIF encoding on JPEG frames without alpha-only filters", async () => { + const { runEncodeStage } = await import("./encodeStage.js"); + const paths = createFramesDir("jpg"); + + await runEncodeStage( + makeInput({ + framesDir: paths.framesDir, + outputPath: join(paths.root, "out.gif"), + videoOnlyPath: join(paths.root, "video-only.mp4"), + isGif: true, + needsAlpha: false, + }), + ); + + expect(runFfmpegMock.mock.calls[0]?.[0]).toContain(join(paths.framesDir, "frame_%06d.jpg")); + expect(runFfmpegMock.mock.calls[0]?.[0]).not.toContain("reserve_transparent"); + expect(runFfmpegMock.mock.calls[1]?.[0]).not.toContain("alpha_threshold"); + }); }); diff --git a/packages/producer/src/services/render/stages/encodeStage.ts b/packages/producer/src/services/render/stages/encodeStage.ts index c66e4bedf..dcebb4a70 100644 --- a/packages/producer/src/services/render/stages/encodeStage.ts +++ b/packages/producer/src/services/render/stages/encodeStage.ts @@ -123,6 +123,7 @@ async function encodeGifFromDir( fps: Fps; loop: number; palettePath: string; + preserveAlpha: boolean; signal?: AbortSignal; timeout: number; }, @@ -148,6 +149,7 @@ async function encodeGifFromDir( outputPath, fps: input.fps, loop: input.loop, + preserveAlpha: input.preserveAlpha, }; try { const paletteResult = await runFfmpeg(buildGifPalettegenArgs(argsInput), { @@ -258,12 +260,14 @@ export async function runEncodeStage(input: EncodeStageInput): Promise { + const result = await runFfmpeg( + ["-y", "-i", gifPath, "-frames:v", "1", "-pix_fmt", "rgba", "-update", "1", outPng], + { timeout: 60_000 }, + ); + if (!result.success) { + throw new Error( + `ffmpeg failed extracting frame 0 from ${gifPath}: ${result.stderr.slice(-400)}`, + ); + } +} + +async function extractFrameAtIndex( + inputPath: string, + frameIndex: number, + outPng: string, +): Promise { + const result = await runFfmpeg( + [ + "-y", + "-i", + inputPath, + "-vf", + `select=eq(n\\,${frameIndex})`, + "-frames:v", + "1", + "-update", + "1", + outPng, + ], + { timeout: 60_000 }, + ); + if (!result.success) { + throw new Error( + `ffmpeg failed extracting frame ${frameIndex} from ${inputPath}: ${result.stderr.slice(-400)}`, + ); + } +} + async function runWebmCheck(workRoot: string): Promise { console.log("\n[webm] rendering transparency-regression …"); const outDir = join(workRoot, "webm"); @@ -141,6 +187,78 @@ async function runWebmCheck(workRoot: string): Promise { console.log("[webm] PASS — transparent + opaque-red pixels verified"); } +async function runGifCheck(workRoot: string): Promise { + console.log("\n[gif] rendering transparency-regression …"); + const outDir = join(workRoot, "gif"); + mkdirSync(outDir, { recursive: true }); + const outPath = join(outDir, "out.gif"); + + const job = createRenderJob({ + fps: { num: 15, den: 1 }, + quality: "draft", + format: "gif", + gifLoop: 0, + }); + + await executeRenderJob(job, FIXTURE_SRC, outPath); + assert.equal(job.status, "complete", `gif render did not complete: status=${job.status}`); + assert.ok(existsSync(outPath), `gif output not written to ${outPath}`); + const size = (await import("node:fs")).statSync(outPath).size; + assert.ok(size > 0, `gif output ${outPath} is empty`); + console.log(`[gif] rendered ${outPath} (${size} bytes)`); + + const framePng = join(outDir, "frame-0.png"); + await extractFirstFrameFromGif(outPath, framePng); + const decoded = decodePng(readFileSync(framePng)); + assertAlphaPixel(decoded, TRANSPARENT_X, TRANSPARENT_Y, "transparent", "gif"); + assertAlphaPixel(decoded, OPAQUE_X, OPAQUE_Y, "opaque-red", "gif"); + console.log("[gif] PASS — transparent + opaque-red pixels verified"); +} + +async function runGifShaderTransitionCheck(workRoot: string): Promise { + console.log("\n[gif-shader] rendering page-side shader transition …"); + const outDir = join(workRoot, "gif-shader"); + mkdirSync(outDir, { recursive: true }); + const outPath = join(outDir, "out.gif"); + + const job = createRenderJob({ + fps: { num: 15, den: 1 }, + quality: "draft", + format: "gif", + gifLoop: 0, + workers: 1, + }); + + await executeRenderJob(job, SHADER_FIXTURE_SRC, outPath); + assert.equal(job.status, "complete", `gif shader render did not complete: status=${job.status}`); + assert.ok(existsSync(outPath), `gif shader output not written to ${outPath}`); + + const gifBefore = join(outDir, "gif-before.png"); + const gifTransition = join(outDir, "gif-transition.png"); + const goldenBefore = join(outDir, "golden-before.png"); + const goldenTransition = join(outDir, "golden-transition.png"); + await Promise.all([ + extractFrameAtIndex(outPath, 7, gifBefore), + extractFrameAtIndex(outPath, 17, gifTransition), + extractFrameAtIndex(SHADER_GOLDEN, 14, goldenBefore), + extractFrameAtIndex(SHADER_GOLDEN, 34, goldenTransition), + ]); + + const beforePsnr = await psnrDb(readFileSync(gifBefore), readFileSync(goldenBefore)); + const transitionPsnr = await psnrDb(readFileSync(gifTransition), readFileSync(goldenTransition)); + assert.ok( + beforePsnr >= 25, + `gif shader control frame expected >=25 dB against the golden, got ${beforePsnr.toFixed(2)} dB`, + ); + assert.ok( + transitionPsnr >= 20, + `gif shader transition expected >=20 dB against the golden, got ${transitionPsnr.toFixed(2)} dB`, + ); + console.log( + `[gif-shader] PASS — control ${beforePsnr.toFixed(2)} dB, transition ${transitionPsnr.toFixed(2)} dB`, + ); +} + async function runPngSequenceCheck(workRoot: string): Promise { console.log("\n[png-sequence] rendering transparency-regression …"); const outDir = join(workRoot, "pngs"); @@ -165,14 +283,14 @@ async function runPngSequenceCheck(workRoot: string): Promise { .sort(); assert.equal( frames.length, - FPS, // 1 second at 30fps = 30 frames - `png-sequence expected ${FPS} frames, got ${frames.length}: ${frames.join(",")}`, + PNG_SEQUENCE_FRAME_COUNT, // 1 second at 30fps = 30 frames + `png-sequence expected ${PNG_SEQUENCE_FRAME_COUNT} frames, got ${frames.length}: ${frames.join(",")}`, ); assert.equal(frames[0], "frame_000001.png", "first frame should be frame_000001.png"); assert.equal( frames[frames.length - 1], - `frame_${String(FPS).padStart(6, "0")}.png`, - `last frame should be frame_${String(FPS).padStart(6, "0")}.png`, + `frame_${String(PNG_SEQUENCE_FRAME_COUNT).padStart(6, "0")}.png`, + `last frame should be frame_${String(PNG_SEQUENCE_FRAME_COUNT).padStart(6, "0")}.png`, ); console.log(`[png-sequence] wrote ${frames.length} frames to ${outDir}`); @@ -188,6 +306,9 @@ async function main(): Promise { if (!existsSync(FIXTURE_SRC)) { throw new Error(`Fixture missing: ${FIXTURE_SRC}`); } + if (!existsSync(SHADER_FIXTURE_SRC) || !existsSync(SHADER_GOLDEN)) { + throw new Error(`Shader fixture or golden missing: ${SHADER_FIXTURE_DIR}`); + } const workRoot = join(tmpdir(), `hf-transparency-${process.pid}-${Date.now()}`); mkdirSync(workRoot, { recursive: true }); const keepWork = process.env.KEEP_TEMP === "1"; @@ -195,6 +316,8 @@ async function main(): Promise { try { await runWebmCheck(workRoot); + await runGifCheck(workRoot); + await runGifShaderTransitionCheck(workRoot); await runPngSequenceCheck(workRoot); console.log("\nAll transparency assertions passed."); } finally { diff --git a/packages/producer/tests/transparency-regression/meta.json b/packages/producer/tests/transparency-regression/meta.json index 31d4eea61..48ecf3ce0 100644 --- a/packages/producer/tests/transparency-regression/meta.json +++ b/packages/producer/tests/transparency-regression/meta.json @@ -1,6 +1,6 @@ { "name": "Transparency Regression", - "description": "Asserts that webm + png-sequence outputs preserve a real alpha channel end-to-end. Exercised by `tsx src/transparency-test.ts`, NOT by the standard regression harness (which compares against a golden MP4).", + "description": "Asserts that webm + gif + png-sequence outputs preserve transparent pixels end-to-end. Exercised by `tsx src/transparency-test.ts`, NOT by the standard regression harness (which compares against a golden MP4).", "tags": ["transparency", "alpha", "smoke"], "renderConfig": { "fps": 30 diff --git a/packages/producer/tests/transparency-regression/src/index.html b/packages/producer/tests/transparency-regression/src/index.html index 526c671b6..24e527554 100644 --- a/packages/producer/tests/transparency-regression/src/index.html +++ b/packages/producer/tests/transparency-regression/src/index.html @@ -34,6 +34,7 @@