fix(engine): pad odd output dimensions up to even for H.264/H.265 encode (#1802)

* fix(engine): pad odd output dimensions up to even for H.264/H.265 encode

A composition with an odd data-width or data-height (e.g. a custom 3:1
canvas at 1080x723) failed to encode to MP4. libx264/libx265 with 4:2:0
chroma subsampling (yuv420p, yuv420p10le) require both dimensions to be
even and abort before writing a packet:

  [libx264] height not divisible by 2 (1080x723)
  Error while opening encoder ... Invalid argument

Both the streaming encoder and the chunk encoder built the software
range-conversion filter ("scale=in_range=pc:out_range=tv") with no
even-dimension enforcement, so any odd-sized canvas reached libx264
unmodified and the whole render failed.

Add a shared withEvenDimensionPad helper that appends
pad=ceil(iw/2)*2:ceil(ih/2)*2 to the filter chain only for 4:2:0 pixel
formats. The pad rounds each odd dimension up by one pixel (a no-op when
already even) without scaling, so content is never resampled. Formats
that accept odd dimensions (ProRes 4444 yuva444p10le, VP9 yuva420p) are
excluded, so transparent/alpha output is untouched.

* fix(engine): extend even-dimension pad to GPU 4:2:0 encode paths

The odd-dimension pad added for libx264/libx265 only covered the software
encoder branches. nvenc, videotoolbox, qsv, and amf feed software frames
straight to the hardware encoder with no -vf chain, so an odd-sized 4:2:0
canvas on --gpu (or an auto-selected hardware encoder) reproduced the same
"height not divisible by 2" abort before any packet was written.

Add the even-dimension pad to the software-side -vf chain for those four
GPU paths in both the chunk and streaming encoders, reusing the shared
withEvenDimensionPad helper (the pad runs on CPU before the encode). vaapi
is left as-is: its existing format=nv12,hwupload conversion already aligns
odd dimensions before upload, so it is not double-padded. ProRes 4444 and
VP9 alpha stay untouched, exactly as the software fix excludes them.

nvenc/videotoolbox/qsv/amf arg construction is logic-tested (the pad filter
is asserted on the built arg list for 8-bit and 10-bit 4:2:0, with alpha
ProRes asserted padless); runtime hardware encode is not exercised here.

packages/engine/src/services/chunkEncoder.test.ts

@@ -904,16 +904,55 @@ describe("buildEncoderArgs color space", () => {
     expect(args[vfIdx + 1]).toBe("scale=in_range=pc:out_range=tv,format=nv12,hwupload");
   });
 
-  it("skips range conversion filter for non-VAAPI GPU encoding", () => {
+  it("pads odd dimensions (no range scale) for non-VAAPI GPU encoding", () => {
+    for (const gpu of ["nvenc", "videotoolbox", "qsv", "amf"] as const) {
+      const args = buildEncoderArgs(
+        { ...baseOptions, codec: "h264", preset: "medium", quality: 23, useGpu: true },
+        inputArgs,
+        "out.mp4",
+        gpu,
+      );
+      const vfIdx = args.indexOf("-vf");
+      // 4:2:0 HW encode still aborts on odd dims, so the pad must be present —
+      // but the range scale belongs to the SW path only.
+      expect(args[vfIdx + 1]).toBe("pad=ceil(iw/2)*2:ceil(ih/2)*2");
+      expect(args[vfIdx + 1]).not.toContain("scale=in_range");
+      // but still has color metadata
+      expect(args).toContain("-colorspace:v");
+    }
+  });
+
+  it("pads odd dimensions for 10-bit (yuv420p10le) GPU HDR encoding", () => {
     const args = buildEncoderArgs(
-      { ...baseOptions, codec: "h264", preset: "medium", quality: 23, useGpu: true },
+      {
+        ...baseOptions,
+        codec: "h265",
+        preset: "medium",
+        quality: 23,
+        useGpu: true,
+        pixelFormat: "yuv420p10le",
+      },
       inputArgs,
       "out.mp4",
       "nvenc",
     );
+    expect(args[args.indexOf("-vf") + 1]).toBe("pad=ceil(iw/2)*2:ceil(ih/2)*2");
+  });
+
+  it("leaves alpha ProRes untouched (no even-dim pad)", () => {
+    const args = buildEncoderArgs(
+      {
+        ...baseOptions,
+        codec: "prores",
+        preset: "4",
+        quality: 23,
+        pixelFormat: "yuva444p10le",
+      },
+      inputArgs,
+      "out.mov",
+    );
     expect(args.indexOf("-vf")).toBe(-1);
-    // but still has color metadata
-    expect(args).toContain("-colorspace:v");
+    expect(args.join(" ")).not.toContain("pad=");
   });
 
   it("does not add color metadata for VP9", () => {

packages/engine/src/services/chunkEncoder.ts

@@ -18,6 +18,7 @@ import {
   mapPresetForGpuEncoder,
 } from "../utils/gpuEncoder.js";
 import { type HdrTransfer, getHdrEncoderColorParams } from "../utils/hdr.js";
+import { withEvenDimensionPad } from "../utils/evenDimensions.js";
 import { formatFfmpegError, runFfmpeg } from "../utils/runFfmpeg.js";
 import { getFfmpegBinary } from "../utils/ffmpegBinaries.js";
 import { extractAudioMetadata } from "../utils/ffprobe.js";
@@ -395,14 +396,25 @@ export function buildEncoderArgs(
 
     // Range conversion: Chrome's full-range RGB → limited/TV range.
     if (gpuEncoder === "vaapi") {
+      // vaapi already runs `format=nv12,hwupload`; the nv12 conversion aligns
+      // odd dimensions before upload, so only prepend the range conversion.
       const vfIdx = args.indexOf("-vf");
       if (vfIdx !== -1) {
         args[vfIdx + 1] = `scale=in_range=pc:out_range=tv,${args[vfIdx + 1]}`;
       }
-    } else if (!shouldUseGpu) {
+    } else if (shouldUseGpu) {
+      // nvenc/videotoolbox/qsv/amf feed software frames straight to the HW
+      // encoder with no `-vf`. They hit the same "height not divisible by 2"
+      // abort as libx264 on an odd-sized 4:2:0 canvas, so pad odd dimensions
+      // up to even on the software side before the encode.
+      const vf = withEvenDimensionPad("", pixelFormat);
+      if (vf) args.push("-vf", vf);
+    } else {
       // Range conversion: Chrome screenshots are full-range RGB.
-      // The scale filter handles both 8-bit and 10-bit correctly.
-      args.push("-vf", "scale=in_range=pc:out_range=tv");
+      // The scale filter handles both 8-bit and 10-bit correctly. Pad odd
+      // dimensions up to even so libx264/libx265 (4:2:0) don't abort with
+      // "height not divisible by 2" on an odd-sized composition canvas.
+      args.push("-vf", withEvenDimensionPad("scale=in_range=pc:out_range=tv", pixelFormat));
     }
 
     // Fixed timescale for consistent A/V timing across platforms.

packages/engine/src/services/streamingEncoder.test.ts

@@ -309,6 +309,24 @@ describe("buildStreamingArgs", () => {
       expect(h265Args[h265Args.indexOf("-c:v") + 1]).toBe("hevc_amf");
       expect(h265Args[h265Args.indexOf("-qp_i") + 1]).toBe("23");
     });
+
+    // 4:2:0 HW encode aborts on odd dims just like libx264, and these paths
+    // feed software frames straight to the encoder with no `-vf`, so the
+    // even-dim pad (and only the pad, not the SW range scale) must be added.
+    it("pads odd dimensions (no range scale) for non-VAAPI GPU encoding", () => {
+      for (const gpu of ["nvenc", "videotoolbox", "qsv", "amf"] as const) {
+        const args = buildStreamingArgs(baseGpu, "/tmp/out.mp4", gpu);
+        const vfIdx = args.indexOf("-vf");
+        expect(args[vfIdx + 1]).toBe("pad=ceil(iw/2)*2:ceil(ih/2)*2");
+        expect(args[vfIdx + 1]).not.toContain("scale=in_range");
+      }
+    });
+
+    it("prepends range conversion to VAAPI chain (nv12 covers even-dim)", () => {
+      const args = buildStreamingArgs(baseGpu, "/tmp/out.mp4", "vaapi");
+      const vfIdx = args.indexOf("-vf");
+      expect(args[vfIdx + 1]).toBe("scale=in_range=pc:out_range=tv,format=nv12,hwupload");
+    });
   });
 });
 

packages/engine/src/services/streamingEncoder.ts

@@ -28,6 +28,7 @@ import {
 import { formatFfmpegError } from "../utils/runFfmpeg.js";
 import { getFfmpegBinary } from "../utils/ffmpegBinaries.js";
 import { getHdrEncoderColorParams } from "../utils/hdr.js";
+import { withEvenDimensionPad } from "../utils/evenDimensions.js";
 import { DEFAULT_CONFIG, type EngineConfig } from "../config.js";
 import { fpsToFfmpegArg, type Fps } from "@hyperframes/core";
 import { appendVp9CpuUsedArg } from "./vp9Options.js";
@@ -350,13 +351,24 @@ export function buildStreamingArgs(
     if (options.rawInputFormat) {
       // No filter needed — PQ data goes straight to encoder
     } else if (gpuEncoder === "vaapi") {
+      // vaapi already runs `format=nv12,hwupload`; the nv12 conversion aligns
+      // odd dimensions before upload, so only prepend the range conversion.
       const vfIdx = args.indexOf("-vf");
       if (vfIdx !== -1) {
         args[vfIdx + 1] = `scale=in_range=pc:out_range=tv,${args[vfIdx + 1]}`;
       }
-    } else if (!shouldUseGpu) {
-      // Range conversion: Chrome screenshots are full-range RGB.
-      args.push("-vf", "scale=in_range=pc:out_range=tv");
+    } else if (shouldUseGpu) {
+      // nvenc/videotoolbox/qsv/amf feed software frames straight to the HW
+      // encoder with no `-vf`. They hit the same "height not divisible by 2"
+      // abort as libx264 on an odd-sized 4:2:0 canvas, so pad odd dimensions
+      // up to even on the software side before the encode.
+      const vf = withEvenDimensionPad("", pixelFormat);
+      if (vf) args.push("-vf", vf);
+    } else {
+      // Range conversion: Chrome screenshots are full-range RGB. Pad odd
+      // dimensions up to even so libx264/libx265 (4:2:0) don't abort with
+      // "height not divisible by 2" on an odd-sized composition canvas.
+      args.push("-vf", withEvenDimensionPad("scale=in_range=pc:out_range=tv", pixelFormat));
     }
 
     // Fixed timescale for consistent A/V timing across platforms.

packages/engine/src/utils/evenDimensions.test.ts

@@ -0,0 +1,39 @@
+import { describe, expect, it } from "vitest";
+import { requiresEvenDimensions, withEvenDimensionPad } from "./evenDimensions.js";
+
+describe("requiresEvenDimensions", () => {
+  it("flags 4:2:0 subsampled formats", () => {
+    expect(requiresEvenDimensions("yuv420p")).toBe(true);
+    expect(requiresEvenDimensions("yuv420p10le")).toBe(true);
+    expect(requiresEvenDimensions("yuvj420p")).toBe(true);
+  });
+
+  it("leaves 4:4:4 / alpha formats alone", () => {
+    expect(requiresEvenDimensions("yuva444p10le")).toBe(false); // ProRes 4444
+    expect(requiresEvenDimensions("yuva420p")).toBe(false); // VP9 alpha (own branch)
+    expect(requiresEvenDimensions("rgb48le")).toBe(false);
+  });
+});
+
+describe("withEvenDimensionPad", () => {
+  it("appends the even-up pad for subsampled output (odd dims bumped to even)", () => {
+    const vf = withEvenDimensionPad("scale=in_range=pc:out_range=tv", "yuv420p");
+    expect(vf).toBe("scale=in_range=pc:out_range=tv,pad=ceil(iw/2)*2:ceil(ih/2)*2");
+  });
+
+  it("returns just the pad when there is no existing filter chain", () => {
+    expect(withEvenDimensionPad("", "yuv420p")).toBe("pad=ceil(iw/2)*2:ceil(ih/2)*2");
+  });
+
+  it("leaves the filter chain unchanged for alpha output (even in, unchanged)", () => {
+    const vf = "scale=in_range=pc:out_range=tv";
+    expect(withEvenDimensionPad(vf, "yuva444p10le")).toBe(vf);
+  });
+
+  it("pad rounds UP to even: ceil(n/2)*2 is a no-op for even and +1 for odd", () => {
+    const evenUp = (n: number) => Math.ceil(n / 2) * 2;
+    expect(evenUp(1080)).toBe(1080); // even in, unchanged
+    expect(evenUp(723)).toBe(724); // odd in, bumped to even
+    expect(evenUp(1)).toBe(2);
+  });
+});

packages/engine/src/utils/evenDimensions.ts

@@ -0,0 +1,45 @@
+/**
+ * Even-dimension normalization for chroma-subsampled encodes.
+ *
+ * libx264 / libx265 with 4:2:0 chroma subsampling (yuv420p, yuv420p10le)
+ * require both width and height to be even. An odd output dimension makes the
+ * encoder abort before writing a single packet:
+ *
+ *   [libx264] height not divisible by 2 (1080x723)
+ *   Error while opening encoder ... Invalid argument
+ *
+ * A composition with an odd data-width / data-height (e.g. a custom 3:1 canvas
+ * at 1080x723) therefore fails to encode to H.264. The fix pads the odd
+ * dimension up by a single pixel inside the encode filter chain, so the
+ * encoder always receives even dimensions. Padding (not scaling) means content
+ * is never resampled — at most one transparent/black row or column is added at
+ * the bottom-right edge.
+ */
+
+// `pad=ceil(iw/2)*2:ceil(ih/2)*2` rounds each dimension UP to the next even
+// value (a no-op when already even) and keeps content at the top-left (x=0,y=0
+// default), so nothing shifts. iw/ih are evaluated by FFmpeg at runtime, so
+// this works whether or not the caller knows the frame size up front.
+const EVEN_DIMENSION_PAD = "pad=ceil(iw/2)*2:ceil(ih/2)*2";
+
+/**
+ * Pixel formats whose chroma subsampling requires even width AND height.
+ * 4:2:0 family only: yuv420p (H.264 SDR), yuv420p10le (H.265 HDR), and the
+ * full-range yuvj420p Chrome screenshots arrive as. ProRes 4444
+ * (yuva444p10le) and other 4:4:4 / alpha formats sample chroma per-pixel and
+ * accept odd dimensions, so they are deliberately excluded — padding them
+ * would needlessly distort transparent output.
+ */
+export function requiresEvenDimensions(pixelFormat: string): boolean {
+  return pixelFormat.startsWith("yuv420") || pixelFormat.startsWith("yuvj420");
+}
+
+/**
+ * Append the even-dimension pad to an FFmpeg `-vf` chain when the target pixel
+ * format requires it. Returns the chain unchanged for formats that accept odd
+ * dimensions, and returns just the pad when there is no existing chain.
+ */
+export function withEvenDimensionPad(vfChain: string, pixelFormat: string): string {
+  if (!requiresEvenDimensions(pixelFormat)) return vfChain;
+  return vfChain ? `${vfChain},${EVEN_DIMENSION_PAD}` : EVEN_DIMENSION_PAD;
+}