fix(engine): accept libx264 preset names with NVENC and QSV (#442)

NVENC rejects the libx264 preset vocabulary (ultrafast / medium / slow /
...) with AVERROR(EINVAL) ("Error applying encoder options: Invalid
argument"), which surfaces as a bare `FFmpeg exited with code -22` from
spawn(). Because ENCODER_PRESETS passes these names straight through to
h264_nvenc / hevc_nvenc, every `--gpu` render using the `draft` tier
failed; `standard` (medium) and `high` (slow) only worked coincidentally
on ffmpeg builds that happened to accept those aliases. QSV has the same
problem on a narrower set (ultrafast / superfast / placebo).

Add `mapPresetForGpuEncoder` in utils/gpuEncoder.ts that translates the
libx264 vocabulary to each encoder's native names:

- nvenc: libx264 -> p1..p7 (already-native pN values pass through);
  unknown values fall back to p4 (medium)
- qsv:   ultrafast / superfast -> veryfast; placebo -> veryslow;
  everything else passes through
- videotoolbox / vaapi / null: unchanged

Both buildEncoderArgs (chunkEncoder.ts) and buildStreamingArgs
(streamingEncoder.ts) now route through the helper before pushing
`-preset` to the ffmpeg arg vector.

To make the next encoder-options failure diagnosable without re-running
ffmpeg by hand, \`formatFfmpegError\` in utils/runFfmpeg.ts now appends
the last 15 non-empty stderr lines to the error string. The four call
sites that previously swallowed stderr (encodeFramesFromDir,
muxVideoWithAudio, applyFaststart, and the streaming encoder exit
handler) have been updated.

Tested end-to-end on an RTX 4080 with ffmpeg 8.1 NVENC across
\`--quality draft|standard|high\` plus \`--video-bitrate\` and \`--crf\`
overrides; the 6 renders were visually equivalent to the CPU baseline.

Co-authored-by: roi32 <75878108+roi32@users.noreply.github.com>

packages/engine/src/services/chunkEncoder.test.ts

@@ -148,6 +148,69 @@ describe("buildEncoderArgs anti-banding", () => {
   });
 });
 
+describe("buildEncoderArgs GPU preset mapping", () => {
+  const baseOptions = { fps: 30, width: 1920, height: 1080 };
+  const inputArgs = ["-framerate", "30", "-i", "frames/%04d.png"];
+
+  function presetArg(args: string[]): string | undefined {
+    const idx = args.indexOf("-preset");
+    return idx === -1 ? undefined : args[idx + 1];
+  }
+
+  // Regression for the "draft quality + --gpu fails with code -22" bug:
+  // NVENC rejects the libx264 preset name `ultrafast` with AVERROR(EINVAL),
+  // so the `draft` quality tier must not forward that value to h264_nvenc.
+  it("translates the draft ultrafast preset to NVENC p1", () => {
+    const args = buildEncoderArgs(
+      { ...baseOptions, codec: "h264", preset: "ultrafast", quality: 28, useGpu: true },
+      inputArgs,
+      "out.mp4",
+      "nvenc",
+    );
+    expect(presetArg(args)).toBe("p1");
+  });
+
+  it("translates the standard medium preset to NVENC p4", () => {
+    const args = buildEncoderArgs(
+      { ...baseOptions, codec: "h264", preset: "medium", quality: 18, useGpu: true },
+      inputArgs,
+      "out.mp4",
+      "nvenc",
+    );
+    expect(presetArg(args)).toBe("p4");
+  });
+
+  it("translates the high slow preset to NVENC p5", () => {
+    const args = buildEncoderArgs(
+      { ...baseOptions, codec: "h264", preset: "slow", quality: 15, useGpu: true },
+      inputArgs,
+      "out.mp4",
+      "nvenc",
+    );
+    expect(presetArg(args)).toBe("p5");
+  });
+
+  it("rewrites QSV's unsupported ultrafast preset to veryfast", () => {
+    const args = buildEncoderArgs(
+      { ...baseOptions, codec: "h264", preset: "ultrafast", quality: 28, useGpu: true },
+      inputArgs,
+      "out.mp4",
+      "qsv",
+    );
+    expect(presetArg(args)).toBe("veryfast");
+  });
+
+  it("passes QSV-supported preset names through unchanged", () => {
+    const args = buildEncoderArgs(
+      { ...baseOptions, codec: "h264", preset: "medium", quality: 23, useGpu: true },
+      inputArgs,
+      "out.mp4",
+      "qsv",
+    );
+    expect(presetArg(args)).toBe("medium");
+  });
+});
+
 describe("buildEncoderArgs color space", () => {
   const baseOptions = { fps: 30, width: 1920, height: 1080 };
   const inputArgs = ["-framerate", "30", "-i", "frames/%04d.png"];

packages/engine/src/services/chunkEncoder.ts

@@ -9,9 +9,14 @@ import { spawn } from "child_process";
 import { copyFileSync, existsSync, mkdirSync, readdirSync, statSync, writeFileSync } from "fs";
 import { join, dirname } from "path";
 import { DEFAULT_CONFIG, type EngineConfig } from "../config.js";
-import { type GpuEncoder, getCachedGpuEncoder, getGpuEncoderName } from "../utils/gpuEncoder.js";
+import {
+  type GpuEncoder,
+  getCachedGpuEncoder,
+  getGpuEncoderName,
+  mapPresetForGpuEncoder,
+} from "../utils/gpuEncoder.js";
 import { type HdrTransfer, getHdrEncoderColorParams } from "../utils/hdr.js";
-import { runFfmpeg } from "../utils/runFfmpeg.js";
+import { formatFfmpegError, runFfmpeg } from "../utils/runFfmpeg.js";
 import type { EncoderOptions, EncodeResult, MuxResult } from "./chunkEncoder.types.js";
 
 export type { EncoderOptions, EncodeResult, MuxResult } from "./chunkEncoder.types.js";
@@ -110,7 +115,7 @@ export function buildEncoderArgs(
 
       switch (gpuEncoder) {
         case "nvenc":
-          args.push("-preset", preset);
+          args.push("-preset", mapPresetForGpuEncoder("nvenc", preset));
           if (bitrate) args.push("-b:v", bitrate);
           else args.push("-cq", String(quality));
           break;
@@ -129,7 +134,7 @@ export function buildEncoderArgs(
           else args.push("-qp", String(quality));
           break;
         case "qsv":
-          args.push("-preset", preset);
+          args.push("-preset", mapPresetForGpuEncoder("qsv", preset));
           if (bitrate) args.push("-b:v", bitrate);
           else args.push("-global_quality", String(quality));
           break;
@@ -320,7 +325,7 @@ export async function encodeFramesFromDir(
           durationMs,
           framesEncoded: 0,
           fileSize: 0,
-          error: `FFmpeg exited with code ${code}`,
+          error: formatFfmpegError(code, stderr),
         });
         return;
       }
@@ -522,11 +527,7 @@ export async function muxVideoWithAudio(
     success: result.success,
     outputPath,
     durationMs: result.durationMs,
-    error: !result.success
-      ? result.exitCode !== null
-        ? `FFmpeg exited with code ${result.exitCode}`
-        : `[FFmpeg] ${result.stderr}`
-      : undefined,
+    error: !result.success ? formatFfmpegError(result.exitCode, result.stderr) : undefined,
   };
 }
 
@@ -559,10 +560,6 @@ export async function applyFaststart(
     success: result.success,
     outputPath,
     durationMs: result.durationMs,
-    error: !result.success
-      ? result.exitCode !== null
-        ? `FFmpeg exited with code ${result.exitCode}`
-        : `[FFmpeg] ${result.stderr}`
-      : undefined,
+    error: !result.success ? formatFfmpegError(result.exitCode, result.stderr) : undefined,
   };
 }

packages/engine/src/services/streamingEncoder.test.ts

@@ -161,6 +161,46 @@ describe("buildStreamingArgs", () => {
       expect(args[args.length - 1]).toBe("/tmp/some-output.mp4");
     });
   });
+
+  describe("GPU preset mapping", () => {
+    const baseGpu: StreamingEncoderOptions = {
+      fps: 30,
+      width: 1920,
+      height: 1080,
+      codec: "h264",
+      preset: "ultrafast",
+      quality: 28,
+      useGpu: true,
+    };
+
+    function presetArg(args: string[]): string | undefined {
+      const idx = args.indexOf("-preset");
+      return idx === -1 ? undefined : args[idx + 1];
+    }
+
+    // Regression for the streaming-encode + --gpu failure: NVENC rejects
+    // libx264 `ultrafast` with AVERROR(EINVAL), which previously surfaced
+    // as a bare "FFmpeg exited with code -22".
+    it("translates ultrafast to NVENC p1", () => {
+      const args = buildStreamingArgs(baseGpu, "/tmp/out.mp4", "nvenc");
+      expect(presetArg(args)).toBe("p1");
+    });
+
+    it("translates medium to NVENC p4", () => {
+      const args = buildStreamingArgs({ ...baseGpu, preset: "medium" }, "/tmp/out.mp4", "nvenc");
+      expect(presetArg(args)).toBe("p4");
+    });
+
+    it("rewrites QSV's unsupported ultrafast preset to veryfast", () => {
+      const args = buildStreamingArgs(baseGpu, "/tmp/out.mp4", "qsv");
+      expect(presetArg(args)).toBe("veryfast");
+    });
+
+    it("passes QSV-supported preset names through unchanged", () => {
+      const args = buildStreamingArgs({ ...baseGpu, preset: "medium" }, "/tmp/out.mp4", "qsv");
+      expect(presetArg(args)).toBe("medium");
+    });
+  });
 });
 
 describe("createFrameReorderBuffer", () => {

packages/engine/src/services/streamingEncoder.ts

@@ -16,7 +16,13 @@ import { spawn, type ChildProcess } from "child_process";
 import { existsSync, mkdirSync, statSync } from "fs";
 import { dirname } from "path";
 
-import { type GpuEncoder, getCachedGpuEncoder, getGpuEncoderName } from "../utils/gpuEncoder.js";
+import {
+  type GpuEncoder,
+  getCachedGpuEncoder,
+  getGpuEncoderName,
+  mapPresetForGpuEncoder,
+} from "../utils/gpuEncoder.js";
+import { formatFfmpegError } from "../utils/runFfmpeg.js";
 import { getHdrEncoderColorParams } from "../utils/hdr.js";
 import { type EncoderOptions } from "./chunkEncoder.types.js";
 import { DEFAULT_CONFIG, type EngineConfig } from "../config.js";
@@ -191,7 +197,7 @@ export function buildStreamingArgs(
 
       switch (gpuEncoder) {
         case "nvenc":
-          args.push("-preset", preset);
+          args.push("-preset", mapPresetForGpuEncoder("nvenc", preset));
           if (bitrate) args.push("-b:v", bitrate);
           else args.push("-cq", String(quality));
           break;
@@ -210,7 +216,7 @@ export function buildStreamingArgs(
           else args.push("-qp", String(quality));
           break;
         case "qsv":
-          args.push("-preset", preset);
+          args.push("-preset", mapPresetForGpuEncoder("qsv", preset));
           if (bitrate) args.push("-b:v", bitrate);
           else args.push("-global_quality", String(quality));
           break;
@@ -439,7 +445,7 @@ export async function spawnStreamingEncoder(
           success: false,
           durationMs,
           fileSize: 0,
-          error: `FFmpeg exited with code ${exitCode}`,
+          error: formatFfmpegError(exitCode, stderr),
         };
       }
 

packages/engine/src/utils/gpuEncoder.test.ts

@@ -0,0 +1,64 @@
+import { describe, expect, it } from "vitest";
+
+import { mapPresetForGpuEncoder } from "./gpuEncoder.js";
+
+describe("mapPresetForGpuEncoder", () => {
+  describe("nvenc", () => {
+    it.each([
+      ["ultrafast", "p1"],
+      ["superfast", "p1"],
+      ["veryfast", "p2"],
+      ["faster", "p3"],
+      ["fast", "p4"],
+      ["medium", "p4"],
+      ["slow", "p5"],
+      ["slower", "p6"],
+      ["veryslow", "p7"],
+      ["placebo", "p7"],
+    ])("maps libx264 preset %s to NVENC %s", (input, expected) => {
+      expect(mapPresetForGpuEncoder("nvenc", input)).toBe(expected);
+    });
+
+    it.each(["p1", "p2", "p3", "p4", "p5", "p6", "p7"])(
+      "passes NVENC-native preset %s through unchanged",
+      (preset) => {
+        expect(mapPresetForGpuEncoder("nvenc", preset)).toBe(preset);
+      },
+    );
+
+    it("falls back to p4 for unknown preset values", () => {
+      expect(mapPresetForGpuEncoder("nvenc", "nonsense")).toBe("p4");
+    });
+  });
+
+  describe("qsv", () => {
+    it.each([
+      ["ultrafast", "veryfast"],
+      ["superfast", "veryfast"],
+      ["placebo", "veryslow"],
+    ])("rewrites libx264-only preset %s to QSV-supported %s", (input, expected) => {
+      expect(mapPresetForGpuEncoder("qsv", input)).toBe(expected);
+    });
+
+    it.each(["veryfast", "faster", "fast", "medium", "slow", "slower", "veryslow"])(
+      "passes supported preset %s through unchanged",
+      (preset) => {
+        expect(mapPresetForGpuEncoder("qsv", preset)).toBe(preset);
+      },
+    );
+  });
+
+  describe("other encoders", () => {
+    it.each(["videotoolbox", "vaapi"] as const)(
+      "passes preset through unchanged for %s",
+      (encoder) => {
+        expect(mapPresetForGpuEncoder(encoder, "medium")).toBe("medium");
+        expect(mapPresetForGpuEncoder(encoder, "ultrafast")).toBe("ultrafast");
+      },
+    );
+
+    it("passes preset through unchanged when encoder is null (CPU)", () => {
+      expect(mapPresetForGpuEncoder(null, "ultrafast")).toBe("ultrafast");
+    });
+  });
+});

packages/engine/src/utils/gpuEncoder.ts

@@ -56,3 +56,54 @@ export function getGpuEncoderName(encoder: GpuEncoder, codec: "h264" | "h265"):
       return codec === "h264" ? "libx264" : "libx265";
   }
 }
+
+// libx264 preset names (ultrafast/superfast/.../placebo) mapped to the
+// equivalent NVENC p1..p7 preset. NVENC rejects libx264 names with
+// AVERROR(EINVAL) ("Error applying encoder options: Invalid argument"),
+// which surfaces as a generic "FFmpeg exited with code -22" — so callers
+// that share a single `preset` field across CPU and GPU paths (e.g. the
+// `draft`/`standard`/`high` quality tiers) must translate before passing
+// the value to h264_nvenc / hevc_nvenc.
+const NVENC_PRESET_MAP: Record<string, string> = {
+  ultrafast: "p1",
+  superfast: "p1",
+  veryfast: "p2",
+  faster: "p3",
+  fast: "p4",
+  medium: "p4",
+  slow: "p5",
+  slower: "p6",
+  veryslow: "p7",
+  placebo: "p7",
+};
+
+// QSV accepts most libx264 preset names but rejects `ultrafast`,
+// `superfast`, and `placebo`. Map those to the nearest supported values.
+const QSV_PRESET_MAP: Record<string, string> = {
+  ultrafast: "veryfast",
+  superfast: "veryfast",
+  placebo: "veryslow",
+};
+
+/**
+ * Translate a libx264-style `-preset` value to one accepted by the given
+ * GPU encoder.
+ *
+ * - `nvenc`: libx264 names → `p1`..`p7`. Already-native `pN` values pass
+ *   through unchanged. Unknown values fall back to `p4` (medium).
+ * - `qsv`:  `ultrafast`/`superfast`/`placebo` → nearest supported name;
+ *   everything else passes through.
+ * - `videotoolbox`, `vaapi`, `null`: no remap (they either ignore `-preset`
+ *   entirely or accept the libx264 vocabulary).
+ */
+export function mapPresetForGpuEncoder(encoder: GpuEncoder, preset: string): string {
+  switch (encoder) {
+    case "nvenc":
+      if (/^p[1-7]$/.test(preset)) return preset;
+      return NVENC_PRESET_MAP[preset] ?? "p4";
+    case "qsv":
+      return QSV_PRESET_MAP[preset] ?? preset;
+    default:
+      return preset;
+  }
+}

packages/engine/src/utils/runFfmpeg.test.ts

@@ -0,0 +1,46 @@
+import { describe, expect, it } from "vitest";
+
+import { formatFfmpegError } from "./runFfmpeg.js";
+
+describe("formatFfmpegError", () => {
+  it("reports exit code alone when stderr is empty", () => {
+    expect(formatFfmpegError(-22, "")).toBe("FFmpeg exited with code -22");
+  });
+
+  it("appends stderr tail when present", () => {
+    const stderr =
+      "ffmpeg version 8.1\nbuilt with gcc 13.2.0\n" +
+      "[h264_nvenc @ 0x7f] Error applying encoder options: Invalid argument\n" +
+      "Error while opening encoder\n";
+    const message = formatFfmpegError(-22, stderr);
+    expect(message).toContain("FFmpeg exited with code -22");
+    expect(message).toContain("ffmpeg stderr (tail):");
+    expect(message).toContain("Error applying encoder options: Invalid argument");
+    expect(message).toContain("Error while opening encoder");
+  });
+
+  it("keeps only the last N non-empty lines in the tail", () => {
+    const lines = Array.from({ length: 30 }, (_, i) => `line-${i}`).join("\n");
+    const message = formatFfmpegError(1, lines, 5);
+    expect(message).toContain("line-29");
+    expect(message).toContain("line-25");
+    expect(message).not.toContain("line-24");
+  });
+
+  it("strips blank lines from the tail so real signal isn't hidden", () => {
+    const stderr = "\n\nError applying encoder options: Invalid argument\n\n\n";
+    const message = formatFfmpegError(-22, stderr);
+    expect(message).toContain("Error applying encoder options: Invalid argument");
+    // Only one non-empty stderr line should appear in the tail.
+    const tailPart = message.split("ffmpeg stderr (tail):\n")[1] ?? "";
+    expect(tailPart.trim().split(/\r?\n/).length).toBe(1);
+  });
+
+  it("falls back to a process-error string when exit code is null and stderr is empty", () => {
+    expect(formatFfmpegError(null, "")).toBe("[FFmpeg] process error");
+  });
+
+  it("wraps stderr in [FFmpeg] prefix when exit code is null (spawn failure)", () => {
+    expect(formatFfmpegError(null, "spawn ffmpeg ENOENT")).toBe("[FFmpeg] spawn ffmpeg ENOENT");
+  });
+});

packages/engine/src/utils/runFfmpeg.ts

@@ -22,6 +22,36 @@ export interface RunFfmpegResult {
 
 const DEFAULT_TIMEOUT = 300_000;
 
+const DEFAULT_STDERR_TAIL_LINES = 15;
+
+/**
+ * Build a user-facing error message for a failed ffmpeg invocation.
+ *
+ * Historically we reported only `FFmpeg exited with code N`, which is useless
+ * for diagnosing encoder-options failures — a rejected `-preset` surfaces as a
+ * bare `code -22` with no hint at which argument ffmpeg objected to. Including
+ * the tail of stderr turns those into a one-line signal (e.g.
+ * `Error applying encoder options: Invalid argument`) that tells the caller
+ * exactly which option to fix.
+ */
+export function formatFfmpegError(
+  exitCode: number | null,
+  stderr: string,
+  tailLines: number = DEFAULT_STDERR_TAIL_LINES,
+): string {
+  const tail = (stderr ?? "")
+    .split(/\r?\n/)
+    .filter((line) => line.length > 0)
+    .slice(-tailLines)
+    .join("\n");
+  if (exitCode === null) {
+    return tail ? `[FFmpeg] ${tail}` : "[FFmpeg] process error";
+  }
+  return tail
+    ? `FFmpeg exited with code ${exitCode}\nffmpeg stderr (tail):\n${tail}`
+    : `FFmpeg exited with code ${exitCode}`;
+}
+
 export async function runFfmpeg(args: string[], opts?: RunFfmpegOptions): Promise<RunFfmpegResult> {
   const startMs = Date.now();
   const signal = opts?.signal;