fix(engine): use supported VideoToolbox quality options (#3687)

packages/engine/src/services/chunkEncoder.test.ts

@@ -935,6 +935,27 @@ describe("buildEncoderArgs GPU preset mapping", () => {
     expect(presetArg(args)).toBe("p5");
   });
 
+  it("uses a supported derived bitrate for high-quality VideoToolbox", () => {
+    const args = buildEncoderArgs(
+      {
+        fps: { num: 24, den: 1 },
+        width: 1920,
+        height: 1080,
+        codec: "h264",
+        preset: "slow",
+        quality: 15,
+        useGpu: true,
+      },
+      inputArgs,
+      "out.mp4",
+      "videotoolbox",
+    );
+
+    expect(args).not.toContain("-q:v");
+    expect(args[args.indexOf("-b:v") + 1]).toBe("12M");
+    expect(args[args.indexOf("-allow_sw") + 1]).toBe("1");
+  });
+
   // hevc_nvenc uses the same p1..p7 preset vocabulary as h264_nvenc, so the
   // mapping must apply to both codecs. Locks in "H.264 and H.265 NVENC share
   // the preset mapping" against a future refactor that might split the path.

packages/engine/src/services/chunkEncoder.ts

@@ -21,13 +21,14 @@ import {
   type GpuEncoder,
   getCachedGpuEncoder,
   getGpuEncoderName,
+  buildVideoToolboxRateControlArgs,
   mapPresetForGpuEncoder,
 } from "../utils/gpuEncoder.js";
 import { type HdrTransfer, getHdrEncoderColorParams } from "../utils/hdr.js";
 import { withEvenDimensionPad } from "../utils/evenDimensions.js";
 import { formatFfmpegError, isExternalFfmpegInterruption, runFfmpeg } from "../utils/runFfmpeg.js";
 import { extractAudioMetadata } from "../utils/ffprobe.js";
-import { type Fps, fpsToFfmpegArg } from "@hyperframes/core";
+import { type Fps, fpsToFfmpegArg, fpsToNumber } from "@hyperframes/core";
 import type { EncoderOptions, EncodeResult, MuxResult } from "./chunkEncoder.types.js";
 import { appendVp9CpuUsedArg } from "./vp9Options.js";
 import { appendRenderProvenanceArgs } from "../utils/renderProvenance.js";
@@ -202,12 +203,15 @@ export function buildEncoderArgs(
           else args.push("-cq", String(quality));
           break;
         case "videotoolbox":
-          if (bitrate) args.push("-b:v", bitrate);
-          else {
-            const vtQuality = Math.max(0, Math.min(100, 100 - quality * 2));
-            args.push("-q:v", String(vtQuality));
-          }
-          args.push("-allow_sw", "1");
+          args.push(
+            ...buildVideoToolboxRateControlArgs({
+              bitrate,
+              width: options.width,
+              height: options.height,
+              fps: fpsToNumber(fps),
+              quality,
+            }),
+          );
           break;
         case "vaapi":
           args.unshift("-vaapi_device", "/dev/dri/renderD128");

packages/engine/src/services/streamingEncoder.test.ts

@@ -271,6 +271,25 @@ describe("buildStreamingArgs", () => {
       expect(presetArg(args)).toBe("p4");
     });
 
+    it("uses a supported derived bitrate for high-quality VideoToolbox", () => {
+      const args = buildStreamingArgs(
+        {
+          ...baseGpu,
+          fps: { num: 24, den: 1 },
+          width: 1920,
+          height: 1080,
+          preset: "slow",
+          quality: 15,
+        },
+        "/tmp/out.mp4",
+        "videotoolbox",
+      );
+
+      expect(args).not.toContain("-q:v");
+      expect(args[args.indexOf("-b:v") + 1]).toBe("12M");
+      expect(args[args.indexOf("-allow_sw") + 1]).toBe("1");
+    });
+
     // Same mapping applies to hevc_nvenc: NVENC's preset vocabulary is
     // codec-agnostic, so the helper must translate for H.265 too.
     it("translates libx264 preset names to NVENC pN for h265 as well", () => {

packages/engine/src/services/streamingEncoder.ts

@@ -27,6 +27,7 @@ import {
   type GpuEncoder,
   getCachedGpuEncoder,
   getGpuEncoderName,
+  buildVideoToolboxRateControlArgs,
   mapPresetForGpuEncoder,
 } from "../utils/gpuEncoder.js";
 import { formatFfmpegError, isExternalFfmpegInterruption } from "../utils/runFfmpeg.js";
@@ -34,7 +35,7 @@ import { getFfmpegBinary } from "../utils/ffmpegBinaries.js";
 import { getHdrEncoderColorParams } from "../utils/hdr.js";
 import { withEvenDimensionPad } from "../utils/evenDimensions.js";
 import { DEFAULT_CONFIG, type EngineConfig } from "../config.js";
-import { fpsToFfmpegArg, type Fps } from "@hyperframes/core";
+import { fpsToFfmpegArg, fpsToNumber, type Fps } from "@hyperframes/core";
 import { appendVp9CpuUsedArg } from "./vp9Options.js";
 import { appendRenderProvenanceArgs } from "../utils/renderProvenance.js";
 
@@ -270,12 +271,15 @@ export function buildStreamingArgs(
           else args.push("-cq", String(quality));
           break;
         case "videotoolbox":
-          if (bitrate) args.push("-b:v", bitrate);
-          else {
-            const vtQuality = Math.max(0, Math.min(100, 100 - quality * 2));
-            args.push("-q:v", String(vtQuality));
-          }
-          args.push("-allow_sw", "1");
+          args.push(
+            ...buildVideoToolboxRateControlArgs({
+              bitrate,
+              width: options.width,
+              height: options.height,
+              fps: fpsToNumber(fps),
+              quality,
+            }),
+          );
           break;
         case "vaapi":
           args.unshift("-vaapi_device", "/dev/dri/renderD128");

packages/engine/src/utils/gpuEncoder.test.ts

@@ -137,4 +137,12 @@ describe("getProbeArgs", () => {
       expect(args).toContain("color=size=320x240:rate=1:duration=1");
     }
   });
+
+  it("probes VideoToolbox with the same supported bitrate option as production", () => {
+    const args = getProbeArgs("videotoolbox");
+
+    expect(args).not.toContain("-q:v");
+    expect(args[args.indexOf("-b:v") + 1]).toBe("1M");
+    expect(args[args.indexOf("-allow_sw") + 1]).toBe("1");
+  });
 });

packages/engine/src/utils/gpuEncoder.ts

@@ -107,6 +107,26 @@ export function getGpuEncoderName(encoder: GpuEncoder, codec: "h264" | "h265"):
   }
 }
 
+export function buildVideoToolboxRateControlArgs(input: {
+  bitrate?: string;
+  width: number;
+  height: number;
+  fps: number;
+  quality: number;
+}): string[] {
+  if (input.bitrate) return ["-b:v", input.bitrate, "-allow_sw", "1"];
+
+  // VideoToolbox rejects FFmpeg's generic qscale path on supported macOS
+  // builds. Derive a bitrate from pixel rate instead: the three CRF-shaped
+  // quality tiers map to roughly 5M / 10M / 12M at 1080p24.
+  const bitsPerPixel = input.quality <= 15 ? 0.24 : input.quality <= 18 ? 0.16 : 0.08;
+  const megabitsPerSecond = Math.max(
+    1,
+    Math.round((input.width * input.height * input.fps * bitsPerPixel) / 1_000_000),
+  );
+  return ["-b:v", `${megabitsPerSecond}M`, "-allow_sw", "1"];
+}
+
 // Minimum probe dimensions must clear every GPU encoder's hardware minimum.
 // NVIDIA data-center SKUs (L4/T4/A10/A100) reject frames below ~257px on
 // either dimension with "Frame Dimension less than the minimum supported
@@ -137,6 +157,17 @@ export function getProbeArgs(encoder: ConcreteGpuEncoder): string[] {
 
   args.push("-c:v", getGpuEncoderName(encoder, "h264"));
 
+  if (encoder === "videotoolbox") {
+    args.push(
+      ...buildVideoToolboxRateControlArgs({
+        width: GPU_PROBE_WIDTH,
+        height: GPU_PROBE_HEIGHT,
+        fps: 1,
+        quality: 23,
+      }),
+    );
+  }
+
   if (encoder === "amf") {
     args.push("-rc", "cqp", "-qp_i", "28", "-qp_p", "28");
   }