fix(engine): encode and tag ProRes mov output as BT.709 (#5382)

* fix(engine): encode and tag ProRes mov output as BT.709

* test(engine): read the ProRes colour tags from the first frame

* test(engine): end the ProRes tag probe's options with --

packages/cli/src/background-removal/pipeline.test.ts

@@ -79,8 +79,15 @@ describe("background-removal/pipeline — buildEncoderArgs", () => {
     },
   );
 
-  it.each(["mov", "png"] as const)("keeps the %s output color path", (format) => {
-    const args = buildEncoderArgs(format, 64, 64, 30, `/tmp/cutout.${format}`);
+  it("converts raw RGBA to the tagged BT.709 matrix for the ProRes mov", () => {
+    const args = buildEncoderArgs("mov", 64, 64, 30, "/tmp/cutout.mov");
+    const filter = args[args.indexOf("-vf") + 1];
+    expect(filter).toContain("out_color_matrix=bt709");
+    expect(filter).toContain("setparams=color_primaries=bt709:color_trc=bt709:colorspace=bt709");
+  });
+
+  it("keeps the png output color path", () => {
+    const args = buildEncoderArgs("png", 64, 64, 30, "/tmp/cutout.png");
     expect(args).not.toContain("-vf");
   });
 

packages/cli/src/background-removal/pipeline.ts

@@ -23,6 +23,7 @@ import { type Device, type ModelId } from "./manager.js";
 import {
   DEFAULT_VP9_CPU_USED,
   SDR_RGB_TO_BT709_FILTER,
+  SDR_RGB_TO_TAGGED_BT709_FILTER,
   renderProvenanceArgs,
 } from "@hyperframes/engine";
 
@@ -192,6 +193,8 @@ export function buildEncoderArgs(
   if (format === "mov") {
     return [
       ...base,
+      "-vf",
+      SDR_RGB_TO_TAGGED_BT709_FILTER,
       "-c:v",
       "prores_ks",
       "-profile:v",

packages/cli/src/background-removal/pipeline.windowsHide.test.ts

@@ -36,6 +36,8 @@ vi.mock("./inference.js", () => ({
 vi.mock("@hyperframes/engine", () => ({
   DEFAULT_VP9_CPU_USED: 4,
   SDR_RGB_TO_BT709_FILTER: "scale=out_color_matrix=bt709:out_range=tv:flags=neighbor",
+  SDR_RGB_TO_TAGGED_BT709_FILTER:
+    "scale=out_color_matrix=bt709:out_range=tv:flags=neighbor,setparams=color_primaries=bt709:color_trc=bt709:colorspace=bt709:range=tv",
   renderProvenanceArgs: () => [],
   extractMediaMetadata: async () => ({ width: 1, height: 1, fps: 1, durationSeconds: 1 }),
 }));

packages/engine/src/index.ts

@@ -61,7 +61,10 @@ export {
   type EngineConfig,
   type ExtractCacheDirResolution,
 } from "./config.js";
-export { SDR_RGB_TO_BT709_FILTER } from "./utils/sdrCaptureColor.js";
+export {
+  SDR_RGB_TO_BT709_FILTER,
+  SDR_RGB_TO_TAGGED_BT709_FILTER,
+} from "./utils/sdrCaptureColor.js";
 export {
   DEFAULT_VP9_CPU_USED,
   MAX_VP9_CPU_USED,

packages/engine/src/services/chunkEncoder.test.ts

@@ -14,8 +14,11 @@ import {
   resolveLockedGopSize,
 } from "./chunkEncoder.js";
 import { renderProvenanceArgs } from "../utils/renderProvenance.js";
-import { SDR_CAPTURE_TO_BT709_FILTER } from "../utils/sdrCaptureColor.js";
-import { getFfmpegBinary } from "../utils/ffmpegBinaries.js";
+import {
+  SDR_CAPTURE_TO_BT709_FILTER,
+  SDR_RGB_TO_TAGGED_BT709_FILTER,
+} from "../utils/sdrCaptureColor.js";
+import { getFfmpegBinary, getFfprobeBinary } from "../utils/ffmpegBinaries.js";
 
 const HAS_FFMPEG = spawnSync(getFfmpegBinary(), ["-version"]).status === 0;
 
@@ -1226,7 +1229,7 @@ describe("buildEncoderArgs color space", () => {
     expect(args[args.indexOf("-vf") + 1]).toBe("pad=ceil(iw/2)*2:ceil(ih/2)*2");
   });
 
-  it("leaves alpha ProRes untouched (no even-dim pad)", () => {
+  it("converts alpha ProRes to BT.709 without an even-dim pad", () => {
     const args = buildEncoderArgs(
       {
         ...baseOptions,
@@ -1238,7 +1241,7 @@ describe("buildEncoderArgs color space", () => {
       inputArgs,
       "out.mov",
     );
-    expect(args.indexOf("-vf")).toBe(-1);
+    expect(args[args.indexOf("-vf") + 1]).toBe(SDR_RGB_TO_TAGGED_BT709_FILTER);
     expect(args.join(" ")).not.toContain("pad=");
   });
 
@@ -1820,26 +1823,27 @@ describe("buildConcatArgs", () => {
 });
 
 describe.skipIf(!HAS_FFMPEG)("buildEncoderArgs SDR colour", () => {
+  const ffmpeg = getFfmpegBinary();
+  const pixelAt = (file: string, decode: string, x: number, format = "rgb24"): number[] => [
+    ...spawnSync(ffmpeg, [
+      "-v",
+      "error",
+      "-i",
+      file,
+      "-vf",
+      `${decode}format=${format},crop=1:1:${x}:8`,
+      "-frames:v",
+      "1",
+      "-f",
+      "rawvideo",
+      "-",
+    ]).stdout,
+  ];
+
   // Chrome captures are BT.601 JPEGs. A direct YUV-to-YUV scale keeps that matrix under the BT.709
   // tag (ffmpeg 7 and older) or tints greys (8 and newer), so the encode goes through RGB.
   it("delivers Chrome's JPEG colours in the BT.709 the mp4 is tagged with", () => {
-    const ffmpeg = getFfmpegBinary();
     const dir = mkdtempSync(join(tmpdir(), "hf-sdr-colour-"));
-    const rgbAt = (file: string, decode: string, x: number): number[] => [
-      ...spawnSync(ffmpeg, [
-        "-v",
-        "error",
-        "-i",
-        file,
-        "-vf",
-        `${decode}format=rgb24,crop=1:1:${x}:8`,
-        "-frames:v",
-        "1",
-        "-f",
-        "rawvideo",
-        "-",
-      ]).stdout,
-    ];
     try {
       for (const color of ["0xC83C28", "0xFE0000", "0x101010", "0x2050E0"]) {
         const jpg = join(dir, "frame.jpg");
@@ -1879,8 +1883,8 @@ describe.skipIf(!HAS_FFMPEG)("buildEncoderArgs SDR colour", () => {
           [8, 2],
           [32, 2],
         ] as const) {
-          const captured = rgbAt(jpg, "", x);
-          const delivered = rgbAt(out, "scale=in_color_matrix=bt709:in_range=tv,", x);
+          const captured = pixelAt(jpg, "", x);
+          const delivered = pixelAt(out, "scale=in_color_matrix=bt709:in_range=tv,", x);
           const worst = Math.max(...delivered.map((v, i) => Math.abs(v - captured[i]!)));
           expect(
             worst,
@@ -1892,4 +1896,70 @@ describe.skipIf(!HAS_FFMPEG)("buildEncoderArgs SDR colour", () => {
       rmSync(dir, { recursive: true, force: true });
     }
   }, 30_000);
+
+  // Untagged HD reads as BT.709 in QuickTime, so ProRes has to be BT.709 too. MOV captures are
+  // RGBA PNGs, and the conversion must leave their alpha plane alone. The tags come from
+  // setparams because prores_ks ignores -color_* (ffmpeg 5.1 writes none, 8.1 only the matrix).
+  it("delivers PNG colours and alpha in the BT.709 the ProRes mov is tagged with", () => {
+    const dir = mkdtempSync(join(tmpdir(), "hf-prores-colour-"));
+    const png = join(dir, "frame.png");
+    const out = join(dir, "out.mov");
+    try {
+      for (const color of ["0xC83C28", "0x00FF00", "0x101010"]) {
+        const synth = spawnSync(ffmpeg, [
+          "-v",
+          "error",
+          "-y",
+          "-f",
+          "lavfi",
+          "-i",
+          `color=c=${color}@0.5:s=64x16,format=rgba`,
+          "-frames:v",
+          "1",
+          png,
+        ]);
+        expect(synth.status).toBe(0);
+        const args = buildEncoderArgs(
+          {
+            fps: { num: 30, den: 1 },
+            width: 64,
+            height: 16,
+            codec: "prores",
+            preset: "4444",
+            quality: 23,
+            pixelFormat: "yuva444p10le",
+          },
+          ["-i", png],
+          out,
+        );
+        expect(spawnSync(ffmpeg, args).status).toBe(0);
+
+        const captured = pixelAt(png, "", 8, "rgba");
+        const delivered = pixelAt(out, "scale=in_color_matrix=bt709:in_range=tv,", 8, "rgba");
+        const worst = Math.max(...delivered.map((v, i) => Math.abs(v - captured[i]!)));
+        expect(worst, `${color}: capture ${captured} delivered ${delivered}`).toBeLessThanOrEqual(
+          2,
+        );
+      }
+      // Read the first frame, not the stream: a prores_ks mov's stream-level range depends on
+      // the ffprobe version (7.1 and 8.0 report unknown), while every version tags the frames.
+      const tags = spawnSync(getFfprobeBinary(), [
+        "-v",
+        "error",
+        "-select_streams",
+        "v:0",
+        "-read_intervals",
+        "%+#1",
+        "-show_entries",
+        "frame=color_range,color_space,color_primaries,color_transfer",
+        "-of",
+        "csv=p=0",
+        "--",
+        out,
+      ]);
+      expect(tags.stdout.toString().trim()).toBe("tv,bt709,bt709,bt709");
+    } finally {
+      rmSync(dir, { recursive: true, force: true });
+    }
+  }, 30_000);
 });

packages/engine/src/services/chunkEncoder.ts

@@ -31,7 +31,10 @@ import {
 } from "../utils/gpuEncoder.js";
 import { type HdrTransfer, getHdrEncoderColorParams } from "../utils/hdr.js";
 import { withEvenDimensionPad } from "../utils/evenDimensions.js";
-import { SDR_CAPTURE_TO_BT709_FILTER } from "../utils/sdrCaptureColor.js";
+import {
+  SDR_CAPTURE_TO_BT709_FILTER,
+  SDR_RGB_TO_TAGGED_BT709_FILTER,
+} from "../utils/sdrCaptureColor.js";
 import {
   describeFfmpegFailure,
   ffmpegStatsReader,
@@ -408,6 +411,7 @@ export function buildEncoderArgs(
     }
   } else if (codec === "prores") {
     args.push("-c:v", "prores_ks", "-profile:v", preset, "-vendor", "apl0");
+    args.push("-vf", SDR_RGB_TO_TAGGED_BT709_FILTER);
     args.push("-pix_fmt", pixelFormat);
     appendRenderProvenanceArgs(args, outputPath);
     return [...args, "-y", outputPath];

packages/engine/src/services/streamingEncoder.test.ts

@@ -26,7 +26,10 @@ import {
 } from "./streamingEncoder.js";
 import { DEFAULT_HDR10_MASTERING } from "../utils/hdr.js";
 import { type GpuEncoder } from "../utils/gpuEncoder.js";
-import { SDR_CAPTURE_TO_BT709_FILTER } from "../utils/sdrCaptureColor.js";
+import {
+  SDR_CAPTURE_TO_BT709_FILTER,
+  SDR_RGB_TO_TAGGED_BT709_FILTER,
+} from "../utils/sdrCaptureColor.js";
 
 const baseHdrPq: StreamingEncoderOptions = {
   fps: { num: 30, den: 1 },
@@ -241,6 +244,15 @@ describe("buildStreamingArgs", () => {
         `${SDR_CAPTURE_TO_BT709_FILTER},pad=ceil(iw/2)*2:ceil(ih/2)*2`,
       );
     });
+
+    it("converts ProRes to the BT.709 it is tagged with and keeps the alpha format", () => {
+      const args = buildStreamingArgs(
+        { ...baseVp9, codec: "prores", preset: "4444", pixelFormat: "yuva444p10le" },
+        "/tmp/out.mov",
+      );
+      expect(args[args.indexOf("-vf") + 1]).toBe(SDR_RGB_TO_TAGGED_BT709_FILTER);
+      expect(args[args.indexOf("-pix_fmt") + 1]).toBe("yuva444p10le");
+    });
   });
 
   describe("output path", () => {

packages/engine/src/services/streamingEncoder.ts

@@ -39,7 +39,11 @@ import {
 import { getFfmpegBinary } from "../utils/ffmpegBinaries.js";
 import { getHdrEncoderColorParams } from "../utils/hdr.js";
 import { withEvenDimensionPad } from "../utils/evenDimensions.js";
-import { SDR_CAPTURE_TO_BT709_FILTER, SDR_RGB_TO_BT709_FILTER } from "../utils/sdrCaptureColor.js";
+import {
+  SDR_CAPTURE_TO_BT709_FILTER,
+  SDR_RGB_TO_BT709_FILTER,
+  SDR_RGB_TO_TAGGED_BT709_FILTER,
+} from "../utils/sdrCaptureColor.js";
 import { DEFAULT_CONFIG, type EngineConfig } from "../config.js";
 import { fpsToFfmpegArg, fpsToNumber, type Fps } from "@hyperframes/core";
 import { appendVp9CpuUsedArg } from "./vp9Options.js";
@@ -385,6 +389,7 @@ export function buildStreamingArgs(
     }
   } else if (codec === "prores") {
     args.push("-c:v", "prores_ks", "-profile:v", preset, "-vendor", "apl0");
+    args.push("-vf", SDR_RGB_TO_TAGGED_BT709_FILTER);
     args.push("-pix_fmt", pixelFormat);
     appendRenderProvenanceArgs(args, outputPath);
     return [...args, "-y", outputPath];

packages/engine/src/utils/sdrCaptureColor.ts

@@ -1,3 +1,5 @@
 export const SDR_RGB_TO_BT709_FILTER = "scale=out_color_matrix=bt709:out_range=tv:flags=neighbor";
 
 export const SDR_CAPTURE_TO_BT709_FILTER = `scale=in_color_matrix=bt601:in_range=pc:flags=neighbor,format=gbrp,${SDR_RGB_TO_BT709_FILTER}`;
+
+export const SDR_RGB_TO_TAGGED_BT709_FILTER = `${SDR_RGB_TO_BT709_FILTER},setparams=color_primaries=bt709:color_trc=bt709:colorspace=bt709:range=tv`;