mirror of
https://github.com/heygen-com/hyperframes.git
synced 2026-09-01 19:42:03 +00:00
fix(engine): add bt709 color space + range conversion to encoding (#223)
## Description Adds proper BT.709 color space metadata and full→limited range conversion to H.264/H.265 encoding. Chrome captures frames in full-range sRGB (BT.709 primaries), but without explicit color tagging, players guess the wrong color space and range — causing color shifts across iOS/Android/desktop and crushed dark values that compound the gradient banding issue fixed in #222. **What changed:** | Setting | Before | After | |---------|--------|-------| | `color_space` | `bt470bg` (guessed) | `bt709` (explicit) | | `color_primaries` | `unknown` | `bt709` | | `color_transfer` | `unknown` | `bt709` | | `color_range` | `pc` (full, wrong for H.264) | `tv` (limited, correct) | | `time_base` | `1/15360` (varies by platform) | `1/90000` (fixed) | **Approach:** - BT.709 VUI params embedded via x264-params/x265-params (`colorprim=bt709:transfer=bt709:colormatrix=bt709`) — ensures the bitstream itself carries color info - FFmpeg-level metadata flags (`-colorspace:v bt709`, etc.) — belt-and-suspenders - `scale=in_range=pc:out_range=tv` filter converts Chrome's full-range output to TV/limited range - VAAPI path chains the range filter with existing `format=nv12,hwupload` - `-video_track_timescale 90000` for consistent cross-platform A/V timing (same as Remotion) - VP9 and ProRes encoding unaffected ## Testing - Verified via ffprobe: all 5 color metadata fields now correct - Directly tested FFmpeg args produce expected output - 40 engine tests pass (8 new: color metadata h264/h265, range filter CPU, VAAPI filter chain, GPU skip, VP9 skip, timescale) - Builds cleanly, lint + format pass
This commit is contained in:
@@ -108,7 +108,7 @@ describe("buildEncoderArgs anti-banding", () => {
|
||||
);
|
||||
const paramIdx = args.indexOf("-x264-params");
|
||||
expect(paramIdx).toBeGreaterThan(-1);
|
||||
expect(args[paramIdx + 1]).toBe("aq-mode=3");
|
||||
expect(args[paramIdx + 1]).toContain("aq-mode=3");
|
||||
expect(args[paramIdx + 1]).not.toContain("deblock");
|
||||
});
|
||||
|
||||
@@ -132,3 +132,103 @@ describe("buildEncoderArgs anti-banding", () => {
|
||||
expect(args.indexOf("-x265-params")).toBe(-1);
|
||||
});
|
||||
});
|
||||
|
||||
describe("buildEncoderArgs color space", () => {
|
||||
const baseOptions = { fps: 30, width: 1920, height: 1080 };
|
||||
const inputArgs = ["-framerate", "30", "-i", "frames/%04d.png"];
|
||||
|
||||
it("adds bt709 color space metadata for h264 CPU encoding", () => {
|
||||
const args = buildEncoderArgs(
|
||||
{ ...baseOptions, codec: "h264", preset: "medium", quality: 23 },
|
||||
inputArgs,
|
||||
"out.mp4",
|
||||
);
|
||||
// FFmpeg-level metadata tags
|
||||
expect(args).toContain("-colorspace:v");
|
||||
expect(args[args.indexOf("-colorspace:v") + 1]).toBe("bt709");
|
||||
expect(args[args.indexOf("-color_primaries:v") + 1]).toBe("bt709");
|
||||
expect(args[args.indexOf("-color_trc:v") + 1]).toBe("bt709");
|
||||
expect(args[args.indexOf("-color_range") + 1]).toBe("tv");
|
||||
// x264-params VUI embedding
|
||||
const paramIdx = args.indexOf("-x264-params");
|
||||
expect(args[paramIdx + 1]).toContain("colorprim=bt709");
|
||||
expect(args[paramIdx + 1]).toContain("transfer=bt709");
|
||||
expect(args[paramIdx + 1]).toContain("colormatrix=bt709");
|
||||
});
|
||||
|
||||
it("adds bt709 color space metadata for h265 CPU encoding", () => {
|
||||
const args = buildEncoderArgs(
|
||||
{ ...baseOptions, codec: "h265", preset: "medium", quality: 23 },
|
||||
inputArgs,
|
||||
"out.mp4",
|
||||
);
|
||||
expect(args).toContain("-colorspace:v");
|
||||
expect(args[args.indexOf("-colorspace:v") + 1]).toBe("bt709");
|
||||
// x265-params VUI embedding
|
||||
const paramIdx = args.indexOf("-x265-params");
|
||||
expect(args[paramIdx + 1]).toContain("colorprim=bt709");
|
||||
});
|
||||
|
||||
it("adds range conversion filter for CPU h264 encoding", () => {
|
||||
const args = buildEncoderArgs(
|
||||
{ ...baseOptions, codec: "h264", preset: "medium", quality: 23 },
|
||||
inputArgs,
|
||||
"out.mp4",
|
||||
);
|
||||
const vfIdx = args.indexOf("-vf");
|
||||
expect(vfIdx).toBeGreaterThan(-1);
|
||||
expect(args[vfIdx + 1]).toContain("scale=in_range=pc:out_range=tv");
|
||||
});
|
||||
|
||||
it("prepends range conversion to VAAPI filter chain", () => {
|
||||
const args = buildEncoderArgs(
|
||||
{ ...baseOptions, codec: "h264", preset: "medium", quality: 23, useGpu: true },
|
||||
inputArgs,
|
||||
"out.mp4",
|
||||
"vaapi",
|
||||
);
|
||||
const vfIdx = args.indexOf("-vf");
|
||||
expect(vfIdx).toBeGreaterThan(-1);
|
||||
expect(args[vfIdx + 1]).toBe("scale=in_range=pc:out_range=tv,format=nv12,hwupload");
|
||||
});
|
||||
|
||||
it("skips range conversion filter for non-VAAPI GPU encoding", () => {
|
||||
const args = buildEncoderArgs(
|
||||
{ ...baseOptions, codec: "h264", preset: "medium", quality: 23, useGpu: true },
|
||||
inputArgs,
|
||||
"out.mp4",
|
||||
"nvenc",
|
||||
);
|
||||
expect(args.indexOf("-vf")).toBe(-1);
|
||||
// but still has color metadata
|
||||
expect(args).toContain("-colorspace:v");
|
||||
});
|
||||
|
||||
it("does not add color metadata for VP9", () => {
|
||||
const args = buildEncoderArgs(
|
||||
{ ...baseOptions, codec: "vp9", preset: "good", quality: 23 },
|
||||
inputArgs,
|
||||
"out.webm",
|
||||
);
|
||||
expect(args).not.toContain("-colorspace:v");
|
||||
});
|
||||
|
||||
it("adds video_track_timescale for h264", () => {
|
||||
const args = buildEncoderArgs(
|
||||
{ ...baseOptions, codec: "h264", preset: "medium", quality: 23 },
|
||||
inputArgs,
|
||||
"out.mp4",
|
||||
);
|
||||
expect(args).toContain("-video_track_timescale");
|
||||
expect(args[args.indexOf("-video_track_timescale") + 1]).toBe("90000");
|
||||
});
|
||||
|
||||
it("does not add timescale for VP9", () => {
|
||||
const args = buildEncoderArgs(
|
||||
{ ...baseOptions, codec: "vp9", preset: "good", quality: 23 },
|
||||
inputArgs,
|
||||
"out.webm",
|
||||
);
|
||||
expect(args).not.toContain("-video_track_timescale");
|
||||
});
|
||||
});
|
||||
|
||||
@@ -100,13 +100,15 @@ export function buildEncoderArgs(
|
||||
if (bitrate) args.push("-b:v", bitrate);
|
||||
else args.push("-crf", String(quality));
|
||||
|
||||
// Anti-banding: aq-mode=3 redistributes bits to dark flat areas (gradients),
|
||||
// deblock smooths quantization boundaries that cause visible bands.
|
||||
// Encoder-specific params: anti-banding + bt709 color space.
|
||||
// aq-mode=3 redistributes bits to dark flat areas (gradients).
|
||||
// colorprim/transfer/colormatrix embed bt709 in the H.264/H.265 VUI.
|
||||
const xParamsFlag = codec === "h264" ? "-x264-params" : "-x265-params";
|
||||
const colorParams = "colorprim=bt709:transfer=bt709:colormatrix=bt709";
|
||||
if (preset === "ultrafast") {
|
||||
args.push(xParamsFlag, "aq-mode=3");
|
||||
args.push(xParamsFlag, `aq-mode=3:${colorParams}`);
|
||||
} else {
|
||||
args.push(xParamsFlag, "aq-mode=3:aq-strength=0.8:deblock=1,1");
|
||||
args.push(xParamsFlag, `aq-mode=3:aq-strength=0.8:deblock=1,1:${colorParams}`);
|
||||
}
|
||||
}
|
||||
} else if (codec === "vp9") {
|
||||
@@ -122,6 +124,36 @@ export function buildEncoderArgs(
|
||||
return [...args, "-y", outputPath];
|
||||
}
|
||||
|
||||
// BT.709 color space metadata — Chrome screenshots are sRGB which maps to bt709.
|
||||
// Tags the output so players interpret colors correctly across devices.
|
||||
if (codec === "h264" || codec === "h265") {
|
||||
args.push(
|
||||
"-colorspace:v",
|
||||
"bt709",
|
||||
"-color_primaries:v",
|
||||
"bt709",
|
||||
"-color_trc:v",
|
||||
"bt709",
|
||||
"-color_range",
|
||||
"tv",
|
||||
);
|
||||
|
||||
// Convert full-range RGB input (Chrome screenshots) to limited/TV range for H.264.
|
||||
// VAAPI already has a -vf chain for hwupload; prepend range conversion to it.
|
||||
if (gpuEncoder === "vaapi") {
|
||||
// Replace the existing VAAPI -vf with one that includes range conversion
|
||||
const vfIdx = args.indexOf("-vf");
|
||||
if (vfIdx !== -1) {
|
||||
args[vfIdx + 1] = `scale=in_range=pc:out_range=tv,${args[vfIdx + 1]}`;
|
||||
}
|
||||
} else if (!shouldUseGpu) {
|
||||
args.push("-vf", "scale=in_range=pc:out_range=tv");
|
||||
}
|
||||
|
||||
// Fixed timescale for consistent A/V timing across platforms.
|
||||
args.push("-video_track_timescale", "90000");
|
||||
}
|
||||
|
||||
if (gpuEncoder !== "vaapi") {
|
||||
args.push("-pix_fmt", pixelFormat);
|
||||
}
|
||||
|
||||
@@ -169,13 +169,15 @@ function buildStreamingArgs(
|
||||
if (bitrate) args.push("-b:v", bitrate);
|
||||
else args.push("-crf", String(quality));
|
||||
|
||||
// Anti-banding: aq-mode=3 redistributes bits to dark flat areas (gradients),
|
||||
// deblock smooths quantization boundaries that cause visible bands.
|
||||
// Encoder-specific params: anti-banding + bt709 color space.
|
||||
// aq-mode=3 redistributes bits to dark flat areas (gradients).
|
||||
// colorprim/transfer/colormatrix embed bt709 in the H.264/H.265 VUI.
|
||||
const xParamsFlag = codec === "h264" ? "-x264-params" : "-x265-params";
|
||||
const colorParams = "colorprim=bt709:transfer=bt709:colormatrix=bt709";
|
||||
if (preset === "ultrafast") {
|
||||
args.push(xParamsFlag, "aq-mode=3");
|
||||
args.push(xParamsFlag, `aq-mode=3:${colorParams}`);
|
||||
} else {
|
||||
args.push(xParamsFlag, "aq-mode=3:aq-strength=0.8:deblock=1,1");
|
||||
args.push(xParamsFlag, `aq-mode=3:aq-strength=0.8:deblock=1,1:${colorParams}`);
|
||||
}
|
||||
}
|
||||
} else if (codec === "vp9") {
|
||||
@@ -191,6 +193,34 @@ function buildStreamingArgs(
|
||||
return [...args, "-y", outputPath];
|
||||
}
|
||||
|
||||
// BT.709 color space metadata — Chrome screenshots are sRGB which maps to bt709.
|
||||
// Tags the output so players interpret colors correctly across devices.
|
||||
if (codec === "h264" || codec === "h265") {
|
||||
args.push(
|
||||
"-colorspace:v",
|
||||
"bt709",
|
||||
"-color_primaries:v",
|
||||
"bt709",
|
||||
"-color_trc:v",
|
||||
"bt709",
|
||||
"-color_range",
|
||||
"tv",
|
||||
);
|
||||
|
||||
// Convert full-range RGB input (Chrome screenshots) to limited/TV range for H.264.
|
||||
if (gpuEncoder === "vaapi") {
|
||||
const vfIdx = args.indexOf("-vf");
|
||||
if (vfIdx !== -1) {
|
||||
args[vfIdx + 1] = `scale=in_range=pc:out_range=tv,${args[vfIdx + 1]}`;
|
||||
}
|
||||
} else if (!shouldUseGpu) {
|
||||
args.push("-vf", "scale=in_range=pc:out_range=tv");
|
||||
}
|
||||
|
||||
// Fixed timescale for consistent A/V timing across platforms.
|
||||
args.push("-video_track_timescale", "90000");
|
||||
}
|
||||
|
||||
if (gpuEncoder !== "vaapi") {
|
||||
args.push("-pix_fmt", pixelFormat);
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user