fix(cli): correct sharp 3-channel mask + BT.709 + quality presets in remove-background

- inference.ts: force `.toColourspace("b-w")` on the resized mask. Sharp upcasts
  the 1-channel raw input to RGB-interleaved during resize, so `fullMask[i]`
  was reading R,G,B,R,G,B... of pixels 0..691199 instead of the alpha for all
  2,073,600 pixels. Visible symptom: horizontal scanline alpha artifact in
  every transparent webm — the avatar appeared semi-transparent throughout.
- pipeline.ts: add BT.709 + limited-range colorspace tags so Chrome's YUV→RGB
  matches the source mp4 (without these, ffmpeg's default RGB→YUV is BT.601
  and skin tones drift visibly when the cutout is overlaid on its source).
- pipeline.ts: add Quality preset type ("fast"/"balanced"/"best" → CRF 30/18/12).
  Default raised from CRF 30 → 18 ("balanced") so the most common pattern
  (text-behind-subject) works out of the box without visible doubling.
- remove-background.ts: wire `--quality` flag with validation, +2 examples.
- Tests: BT.709 tags present, quality preset → CRF mapping, default is balanced.

Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
This commit is contained in:
James
2026-05-04 20:27:56 -07:00
co-authored by Claude Opus 4.7
parent 06f5422d34
commit 688052d368
4 changed files with 102 additions and 4 deletions
@@ -159,10 +159,16 @@ async function postprocess(
}
// lanczos3 keeps soft edges; nearest leaves visible jaggies on hair.
// Sharp upcasts the single-channel raw input to a 3-channel buffer during
// resize, so the output is laid out as RGB-interleaved (R0,G0,B0,R1,G1,B1,...)
// even though all three channels carry the same grayscale value. Force the
// output back to single channel with toColourspace("b-w") so we can index
// it linearly as a mask.
const fullMask = await sharp(maskBuf, {
raw: { width: INPUT_SIZE, height: INPUT_SIZE, channels: 1 },
})
.resize(width, height, { kernel: "lanczos3", fit: "fill" })
.toColourspace("b-w")
.raw()
.toBuffer();
@@ -46,6 +46,35 @@ describe("background-removal/pipeline — buildEncoderArgs", () => {
expect(args[args.length - 1]).toBe("/tmp/out.webm");
});
it("webm preset tags BT.709 colorspace + limited range", () => {
// Without these tags, ffmpeg's RGB→YUV conversion uses the BT.601 default,
// and Chrome's YUV→RGB pass on the resulting webm produces a different
// RGB triple than the source mp4 (visible color shift on overlay). Pin
// BT.709 limited-range so the cutout matches modern Rec.709 sources.
const args = buildEncoderArgs("webm", 1920, 1080, 30, "/tmp/out.webm");
const csIdx = args.indexOf("-colorspace");
expect(csIdx).toBeGreaterThan(-1);
expect(args[csIdx + 1]).toBe("bt709");
const rangeIdx = args.indexOf("-color_range");
expect(rangeIdx).toBeGreaterThan(-1);
expect(args[rangeIdx + 1]).toBe("tv");
});
it("webm quality presets map to crf 30/18/12", () => {
const fast = buildEncoderArgs("webm", 1920, 1080, 30, "/tmp/o.webm", "fast");
const balanced = buildEncoderArgs("webm", 1920, 1080, 30, "/tmp/o.webm", "balanced");
const best = buildEncoderArgs("webm", 1920, 1080, 30, "/tmp/o.webm", "best");
const crf = (args: string[]) => args[args.indexOf("-crf") + 1];
expect(crf(fast)).toBe("30");
expect(crf(balanced)).toBe("18");
expect(crf(best)).toBe("12");
});
it("webm default quality is balanced (crf 18)", () => {
const args = buildEncoderArgs("webm", 1920, 1080, 30, "/tmp/o.webm");
expect(args[args.indexOf("-crf") + 1]).toBe("18");
});
it("mov preset emits ProRes 4444 + yuva444p10le", () => {
const args = buildEncoderArgs("mov", 1920, 1080, 30, "/tmp/out.mov");
expect(args).toContain("prores_ks");
@@ -20,11 +20,28 @@ import { type Device, type ModelId } from "./manager.js";
export type OutputFormat = "webm" | "mov" | "png";
export type Quality = "fast" | "balanced" | "best";
export const QUALITIES: readonly Quality[] = ["fast", "balanced", "best"] as const;
export const QUALITY_CRF: Record<Quality, number> = {
fast: 30,
balanced: 18,
best: 12,
};
export const DEFAULT_QUALITY: Quality = "balanced";
export const isQuality = (v: unknown): v is Quality =>
typeof v === "string" && (QUALITIES as readonly string[]).includes(v);
export interface RenderOptions {
inputPath: string;
outputPath: string;
device?: Device;
model?: ModelId;
/** Encoder CRF preset for `.webm`. See `QUALITY_CRF`. Ignored for `.mov`/`.png`. */
quality?: Quality;
onProgress?: (event: ProgressEvent) => void;
}
@@ -100,6 +117,7 @@ export function buildEncoderArgs(
height: number,
fps: number,
outputPath: string,
quality: Quality = DEFAULT_QUALITY,
): string[] {
const base = [
"-y",
@@ -123,7 +141,7 @@ export function buildEncoderArgs(
"-b:v",
"0",
"-crf",
"30",
String(QUALITY_CRF[quality]),
"-deadline",
"good",
"-row-mt",
@@ -132,6 +150,19 @@ export function buildEncoderArgs(
"0",
"-pix_fmt",
"yuva420p",
// Tag the output as BT.709 limited range so browsers use the same
// YUV→RGB matrix the source video was encoded with. Without these tags
// ffmpeg's default RGB→YUV conversion is BT.601, which causes a visible
// color shift (red/skin tones in particular) when the matted overlay is
// composited over the original mp4.
"-colorspace",
"bt709",
"-color_primaries",
"bt709",
"-color_trc",
"bt709",
"-color_range",
"tv",
"-metadata:s:v:0",
"alpha_mode=1",
"-an",
@@ -250,9 +281,20 @@ async function runPipeline(
});
const decoderExit = waitForExit(decoder, "ffmpeg decoder", () => decoderStderr);
const encoder = spawn("ffmpeg", buildEncoderArgs(format, width, height, fps || 30, outputPath), {
stdio: ["pipe", "ignore", "pipe"],
});
const encoder = spawn(
"ffmpeg",
buildEncoderArgs(
format,
width,
height,
fps || 30,
outputPath,
options.quality ?? DEFAULT_QUALITY,
),
{
stdio: ["pipe", "ignore", "pipe"],
},
);
let encoderStderr = "";
encoder.stderr?.on("data", (d: Buffer) => {
encoderStderr += d.toString();
@@ -4,6 +4,7 @@ import { existsSync } from "node:fs";
import * as clack from "@clack/prompts";
import { c } from "../ui/colors.js";
import { isDevice, DEVICES } from "../background-removal/manager.js";
import { DEFAULT_QUALITY, QUALITIES, isQuality } from "../background-removal/pipeline.js";
import type { Example } from "./_examples.js";
export const examples: Example[] = [
@@ -23,6 +24,14 @@ export const examples: Example[] = [
"Force CPU (skip CoreML/CUDA)",
"hyperframes remove-background avatar.mp4 -o transparent.webm --device cpu",
],
[
"Smaller file at the cost of color match (text-behind-subject won't blend as cleanly)",
"hyperframes remove-background avatar.mp4 -o transparent.webm --quality fast",
],
[
"Visually-lossless WebM (master / re-encode source)",
"hyperframes remove-background avatar.mp4 -o transparent.webm --quality best",
],
["Show detected providers without rendering", "hyperframes remove-background --info"],
];
@@ -48,6 +57,11 @@ export default defineCommand({
description: `Execution provider: ${DEVICES.join(", ")}`,
default: "auto",
},
quality: {
type: "string",
description: `Encoder quality preset for .webm output: ${QUALITIES.join(", ")} (default: ${DEFAULT_QUALITY}). Higher quality = closer color match when overlaying on the source mp4, larger file. Ignored for .mov / .png.`,
default: DEFAULT_QUALITY,
},
info: {
type: "boolean",
description: "Print detected execution providers and exit (no render)",
@@ -81,6 +95,12 @@ export default defineCommand({
);
process.exit(1);
}
if (!isQuality(args.quality)) {
console.error(
c.error(`Invalid --quality '${String(args.quality)}'. Use: ${QUALITIES.join(", ")}.`),
);
process.exit(1);
}
const inputPath = resolve(args.input);
const outputPath = resolve(args.output);
@@ -95,6 +115,7 @@ export default defineCommand({
inputPath,
outputPath,
device: args.device,
quality: args.quality,
onProgress: (event) => {
if (event.kind === "info") {
spin?.message(event.message);