diff --git a/packages/cli/src/background-removal/inference.ts b/packages/cli/src/background-removal/inference.ts index ced409846..d0b3d8adb 100644 --- a/packages/cli/src/background-removal/inference.ts +++ b/packages/cli/src/background-removal/inference.ts @@ -159,10 +159,16 @@ async function postprocess( } // lanczos3 keeps soft edges; nearest leaves visible jaggies on hair. + // Sharp upcasts the single-channel raw input to a 3-channel buffer during + // resize, so the output is laid out as RGB-interleaved (R0,G0,B0,R1,G1,B1,...) + // even though all three channels carry the same grayscale value. Force the + // output back to single channel with toColourspace("b-w") so we can index + // it linearly as a mask. const fullMask = await sharp(maskBuf, { raw: { width: INPUT_SIZE, height: INPUT_SIZE, channels: 1 }, }) .resize(width, height, { kernel: "lanczos3", fit: "fill" }) + .toColourspace("b-w") .raw() .toBuffer(); diff --git a/packages/cli/src/background-removal/pipeline.test.ts b/packages/cli/src/background-removal/pipeline.test.ts index 93a870c55..7bc5488a9 100644 --- a/packages/cli/src/background-removal/pipeline.test.ts +++ b/packages/cli/src/background-removal/pipeline.test.ts @@ -46,6 +46,35 @@ describe("background-removal/pipeline — buildEncoderArgs", () => { expect(args[args.length - 1]).toBe("/tmp/out.webm"); }); + it("webm preset tags BT.709 colorspace + limited range", () => { + // Without these tags, ffmpeg's RGB→YUV conversion uses the BT.601 default, + // and Chrome's YUV→RGB pass on the resulting webm produces a different + // RGB triple than the source mp4 (visible color shift on overlay). Pin + // BT.709 limited-range so the cutout matches modern Rec.709 sources. + const args = buildEncoderArgs("webm", 1920, 1080, 30, "/tmp/out.webm"); + const csIdx = args.indexOf("-colorspace"); + expect(csIdx).toBeGreaterThan(-1); + expect(args[csIdx + 1]).toBe("bt709"); + const rangeIdx = args.indexOf("-color_range"); + expect(rangeIdx).toBeGreaterThan(-1); + expect(args[rangeIdx + 1]).toBe("tv"); + }); + + it("webm quality presets map to crf 30/18/12", () => { + const fast = buildEncoderArgs("webm", 1920, 1080, 30, "/tmp/o.webm", "fast"); + const balanced = buildEncoderArgs("webm", 1920, 1080, 30, "/tmp/o.webm", "balanced"); + const best = buildEncoderArgs("webm", 1920, 1080, 30, "/tmp/o.webm", "best"); + const crf = (args: string[]) => args[args.indexOf("-crf") + 1]; + expect(crf(fast)).toBe("30"); + expect(crf(balanced)).toBe("18"); + expect(crf(best)).toBe("12"); + }); + + it("webm default quality is balanced (crf 18)", () => { + const args = buildEncoderArgs("webm", 1920, 1080, 30, "/tmp/o.webm"); + expect(args[args.indexOf("-crf") + 1]).toBe("18"); + }); + it("mov preset emits ProRes 4444 + yuva444p10le", () => { const args = buildEncoderArgs("mov", 1920, 1080, 30, "/tmp/out.mov"); expect(args).toContain("prores_ks"); diff --git a/packages/cli/src/background-removal/pipeline.ts b/packages/cli/src/background-removal/pipeline.ts index c5a7e774e..03ce3cb63 100644 --- a/packages/cli/src/background-removal/pipeline.ts +++ b/packages/cli/src/background-removal/pipeline.ts @@ -20,11 +20,28 @@ import { type Device, type ModelId } from "./manager.js"; export type OutputFormat = "webm" | "mov" | "png"; +export type Quality = "fast" | "balanced" | "best"; + +export const QUALITIES: readonly Quality[] = ["fast", "balanced", "best"] as const; + +export const QUALITY_CRF: Record = { + fast: 30, + balanced: 18, + best: 12, +}; + +export const DEFAULT_QUALITY: Quality = "balanced"; + +export const isQuality = (v: unknown): v is Quality => + typeof v === "string" && (QUALITIES as readonly string[]).includes(v); + export interface RenderOptions { inputPath: string; outputPath: string; device?: Device; model?: ModelId; + /** Encoder CRF preset for `.webm`. See `QUALITY_CRF`. Ignored for `.mov`/`.png`. */ + quality?: Quality; onProgress?: (event: ProgressEvent) => void; } @@ -100,6 +117,7 @@ export function buildEncoderArgs( height: number, fps: number, outputPath: string, + quality: Quality = DEFAULT_QUALITY, ): string[] { const base = [ "-y", @@ -123,7 +141,7 @@ export function buildEncoderArgs( "-b:v", "0", "-crf", - "30", + String(QUALITY_CRF[quality]), "-deadline", "good", "-row-mt", @@ -132,6 +150,19 @@ export function buildEncoderArgs( "0", "-pix_fmt", "yuva420p", + // Tag the output as BT.709 limited range so browsers use the same + // YUV→RGB matrix the source video was encoded with. Without these tags + // ffmpeg's default RGB→YUV conversion is BT.601, which causes a visible + // color shift (red/skin tones in particular) when the matted overlay is + // composited over the original mp4. + "-colorspace", + "bt709", + "-color_primaries", + "bt709", + "-color_trc", + "bt709", + "-color_range", + "tv", "-metadata:s:v:0", "alpha_mode=1", "-an", @@ -250,9 +281,20 @@ async function runPipeline( }); const decoderExit = waitForExit(decoder, "ffmpeg decoder", () => decoderStderr); - const encoder = spawn("ffmpeg", buildEncoderArgs(format, width, height, fps || 30, outputPath), { - stdio: ["pipe", "ignore", "pipe"], - }); + const encoder = spawn( + "ffmpeg", + buildEncoderArgs( + format, + width, + height, + fps || 30, + outputPath, + options.quality ?? DEFAULT_QUALITY, + ), + { + stdio: ["pipe", "ignore", "pipe"], + }, + ); let encoderStderr = ""; encoder.stderr?.on("data", (d: Buffer) => { encoderStderr += d.toString(); diff --git a/packages/cli/src/commands/remove-background.ts b/packages/cli/src/commands/remove-background.ts index e5258a7d1..cc942f67e 100644 --- a/packages/cli/src/commands/remove-background.ts +++ b/packages/cli/src/commands/remove-background.ts @@ -4,6 +4,7 @@ import { existsSync } from "node:fs"; import * as clack from "@clack/prompts"; import { c } from "../ui/colors.js"; import { isDevice, DEVICES } from "../background-removal/manager.js"; +import { DEFAULT_QUALITY, QUALITIES, isQuality } from "../background-removal/pipeline.js"; import type { Example } from "./_examples.js"; export const examples: Example[] = [ @@ -23,6 +24,14 @@ export const examples: Example[] = [ "Force CPU (skip CoreML/CUDA)", "hyperframes remove-background avatar.mp4 -o transparent.webm --device cpu", ], + [ + "Smaller file at the cost of color match (text-behind-subject won't blend as cleanly)", + "hyperframes remove-background avatar.mp4 -o transparent.webm --quality fast", + ], + [ + "Visually-lossless WebM (master / re-encode source)", + "hyperframes remove-background avatar.mp4 -o transparent.webm --quality best", + ], ["Show detected providers without rendering", "hyperframes remove-background --info"], ]; @@ -48,6 +57,11 @@ export default defineCommand({ description: `Execution provider: ${DEVICES.join(", ")}`, default: "auto", }, + quality: { + type: "string", + description: `Encoder quality preset for .webm output: ${QUALITIES.join(", ")} (default: ${DEFAULT_QUALITY}). Higher quality = closer color match when overlaying on the source mp4, larger file. Ignored for .mov / .png.`, + default: DEFAULT_QUALITY, + }, info: { type: "boolean", description: "Print detected execution providers and exit (no render)", @@ -81,6 +95,12 @@ export default defineCommand({ ); process.exit(1); } + if (!isQuality(args.quality)) { + console.error( + c.error(`Invalid --quality '${String(args.quality)}'. Use: ${QUALITIES.join(", ")}.`), + ); + process.exit(1); + } const inputPath = resolve(args.input); const outputPath = resolve(args.output); @@ -95,6 +115,7 @@ export default defineCommand({ inputPath, outputPath, device: args.device, + quality: args.quality, onProgress: (event) => { if (event.kind === "info") { spin?.message(event.message);