mirror of
https://github.com/heygen-com/hyperframes.git
synced 2026-09-03 04:38:33 +00:00
fix(cli): correct sharp 3-channel mask + BT.709 + quality presets in remove-background
- inference.ts: force `.toColourspace("b-w")` on the resized mask. Sharp upcasts
the 1-channel raw input to RGB-interleaved during resize, so `fullMask[i]`
was reading R,G,B,R,G,B... of pixels 0..691199 instead of the alpha for all
2,073,600 pixels. Visible symptom: horizontal scanline alpha artifact in
every transparent webm — the avatar appeared semi-transparent throughout.
- pipeline.ts: add BT.709 + limited-range colorspace tags so Chrome's YUV→RGB
matches the source mp4 (without these, ffmpeg's default RGB→YUV is BT.601
and skin tones drift visibly when the cutout is overlaid on its source).
- pipeline.ts: add Quality preset type ("fast"/"balanced"/"best" → CRF 30/18/12).
Default raised from CRF 30 → 18 ("balanced") so the most common pattern
(text-behind-subject) works out of the box without visible doubling.
- remove-background.ts: wire `--quality` flag with validation, +2 examples.
- Tests: BT.709 tags present, quality preset → CRF mapping, default is balanced.
Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
This commit is contained in:
@@ -159,10 +159,16 @@ async function postprocess(
|
||||
}
|
||||
|
||||
// lanczos3 keeps soft edges; nearest leaves visible jaggies on hair.
|
||||
// Sharp upcasts the single-channel raw input to a 3-channel buffer during
|
||||
// resize, so the output is laid out as RGB-interleaved (R0,G0,B0,R1,G1,B1,...)
|
||||
// even though all three channels carry the same grayscale value. Force the
|
||||
// output back to single channel with toColourspace("b-w") so we can index
|
||||
// it linearly as a mask.
|
||||
const fullMask = await sharp(maskBuf, {
|
||||
raw: { width: INPUT_SIZE, height: INPUT_SIZE, channels: 1 },
|
||||
})
|
||||
.resize(width, height, { kernel: "lanczos3", fit: "fill" })
|
||||
.toColourspace("b-w")
|
||||
.raw()
|
||||
.toBuffer();
|
||||
|
||||
|
||||
@@ -46,6 +46,35 @@ describe("background-removal/pipeline — buildEncoderArgs", () => {
|
||||
expect(args[args.length - 1]).toBe("/tmp/out.webm");
|
||||
});
|
||||
|
||||
it("webm preset tags BT.709 colorspace + limited range", () => {
|
||||
// Without these tags, ffmpeg's RGB→YUV conversion uses the BT.601 default,
|
||||
// and Chrome's YUV→RGB pass on the resulting webm produces a different
|
||||
// RGB triple than the source mp4 (visible color shift on overlay). Pin
|
||||
// BT.709 limited-range so the cutout matches modern Rec.709 sources.
|
||||
const args = buildEncoderArgs("webm", 1920, 1080, 30, "/tmp/out.webm");
|
||||
const csIdx = args.indexOf("-colorspace");
|
||||
expect(csIdx).toBeGreaterThan(-1);
|
||||
expect(args[csIdx + 1]).toBe("bt709");
|
||||
const rangeIdx = args.indexOf("-color_range");
|
||||
expect(rangeIdx).toBeGreaterThan(-1);
|
||||
expect(args[rangeIdx + 1]).toBe("tv");
|
||||
});
|
||||
|
||||
it("webm quality presets map to crf 30/18/12", () => {
|
||||
const fast = buildEncoderArgs("webm", 1920, 1080, 30, "/tmp/o.webm", "fast");
|
||||
const balanced = buildEncoderArgs("webm", 1920, 1080, 30, "/tmp/o.webm", "balanced");
|
||||
const best = buildEncoderArgs("webm", 1920, 1080, 30, "/tmp/o.webm", "best");
|
||||
const crf = (args: string[]) => args[args.indexOf("-crf") + 1];
|
||||
expect(crf(fast)).toBe("30");
|
||||
expect(crf(balanced)).toBe("18");
|
||||
expect(crf(best)).toBe("12");
|
||||
});
|
||||
|
||||
it("webm default quality is balanced (crf 18)", () => {
|
||||
const args = buildEncoderArgs("webm", 1920, 1080, 30, "/tmp/o.webm");
|
||||
expect(args[args.indexOf("-crf") + 1]).toBe("18");
|
||||
});
|
||||
|
||||
it("mov preset emits ProRes 4444 + yuva444p10le", () => {
|
||||
const args = buildEncoderArgs("mov", 1920, 1080, 30, "/tmp/out.mov");
|
||||
expect(args).toContain("prores_ks");
|
||||
|
||||
@@ -20,11 +20,28 @@ import { type Device, type ModelId } from "./manager.js";
|
||||
|
||||
export type OutputFormat = "webm" | "mov" | "png";
|
||||
|
||||
export type Quality = "fast" | "balanced" | "best";
|
||||
|
||||
export const QUALITIES: readonly Quality[] = ["fast", "balanced", "best"] as const;
|
||||
|
||||
export const QUALITY_CRF: Record<Quality, number> = {
|
||||
fast: 30,
|
||||
balanced: 18,
|
||||
best: 12,
|
||||
};
|
||||
|
||||
export const DEFAULT_QUALITY: Quality = "balanced";
|
||||
|
||||
export const isQuality = (v: unknown): v is Quality =>
|
||||
typeof v === "string" && (QUALITIES as readonly string[]).includes(v);
|
||||
|
||||
export interface RenderOptions {
|
||||
inputPath: string;
|
||||
outputPath: string;
|
||||
device?: Device;
|
||||
model?: ModelId;
|
||||
/** Encoder CRF preset for `.webm`. See `QUALITY_CRF`. Ignored for `.mov`/`.png`. */
|
||||
quality?: Quality;
|
||||
onProgress?: (event: ProgressEvent) => void;
|
||||
}
|
||||
|
||||
@@ -100,6 +117,7 @@ export function buildEncoderArgs(
|
||||
height: number,
|
||||
fps: number,
|
||||
outputPath: string,
|
||||
quality: Quality = DEFAULT_QUALITY,
|
||||
): string[] {
|
||||
const base = [
|
||||
"-y",
|
||||
@@ -123,7 +141,7 @@ export function buildEncoderArgs(
|
||||
"-b:v",
|
||||
"0",
|
||||
"-crf",
|
||||
"30",
|
||||
String(QUALITY_CRF[quality]),
|
||||
"-deadline",
|
||||
"good",
|
||||
"-row-mt",
|
||||
@@ -132,6 +150,19 @@ export function buildEncoderArgs(
|
||||
"0",
|
||||
"-pix_fmt",
|
||||
"yuva420p",
|
||||
// Tag the output as BT.709 limited range so browsers use the same
|
||||
// YUV→RGB matrix the source video was encoded with. Without these tags
|
||||
// ffmpeg's default RGB→YUV conversion is BT.601, which causes a visible
|
||||
// color shift (red/skin tones in particular) when the matted overlay is
|
||||
// composited over the original mp4.
|
||||
"-colorspace",
|
||||
"bt709",
|
||||
"-color_primaries",
|
||||
"bt709",
|
||||
"-color_trc",
|
||||
"bt709",
|
||||
"-color_range",
|
||||
"tv",
|
||||
"-metadata:s:v:0",
|
||||
"alpha_mode=1",
|
||||
"-an",
|
||||
@@ -250,9 +281,20 @@ async function runPipeline(
|
||||
});
|
||||
const decoderExit = waitForExit(decoder, "ffmpeg decoder", () => decoderStderr);
|
||||
|
||||
const encoder = spawn("ffmpeg", buildEncoderArgs(format, width, height, fps || 30, outputPath), {
|
||||
stdio: ["pipe", "ignore", "pipe"],
|
||||
});
|
||||
const encoder = spawn(
|
||||
"ffmpeg",
|
||||
buildEncoderArgs(
|
||||
format,
|
||||
width,
|
||||
height,
|
||||
fps || 30,
|
||||
outputPath,
|
||||
options.quality ?? DEFAULT_QUALITY,
|
||||
),
|
||||
{
|
||||
stdio: ["pipe", "ignore", "pipe"],
|
||||
},
|
||||
);
|
||||
let encoderStderr = "";
|
||||
encoder.stderr?.on("data", (d: Buffer) => {
|
||||
encoderStderr += d.toString();
|
||||
|
||||
@@ -4,6 +4,7 @@ import { existsSync } from "node:fs";
|
||||
import * as clack from "@clack/prompts";
|
||||
import { c } from "../ui/colors.js";
|
||||
import { isDevice, DEVICES } from "../background-removal/manager.js";
|
||||
import { DEFAULT_QUALITY, QUALITIES, isQuality } from "../background-removal/pipeline.js";
|
||||
import type { Example } from "./_examples.js";
|
||||
|
||||
export const examples: Example[] = [
|
||||
@@ -23,6 +24,14 @@ export const examples: Example[] = [
|
||||
"Force CPU (skip CoreML/CUDA)",
|
||||
"hyperframes remove-background avatar.mp4 -o transparent.webm --device cpu",
|
||||
],
|
||||
[
|
||||
"Smaller file at the cost of color match (text-behind-subject won't blend as cleanly)",
|
||||
"hyperframes remove-background avatar.mp4 -o transparent.webm --quality fast",
|
||||
],
|
||||
[
|
||||
"Visually-lossless WebM (master / re-encode source)",
|
||||
"hyperframes remove-background avatar.mp4 -o transparent.webm --quality best",
|
||||
],
|
||||
["Show detected providers without rendering", "hyperframes remove-background --info"],
|
||||
];
|
||||
|
||||
@@ -48,6 +57,11 @@ export default defineCommand({
|
||||
description: `Execution provider: ${DEVICES.join(", ")}`,
|
||||
default: "auto",
|
||||
},
|
||||
quality: {
|
||||
type: "string",
|
||||
description: `Encoder quality preset for .webm output: ${QUALITIES.join(", ")} (default: ${DEFAULT_QUALITY}). Higher quality = closer color match when overlaying on the source mp4, larger file. Ignored for .mov / .png.`,
|
||||
default: DEFAULT_QUALITY,
|
||||
},
|
||||
info: {
|
||||
type: "boolean",
|
||||
description: "Print detected execution providers and exit (no render)",
|
||||
@@ -81,6 +95,12 @@ export default defineCommand({
|
||||
);
|
||||
process.exit(1);
|
||||
}
|
||||
if (!isQuality(args.quality)) {
|
||||
console.error(
|
||||
c.error(`Invalid --quality '${String(args.quality)}'. Use: ${QUALITIES.join(", ")}.`),
|
||||
);
|
||||
process.exit(1);
|
||||
}
|
||||
|
||||
const inputPath = resolve(args.input);
|
||||
const outputPath = resolve(args.output);
|
||||
@@ -95,6 +115,7 @@ export default defineCommand({
|
||||
inputPath,
|
||||
outputPath,
|
||||
device: args.device,
|
||||
quality: args.quality,
|
||||
onProgress: (event) => {
|
||||
if (event.kind === "info") {
|
||||
spin?.message(event.message);
|
||||
|
||||
Reference in New Issue
Block a user