feat(cli): add --background-output to remove-background

Emit an inverse-alpha background plate alongside the cutout in a single
inference pass. Same source RGB, alpha = 255 − mask. Dual-encoder pipeline
runs in parallel; both outputs share the same --quality preset.

This is a hole-cut plate (subject region transparent), not an inpainted
clean plate — composite something opaque under it to fill the hole.
Docs and skill cover when each is the right tool.

Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>

packages/cli/src/background-removal/inference.ts

@@ -24,10 +24,24 @@ interface OrtModule {
   Tensor: typeof Tensor;
 }
 
+export interface SessionResult {
+  /** Subject opaque, background fully transparent. */
+  fg: Buffer;
+  /** Inverse-alpha plate: same RGB, alpha is `255 − mask`. Null unless `withBackground` was true. */
+  bg: Buffer | null;
+}
+
 export interface Session {
-  /** Run inference on one RGB frame, return RGBA bytes (H*W*4). */
-  process(rgb: Buffer, width: number, height: number): Promise<Buffer>;
-  /** ORT EP that was actually selected. */
+  /**
+   * Both `fg` and `bg` (when requested) are session-owned buffers reused on the
+   * next call — drain the encoder's stdin before invoking `process` again.
+   */
+  process(
+    rgb: Buffer,
+    width: number,
+    height: number,
+    withBackground?: boolean,
+  ): Promise<SessionResult>;
   provider: string;
   close(): Promise<void>;
 }
@@ -73,16 +87,15 @@ export async function createSession(options: CreateSessionOptions = {}): Promise
     throw new Error("ONNX session is missing input or output bindings");
   }
 
-  // Pre-allocated per-frame buffers reused across every process() call.
-  // At 1080p this saves ~9 MB of allocations per frame. rgbaBuf is sized
-  // lazily on the first call (we don't know W/H until then).
+  // Reused across calls; sized lazily on first frame. Saves ~9 MB/frame at 1080p.
   const inputData = new Float32Array(3 * INPUT_PLANE);
   const maskBuf = Buffer.allocUnsafe(INPUT_PLANE);
   let rgbaBuf: Buffer | null = null;
+  let rgbaBgBuf: Buffer | null = null;
 
   return {
     provider: providerUsed,
-    async process(rgb, width, height) {
+    async process(rgb, width, height, withBackground = false) {
       const tensor = await preprocess(sharp, ort, rgb, width, height, inputData);
       const outputs = await session.run({ [inputName]: tensor });
       const output = outputs[outputName];
@@ -91,7 +104,21 @@ export async function createSession(options: CreateSessionOptions = {}): Promise
       if (!rgbaBuf || rgbaBuf.length !== expectedBytes) {
         rgbaBuf = Buffer.allocUnsafe(expectedBytes);
       }
-      return await postprocess(sharp, output, rgb, width, height, maskBuf, rgbaBuf);
+      if (withBackground) {
+        if (!rgbaBgBuf || rgbaBgBuf.length !== expectedBytes) {
+          rgbaBgBuf = Buffer.allocUnsafe(expectedBytes);
+        }
+      }
+      return await postprocess(
+        sharp,
+        output,
+        rgb,
+        width,
+        height,
+        maskBuf,
+        rgbaBuf,
+        withBackground ? rgbaBgBuf : null,
+      );
     },
     async close() {
       await session.release();
@@ -141,7 +168,8 @@ async function postprocess(
   height: number,
   maskBuf: Buffer,
   rgbaBuf: Buffer,
-): Promise<Buffer> {
+  rgbaBgBuf: Buffer | null,
+): Promise<SessionResult> {
   const raw = output.data as Float32Array;
 
   let lo = Infinity;
@@ -172,11 +200,30 @@ async function postprocess(
     .raw()
     .toBuffer();
 
-  for (let i = 0; i < width * height; i++) {
+  const pixels = width * height;
+  if (rgbaBgBuf) {
+    for (let i = 0; i < pixels; i++) {
+      const r = rgb[i * 3]!;
+      const g = rgb[i * 3 + 1]!;
+      const b = rgb[i * 3 + 2]!;
+      const m = fullMask[i]!;
+      const o = i * 4;
+      rgbaBuf[o] = r;
+      rgbaBuf[o + 1] = g;
+      rgbaBuf[o + 2] = b;
+      rgbaBuf[o + 3] = m;
+      rgbaBgBuf[o] = r;
+      rgbaBgBuf[o + 1] = g;
+      rgbaBgBuf[o + 2] = b;
+      rgbaBgBuf[o + 3] = 255 - m;
+    }
+    return { fg: rgbaBuf, bg: rgbaBgBuf };
+  }
+  for (let i = 0; i < pixels; i++) {
     rgbaBuf[i * 4] = rgb[i * 3]!;
     rgbaBuf[i * 4 + 1] = rgb[i * 3 + 1]!;
     rgbaBuf[i * 4 + 2] = rgb[i * 3 + 2]!;
     rgbaBuf[i * 4 + 3] = fullMask[i]!;
   }
-  return rgbaBuf;
+  return { fg: rgbaBuf, bg: null };
 }

packages/cli/src/background-removal/pipeline.test.ts

@@ -1,7 +1,13 @@
 import { describe, expect, it } from "vitest";
 import { EventEmitter } from "node:events";
 import type { spawn } from "node:child_process";
-import { inferOutputFormat, inferInputKind, buildEncoderArgs, waitForExit } from "./pipeline.js";
+import {
+  inferOutputFormat,
+  inferInputKind,
+  buildEncoderArgs,
+  resolveRenderTargets,
+  waitForExit,
+} from "./pipeline.js";
 
 describe("background-removal/pipeline — inferOutputFormat", () => {
   it("maps .webm → webm", () => {
@@ -97,6 +103,54 @@ describe("background-removal/pipeline — buildEncoderArgs", () => {
   });
 });
 
+describe("background-removal/pipeline — resolveRenderTargets", () => {
+  it("resolves a normal video → webm render", () => {
+    const t = resolveRenderTargets("/tmp/clip.mp4", "/tmp/cutout.webm");
+    expect(t.format).toBe("webm");
+    expect(t.inputKind).toBe("video");
+    expect(t.bgFormat).toBeUndefined();
+  });
+
+  it("resolves an image → png render", () => {
+    const t = resolveRenderTargets("/tmp/portrait.jpg", "/tmp/cutout.png");
+    expect(t.format).toBe("png");
+    expect(t.inputKind).toBe("image");
+  });
+
+  it("rejects image input with a video output extension", () => {
+    expect(() => resolveRenderTargets("/tmp/portrait.jpg", "/tmp/cutout.webm")).toThrow(
+      /Image input requires a \.png output/,
+    );
+  });
+
+  it("rejects video input with a .png output", () => {
+    expect(() => resolveRenderTargets("/tmp/clip.mp4", "/tmp/cutout.png")).toThrow(
+      /Video input requires a \.webm or \.mov output/,
+    );
+  });
+
+  it("threads background-output format through when valid", () => {
+    const t = resolveRenderTargets("/tmp/clip.mp4", "/tmp/fg.webm", "/tmp/bg.webm");
+    expect(t.bgFormat).toBe("webm");
+    const tMov = resolveRenderTargets("/tmp/clip.mp4", "/tmp/fg.webm", "/tmp/bg.mov");
+    expect(tMov.bgFormat).toBe("mov");
+  });
+
+  it("rejects --background-output for image inputs (no temporal pairing to do)", () => {
+    expect(() =>
+      resolveRenderTargets("/tmp/portrait.jpg", "/tmp/cutout.png", "/tmp/bg.png"),
+    ).toThrow(/--background-output is not supported for image inputs/);
+  });
+
+  it("rejects .png as the --background-output extension", () => {
+    // .png is only valid for single-image inputs, and image inputs themselves
+    // can't have a background-output anyway. So .png here is always a misuse.
+    expect(() => resolveRenderTargets("/tmp/clip.mp4", "/tmp/fg.webm", "/tmp/bg.png")).toThrow(
+      /--background-output must be \.webm or \.mov/,
+    );
+  });
+});
+
 // Regression: a previous version of waitForExit treated `code === null` as
 // success. Per Node's child_process docs, that's the signal-killed case —
 // reporting it as success means a SIGTERM/SIGKILL'd ffmpeg encoder produces

packages/cli/src/background-removal/pipeline.ts

@@ -38,6 +38,16 @@ export const isQuality = (v: unknown): v is Quality =>
 export interface RenderOptions {
   inputPath: string;
   outputPath: string;
+  /**
+   * Optional second output: an inverse-alpha background plate (same source
+   * RGB, transparent where the subject was). Only valid for video inputs and
+   * .webm/.mov outputs — not allowed alongside a .png output. The plate's
+   * format is inferred from this path independently of the foreground's.
+   *
+   * NOTE: this is a hole-cut plate, not an inpainted clean plate. Composite
+   * something opaque (graphics, blur, scene) under it to fill the hole.
+   */
+  backgroundOutputPath?: string;
   device?: Device;
   model?: ModelId;
   /** Encoder CRF preset for `.webm`. See `QUALITY_CRF`. Ignored for `.mov`/`.png`. */
@@ -52,6 +62,8 @@ export type ProgressEvent =
 
 export interface RenderResult {
   outputPath: string;
+  /** Present only when `backgroundOutputPath` was set. */
+  backgroundOutputPath?: string;
   framesProcessed: number;
   durationSeconds: number;
   avgMsPerFrame: number;
@@ -203,17 +215,27 @@ async function* readFrames(
   }
 }
 
-export async function render(options: RenderOptions): Promise<RenderResult> {
-  if (!hasFFmpeg() || !hasFFprobe()) {
-    throw new Error("ffmpeg and ffprobe are required. Install: brew install ffmpeg");
-  }
+export interface RenderTargets {
+  format: OutputFormat;
+  inputKind: "video" | "image";
+  bgFormat: OutputFormat | undefined;
+}
 
-  const format = inferOutputFormat(options.outputPath);
-  const inputKind = inferInputKind(options.inputPath);
+/**
+ * Resolve and validate the input/output combination before any I/O. Pure;
+ * exported so unit tests can pin the error messages without spawning ffmpeg.
+ */
+export function resolveRenderTargets(
+  inputPath: string,
+  outputPath: string,
+  backgroundOutputPath?: string,
+): RenderTargets {
+  const format = inferOutputFormat(outputPath);
+  const inputKind = inferInputKind(inputPath);
 
   if (inputKind === "image" && format !== "png") {
     throw new Error(
-      `Image input requires a .png output (got ${extname(options.outputPath)}). Use a video input for .webm/.mov.`,
+      `Image input requires a .png output (got ${extname(outputPath)}). Use a video input for .webm/.mov.`,
     );
   }
   if (inputKind === "video" && format === "png") {
@@ -222,6 +244,35 @@ export async function render(options: RenderOptions): Promise<RenderResult> {
     );
   }
 
+  let bgFormat: OutputFormat | undefined;
+  if (backgroundOutputPath) {
+    if (inputKind === "image") {
+      throw new Error(
+        "--background-output is not supported for image inputs. Use a video input (mp4/mov/webm) to produce both a cutout and a background plate.",
+      );
+    }
+    bgFormat = inferOutputFormat(backgroundOutputPath);
+    if (bgFormat === "png") {
+      throw new Error(
+        "--background-output must be .webm or .mov; .png is only valid for single-image inputs.",
+      );
+    }
+  }
+
+  return { format, inputKind, bgFormat };
+}
+
+export async function render(options: RenderOptions): Promise<RenderResult> {
+  if (!hasFFmpeg() || !hasFFprobe()) {
+    throw new Error("ffmpeg and ffprobe are required. Install: brew install ffmpeg");
+  }
+
+  const { format, bgFormat } = resolveRenderTargets(
+    options.inputPath,
+    options.outputPath,
+    options.backgroundOutputPath,
+  );
+
   const media = await probeMedia(options.inputPath);
 
   options.onProgress?.({
@@ -240,12 +291,13 @@ export async function render(options: RenderOptions): Promise<RenderResult> {
 
   try {
     const start = Date.now();
-    const framesProcessed = await runPipeline(options, session, media, format);
+    const framesProcessed = await runPipeline(options, session, media, format, bgFormat);
     const durationSeconds = (Date.now() - start) / 1000;
     const avgMsPerFrame = framesProcessed ? (durationSeconds * 1000) / framesProcessed : 0;
 
     return {
       outputPath: options.outputPath,
+      backgroundOutputPath: options.backgroundOutputPath,
       framesProcessed,
       durationSeconds,
       avgMsPerFrame,
@@ -259,61 +311,68 @@ export async function render(options: RenderOptions): Promise<RenderResult> {
 
 const RECENT_WINDOW = 30;
 
+interface FfmpegProc {
+  proc: ReturnType<typeof spawn>;
+  exit: Promise<void>;
+  /** Tail of stderr, captured for inclusion in error messages. */
+  getStderr: () => string;
+}
+
+function spawnFfmpeg(args: string[], label: string, stdio: ("ignore" | "pipe")[]): FfmpegProc {
+  const proc = spawn("ffmpeg", args, { stdio });
+  let stderrBuf = "";
+  proc.stderr?.on("data", (d: Buffer) => {
+    stderrBuf += d.toString();
+  });
+  const exit = waitForExit(proc, label, () => stderrBuf);
+  return { proc, exit, getStderr: () => stderrBuf };
+}
+
 async function runPipeline(
   options: RenderOptions,
   session: Session,
   media: MediaInfo,
   format: OutputFormat,
+  bgFormat: OutputFormat | undefined,
 ): Promise<number> {
-  const { inputPath, outputPath } = options;
+  const { inputPath, outputPath, backgroundOutputPath } = options;
   const { width, height, fps, frameCount } = media;
   const frameBytes = width * height * 3;
+  const quality = options.quality ?? DEFAULT_QUALITY;
 
-  const decoder = spawn(
-    "ffmpeg",
+  const decoder = spawnFfmpeg(
     ["-loglevel", "error", "-i", inputPath, "-f", "rawvideo", "-pix_fmt", "rgb24", "-an", "-"],
-    { stdio: ["ignore", "pipe", "pipe"] },
+    "ffmpeg decoder",
+    ["ignore", "pipe", "pipe"],
   );
 
-  let decoderStderr = "";
-  decoder.stderr?.on("data", (d: Buffer) => {
-    decoderStderr += d.toString();
-  });
-  const decoderExit = waitForExit(decoder, "ffmpeg decoder", () => decoderStderr);
-
-  const encoder = spawn(
-    "ffmpeg",
-    buildEncoderArgs(
-      format,
-      width,
-      height,
-      fps || 30,
-      outputPath,
-      options.quality ?? DEFAULT_QUALITY,
-    ),
-    {
-      stdio: ["pipe", "ignore", "pipe"],
-    },
+  const fg = spawnFfmpeg(
+    buildEncoderArgs(format, width, height, fps || 30, outputPath, quality),
+    "ffmpeg encoder",
+    ["pipe", "ignore", "pipe"],
   );
-  let encoderStderr = "";
-  encoder.stderr?.on("data", (d: Buffer) => {
-    encoderStderr += d.toString();
-  });
-  const encoderExit = waitForExit(encoder, "ffmpeg encoder", () => encoderStderr);
+
+  const bg =
+    backgroundOutputPath && bgFormat
+      ? spawnFfmpeg(
+          buildEncoderArgs(bgFormat, width, height, fps || 30, backgroundOutputPath, quality),
+          "ffmpeg background encoder",
+          ["pipe", "ignore", "pipe"],
+        )
+      : null;
 
   let processed = 0;
   const total = frameCount;
 
-  // Running average over the last RECENT_WINDOW frames.
   const recentMs = new Array<number>(RECENT_WINDOW).fill(0);
   let recentSum = 0;
   let recentSlot = 0;
   let recentCount = 0;
 
   try {
-    for await (const rgb of readFrames(decoder.stdout!, frameBytes)) {
+    for await (const rgb of readFrames(decoder.proc.stdout!, frameBytes)) {
       const t0 = Date.now();
-      const rgba = await session.process(rgb, width, height);
+      const result = await session.process(rgb, width, height, bg !== null);
       const elapsed = Date.now() - t0;
 
       recentSum += elapsed - recentMs[recentSlot]!;
@@ -321,8 +380,24 @@ async function runPipeline(
       recentSlot = (recentSlot + 1) % RECENT_WINDOW;
       if (recentCount < RECENT_WINDOW) recentCount++;
 
-      if (!encoder.stdin!.write(rgba)) {
-        await new Promise<void>((resolve) => encoder.stdin!.once("drain", () => resolve()));
+      // Issue both writes before any await so a slow encoder doesn't block
+      // the other. Drain anything that returned false before the next
+      // session.process() — its output buffers are reused per frame.
+      const fgWroteFully = fg.proc.stdin!.write(result.fg);
+      const bgWroteFully = bg && result.bg ? bg.proc.stdin!.write(result.bg) : true;
+      if (!fgWroteFully || !bgWroteFully) {
+        const drains: Promise<void>[] = [];
+        if (!fgWroteFully) {
+          drains.push(
+            new Promise<void>((resolve) => fg.proc.stdin!.once("drain", () => resolve())),
+          );
+        }
+        if (!bgWroteFully && bg) {
+          drains.push(
+            new Promise<void>((resolve) => bg.proc.stdin!.once("drain", () => resolve())),
+          );
+        }
+        await Promise.all(drains);
       }
 
       processed++;
@@ -334,17 +409,21 @@ async function runPipeline(
       });
     }
   } catch (err) {
-    decoder.kill("SIGKILL");
-    encoder.kill("SIGKILL");
+    decoder.proc.kill("SIGKILL");
+    fg.proc.kill("SIGKILL");
+    bg?.proc.kill("SIGKILL");
     throw err;
   }
 
-  encoder.stdin!.end();
-  await Promise.all([decoderExit, encoderExit]);
+  fg.proc.stdin!.end();
+  bg?.proc.stdin!.end();
+  const exits: Promise<void>[] = [decoder.exit, fg.exit];
+  if (bg) exits.push(bg.exit);
+  await Promise.all(exits);
 
   if (processed === 0) {
     throw new Error(
-      `No frames produced from ${inputPath}. Decoder stderr:\n${decoderStderr.slice(-400)}`,
+      `No frames produced from ${inputPath}. Decoder stderr:\n${decoder.getStderr().slice(-400)}`,
     );
   }
 

packages/cli/src/commands/remove-background.ts

@@ -20,6 +20,10 @@ export const examples: Example[] = [
     "Remove background from a single image, output transparent PNG",
     "hyperframes remove-background portrait.jpg -o cutout.png",
   ],
+  [
+    "Separate the layers — emit both the cutout and an inverse-alpha background plate (subject region transparent)",
+    "hyperframes remove-background avatar.mp4 -o subject.webm --background-output plate.webm",
+  ],
   [
     "Force CPU (skip CoreML/CUDA)",
     "hyperframes remove-background avatar.mp4 -o transparent.webm --device cpu",
@@ -52,6 +56,12 @@ export default defineCommand({
       description: "Output path. Format inferred from extension: .webm (default), .mov, .png",
       alias: "o",
     },
+    "background-output": {
+      type: "string",
+      description:
+        "Optional second output path for the inverse-alpha background plate (subject region transparent, original surroundings opaque). Hole-cut, not inpainted — composite something underneath to fill the hole. Must be .webm or .mov; not allowed for image inputs.",
+      alias: "b",
+    },
     device: {
       type: "string",
       description: `Execution provider: ${DEVICES.join(", ")}`,
@@ -104,6 +114,8 @@ export default defineCommand({
 
     const inputPath = resolve(args.input);
     const outputPath = resolve(args.output);
+    const backgroundOutputArg = args["background-output"];
+    const backgroundOutputPath = backgroundOutputArg ? resolve(backgroundOutputArg) : undefined;
 
     const { render } = await import("../background-removal/pipeline.js");
 
@@ -114,6 +126,7 @@ export default defineCommand({
       const result = await render({
         inputPath,
         outputPath,
+        backgroundOutputPath,
         device: args.device,
         quality: args.quality,
         onProgress: (event) => {
@@ -137,6 +150,9 @@ export default defineCommand({
           JSON.stringify({
             ok: true,
             outputPath: result.outputPath,
+            ...(result.backgroundOutputPath
+              ? { backgroundOutputPath: result.backgroundOutputPath }
+              : {}),
             framesProcessed: result.framesProcessed,
             durationSeconds: Number(result.durationSeconds.toFixed(2)),
             avgMsPerFrame: Number(result.avgMsPerFrame.toFixed(1)),
@@ -148,9 +164,12 @@ export default defineCommand({
         const fpsThroughput = result.durationSeconds
           ? (result.framesProcessed / result.durationSeconds).toFixed(1)
           : "n/a";
+        const outputs = result.backgroundOutputPath
+          ? `${c.accent(result.outputPath)} + ${c.accent(result.backgroundOutputPath)}`
+          : c.accent(result.outputPath);
         spin?.stop(
           c.success(
-            `Removed background from ${c.accent(String(result.framesProcessed))} frames in ${result.durationSeconds.toFixed(1)}s (${fpsThroughput} fps, ${c.accent(result.provider)}) → ${c.accent(result.outputPath)}`,
+            `Removed background from ${c.accent(String(result.framesProcessed))} frames in ${result.durationSeconds.toFixed(1)}s (${fpsThroughput} fps, ${c.accent(result.provider)}) → ${outputs}`,
           ),
         );
       }