fix: resolve merge conflict in shader-transitions capture.ts

Merges main's refactored capture (CaptureSceneOptions, forceVisible,
stabilizeTransformedBoxShadows, foreignObjectRendering fallback) with
our HTML-in-Canvas drawElementImage capture path. The native capture
tries first and falls back to html2canvas on failure.
This commit is contained in:
Miguel Ángel
2026-05-06 07:49:31 -07:00
47 changed files with 4593 additions and 565 deletions
@@ -1,5 +1,5 @@
import { describe, expect, it } from "vitest";
import { MEAN, STD } from "./inference.js";
import { MEAN, STD, applyMask } from "./inference.js";
// Regression: the u2net_human_seg model was trained with ImageNet
// normalization. Drifting away from these exact values changes the input
@@ -16,3 +16,112 @@ describe("background-removal/inference — rembg u2net_human_seg parity", () =>
expect(STD).toEqual([0.229, 0.224, 0.225]);
});
});
// These tests pin the contract that `--background-output` is built on:
// fg.alpha + bg.alpha === 255 per pixel, and the RGB plane is byte-identical
// between fg and bg. A future change to the postprocess loop (different mask
// threshold, premultiplied alpha, gamma-corrected compositing) that breaks
// either invariant should fail here loudly.
describe("background-removal/inference — applyMask invariants", () => {
function makeRgb(pixels: number): Buffer {
// Deterministic but non-trivial RGB so byte equality is meaningful.
const buf = Buffer.allocUnsafe(pixels * 3);
for (let i = 0; i < pixels; i++) {
buf[i * 3] = (i * 7) & 0xff;
buf[i * 3 + 1] = (i * 13 + 31) & 0xff;
buf[i * 3 + 2] = (i * 19 + 61) & 0xff;
}
return buf;
}
function makeMask(pixels: number): Buffer {
// Hit the saturation endpoints (0, 255) and a few mid-tone values so the
// 255-m inversion is exercised across the full byte range.
const buf = Buffer.allocUnsafe(pixels);
for (let i = 0; i < pixels; i++) buf[i] = (i * 37) & 0xff;
return buf;
}
it("dual-output: fg.alpha + bg.alpha === 255 for every pixel", () => {
const pixels = 64;
const rgb = makeRgb(pixels);
const mask = makeMask(pixels);
const fg = Buffer.allocUnsafe(pixels * 4);
const bg = Buffer.allocUnsafe(pixels * 4);
const result = applyMask(rgb, mask, fg, bg, pixels);
expect(result.fg).toBe(fg);
expect(result.bg).toBe(bg);
for (let i = 0; i < pixels; i++) {
const sum = fg[i * 4 + 3]! + bg[i * 4 + 3]!;
expect(sum).toBe(255);
}
});
it("dual-output: RGB triples are byte-identical between fg and bg", () => {
const pixels = 64;
const rgb = makeRgb(pixels);
const mask = makeMask(pixels);
const fg = Buffer.allocUnsafe(pixels * 4);
const bg = Buffer.allocUnsafe(pixels * 4);
applyMask(rgb, mask, fg, bg, pixels);
for (let i = 0; i < pixels; i++) {
expect(fg[i * 4]).toBe(bg[i * 4]);
expect(fg[i * 4 + 1]).toBe(bg[i * 4 + 1]);
expect(fg[i * 4 + 2]).toBe(bg[i * 4 + 2]);
// And both match the source.
expect(fg[i * 4]).toBe(rgb[i * 3]);
expect(fg[i * 4 + 1]).toBe(rgb[i * 3 + 1]);
expect(fg[i * 4 + 2]).toBe(rgb[i * 3 + 2]);
}
});
it("dual-output: fg.alpha equals the input mask", () => {
const pixels = 32;
const rgb = makeRgb(pixels);
const mask = makeMask(pixels);
const fg = Buffer.allocUnsafe(pixels * 4);
const bg = Buffer.allocUnsafe(pixels * 4);
applyMask(rgb, mask, fg, bg, pixels);
for (let i = 0; i < pixels; i++) {
expect(fg[i * 4 + 3]).toBe(mask[i]);
}
});
it("single-output: bg=null returns bg=null and writes only fg", () => {
const pixels = 32;
const rgb = makeRgb(pixels);
const mask = makeMask(pixels);
const fg = Buffer.allocUnsafe(pixels * 4);
const result = applyMask(rgb, mask, fg, null, pixels);
expect(result.bg).toBeNull();
expect(result.fg).toBe(fg);
for (let i = 0; i < pixels; i++) {
expect(fg[i * 4]).toBe(rgb[i * 3]);
expect(fg[i * 4 + 3]).toBe(mask[i]);
}
});
it("saturates correctly at mask=0 and mask=255", () => {
// mask=0 → fg.alpha=0 (transparent subject), bg.alpha=255 (fully opaque plate)
// mask=255 → fg.alpha=255 (fully opaque subject), bg.alpha=0 (transparent plate)
const rgb = Buffer.from([10, 20, 30, 40, 50, 60]);
const mask = Buffer.from([0, 255]);
const fg = Buffer.allocUnsafe(8);
const bg = Buffer.allocUnsafe(8);
applyMask(rgb, mask, fg, bg, 2);
expect(fg[3]).toBe(0);
expect(bg[3]).toBe(255);
expect(fg[7]).toBe(255);
expect(bg[7]).toBe(0);
});
});
@@ -24,10 +24,24 @@ interface OrtModule {
Tensor: typeof Tensor;
}
export interface SessionResult {
/** Subject opaque, background fully transparent. */
fg: Buffer;
/** Inverse-alpha plate: same RGB, alpha is `255 mask`. Null unless `withBackground` was true. */
bg: Buffer | null;
}
export interface Session {
/** Run inference on one RGB frame, return RGBA bytes (H*W*4). */
process(rgb: Buffer, width: number, height: number): Promise<Buffer>;
/** ORT EP that was actually selected. */
/**
* Both `fg` and `bg` (when requested) are session-owned buffers reused on the
* next call — drain the encoder's stdin before invoking `process` again.
*/
process(
rgb: Buffer,
width: number,
height: number,
withBackground?: boolean,
): Promise<SessionResult>;
provider: string;
close(): Promise<void>;
}
@@ -73,16 +87,15 @@ export async function createSession(options: CreateSessionOptions = {}): Promise
throw new Error("ONNX session is missing input or output bindings");
}
// Pre-allocated per-frame buffers reused across every process() call.
// At 1080p this saves ~9 MB of allocations per frame. rgbaBuf is sized
// lazily on the first call (we don't know W/H until then).
// Reused across calls; sized lazily on first frame. Saves ~9 MB/frame at 1080p.
const inputData = new Float32Array(3 * INPUT_PLANE);
const maskBuf = Buffer.allocUnsafe(INPUT_PLANE);
let rgbaBuf: Buffer | null = null;
let rgbaBgBuf: Buffer | null = null;
return {
provider: providerUsed,
async process(rgb, width, height) {
async process(rgb, width, height, withBackground = false) {
const tensor = await preprocess(sharp, ort, rgb, width, height, inputData);
const outputs = await session.run({ [inputName]: tensor });
const output = outputs[outputName];
@@ -91,7 +104,21 @@ export async function createSession(options: CreateSessionOptions = {}): Promise
if (!rgbaBuf || rgbaBuf.length !== expectedBytes) {
rgbaBuf = Buffer.allocUnsafe(expectedBytes);
}
return await postprocess(sharp, output, rgb, width, height, maskBuf, rgbaBuf);
if (withBackground) {
if (!rgbaBgBuf || rgbaBgBuf.length !== expectedBytes) {
rgbaBgBuf = Buffer.allocUnsafe(expectedBytes);
}
}
return await postprocess(
sharp,
output,
rgb,
width,
height,
maskBuf,
rgbaBuf,
withBackground ? rgbaBgBuf : null,
);
},
async close() {
await session.release();
@@ -141,7 +168,8 @@ async function postprocess(
height: number,
maskBuf: Buffer,
rgbaBuf: Buffer,
): Promise<Buffer> {
rgbaBgBuf: Buffer | null,
): Promise<SessionResult> {
const raw = output.data as Float32Array;
let lo = Infinity;
@@ -172,11 +200,50 @@ async function postprocess(
.raw()
.toBuffer();
for (let i = 0; i < width * height; i++) {
rgbaBuf[i * 4] = rgb[i * 3]!;
rgbaBuf[i * 4 + 1] = rgb[i * 3 + 1]!;
rgbaBuf[i * 4 + 2] = rgb[i * 3 + 2]!;
rgbaBuf[i * 4 + 3] = fullMask[i]!;
return applyMask(rgb, fullMask, rgbaBuf, rgbaBgBuf, width * height);
}
/**
* Composite the RGB source frame with the segmentation mask into one or two
* RGBA buffers. The contract this PR is built on:
* - `fg`'s alpha is the mask, `bg`'s alpha (when provided) is `255 mask`,
* so `fg.alpha + bg.alpha === 255` for every pixel.
* - RGB triples are byte-identical between `fg` and `bg`.
* - When `bg` is null, only `fg` is touched.
*
* Exported for direct unit testing of the invariants above without spinning
* up an ONNX session.
*/
export function applyMask(
rgb: Buffer,
mask: Buffer,
fg: Buffer,
bg: Buffer | null,
pixels: number,
): SessionResult {
if (bg) {
for (let i = 0; i < pixels; i++) {
const r = rgb[i * 3]!;
const g = rgb[i * 3 + 1]!;
const b = rgb[i * 3 + 2]!;
const m = mask[i]!;
const o = i * 4;
fg[o] = r;
fg[o + 1] = g;
fg[o + 2] = b;
fg[o + 3] = m;
bg[o] = r;
bg[o + 1] = g;
bg[o + 2] = b;
bg[o + 3] = 255 - m;
}
return { fg, bg };
}
return rgbaBuf;
for (let i = 0; i < pixels; i++) {
fg[i * 4] = rgb[i * 3]!;
fg[i * 4 + 1] = rgb[i * 3 + 1]!;
fg[i * 4 + 2] = rgb[i * 3 + 2]!;
fg[i * 4 + 3] = mask[i]!;
}
return { fg, bg: null };
}
@@ -1,7 +1,13 @@
import { describe, expect, it } from "vitest";
import { EventEmitter } from "node:events";
import type { spawn } from "node:child_process";
import { inferOutputFormat, inferInputKind, buildEncoderArgs, waitForExit } from "./pipeline.js";
import {
inferOutputFormat,
inferInputKind,
buildEncoderArgs,
resolveRenderTargets,
waitForExit,
} from "./pipeline.js";
describe("background-removal/pipeline — inferOutputFormat", () => {
it("maps .webm → webm", () => {
@@ -97,6 +103,54 @@ describe("background-removal/pipeline — buildEncoderArgs", () => {
});
});
describe("background-removal/pipeline — resolveRenderTargets", () => {
it("resolves a normal video → webm render", () => {
const t = resolveRenderTargets("/tmp/clip.mp4", "/tmp/cutout.webm");
expect(t.format).toBe("webm");
expect(t.inputKind).toBe("video");
expect(t.bgFormat).toBeUndefined();
});
it("resolves an image → png render", () => {
const t = resolveRenderTargets("/tmp/portrait.jpg", "/tmp/cutout.png");
expect(t.format).toBe("png");
expect(t.inputKind).toBe("image");
});
it("rejects image input with a video output extension", () => {
expect(() => resolveRenderTargets("/tmp/portrait.jpg", "/tmp/cutout.webm")).toThrow(
/Image input requires a \.png output/,
);
});
it("rejects video input with a .png output", () => {
expect(() => resolveRenderTargets("/tmp/clip.mp4", "/tmp/cutout.png")).toThrow(
/Video input requires a \.webm or \.mov output/,
);
});
it("threads background-output format through when valid", () => {
const t = resolveRenderTargets("/tmp/clip.mp4", "/tmp/fg.webm", "/tmp/bg.webm");
expect(t.bgFormat).toBe("webm");
const tMov = resolveRenderTargets("/tmp/clip.mp4", "/tmp/fg.webm", "/tmp/bg.mov");
expect(tMov.bgFormat).toBe("mov");
});
it("rejects --background-output for image inputs (no temporal pairing to do)", () => {
expect(() =>
resolveRenderTargets("/tmp/portrait.jpg", "/tmp/cutout.png", "/tmp/bg.png"),
).toThrow(/--background-output is not supported for image inputs/);
});
it("rejects .png as the --background-output extension", () => {
// .png is only valid for single-image inputs, and image inputs themselves
// can't have a background-output anyway. So .png here is always a misuse.
expect(() => resolveRenderTargets("/tmp/clip.mp4", "/tmp/fg.webm", "/tmp/bg.png")).toThrow(
/--background-output must be \.webm or \.mov/,
);
});
});
// Regression: a previous version of waitForExit treated `code === null` as
// success. Per Node's child_process docs, that's the signal-killed case —
// reporting it as success means a SIGTERM/SIGKILL'd ffmpeg encoder produces
+141 -46
View File
@@ -38,6 +38,16 @@ export const isQuality = (v: unknown): v is Quality =>
export interface RenderOptions {
inputPath: string;
outputPath: string;
/**
* Optional second output: an inverse-alpha background plate (same source
* RGB, transparent where the subject was). Only valid for video inputs and
* .webm/.mov outputs not allowed alongside a .png output. The plate's
* format is inferred from this path independently of the foreground's.
*
* NOTE: this is a hole-cut plate, not an inpainted clean plate. Composite
* something opaque (graphics, blur, scene) under it to fill the hole.
*/
backgroundOutputPath?: string;
device?: Device;
model?: ModelId;
/** Encoder CRF preset for `.webm`. See `QUALITY_CRF`. Ignored for `.mov`/`.png`. */
@@ -52,6 +62,8 @@ export type ProgressEvent =
export interface RenderResult {
outputPath: string;
/** Present only when `backgroundOutputPath` was set. */
backgroundOutputPath?: string;
framesProcessed: number;
durationSeconds: number;
avgMsPerFrame: number;
@@ -203,17 +215,27 @@ async function* readFrames(
}
}
export async function render(options: RenderOptions): Promise<RenderResult> {
if (!hasFFmpeg() || !hasFFprobe()) {
throw new Error("ffmpeg and ffprobe are required. Install: brew install ffmpeg");
}
export interface RenderTargets {
format: OutputFormat;
inputKind: "video" | "image";
bgFormat: OutputFormat | undefined;
}
const format = inferOutputFormat(options.outputPath);
const inputKind = inferInputKind(options.inputPath);
/**
* Resolve and validate the input/output combination before any I/O. Pure;
* exported so unit tests can pin the error messages without spawning ffmpeg.
*/
export function resolveRenderTargets(
inputPath: string,
outputPath: string,
backgroundOutputPath?: string,
): RenderTargets {
const format = inferOutputFormat(outputPath);
const inputKind = inferInputKind(inputPath);
if (inputKind === "image" && format !== "png") {
throw new Error(
`Image input requires a .png output (got ${extname(options.outputPath)}). Use a video input for .webm/.mov.`,
`Image input requires a .png output (got ${extname(outputPath)}). Use a video input for .webm/.mov.`,
);
}
if (inputKind === "video" && format === "png") {
@@ -222,6 +244,35 @@ export async function render(options: RenderOptions): Promise<RenderResult> {
);
}
let bgFormat: OutputFormat | undefined;
if (backgroundOutputPath) {
if (inputKind === "image") {
throw new Error(
"--background-output is not supported for image inputs. Use a video input (mp4/mov/webm) to produce both a cutout and a background plate.",
);
}
bgFormat = inferOutputFormat(backgroundOutputPath);
if (bgFormat === "png") {
throw new Error(
"--background-output must be .webm or .mov; .png is only valid for single-image inputs.",
);
}
}
return { format, inputKind, bgFormat };
}
export async function render(options: RenderOptions): Promise<RenderResult> {
if (!hasFFmpeg() || !hasFFprobe()) {
throw new Error("ffmpeg and ffprobe are required. Install: brew install ffmpeg");
}
const { format, bgFormat } = resolveRenderTargets(
options.inputPath,
options.outputPath,
options.backgroundOutputPath,
);
const media = await probeMedia(options.inputPath);
options.onProgress?.({
@@ -240,12 +291,13 @@ export async function render(options: RenderOptions): Promise<RenderResult> {
try {
const start = Date.now();
const framesProcessed = await runPipeline(options, session, media, format);
const framesProcessed = await runPipeline(options, session, media, format, bgFormat);
const durationSeconds = (Date.now() - start) / 1000;
const avgMsPerFrame = framesProcessed ? (durationSeconds * 1000) / framesProcessed : 0;
return {
outputPath: options.outputPath,
backgroundOutputPath: options.backgroundOutputPath,
framesProcessed,
durationSeconds,
avgMsPerFrame,
@@ -259,61 +311,77 @@ export async function render(options: RenderOptions): Promise<RenderResult> {
const RECENT_WINDOW = 30;
interface FfmpegProc {
proc: ReturnType<typeof spawn>;
exit: Promise<void>;
/** Tail of stderr, captured for inclusion in error messages. */
getStderr: () => string;
}
type StdioFd = "ignore" | "pipe";
type StdioTuple = [StdioFd, StdioFd, StdioFd];
function spawnFfmpeg(args: string[], label: string, stdio: StdioTuple): FfmpegProc {
const proc = spawn("ffmpeg", args, { stdio });
let stderrBuf = "";
proc.stderr?.on("data", (d: Buffer) => {
stderrBuf += d.toString();
});
// If the encoder dies mid-render, the next .write() to its stdin emits an
// 'error' event on the writable. Without a listener, Node treats it as
// unhandled and crashes the CLI before waitForExit's reject path can
// surface the real cause (encoder stderr tail). Swallowing here is safe —
// the process exit is the source of truth.
proc.stdin?.on("error", () => {});
const exit = waitForExit(proc, label, () => stderrBuf);
return { proc, exit, getStderr: () => stderrBuf };
}
async function runPipeline(
options: RenderOptions,
session: Session,
media: MediaInfo,
format: OutputFormat,
bgFormat: OutputFormat | undefined,
): Promise<number> {
const { inputPath, outputPath } = options;
const { inputPath, outputPath, backgroundOutputPath } = options;
const { width, height, fps, frameCount } = media;
const frameBytes = width * height * 3;
const quality = options.quality ?? DEFAULT_QUALITY;
const decoder = spawn(
"ffmpeg",
const decoder = spawnFfmpeg(
["-loglevel", "error", "-i", inputPath, "-f", "rawvideo", "-pix_fmt", "rgb24", "-an", "-"],
{ stdio: ["ignore", "pipe", "pipe"] },
"ffmpeg decoder",
["ignore", "pipe", "pipe"],
);
let decoderStderr = "";
decoder.stderr?.on("data", (d: Buffer) => {
decoderStderr += d.toString();
});
const decoderExit = waitForExit(decoder, "ffmpeg decoder", () => decoderStderr);
const encoder = spawn(
"ffmpeg",
buildEncoderArgs(
format,
width,
height,
fps || 30,
outputPath,
options.quality ?? DEFAULT_QUALITY,
),
{
stdio: ["pipe", "ignore", "pipe"],
},
const fg = spawnFfmpeg(
buildEncoderArgs(format, width, height, fps || 30, outputPath, quality),
"ffmpeg encoder",
["pipe", "ignore", "pipe"],
);
let encoderStderr = "";
encoder.stderr?.on("data", (d: Buffer) => {
encoderStderr += d.toString();
});
const encoderExit = waitForExit(encoder, "ffmpeg encoder", () => encoderStderr);
const bg =
backgroundOutputPath && bgFormat
? spawnFfmpeg(
buildEncoderArgs(bgFormat, width, height, fps || 30, backgroundOutputPath, quality),
"ffmpeg background encoder",
["pipe", "ignore", "pipe"],
)
: null;
let processed = 0;
const total = frameCount;
// Running average over the last RECENT_WINDOW frames.
const recentMs = new Array<number>(RECENT_WINDOW).fill(0);
let recentSum = 0;
let recentSlot = 0;
let recentCount = 0;
try {
for await (const rgb of readFrames(decoder.stdout!, frameBytes)) {
for await (const rgb of readFrames(decoder.proc.stdout!, frameBytes)) {
const t0 = Date.now();
const rgba = await session.process(rgb, width, height);
const result = await session.process(rgb, width, height, bg !== null);
const elapsed = Date.now() - t0;
recentSum += elapsed - recentMs[recentSlot]!;
@@ -321,8 +389,31 @@ async function runPipeline(
recentSlot = (recentSlot + 1) % RECENT_WINDOW;
if (recentCount < RECENT_WINDOW) recentCount++;
if (!encoder.stdin!.write(rgba)) {
await new Promise<void>((resolve) => encoder.stdin!.once("drain", () => resolve()));
// Issue both writes before any await so a slow encoder doesn't block
// the other. Drain anything that returned false before the next
// session.process() — its output buffers are reused per frame.
//
// Subtlety: write() returning true means "highWaterMark not exceeded,"
// NOT "libuv has flushed the chunk." The buffer reference is held by
// libuv until the underlying syscall completes. Reusing the session's
// output buffer is safe because the next session.process() call takes
// ~1050ms (ORT inference) — plenty of event-loop turns for libuv to
// drain. If that ever stops being true, we'd need to copy here.
const fgWroteFully = fg.proc.stdin!.write(result.fg);
const bgWroteFully = bg && result.bg ? bg.proc.stdin!.write(result.bg) : true;
if (!fgWroteFully || !bgWroteFully) {
const drains: Promise<void>[] = [];
if (!fgWroteFully) {
drains.push(
new Promise<void>((resolve) => fg.proc.stdin!.once("drain", () => resolve())),
);
}
if (!bgWroteFully && bg) {
drains.push(
new Promise<void>((resolve) => bg.proc.stdin!.once("drain", () => resolve())),
);
}
await Promise.all(drains);
}
processed++;
@@ -334,17 +425,21 @@ async function runPipeline(
});
}
} catch (err) {
decoder.kill("SIGKILL");
encoder.kill("SIGKILL");
decoder.proc.kill("SIGKILL");
fg.proc.kill("SIGKILL");
bg?.proc.kill("SIGKILL");
throw err;
}
encoder.stdin!.end();
await Promise.all([decoderExit, encoderExit]);
fg.proc.stdin!.end();
bg?.proc.stdin!.end();
const exits: Promise<void>[] = [decoder.exit, fg.exit];
if (bg) exits.push(bg.exit);
await Promise.all(exits);
if (processed === 0) {
throw new Error(
`No frames produced from ${inputPath}. Decoder stderr:\n${decoderStderr.slice(-400)}`,
`No frames produced from ${inputPath}. Decoder stderr:\n${decoder.getStderr().slice(-400)}`,
);
}
+4 -21
View File
@@ -1,6 +1,6 @@
import { defineCommand } from "citty";
import { existsSync, readFileSync } from "node:fs";
import { dirname, join, resolve } from "node:path";
import { dirname, join } from "node:path";
import { fileURLToPath } from "node:url";
import type { Example } from "./_examples.js";
import { c } from "../ui/colors.js";
@@ -107,27 +107,10 @@ async function seekTo(page: import("puppeteer-core").Page, time: number): Promis
}
async function bundleProjectHtml(projectDir: string): Promise<string> {
// `bundleToSingleHtml` now inlines the runtime IIFE by default, so the
// previous post-bundle runtime substitution is no longer needed.
const { bundleToSingleHtml } = await import("@hyperframes/core/compiler");
let html = await bundleToSingleHtml(projectDir);
const runtimePath = resolve(
__dirname,
"..",
"..",
"..",
"core",
"dist",
"hyperframe.runtime.iife.js",
);
if (existsSync(runtimePath)) {
const runtimeSource = readFileSync(runtimePath, "utf-8");
html = html.replace(
/<script[^>]*data-hyperframes-preview-runtime[^>]*src="[^"]*"[^>]*><\/script>/,
() => `<script data-hyperframes-preview-runtime="1">${runtimeSource}</script>`,
);
}
return html;
return bundleToSingleHtml(projectDir);
}
async function alignViewportToComposition(
+20 -1
View File
@@ -20,6 +20,10 @@ export const examples: Example[] = [
"Remove background from a single image, output transparent PNG",
"hyperframes remove-background portrait.jpg -o cutout.png",
],
[
"Separate the layers — emit both the cutout and an inverse-alpha background plate (subject region transparent)",
"hyperframes remove-background avatar.mp4 -o subject.webm --background-output plate.webm",
],
[
"Force CPU (skip CoreML/CUDA)",
"hyperframes remove-background avatar.mp4 -o transparent.webm --device cpu",
@@ -52,6 +56,12 @@ export default defineCommand({
description: "Output path. Format inferred from extension: .webm (default), .mov, .png",
alias: "o",
},
"background-output": {
type: "string",
description:
"Optional second output path for the inverse-alpha background plate (subject region transparent, original surroundings opaque). Hole-cut, not inpainted — composite something underneath to fill the hole. Must be .webm or .mov; not allowed for image inputs.",
alias: "b",
},
device: {
type: "string",
description: `Execution provider: ${DEVICES.join(", ")}`,
@@ -104,6 +114,8 @@ export default defineCommand({
const inputPath = resolve(args.input);
const outputPath = resolve(args.output);
const backgroundOutputArg = args["background-output"];
const backgroundOutputPath = backgroundOutputArg ? resolve(backgroundOutputArg) : undefined;
const { render } = await import("../background-removal/pipeline.js");
@@ -114,6 +126,7 @@ export default defineCommand({
const result = await render({
inputPath,
outputPath,
backgroundOutputPath,
device: args.device,
quality: args.quality,
onProgress: (event) => {
@@ -137,6 +150,9 @@ export default defineCommand({
JSON.stringify({
ok: true,
outputPath: result.outputPath,
...(result.backgroundOutputPath
? { backgroundOutputPath: result.backgroundOutputPath }
: {}),
framesProcessed: result.framesProcessed,
durationSeconds: Number(result.durationSeconds.toFixed(2)),
avgMsPerFrame: Number(result.avgMsPerFrame.toFixed(1)),
@@ -148,9 +164,12 @@ export default defineCommand({
const fpsThroughput = result.durationSeconds
? (result.framesProcessed / result.durationSeconds).toFixed(1)
: "n/a";
const outputs = result.backgroundOutputPath
? `${c.accent(result.outputPath)} + ${c.accent(result.backgroundOutputPath)}`
: c.accent(result.outputPath);
spin?.stop(
c.success(
`Removed background from ${c.accent(String(result.framesProcessed))} frames in ${result.durationSeconds.toFixed(1)}s (${fpsThroughput} fps, ${c.accent(result.provider)}) → ${c.accent(result.outputPath)}`,
`Removed background from ${c.accent(String(result.framesProcessed))} frames in ${result.durationSeconds.toFixed(1)}s (${fpsThroughput} fps, ${c.accent(result.provider)}) → ${outputs}`,
),
);
}
+3 -14
View File
@@ -97,20 +97,9 @@ async function captureSnapshots(
const numFrames = opts.frames ?? 5;
// 1. Bundle
let html = await bundleToSingleHtml(projectDir);
// Inject local runtime if available.
// Uses the same multi-strategy resolver as the studio preview server
// (runtimeSource.ts) so snapshot works in dev (tsx), built CLI, and npx.
const { loadRuntimeSource } = await import("../server/runtimeSource.js");
const runtimeSource = await loadRuntimeSource();
if (runtimeSource) {
html = html.replace(
/<script[^>]*data-hyperframes-preview-runtime[^>]*src="[^"]*"[^>]*><\/script>/,
() => `<script data-hyperframes-preview-runtime="1">${runtimeSource}</script>`,
);
}
// 1. Bundle. `bundleToSingleHtml` now inlines the runtime IIFE by default,
// so the previous post-bundle runtime substitution is no longer needed.
const html = await bundleToSingleHtml(projectDir);
const server = await serveStaticProjectHtml(projectDir, html);
+5 -19
View File
@@ -1,6 +1,6 @@
import { defineCommand } from "citty";
import { existsSync, readFileSync } from "node:fs";
import { resolve, join, dirname } from "node:path";
import { join, dirname } from "node:path";
import { fileURLToPath } from "node:url";
import { resolveProject } from "../utils/project.js";
import { resolveCompositionViewportFromHtml } from "../utils/compositionViewport.js";
@@ -113,24 +113,10 @@ async function validateInBrowser(
const { bundleToSingleHtml } = await import("@hyperframes/core/compiler");
const { ensureBrowser } = await import("../browser/manager.js");
let html = await bundleToSingleHtml(projectDir);
const runtimePath = resolve(
__dirname,
"..",
"..",
"..",
"core",
"dist",
"hyperframe.runtime.iife.js",
);
if (existsSync(runtimePath)) {
const runtimeSource = readFileSync(runtimePath, "utf-8");
html = html.replace(
/<script[^>]*data-hyperframes-preview-runtime[^>]*src="[^"]*"[^>]*><\/script>/,
() => `<script data-hyperframes-preview-runtime="1">${runtimeSource}</script>`,
);
}
// `bundleToSingleHtml` now inlines the runtime IIFE by default, so the
// previous post-bundle regex substitution (which matched `src="..."` on the
// runtime tag) is no longer needed — there's no `src` attribute to match.
const html = await bundleToSingleHtml(projectDir);
const { createServer } = await import("node:http");
const { getMimeType } = await import("@hyperframes/core/studio-api");
@@ -0,0 +1,19 @@
import { describe, expect, it } from "vitest";
import { shouldWatchProjectFile } from "./fileWatcher.js";
describe("shouldWatchProjectFile", () => {
it("watches files that can affect the project signature", () => {
expect(shouldWatchProjectFile("index.html")).toBe(true);
expect(shouldWatchProjectFile("src/scene.tsx")).toBe(true);
expect(shouldWatchProjectFile("assets/hero.png")).toBe(true);
expect(shouldWatchProjectFile("Dockerfile")).toBe(true);
});
it("skips generated and dependency directories excluded from signatures", () => {
expect(shouldWatchProjectFile("node_modules/pkg/index.js")).toBe(false);
expect(shouldWatchProjectFile("renders/output.mp4")).toBe(false);
expect(shouldWatchProjectFile("dist/index.html")).toBe(false);
expect(shouldWatchProjectFile(".hyperframes/cache.json")).toBe(false);
});
});
+22 -4
View File
@@ -8,9 +8,27 @@ export interface ProjectWatcher {
close(): void;
}
const WATCHED_EXTENSIONS = new Set([".html", ".css", ".js", ".json"]);
const WATCHER_EXCLUDED_DIRS = new Set([
".cache",
".git",
".hyperframes",
".next",
".vite",
"build",
"coverage",
"dist",
"node_modules",
"outputs",
"renders",
]);
const DEBOUNCE_MS = 300;
export function shouldWatchProjectFile(filename: string): boolean {
if (!filename) return false;
const parts = filename.split(/[\\/]+/);
return !parts.some((part) => WATCHER_EXCLUDED_DIRS.has(part));
}
export function createProjectWatcher(projectDir: string): ProjectWatcher {
const listeners = new Set<FileChangeListener>();
let debounceTimer: ReturnType<typeof setTimeout> | null = null;
@@ -19,13 +37,13 @@ export function createProjectWatcher(projectDir: string): ProjectWatcher {
try {
watcher = watch(projectDir, { recursive: true }, (_event, filename) => {
if (!filename) return;
const ext = "." + filename.split(".").pop()?.toLowerCase();
if (!WATCHED_EXTENSIONS.has(ext)) return;
const relativePath = filename.toString();
if (!shouldWatchProjectFile(relativePath)) return;
if (debounceTimer) clearTimeout(debounceTimer);
debounceTimer = setTimeout(() => {
for (const fn of listeners) {
fn(filename);
fn(relativePath);
}
}, DEBOUNCE_MS);
});
+16 -2
View File
@@ -14,6 +14,7 @@ import { loadRuntimeSource } from "./runtimeSource.js";
import { VERSION as version } from "../version.js";
import {
createStudioApi,
createProjectSignature,
getMimeType,
type StudioApiAdapter,
type ResolvedProject,
@@ -144,6 +145,10 @@ export function createStudioServer(options: StudioServerOptions): StudioServer {
// ── CLI adapter for the shared studio API ──────────────────────────────
const project: ResolvedProject = { id: projectId, dir: projectDir, title: projectId };
let cachedProjectSignature: string | null = null;
watcher.addListener(() => {
cachedProjectSignature = null;
});
const adapter: StudioApiAdapter = {
listProjects: () => [project],
@@ -153,8 +158,11 @@ export function createStudioServer(options: StudioServerOptions): StudioServer {
async bundle(dir: string): Promise<string | null> {
try {
const { bundleToSingleHtml } = await import("@hyperframes/core/compiler");
let html = await bundleToSingleHtml(dir);
// Fix empty runtime src from bundler — point to the local runtime endpoint
// Studio dev server: ask the bundler for an empty `src=""` placeholder so
// we can point it at our hot-reloadable local runtime endpoint. Inlining
// ~150 KB of runtime body on every preview render would defeat browser
// caching across composition edits.
let html = await bundleToSingleHtml(dir, { runtime: "placeholder" });
html = html.replace(
'data-hyperframes-preview-runtime="1" src=""',
'data-hyperframes-preview-runtime="1" src="/api/runtime.js"',
@@ -166,6 +174,12 @@ export function createStudioServer(options: StudioServerOptions): StudioServer {
}
},
getProjectSignature(dir: string): string {
if (resolve(dir) !== resolve(projectDir)) return createProjectSignature(dir);
cachedProjectSignature ??= createProjectSignature(projectDir);
return cachedProjectSignature;
},
async lint(html: string, opts?: { filePath?: string }) {
const { lintHyperframeHtml } = await import("@hyperframes/core/lint");
return lintHyperframeHtml(html, opts);
@@ -7,6 +7,7 @@ import {
createPublishArchive,
getPublishApiBaseUrl,
publishProjectArchive,
uploadTimeoutMs,
} from "./publishProject.js";
function makeProjectDir(): string {
@@ -35,6 +36,22 @@ describe("createPublishArchive", () => {
});
});
describe("uploadTimeoutMs", () => {
it("returns the minimum timeout for small files", () => {
expect(uploadTimeoutMs(0)).toBe(120_000);
expect(uploadTimeoutMs(50 * 1024 * 1024)).toBe(120_000);
});
it("scales above the floor for large files", () => {
expect(uploadTimeoutMs(64 * 1024 * 1024)).toBeGreaterThan(120_000);
expect(uploadTimeoutMs(500 * 1024 * 1024)).toBeGreaterThan(900_000);
});
it("returns an integer", () => {
expect(Number.isInteger(uploadTimeoutMs(123_456))).toBe(true);
});
});
describe("publishProjectArchive", () => {
beforeEach(() => {
vi.stubEnv("HYPERFRAMES_PUBLISHED_PROJECTS_API_URL", "");
+23 -5
View File
@@ -5,7 +5,11 @@ import AdmZip from "adm-zip";
const IGNORED_DIRS = new Set([".git", "node_modules", "dist", ".next", "coverage"]);
const IGNORED_FILES = new Set([".DS_Store", "Thumbs.db"]);
const PUBLISH_CONTENT_TYPE = "application/zip";
const PUBLISH_REQUEST_TIMEOUT_MS = 30_000;
const PUBLISH_METADATA_TIMEOUT_MS = 30_000;
const PUBLISH_UPLOAD_MIN_TIMEOUT_MS = 120_000;
// Conservative floor — most connections are faster, but this prevents
// premature aborts on slow/unstable networks (hotel wifi, tethering).
const PUBLISH_UPLOAD_BYTES_PER_SECOND = 500_000;
export interface PublishArchiveResult {
buffer: Buffer;
@@ -25,6 +29,7 @@ interface StagedUploadResponse {
uploadKey: string;
contentType: string;
uploadHeaders: Record<string, string>;
expiresInSeconds: number;
}
type JsonRecord = Record<string, unknown>;
@@ -73,11 +78,14 @@ function parseStagedUploadResponse(
const uploadKey = stringField(data, "upload_key");
const contentType = stringField(data, "content_type") || PUBLISH_CONTENT_TYPE;
if (!uploadUrl || !uploadKey) return null;
const rawExpires = data["expires_in_seconds"];
const expiresInSeconds = typeof rawExpires === "number" && rawExpires > 0 ? rawExpires : 1800;
return {
uploadUrl,
uploadKey,
contentType,
uploadHeaders: getUploadHeaders(data, uploadUrl, contentType, archiveByteLength),
expiresInSeconds,
};
}
@@ -142,6 +150,13 @@ async function readErrorMessage(response: Response, fallback: string): Promise<s
return text.trim() ? `${fallback}: ${text.trim().slice(0, 180)}` : fallback;
}
export function uploadTimeoutMs(byteLength: number): number {
return Math.max(
PUBLISH_UPLOAD_MIN_TIMEOUT_MS,
Math.ceil((byteLength / PUBLISH_UPLOAD_BYTES_PER_SECOND) * 1000),
);
}
function shouldIgnoreSegment(segment: string): boolean {
return segment.startsWith(".") || IGNORED_DIRS.has(segment) || IGNORED_FILES.has(segment);
}
@@ -214,7 +229,7 @@ async function publishProjectArchiveDirect(
method: "POST",
body,
headers,
signal: AbortSignal.timeout(PUBLISH_REQUEST_TIMEOUT_MS),
signal: AbortSignal.timeout(uploadTimeoutMs(archive.buffer.byteLength)),
});
const payload = await readJson(response);
@@ -243,7 +258,7 @@ async function publishProjectArchiveStaged(
"content-type": "application/json",
heygen_route: "canary",
},
signal: AbortSignal.timeout(PUBLISH_REQUEST_TIMEOUT_MS),
signal: AbortSignal.timeout(PUBLISH_METADATA_TIMEOUT_MS),
});
if (uploadResponse.status === 404 || uploadResponse.status === 405) {
@@ -256,11 +271,14 @@ async function publishProjectArchiveStaged(
throw new Error(await readErrorMessage(uploadResponse, "Failed to prepare project upload"));
}
const presignedUrlTtlMs = stagedUpload.expiresInSeconds * 1000 - PUBLISH_METADATA_TIMEOUT_MS;
const s3Response = await fetch(stagedUpload.uploadUrl, {
method: "PUT",
body: new Blob([archiveArrayBuffer(archive)], { type: stagedUpload.contentType }),
headers: stagedUpload.uploadHeaders,
signal: AbortSignal.timeout(PUBLISH_REQUEST_TIMEOUT_MS),
signal: AbortSignal.timeout(
Math.min(uploadTimeoutMs(archive.buffer.byteLength), presignedUrlTtlMs),
),
});
if (!s3Response.ok) {
throw new Error(await readErrorMessage(s3Response, "Failed to upload project archive"));
@@ -277,7 +295,7 @@ async function publishProjectArchiveStaged(
"content-type": "application/json",
heygen_route: "canary",
},
signal: AbortSignal.timeout(PUBLISH_REQUEST_TIMEOUT_MS),
signal: AbortSignal.timeout(uploadTimeoutMs(archive.buffer.byteLength)),
});
const completePayload = await readJson(completeResponse);