From 15f6b86ac0b1fb1d235dc322e6cc1ebdd7716697 Mon Sep 17 00:00:00 2001 From: James Date: Fri, 27 Mar 2026 02:12:39 +0000 Subject: [PATCH] feat(render): add WebM output with VP9 alpha transparency Support rendering compositions with transparent backgrounds via `--format webm`. VP9+alpha is the standard format for overlayable video (captions, lower thirds, overlays). Changes by layer: - CLI: `--format mp4|webm` flag on render command - Producer: threads format through RenderConfig, switches to PNG capture and VP9 encoding when webm - Engine: getEncoderPreset() returns VP9 config with yuva420p; transparent page background via CDP when capturing PNG; mux uses Opus audio for WebM; VP9 flags from production: -row-mt 1, -auto-alt-ref 0, alpha_mode=1 metadata - Frame capture: Emulation.setDefaultBackgroundColorOverride a=0 Co-Authored-By: Claude Opus 4.6 (1M context) --- packages/cli/src/commands/render.ts | 34 ++++++++++- packages/engine/src/index.ts | 1 + packages/engine/src/services/chunkEncoder.ts | 61 +++++++++++++------ packages/engine/src/services/frameCapture.ts | 9 +++ .../engine/src/services/streamingEncoder.ts | 5 ++ .../src/services/renderOrchestrator.ts | 57 +++++++++-------- 6 files changed, 119 insertions(+), 48 deletions(-) diff --git a/packages/cli/src/commands/render.ts b/packages/cli/src/commands/render.ts index 369dfc291..6d2990bdd 100644 --- a/packages/cli/src/commands/render.ts +++ b/packages/cli/src/commands/render.ts @@ -10,14 +10,20 @@ import { trackRenderComplete, trackRenderError } from "../telemetry/events.js"; const VALID_FPS = new Set([24, 30, 60]); const VALID_QUALITY = new Set(["draft", "standard", "high"]); +const VALID_FORMAT = new Set(["mp4", "webm"]); export default defineCommand({ - meta: { name: "render", description: "Render a composition to MP4" }, + meta: { name: "render", description: "Render a composition to MP4 or WebM" }, args: { dir: { type: "positional", description: "Project directory", required: false }, output: { type: "string", description: "Output path (default: renders/.mp4)" }, fps: { type: "string", description: "Frame rate: 24, 30, 60", default: "30" }, quality: { type: "string", description: "Quality: draft, standard, high", default: "standard" }, + format: { + type: "string", + description: "Output format: mp4, webm (WebM renders with transparency)", + default: "mp4", + }, workers: { type: "string", description: "Parallel workers 1-8" }, docker: { type: "boolean", description: "Use Docker for deterministic render", default: false }, gpu: { type: "boolean", description: "Use GPU encoding", default: false }, @@ -43,6 +49,14 @@ export default defineCommand({ } const quality = qualityRaw as "draft" | "standard" | "high"; + // ── Validate format ───────────────────────────────────────────────── + const formatRaw = args.format ?? "mp4"; + if (!VALID_FORMAT.has(formatRaw)) { + errorBox("Invalid format", `Got "${formatRaw}". Must be mp4 or webm.`); + process.exit(1); + } + const format = formatRaw as "mp4" | "webm"; + // ── Validate workers ────────────────────────────────────────────────── let workers: number | undefined; if (args.workers != null) { @@ -56,7 +70,10 @@ export default defineCommand({ // ── Resolve output path ─────────────────────────────────────────────── const rendersDir = resolve("renders"); - const outputPath = args.output ? resolve(args.output) : join(rendersDir, `${project.name}.mp4`); + const ext = format === "webm" ? ".webm" : ".mp4"; + const outputPath = args.output + ? resolve(args.output) + : join(rendersDir, `${project.name}${ext}`); // Ensure output directory exists const outputDir = dirname(outputPath); @@ -129,11 +146,19 @@ export default defineCommand({ // ── Render ──────────────────────────────────────────────────────────── if (useDocker) { - await renderDocker(project.dir, outputPath, { fps, quality, workers, gpu: useGpu, quiet }); + await renderDocker(project.dir, outputPath, { + fps, + quality, + format, + workers, + gpu: useGpu, + quiet, + }); } else { await renderLocal(project.dir, outputPath, { fps, quality, + format, workers, gpu: useGpu, quiet, @@ -146,6 +171,7 @@ export default defineCommand({ interface RenderOptions { fps: 24 | 30 | 60; quality: "draft" | "standard" | "high"; + format: "mp4" | "webm"; workers?: number; gpu: boolean; quiet: boolean; @@ -164,6 +190,7 @@ async function renderDocker( const job = producer.createRenderJob({ fps: options.fps, quality: options.quality, + format: options.format, workers: options.workers, useGpu: options.gpu, }); @@ -206,6 +233,7 @@ async function renderLocal( const job = producer.createRenderJob({ fps: options.fps, quality: options.quality, + format: options.format, workers: options.workers, useGpu: options.gpu, }); diff --git a/packages/engine/src/index.ts b/packages/engine/src/index.ts index a91b6d3c4..a29347b1b 100644 --- a/packages/engine/src/index.ts +++ b/packages/engine/src/index.ts @@ -88,6 +88,7 @@ export { applyFaststart, detectGpuEncoder, ENCODER_PRESETS, + getEncoderPreset, type GpuEncoder, } from "./services/chunkEncoder.js"; export type { EncoderOptions, EncodeResult, MuxResult } from "./services/chunkEncoder.types.js"; diff --git a/packages/engine/src/services/chunkEncoder.ts b/packages/engine/src/services/chunkEncoder.ts index 273704fb0..fc2f95203 100644 --- a/packages/engine/src/services/chunkEncoder.ts +++ b/packages/engine/src/services/chunkEncoder.ts @@ -21,6 +21,26 @@ export const ENCODER_PRESETS = { high: { preset: "slow", quality: 18, codec: "h264" as const }, }; +/** + * Get encoder preset for a given quality and output format. + * WebM uses VP9 with alpha-capable pixel format; MP4 uses h264. + */ +export function getEncoderPreset( + quality: "draft" | "standard" | "high", + format: "mp4" | "webm" = "mp4", +): { preset: string; quality: number; codec: "h264" | "vp9"; pixelFormat: string } { + const base = ENCODER_PRESETS[quality]; + if (format === "webm") { + return { + preset: base.preset === "ultrafast" ? "realtime" : "good", + quality: base.quality, + codec: "vp9", + pixelFormat: "yuva420p", + }; + } + return { ...base, pixelFormat: "yuv420p" }; +} + // Re-export GPU utilities so existing consumers that import from chunkEncoder still work. export { detectGpuEncoder, type GpuEncoder } from "../utils/gpuEncoder.js"; @@ -83,6 +103,11 @@ function buildEncoderArgs( } else if (codec === "vp9") { args.push("-c:v", "libvpx-vp9", "-b:v", bitrate || "0", "-crf", String(quality)); args.push("-deadline", preset === "ultrafast" ? "realtime" : "good"); + args.push("-row-mt", "1"); + if (pixelFormat === "yuva420p") { + args.push("-auto-alt-ref", "0"); + args.push("-metadata:s:v:0", "alpha_mode=1"); + } } else if (codec === "prores") { args.push("-c:v", "prores_ks", "-profile:v", preset, "-vendor", "apl0"); return [...args, "-y", outputPath]; @@ -243,7 +268,8 @@ export async function encodeFramesChunkedConcat( } const startNumber = i * chunkSize; const framesInChunk = Math.min(chunkSize, files.length - startNumber); - const chunkPath = join(chunkDir, `chunk_${String(i).padStart(4, "0")}.mp4`); + const ext = outputPath.endsWith(".webm") ? ".webm" : ".mp4"; + const chunkPath = join(chunkDir, `chunk_${String(i).padStart(4, "0")}${ext}`); const inputPath = join(framesDir, framePattern); const inputArgs = [ "-framerate", @@ -347,23 +373,15 @@ export async function muxVideoWithAudio( const outputDir = dirname(outputPath); if (!existsSync(outputDir)) mkdirSync(outputDir, { recursive: true }); - const args = [ - "-i", - videoPath, - "-i", - audioPath, - "-c:v", - "copy", - "-c:a", - "aac", - "-b:a", - "192k", - "-shortest", - "-movflags", - "+faststart", - "-y", - outputPath, - ]; + const isWebm = outputPath.endsWith(".webm"); + const args = ["-i", videoPath, "-i", audioPath, "-c:v", "copy"]; + + if (isWebm) { + args.push("-c:a", "libopus", "-b:a", "128k"); + } else { + args.push("-c:a", "aac", "-b:a", "192k", "-movflags", "+faststart"); + } + args.push("-shortest", "-y", outputPath); const processTimeout = config?.ffmpegProcessTimeout ?? DEFAULT_CONFIG.ffmpegProcessTimeout; const result = await runFfmpeg(args, { signal, timeout: processTimeout }); @@ -394,6 +412,13 @@ export async function applyFaststart( signal?: AbortSignal, config?: Partial>, ): Promise { + // faststart is MP4-only (moves moov atom to file start for streaming) + if (outputPath.endsWith(".webm")) { + // For WebM, just copy the file as-is + const { copyFileSync } = await import("fs"); + if (inputPath !== outputPath) copyFileSync(inputPath, outputPath); + return { success: true, outputPath, durationMs: 0 }; + } const args = ["-i", inputPath, "-c", "copy", "-movflags", "+faststart", "-y", outputPath]; const processTimeout = config?.ffmpegProcessTimeout ?? DEFAULT_CONFIG.ffmpegProcessTimeout; diff --git a/packages/engine/src/services/frameCapture.ts b/packages/engine/src/services/frameCapture.ts index 54b755c15..d164253b1 100644 --- a/packages/engine/src/services/frameCapture.ts +++ b/packages/engine/src/services/frameCapture.ts @@ -107,6 +107,15 @@ export async function createCaptureSession( }; await page.setViewport(viewport); + // For PNG capture (used by WebM/transparency), make the page background transparent + // so Chrome's screenshot captures alpha channel data. + if (options.format === "png") { + const cdp = await page.createCDPSession(); + await cdp.send("Emulation.setDefaultBackgroundColorOverride", { + color: { r: 0, g: 0, b: 0, a: 0 }, + }); + } + return { browser, page, diff --git a/packages/engine/src/services/streamingEncoder.ts b/packages/engine/src/services/streamingEncoder.ts index 0a3e9fe65..7a99fa115 100644 --- a/packages/engine/src/services/streamingEncoder.ts +++ b/packages/engine/src/services/streamingEncoder.ts @@ -172,6 +172,11 @@ function buildStreamingArgs( } else if (codec === "vp9") { args.push("-c:v", "libvpx-vp9", "-b:v", bitrate || "0", "-crf", String(quality)); args.push("-deadline", preset === "ultrafast" ? "realtime" : "good"); + args.push("-row-mt", "1"); + if (pixelFormat === "yuva420p") { + args.push("-auto-alt-ref", "0"); + args.push("-metadata:s:v:0", "alpha_mode=1"); + } } else if (codec === "prores") { args.push("-c:v", "prores_ks", "-profile:v", preset, "-vendor", "apl0"); return [...args, "-y", outputPath]; diff --git a/packages/producer/src/services/renderOrchestrator.ts b/packages/producer/src/services/renderOrchestrator.ts index 1205e8715..441e249e3 100644 --- a/packages/producer/src/services/renderOrchestrator.ts +++ b/packages/producer/src/services/renderOrchestrator.ts @@ -35,7 +35,7 @@ import { encodeFramesChunkedConcat, muxVideoWithAudio, applyFaststart, - ENCODER_PRESETS, + getEncoderPreset, processCompositionAudio, type AudioElement, calculateOptimalWorkers, @@ -90,6 +90,8 @@ export type RenderStatus = export interface RenderConfig { fps: 24 | 30 | 60; quality: "draft" | "standard" | "high"; + /** Output container format. WebM uses VP9+alpha for transparency. */ + format?: "mp4" | "webm"; workers?: number; useGpu?: boolean; debug?: boolean; @@ -357,12 +359,13 @@ export async function executeRenderJob( }); assertNotAborted(); + const isWebm = job.config.format === "webm"; const captureOpts: CaptureOptions = { width, height, fps: job.config.fps, - format: "jpeg", - quality: 80, + format: isWebm ? "png" : "jpeg", + quality: isWebm ? undefined : 80, }; probeSession = await createCaptureSession( fileServer.url, @@ -593,18 +596,21 @@ export async function executeRenderJob( const framesDir = join(workDir, "captured-frames"); if (!existsSync(framesDir)) mkdirSync(framesDir, { recursive: true }); + const outputFormat = job.config.format ?? "mp4"; + const isWebmRender = outputFormat === "webm"; const captureOptions: CaptureOptions = { width, height, fps: job.config.fps, - format: "jpeg", - quality: job.config.quality === "draft" ? 80 : 95, + format: isWebmRender ? "png" : "jpeg", + quality: isWebmRender ? undefined : job.config.quality === "draft" ? 80 : 95, }; const workerCount = calculateOptimalWorkers(job.totalFrames!, job.config.workers, cfg); - const videoOnlyPath = join(workDir, "video-only.mp4"); - const preset = ENCODER_PRESETS[job.config.quality]; + const videoExt = isWebmRender ? ".webm" : ".mp4"; + const videoOnlyPath = join(workDir, `video-only${videoExt}`); + const preset = getEncoderPreset(job.config.quality, outputFormat); job.framesRendered = 0; @@ -622,6 +628,7 @@ export async function executeRenderJob( codec: preset.codec, preset: preset.preset, quality: preset.quality, + pixelFormat: preset.pixelFormat, useGpu: job.config.useGpu, imageFormat: captureOptions.format || "jpeg", }, @@ -835,36 +842,32 @@ export async function executeRenderJob( const stage5Start = Date.now(); updateJobStatus(job, "encoding", "Encoding video", 75, onProgress); + const frameExt = isWebmRender ? "png" : "jpg"; + const framePattern = `frame_%06d.${frameExt}`; + const encoderOpts = { + fps: job.config.fps, + width, + height, + codec: preset.codec, + preset: preset.preset, + quality: preset.quality, + pixelFormat: preset.pixelFormat, + useGpu: job.config.useGpu, + }; const encodeResult = enableChunkedEncode ? await encodeFramesChunkedConcat( framesDir, - "frame_%06d.jpg", + framePattern, videoOnlyPath, - { - fps: job.config.fps, - width, - height, - codec: preset.codec, - preset: preset.preset, - quality: preset.quality, - useGpu: job.config.useGpu, - }, + encoderOpts, chunkedEncodeSize, abortSignal, ) : await encodeFramesFromDir( framesDir, - "frame_%06d.jpg", + framePattern, videoOnlyPath, - { - fps: job.config.fps, - width, - height, - codec: preset.codec, - preset: preset.preset, - quality: preset.quality, - useGpu: job.config.useGpu, - }, + encoderOpts, abortSignal, ); assertNotAborted();