feat(engine): add HDR video output pipeline (#265)

## Summary Adds the ability to render HDR video output (H.265 10-bit, BT.2020) from HyperFrames compositions. When the renderer detects HDR source video, it automatically switches to the HDR output pipeline — no flags needed. ## What it does - **Auto-detection** — Probes each video source with `ffprobe`. If any has bt2020/PQ/HLG color metadata, the output switches to H.265 10-bit with correct color tags. SDR-only compositions are unaffected (H.264, bt709). - **HLG pass-through** — Native HLG pixels from FFmpeg extraction are piped directly to the encoder without conversion. This avoids brightness loss from HLG→linear→PQ conversion (which requires an OOTF system gamma we can't reliably apply). - **Encoder HDR support** — Both chunk and streaming encoders accept HDR presets: `libx265`, `yuv420p10le`, BT.2020 color primaries, `hvc1` codec tag (required for Apple playback). - **WebGPU HDR capture (gated)** — A complete WebGPU float16 readback pipeline is implemented and tested but gated behind headed Chrome (headless doesn't expose WebGPU). Ready for future use with WebGPU canvas content. - **HDR utilities** — `detectTransfer()` (PQ vs HLG), `getHdrEncoderColorParams()`, `analyzeCompositionHdr()`. 15 unit tests. ## Key design decisions | Decision | Why | |----------|-----| | No `--hdr` flag | SDR content encoded as HDR causes orange shift in browsers. Auto-detect eliminates this. | | HLG pass-through (not HLG→PQ) | Conversion loses brightness without OOTF. Pass-through matches source exactly. | | `hvc1` codec tag | Apple QuickTime requires `hvc1` (not `hev1`) for HEVC playback. | | 1-hour streaming timeout | HDR capture at ~6fps needs more time than the default 10-minute FFmpeg timeout. | ## Files changed | File | What changed | |------|-------------| | `packages/engine/src/utils/hdr.ts` | **NEW** — HDR detection, transfer types, encoder params (15 tests) | | `packages/engine/src/services/hdrCapture.ts` | **NEW** — WebGPU readback, HLG conversion, PQ encode | | `packages/engine/src/services/streamingEncoder.ts` | HDR presets, raw rgb48le input, color tags | | `packages/engine/src/services/chunkEncoder.ts` | HDR presets, conditional color tags | | `packages/producer/src/services/renderOrchestrator.ts` | Auto-detection loop, HDR pass-through capture path | ## How to test Render a composition with an HDR video source. The output should be H.265 10-bit with HDR metadata visible in `ffprobe` (bt2020, arib-std-b67 or smpte2084). Plays correctly in QuickTime and on HDR displays. ## Stack position **2 of 6** — Stacked on #258 (SDR/HDR normalization). Provides the encoder infrastructure that phases 1-5 build on. 🤖 Generated with [Claude Code](https://claude.com/claude-code)
2026-09-07 10:06:21 +00:00 · 2026-04-19 15:10:59 -07:00
parent d1f992570a
commit 5a3fde19d4
19 changed files with 1794 additions and 318 deletions
@@ -10,7 +10,9 @@ import { existsSync, mkdirSync, readdirSync, rmSync } from "fs";
 import { join } from "path";
 import { parseHTML } from "linkedom";
 import { extractVideoMetadata, type VideoMetadata } from "../utils/ffprobe.js";
+import { isHdrColorSpace as isHdrColorSpaceUtil } from "../utils/hdr.js";
 import { downloadToTemp, isHttpUrl } from "../utils/urlDownloader.js";
+import { runFfmpeg } from "../utils/runFfmpeg.js";
 import { DEFAULT_CONFIG, type EngineConfig } from "../config.js";

 export interface VideoElement {
@@ -114,18 +116,28 @@ export async function extractVideoFramesRange(
  const framePattern = `frame_%05d.${format}`;
  const outputPattern = join(videoOutputDir, framePattern);

-  const args: string[] = [
-    "-ss",
-    String(startTime),
-    "-i",
-    videoPath,
-    "-t",
-    String(duration),
-    "-vf",
-    `fps=${fps}`,
-    "-q:v",
-    format === "jpg" ? String(Math.ceil((100 - quality) / 3)) : "0",
-  ];
+  // When extracting from HDR source, tone-map to SDR in FFmpeg rather than
+  // letting Chrome's uncontrollable tone-mapper handle it (which washes out).
+  // macOS: VideoToolbox hardware decoder does HDR→SDR natively on Apple Silicon.
+  // Linux: zscale filter (when available) or colorspace filter as fallback.
+  const isHdr = isHdrColorSpaceUtil(metadata.colorSpace);
+  const isMacOS = process.platform === "darwin";
+
+  const args: string[] = [];
+  if (isHdr && isMacOS) {
+    args.push("-hwaccel", "videotoolbox");
+  }
+  args.push("-ss", String(startTime), "-i", videoPath, "-t", String(duration));
+
+  const vfFilters: string[] = [];
+  if (isHdr && isMacOS) {
+    // VideoToolbox tone-maps during decode; force output to bt709 SDR format
+    vfFilters.push("format=nv12");
+  }
+  vfFilters.push(`fps=${fps}`);
+  args.push("-vf", vfFilters.join(","));
+
+  args.push("-q:v", format === "jpg" ? String(Math.ceil((100 - quality) / 3)) : "0");
  if (format === "png") args.push("-compression_level", "6");
  args.push("-y", outputPattern);

@@ -195,6 +207,53 @@ export async function extractVideoFramesRange(
  });
 }

+/**
+ * Convert an SDR video to HDR color space (HLG / BT.2020) so it can be
+ * composited alongside HDR content without looking washed out.
+ *
+ * Uses zscale for color space conversion with a nominal peak luminance of
+ * 600 nits — high enough that SDR content doesn't appear too dark next to
+ * HDR, matching the approach used by HeyGen's Rio pipeline.
+ */
+async function convertSdrToHdr(
+  inputPath: string,
+  outputPath: string,
+  signal?: AbortSignal,
+  config?: Partial<Pick<EngineConfig, "ffmpegProcessTimeout">>,
+): Promise<void> {
+  const timeout = config?.ffmpegProcessTimeout ?? DEFAULT_CONFIG.ffmpegProcessTimeout;
+
+  const args = [
+    "-i",
+    inputPath,
+    "-vf",
+    "colorspace=all=bt2020:iall=bt709:range=tv",
+    "-color_primaries",
+    "bt2020",
+    "-color_trc",
+    "arib-std-b67",
+    "-colorspace",
+    "bt2020nc",
+    "-c:v",
+    "libx264",
+    "-preset",
+    "fast",
+    "-crf",
+    "16",
+    "-c:a",
+    "copy",
+    "-y",
+    outputPath,
+  ];
+
+  const result = await runFfmpeg(args, { signal, timeout });
+  if (!result.success) {
+    throw new Error(
+      `SDR→HDR conversion failed (exit ${result.exitCode}): ${result.stderr.slice(-300)}`,
+    );
+  }
+}
+
 export async function extractAllVideoFrames(
  videos: VideoElement[],
  baseDir: string,
@@ -208,30 +267,75 @@ export async function extractAllVideoFrames(
  const errors: Array<{ videoId: string; error: string }> = [];
  let totalFramesExtracted = 0;

-  // Process videos in parallel for better performance
+  // Phase 1: Resolve paths and download remote videos
+  const resolvedVideos: Array<{ video: VideoElement; videoPath: string }> = [];
+  for (const video of videos) {
+    if (signal?.aborted) break;
+    try {
+      let videoPath = video.src;
+      if (!videoPath.startsWith("/") && !isHttpUrl(videoPath)) {
+        const fromCompiled = compiledDir ? join(compiledDir, videoPath) : null;
+        videoPath =
+          fromCompiled && existsSync(fromCompiled) ? fromCompiled : join(baseDir, videoPath);
+      }
+
+      if (isHttpUrl(videoPath)) {
+        const downloadDir = join(options.outputDir, "_downloads");
+        mkdirSync(downloadDir, { recursive: true });
+        videoPath = await downloadToTemp(videoPath, downloadDir);
+      }
+
+      if (!existsSync(videoPath)) {
+        errors.push({ videoId: video.id, error: `Video file not found: ${videoPath}` });
+        continue;
+      }
+      resolvedVideos.push({ video, videoPath });
+    } catch (err) {
+      errors.push({ videoId: video.id, error: err instanceof Error ? err.message : String(err) });
+    }
+  }
+
+  // Phase 2: Probe color spaces and normalize if mixed HDR/SDR
+  const videoColorSpaces = await Promise.all(
+    resolvedVideos.map(async ({ videoPath }) => {
+      const metadata = await extractVideoMetadata(videoPath);
+      return metadata.colorSpace;
+    }),
+  );
+
+  const hasAnyHdr = videoColorSpaces.some(isHdrColorSpaceUtil);
+  if (hasAnyHdr) {
+    const convertDir = join(options.outputDir, "_hdr_normalized");
+    mkdirSync(convertDir, { recursive: true });
+
+    for (let i = 0; i < resolvedVideos.length; i++) {
+      if (signal?.aborted) break;
+      const cs = videoColorSpaces[i] ?? null;
+      if (!isHdrColorSpaceUtil(cs)) {
+        // SDR video in a mixed timeline — convert to HDR color space
+        const entry = resolvedVideos[i];
+        if (!entry) continue;
+        const convertedPath = join(convertDir, `${entry.video.id}_hdr.mp4`);
+        try {
+          await convertSdrToHdr(entry.videoPath, convertedPath, signal, config);
+          entry.videoPath = convertedPath;
+        } catch (err) {
+          errors.push({
+            videoId: entry.video.id,
+            error: `SDR→HDR conversion failed: ${err instanceof Error ? err.message : String(err)}`,
+          });
+        }
+      }
+    }
+  }
+
+  // Phase 3: Extract frames (parallel)
  const results = await Promise.all(
-    videos.map(async (video) => {
+    resolvedVideos.map(async ({ video, videoPath }) => {
      if (signal?.aborted) {
        throw new Error("Video frame extraction cancelled");
      }
      try {
-        let videoPath = video.src;
-        if (!videoPath.startsWith("/") && !isHttpUrl(videoPath)) {
-          const fromCompiled = compiledDir ? join(compiledDir, videoPath) : null;
-          videoPath =
-            fromCompiled && existsSync(fromCompiled) ? fromCompiled : join(baseDir, videoPath);
-        }
-
-        if (isHttpUrl(videoPath)) {
-          const downloadDir = join(options.outputDir, "_downloads");
-          mkdirSync(downloadDir, { recursive: true });
-          videoPath = await downloadToTemp(videoPath, downloadDir);
-        }
-
-        if (!existsSync(videoPath)) {
-          return { error: { videoId: video.id, error: `Video file not found: ${videoPath}` } };
-        }
-
        let videoDuration = video.end - video.start;

        // Fallback: if no data-duration/data-end was specified (end is Infinity or 0),