fix(engine): tone-map HDR frames for forced SDR (#3981)

Co-authored-by: heygengenesis[bot] <262951085+heygengenesis[bot]@users.noreply.github.com>
Co-authored-by: miguel.sierra <229591595+miguel-heygen@users.noreply.github.com>

packages/engine/src/services/videoFrameExtractor.test.ts

@@ -2709,6 +2709,115 @@ describe.skipIf(!HAS_FFMPEG)("extractAllVideoFrames on a VFR source", () => {
   }, 60_000);
 });
 
+describe.skipIf(!HAS_FFMPEG || process.platform === "darwin")("forced-SDR HDR extraction", () => {
+  let fixtureDir = "";
+
+  beforeAll(() => {
+    fixtureDir = mkdtempSync(join(tmpdir(), "hf-forced-sdr-tonemap-test-"));
+  });
+
+  afterAll(() => {
+    rmSync(fixtureDir, { recursive: true, force: true });
+  });
+
+  it("matches Studio's HLG tone map and isolates transformed cache entries", async () => {
+    const source = join(fixtureDir, "hlg-warm.mp4");
+    const synthesized = await runFfmpeg([
+      "-y",
+      "-hide_banner",
+      "-loglevel",
+      "error",
+      "-f",
+      "lavfi",
+      "-i",
+      "color=c=0xe0b080:s=64x64:r=1:d=1",
+      "-vf",
+      "zscale=pin=bt709:tin=bt709:min=bt709:p=bt2020:t=arib-std-b67:m=bt2020nc:r=tv,format=yuv420p",
+      "-c:v",
+      "libx264",
+      "-preset",
+      "ultrafast",
+      "-color_primaries",
+      "bt2020",
+      "-color_trc",
+      "arib-std-b67",
+      "-colorspace",
+      "bt2020nc",
+      "-bsf:v",
+      "h264_metadata=colour_primaries=9:transfer_characteristics=18:matrix_coefficients=9",
+      source,
+    ]);
+    if (!synthesized.success) {
+      throw new Error(`HLG fixture synthesis failed: ${synthesized.stderr.slice(-400)}`);
+    }
+
+    const reference = join(fixtureDir, "studio-reference.png");
+    const referenceResult = await runFfmpeg([
+      "-y",
+      "-ss",
+      "0",
+      "-i",
+      source,
+      "-t",
+      "1",
+      "-vf",
+      "fps=1,zscale=t=linear:npl=100,tonemap=hable:desat=0,zscale=p=bt709:t=bt709:m=bt709:r=tv",
+      "-q:v",
+      "0",
+      "-compression_level",
+      "1",
+      reference,
+    ]);
+    if (!referenceResult.success) {
+      throw new Error(`Studio reference extraction failed: ${referenceResult.stderr.slice(-400)}`);
+    }
+
+    const cacheDir = join(fixtureDir, "cache");
+    const video = (id: string): VideoElement => ({
+      id,
+      src: source,
+      start: 0,
+      end: 1,
+      mediaStart: 0,
+      loop: false,
+      hasAudio: false,
+    });
+    const extract = (id: string, toneMapHdrToSdr = false) =>
+      extractAllVideoFrames(
+        [video(id)],
+        fixtureDir,
+        {
+          fps: 1,
+          outputDir: join(fixtureDir, id),
+          format: "png",
+          toneMapHdrToSdr,
+        },
+        undefined,
+        { extractCacheDir: cacheDir },
+      );
+
+    const plain = await extract("plain");
+    const toneMapped = await extract("tone-mapped", true);
+    const toneMappedAgain = await extract("tone-mapped-again", true);
+
+    expect(plain.errors).toEqual([]);
+    expect(toneMapped.errors).toEqual([]);
+    expect(toneMapped.phaseBreakdown.cacheHits).toBe(0);
+    expect(toneMapped.phaseBreakdown.cacheMisses).toBe(1);
+    expect(toneMappedAgain.phaseBreakdown.cacheHits).toBe(1);
+    expect(readdirSync(cacheDir).filter((name) => name.startsWith(SCHEMA_PREFIX))).toHaveLength(2);
+
+    const frame = (result: ExtractionResult): Buffer => {
+      const path = result.extracted[0]?.framePaths.get(0);
+      if (!path) throw new Error("expected extracted frame");
+      return readFileSync(path);
+    };
+    expect(frame(toneMapped)).toEqual(readFileSync(reference));
+    expect(frame(plain)).not.toEqual(frame(toneMapped));
+    expect(frame(toneMappedAgain)).toEqual(frame(toneMapped));
+  }, 60_000);
+});
+
 describe("getFrameAtTime — IEEE 754 boundary precision", () => {
   function makeExtracted(fps: number, totalFrames: number): ExtractedFrames {
     const framePaths = new Map<number, string>();

packages/engine/src/services/videoFrameExtractor.ts

@@ -182,6 +182,7 @@ export interface ExtractionOptions {
   quality?: number;
   format?: VideoFrameFormat;
   sdrToHdrTransfer?: HdrTransfer;
+  toneMapHdrToSdr?: boolean;
   /** Extract exactly one frame at `startTime`. Used only after ffprobe has
    *  resolved the actual final decoded-frame timestamp for a held tail. */
   finalFrameOnly?: boolean;
@@ -207,6 +208,9 @@ export interface ExtractionOptions {
 const EXTRACT_CACHE_MIN_AGE_MS = 60 * 60 * 1000;
 const GC_STALENESS_MS = 24 * 60 * 60 * 1000;
 const SDR_TO_HDR_COLORSPACE_FILTER = "colorspace=all=bt2020:iall=bt709:range=tv";
+const HDR_TO_SDR_TONEMAP_FILTER =
+  "zscale=t=linear:npl=100,tonemap=hable:desat=0,zscale=p=bt709:t=bt709:m=bt709:r=tv";
+const HDR_TO_SDR_TRANSFORM_KEY = "hdr2sdr-hable-bt709";
 
 function sdrToHdrTransformKey(transfer: HdrTransfer): string {
   return `sdr2hdr-${transfer}`;
@@ -792,10 +796,9 @@ export async function extractVideoFramesRange(
   const framePattern = `${FRAME_FILENAME_PREFIX}%05d.${format}`;
   const outputPattern = join(videoOutputDir, framePattern);
 
-  // When extracting from HDR source, tone-map to SDR in FFmpeg rather than
-  // letting Chrome's uncontrollable tone-mapper handle it (which washes out).
+  // Forced-SDR extraction tone-maps HDR before the intermediate frames reach Chrome.
   // macOS: VideoToolbox hardware decoder does HDR→SDR natively on Apple Silicon.
-  // Linux: zscale filter (when available) or colorspace filter as fallback.
+  // Linux: use the same zscale/tonemap policy as Studio proxies.
   const isHdr = isHdrColorSpaceUtil(metadata.colorSpace);
   const isMacOS = process.platform === "darwin";
 
@@ -843,6 +846,9 @@ export async function extractVideoFramesRange(
     // remap only applies to SDR sources, nv12 only to HDR sources).
     vfFilters.push(SDR_TO_HDR_COLORSPACE_FILTER);
   }
+  if (options.toneMapHdrToSdr && isHdr && !isMacOS) {
+    vfFilters.push(HDR_TO_SDR_TONEMAP_FILTER);
+  }
   if (vfFilters.length > 0) args.push("-vf", vfFilters.join(","));
   if (!options.finalFrameOnly && metadata.isVFR) {
     args.push("-fps_mode", "cfr", "-r", ffmpegFps);
@@ -1276,6 +1282,7 @@ type PreparedExtraction = {
   finalFrameOnly: boolean;
   format: CacheFrameFormat;
   sdrToHdrTransfer?: HdrTransfer;
+  toneMapHdrToSdr: boolean;
   dedupeKey: string;
 };
 
@@ -1339,6 +1346,7 @@ function supersetGroupingKey(work: PreparedExtraction, fps: number): string {
     String(fps),
     work.format,
     work.sdrToHdrTransfer ?? "",
+    work.toneMapHdrToSdr ? HDR_TO_SDR_TRANSFORM_KEY : "",
     work.finalFrameOnly ? "final" : "range",
   ].join("\0");
 }
@@ -1836,6 +1844,7 @@ export async function extractAllVideoFrames(
       ...options,
       format: work.format,
       sdrToHdrTransfer: work.sdrToHdrTransfer,
+      toneMapHdrToSdr: work.toneMapHdrToSdr,
       finalFrameOnly: work.finalFrameOnly,
     };
   }
@@ -1857,6 +1866,7 @@ export async function extractAllVideoFrames(
     if (!keyInput) return { work };
     const transformParts = [
       work.sdrToHdrTransfer ? sdrToHdrTransformKey(work.sdrToHdrTransfer) : undefined,
+      work.toneMapHdrToSdr ? HDR_TO_SDR_TRANSFORM_KEY : undefined,
       work.finalFrameOnly ? "final-frame" : undefined,
     ].filter((part): part is string => part !== undefined);
     const transform = transformParts.length > 0 ? transformParts.join("+") : undefined;
@@ -2082,8 +2092,10 @@ export async function extractAllVideoFrames(
 
         const format = resolveFrameFormat(metadata, options.format);
         const sdrToHdrTransfer = sdrToHdrTransfers[index];
+        const toneMapHdrToSdr =
+          options.toneMapHdrToSdr === true && isHdrColorSpaceUtil(metadata.colorSpace);
         const finalFrameOnly = window.finalFrameOnly === true;
-        const dedupeKey = `${videoPath}\0${extractionMediaStart}\0${videoDuration}\0${fpsKey}\0${format}\0${sdrToHdrTransfer ?? ""}\0${finalFrameOnly ? "final" : "range"}`;
+        const dedupeKey = `${videoPath}\0${extractionMediaStart}\0${videoDuration}\0${fpsKey}\0${format}\0${sdrToHdrTransfer ?? ""}\0${toneMapHdrToSdr ? HDR_TO_SDR_TRANSFORM_KEY : ""}\0${finalFrameOnly ? "final" : "range"}`;
 
         return {
           work: {
@@ -2096,6 +2108,7 @@ export async function extractAllVideoFrames(
             finalFrameOnly,
             format,
             sdrToHdrTransfer,
+            toneMapHdrToSdr,
             dedupeKey,
           },
         };

packages/producer/src/services/render/stages/extractVideosStage.timelineBound.test.ts

@@ -10,6 +10,7 @@ import type { ProducerLogger } from "../../../logger.js";
 const extractionCalls = vi.hoisted(
   () => new Array<{ timelineEnd: number | undefined; durationSeconds: number }>(),
 );
+const toneMapHdrToSdrCalls = vi.hoisted(() => new Array<boolean | undefined>());
 const fixtureState = vi.hoisted(() => ({ sourceDurationSeconds: 60 }));
 
 vi.mock("@hyperframes/engine", async (importOriginal) => {
@@ -19,7 +20,7 @@ vi.mock("@hyperframes/engine", async (importOriginal) => {
     extractAllVideoFrames: async (
       videos: VideoElement[],
       _baseDir: string,
-      options: { timelineEnd?: number },
+      options: { timelineEnd?: number; toneMapHdrToSdr?: boolean },
     ): Promise<ExtractionResult> => {
       const sourceDurationSeconds = fixtureState.sourceDurationSeconds;
       const video = videos[0];
@@ -36,6 +37,7 @@ vi.mock("@hyperframes/engine", async (importOriginal) => {
           : Math.min(resolvedDuration, Math.max(0, options.timelineEnd - video.start));
       video.end = video.start + durationSeconds;
       extractionCalls.push({ timelineEnd: options.timelineEnd, durationSeconds });
+      toneMapHdrToSdrCalls.push(options.toneMapHdrToSdr);
       return {
         success: true,
         extracted: [],
@@ -113,6 +115,7 @@ describe.each([
 ] as const)("%s video extraction timeline bound", (_mode, materializeSymlinks) => {
   it("caps an open 60-second source to a two-second composition", async () => {
     extractionCalls.splice(0);
+    toneMapHdrToSdrCalls.splice(0);
     fixtureState.sourceDurationSeconds = 60;
 
     await runStage(2, materializeSymlinks);
@@ -122,12 +125,22 @@ describe.each([
 
   it("keeps a two-second natural source inside a ten-second composition", async () => {
     extractionCalls.splice(0);
+    toneMapHdrToSdrCalls.splice(0);
     fixtureState.sourceDurationSeconds = 2;
 
     await runStage(10, materializeSymlinks);
 
     expect(extractionCalls).toEqual([{ timelineEnd: 10, durationSeconds: 2 }]);
   });
+
+  it("requests HDR-to-SDR tone mapping for forced-SDR extraction", async () => {
+    extractionCalls.splice(0);
+    toneMapHdrToSdrCalls.splice(0);
+
+    await runStage(2, materializeSymlinks);
+
+    expect(toneMapHdrToSdrCalls).toEqual([true]);
+  });
 });
 
 describe("video extraction source logging", () => {

packages/producer/src/services/render/stages/extractVideosStage.ts

@@ -496,6 +496,7 @@ export async function runExtractVideosStage(
         fps: job.config.fps,
         outputDir: join(compiledDir, "__hyperframes_video_frames"),
         format: job.config.videoFrameFormat ?? "auto",
+        toneMapHdrToSdr: job.config.hdrMode === "force-sdr",
         timelineEnd: composition.duration,
         maxTransientRetries: extractionPolicy.maxTransientRetries,
         collectProbeFailures: extractionPolicy.failureMode === "enforce",