fix(studio-server): an HDR clip previews and snapshots the way the SDR export renders it (#6537)

* fix(studio-server): an HDR clip previews and snapshots the way the SDR export renders it

Footage tagged HLG or PQ played raw in the preview, where the browser tone-maps it its
own way, while the SDR export tone-maps it with zscale and hable: the same clip came
out lighter and with different reds and blues. HDR video now takes the tone-mapped
preview copy (the filter the export uses), and a snapshot still of HDR footage runs
the same tone map. Without an ffmpeg that has zscale and tonemap the clip plays
natively, with a warning that names the missing library.

* fix(studio-server): an HDR clip is never playable for preview, whatever its codec

Only HDR H.264 and ProRes took the tone-mapped copy: an HDR HEVC, VP9 or AV1 clip kept
its codec's test type, so a browser that can decode it was judged able to play it and
the original played with the browser's own mapping. An HDR clip now has no test type, so
the swap always fires. The missing-library warning no longer names the file.

* test(cli): the HDR still test ends ffprobe's options before its input

The repo's ffprobe argv contract requires "--" immediately before the input path; the new
test passed the still's path without it.

packages/cli/src/commands/snapshot.ts

@@ -3,6 +3,7 @@ import {
   buildZeroDurationDiagnostic,
   evaluateHfDiagnostic,
   extractMediaMetadata,
+  isHdrColorSpace,
   sdrCanvasColourFilters,
 } from "@hyperframes/engine";
 import { parseTimeoutMs } from "../utils/timeoutArg.js";
@@ -57,8 +58,11 @@ import {
   type ZoomTarget,
 } from "../capture/captureCompositionFrame.js";
 import {
+  firstFrameColourArgs,
+  hdrToSdrToneMapFilter,
   isClipVisibleAt,
   isInClipWindow,
+  parseFirstFrameColour,
   readElementRateSpec,
   sourceTimeAt,
   timeAtSourceTime,
@@ -459,6 +463,16 @@ function warnColourUnknownOnce(): void {
   );
 }
 
+let warnedNoHdrFilters = false;
+
+function warnNoHdrFiltersOnce(): void {
+  if (warnedNoHdrFilters) return;
+  warnedNoHdrFilters = true;
+  console.warn(
+    `   ${c.warn("⚠")} A video in this project is HDR, but this ffmpeg has no zscale/tonemap (libzimg), so the still shows it as the browser does and will not match an SDR export. Install an ffmpeg with libzimg.`,
+  );
+}
+
 class FrameExtractTimeoutError extends Error {}
 
 export async function extractVideoFrameToBuffer(
@@ -478,24 +492,50 @@ export async function extractVideoFrameToBuffer(
       holdLastFrame,
     );
     if (seek === null) return null;
-    const args = ["-hide_banner", "-loglevel", "error"];
-    if (useVp9AlphaDecoder) args.push("-c:v", "libvpx-vp9");
-    if (seek.seek !== undefined) args.push("-ss", String(seek.seek));
-    args.push("-i", videoPath, "-map", "0:v:0");
-    const filters: string[] = [];
-    if (seek.select !== undefined) filters.push(`select=gte(t-start_t\\,${seek.select})`);
     const metadata = findFFprobe() ? await extractMediaMetadata(videoPath) : null;
-    if (metadata) filters.push(...sdrCanvasColourFilters(metadata));
-    else warnColourUnknownOnce();
-    // Shrink so the short side still covers the displayed box at any rotation or fit; never enlarge.
-    if (displayedPixels > 0) {
-      filters.push(
-        `scale='min(iw,${displayedPixels})':'min(ih,${displayedPixels})':force_original_aspect_ratio=increase`,
-      );
+    if (!metadata) warnColourUnknownOnce();
+    // HDR footage takes the tone map the SDR export uses, so a still matches the render.
+    const toneMap =
+      metadata && isHdrColorSpace(metadata.colorSpace)
+        ? hdrToSdrToneMapFilter(
+            metadata.colorSpace ?? {},
+            parseFirstFrameColour(
+              (
+                await runFfmpegOnce(
+                  ffmpegPath,
+                  firstFrameColourArgs(videoPath),
+                  FFMPEG_EXTRACT_TIMEOUT_MS,
+                )
+              ).stderr,
+            ),
+          )
+        : null;
+    const frameArgs = (toneMapFilter: string | null): string[] => {
+      const args = ["-hide_banner", "-loglevel", "error"];
+      if (useVp9AlphaDecoder) args.push("-c:v", "libvpx-vp9");
+      if (seek.seek !== undefined) args.push("-ss", String(seek.seek));
+      args.push("-i", videoPath, "-map", "0:v:0");
+      const filters: string[] = [];
+      if (seek.select !== undefined) filters.push(`select=gte(t-start_t\\,${seek.select})`);
+      if (metadata) {
+        if (toneMapFilter) filters.push(toneMapFilter);
+        filters.push(...sdrCanvasColourFilters(metadata, toneMapFilter !== null));
+      }
+      // Shrink so the short side still covers the displayed box at any rotation or fit; never enlarge.
+      if (displayedPixels > 0) {
+        filters.push(
+          `scale='min(iw,${displayedPixels})':'min(ih,${displayedPixels})':force_original_aspect_ratio=increase`,
+        );
+      }
+      if (filters.length > 0) args.push("-vf", filters.join(","));
+      args.push("-frames:v", "1", "-q:v", "2", "-y", outPath);
+      return args;
+    };
+    let result = await runFfmpegOnce(ffmpegPath, frameArgs(toneMap), FFMPEG_EXTRACT_TIMEOUT_MS);
+    if (toneMap && result.code !== 0 && /no such filter/i.test(result.stderr)) {
+      warnNoHdrFiltersOnce();
+      result = await runFfmpegOnce(ffmpegPath, frameArgs(null), FFMPEG_EXTRACT_TIMEOUT_MS);
     }
-    if (filters.length > 0) args.push("-vf", filters.join(","));
-    args.push("-frames:v", "1", "-q:v", "2", "-y", outPath);
-    const result = await runFfmpegOnce(ffmpegPath, args, FFMPEG_EXTRACT_TIMEOUT_MS);
     if (result.timedOut) {
       throw new FrameExtractTimeoutError(
         `FFmpeg timed out after ${FFMPEG_EXTRACT_TIMEOUT_MS / 1000}s extracting the frame of ${videoPath} at ${timeSeconds.toFixed(2)}s`,

packages/cli/src/commands/snapshotHdrFrame.test.ts

@@ -0,0 +1,53 @@
+import { spawnSync } from "node:child_process";
+import { mkdtempSync, writeFileSync } from "node:fs";
+import { tmpdir } from "node:os";
+import { join } from "node:path";
+import { describe, expect, it } from "vitest";
+import { extractVideoFrameToBuffer } from "./snapshot.js";
+
+const hasZscale =
+  spawnSync("ffmpeg", ["-hide_banner", "-filters"], { encoding: "utf8" }).stdout?.includes(
+    "zscale",
+  ) === true;
+
+function hlgClip(dir: string): string {
+  const clip = join(dir, "clip.mp4");
+  const made = spawnSync(
+    "ffmpeg",
+    ["-v", "error", "-y", "-f", "lavfi", "-i", "smptehdbars=size=320x180:rate=10:duration=1"]
+      .concat(["-pix_fmt", "yuv420p", "-c:v", "libx264", "-crf", "12"])
+      .concat(["-color_primaries", "bt2020", "-color_trc", "arib-std-b67"])
+      .concat(["-colorspace", "bt2020nc", "-color_range", "tv", clip]),
+  );
+  expect(made.status).toBe(0);
+  return clip;
+}
+
+describe.runIf(hasZscale)("a snapshot still of HDR footage", () => {
+  it("is tone-mapped to SDR like the export, not left tagged HLG for the browser to map", async () => {
+    const dir = mkdtempSync(join(tmpdir(), "hf-snapshot-hdr-"));
+    const buffer = await extractVideoFrameToBuffer(hlgClip(dir), 0.5);
+    expect(buffer).not.toBeNull();
+    const png = join(dir, "still.png");
+    writeFileSync(png, buffer!);
+    const tags = spawnSync(
+      "ffprobe",
+      ["-v", "error", "-show_entries", "stream=color_transfer", "-of", "csv=p=0", "--", png],
+      { encoding: "utf8" },
+    ).stdout.trim();
+    expect(tags).not.toBe("arib-std-b67");
+    const rgb = spawnSync(
+      "ffmpeg",
+      ["-v", "error", "-i", png, "-vf", "crop=1:1:25:75,format=rgb24"].concat([
+        "-frames:v",
+        "1",
+        "-f",
+        "rawvideo",
+        "-",
+      ]),
+    ).stdout;
+    // The 40% grey bar: the export's tone map gives about 102 (an untouched HLG decode gives the same
+    // number but stays tagged HLG, which the browser then maps darker).
+    expect(Math.abs(rgb[0]! - 102)).toBeLessThanOrEqual(8);
+  }, 60_000);
+});

packages/studio-server/src/helpers/hdrFilters.ts

@@ -0,0 +1,25 @@
+import { spawn } from "node:child_process";
+
+const checks = new Map<string, Promise<boolean>>();
+
+/** Whether this ffmpeg has the zscale and tonemap filters (libzimg) that the HDR to SDR copy needs. */
+export function ffmpegHasHdrFilters(ffmpegPath: string): Promise<boolean> {
+  const known = checks.get(ffmpegPath);
+  if (known) return known;
+  const check = new Promise<boolean>((resolveCheck) => {
+    const proc = spawn(ffmpegPath, ["-hide_banner", "-filters"], {
+      stdio: ["ignore", "pipe", "pipe"],
+      windowsHide: true,
+    });
+    let stdout = "";
+    proc.stdout?.on("data", (chunk: Buffer) => {
+      stdout += chunk.toString();
+    });
+    proc.on("error", () => resolveCheck(false));
+    proc.on("close", (code) =>
+      resolveCheck(code === 0 && /\bzscale\b/.test(stdout) && /\btonemap\b/.test(stdout)),
+    );
+  });
+  checks.set(ffmpegPath, check);
+  return check;
+}

packages/studio-server/src/helpers/mediaCodecMap.test.ts

@@ -1,4 +1,4 @@
-import { afterEach, beforeEach, describe, expect, it } from "vitest";
+import { afterEach, beforeEach, describe, expect, it, vi } from "vitest";
 import { mkdirSync, mkdtempSync, rmSync, utimesSync, writeFileSync } from "node:fs";
 import { join } from "node:path";
 import { tmpdir } from "node:os";
@@ -32,6 +32,7 @@ function makeRunner(
         tags?: Record<string, string>;
         size?: [number, number];
         bitRate?: string;
+        colorTransfer?: string;
       }
   >,
 ): FfprobeRunner {
@@ -49,6 +50,8 @@ function makeRunner(
             width: normalized.size?.[0],
             height: normalized.size?.[1],
             bit_rate: normalized.bitRate,
+            color_transfer: normalized.colorTransfer,
+            color_primaries: normalized.colorTransfer ? "bt2020" : undefined,
           },
         ]
       : [];
@@ -558,3 +561,57 @@ describe("scanProjectMediaCodecMap", () => {
     expect(cache.has(join(project, "clip-512.mp4"))).toBe(true);
   });
 });
+
+describe("HDR video previews through the SDR tone-mapped copy", () => {
+  const hlgClip = { codecName: "h264", pixFmt: "yuv420p", colorTransfer: "arib-std-b67" };
+
+  async function probe(entry: Parameters<typeof makeRunner>[0][string], hasFilters: boolean) {
+    vi.resetModules();
+    vi.doMock("./hdrFilters.js", () => ({ ffmpegHasHdrFilters: async () => hasFilters }));
+    vi.doMock("@hyperframes/parsers/ff-binaries", () => ({
+      findFfBinary: (name: string) => (name === "ffprobe" ? FAKE_FFPROBE_PATH : "/fake/ffmpeg"),
+    }));
+    process.env.HYPERFRAMES_FFPROBE_PATH = FAKE_FFPROBE_PATH;
+    const fresh = await import("./mediaCodecMap.js");
+    const filePath = join(mkdtempSync(join(tmpdir(), "hf-hdr-clip-")), "clip.mp4");
+    writeFileSync(filePath, "x");
+    return fresh.probeAssetCodec(
+      filePath,
+      makeRunner({ [filePath]: entry }),
+      fresh.createMediaCodecProbeCache(),
+    );
+  }
+
+  afterEach(() => {
+    vi.doUnmock("./hdrFilters.js");
+    vi.doUnmock("@hyperframes/parsers/ff-binaries");
+    vi.restoreAllMocks();
+  });
+
+  it("marks an HLG H.264 clip browser-hostile so it gets the same copy the export uses", async () => {
+    const facts = await probe(hlgClip, true);
+    expect(facts?.browserHostile).toBe(true);
+    expect(decideMediaProxyEligibility(facts)).toEqual({ eligible: true });
+  });
+
+  it("treats an HDR clip as not playable whatever its codec, so a browser that decodes HEVC still swaps it", async () => {
+    const facts = await probe({ ...hlgClip, codecName: "hevc" }, true);
+    expect(facts?.browserHostile).toBe(true);
+    expect(facts?.representativeMime).toBeNull();
+    const vp9 = await probe({ ...hlgClip, codecName: "vp9" }, true);
+    expect(vp9?.representativeMime).toBeNull();
+  });
+
+  it("leaves an SDR H.264 clip native", async () => {
+    const facts = await probe({ codecName: "h264", pixFmt: "yuv420p" }, true);
+    expect(facts?.browserHostile).toBe(false);
+  });
+
+  it("plays the clip natively and says why once when ffmpeg has no zscale/tonemap", async () => {
+    const warn = vi.spyOn(console, "warn").mockImplementation(() => undefined);
+    const facts = await probe(hlgClip, false);
+    expect(facts?.browserHostile).toBe(false);
+    expect(warn).toHaveBeenCalledOnce();
+    expect(warn.mock.calls[0]![0]).toContain("libzimg");
+  });
+});

packages/studio-server/src/helpers/mediaCodecMap.ts

@@ -2,6 +2,8 @@ import { scanHtmlOpeningTags, decodeAuthoredAttribute } from "@hyperframes/parse
 import { statSync } from "node:fs";
 import { realpath } from "./safePath.js";
 import { sep } from "node:path";
+import { findFfBinary } from "@hyperframes/parsers/ff-binaries";
+import { ffmpegHasHdrFilters } from "./hdrFilters.js";
 import { rewriteAssetPath } from "@hyperframes/parsers/asset-paths";
 import {
   cleanAssetUrl,
@@ -203,12 +205,14 @@ function codecFactsFor(
   codecName: string,
   hasAlpha: boolean,
   source?: PreviewSourceFacts,
+  toneMapsToSdr = false,
 ): AssetCodecFacts {
   const hostile = hostileCodecEntry(codecName);
   return {
     codecName,
-    browserHostile: hostile !== undefined,
-    representativeMime: hostile?.representativeMime ?? null,
+    browserHostile: hostile !== undefined || toneMapsToSdr,
+    // An HDR clip is never 'playable' for preview, whatever its codec: it takes the SDR copy the export uses.
+    representativeMime: toneMapsToSdr ? null : (hostile?.representativeMime ?? null),
     hasAlpha,
     ...(source && { source }),
   };
@@ -226,7 +230,25 @@ async function probeCodecFacts(
   if (metadata.kind !== "video") return null;
   const codecName = metadata.color.codecName;
   if (!codecName) return null;
-  return codecFactsFor(codecName, metadata.hasAlpha === true, metadata.source);
+  const hasAlpha = metadata.hasAlpha === true;
+  const toneMaps = !hasAlpha && (await toneMapsHdr(metadata.color.hdrTransfer));
+  return codecFactsFor(codecName, hasAlpha, metadata.source, toneMaps);
+}
+
+let warnedNoHdrFilters = false;
+
+/** An HDR video previews through the same tone-mapped copy the SDR export uses, so the two match. */
+async function toneMapsHdr(hdrTransfer: string | null): Promise<boolean> {
+  if (hdrTransfer !== "pq" && hdrTransfer !== "hlg") return false;
+  const ffmpegPath = findFfBinary("ffmpeg");
+  if (ffmpegPath && (await ffmpegHasHdrFilters(ffmpegPath))) return true;
+  if (!warnedNoHdrFilters) {
+    warnedNoHdrFilters = true;
+    console.warn(
+      `[hyperframes] A video in this project is HDR, but this ffmpeg has no zscale/tonemap (libzimg), so the preview plays it as the browser shows it and will not match an SDR export. Install an ffmpeg with libzimg.`,
+    );
+  }
+  return false;
 }
 
 interface CachedAssetProbe {

packages/studio-server/src/helpers/proxyTranscoder.ts

@@ -14,6 +14,7 @@ import { basename, dirname, extname, isAbsolute, join, relative, resolve, sep }
 import { hdrToSdrToneMapFilter, type PreviewProxyBox } from "@hyperframes/core";
 import { findFfBinary } from "@hyperframes/parsers/ff-binaries";
 import { alphaDecoderFor } from "@hyperframes/parsers/media-alpha";
+import { ffmpegHasHdrFilters } from "./hdrFilters.js";
 import { probeFirstFrameColour, probeMediaMetadata } from "./mediaMetadata.js";
 import { cleanupProxyCache } from "./proxyCache.js";
 import {
@@ -352,30 +353,8 @@ interface RememberedFailure {
 
 const failedTranscodes = new Map<string, RememberedFailure>();
 
-let hdrFilterCheck: { ffmpegPath: string; promise: Promise<void> } | undefined;
-
-function ensureHdrFilters(ffmpegPath: string): Promise<void> {
-  if (hdrFilterCheck?.ffmpegPath === ffmpegPath) return hdrFilterCheck.promise;
-  const promise = new Promise<void>((resolveCheck, rejectCheck) => {
-    const proc = spawn(ffmpegPath, ["-hide_banner", "-filters"], {
-      stdio: ["ignore", "pipe", "pipe"],
-      windowsHide: true,
-    });
-    let stdout = "";
-    proc.stdout?.on("data", (chunk: Buffer) => {
-      stdout += chunk.toString();
-    });
-    proc.on("error", () => rejectCheck(new FfmpegMissingFilterError()));
-    proc.on("close", (code) => {
-      if (code !== 0 || !/\bzscale\b/.test(stdout) || !/\btonemap\b/.test(stdout)) {
-        rejectCheck(new FfmpegMissingFilterError());
-      } else {
-        resolveCheck();
-      }
-    });
-  });
-  hdrFilterCheck = { ffmpegPath, promise };
-  return promise;
+async function ensureHdrFilters(ffmpegPath: string): Promise<void> {
+  if (!(await ffmpegHasHdrFilters(ffmpegPath))) throw new FfmpegMissingFilterError();
 }
 
 function rememberFailure(cachePath: string, error: unknown): void {