diff --git a/packages/cli/src/commands/snapshot.ts b/packages/cli/src/commands/snapshot.ts index 5dd517e22e..1422e098f8 100644 --- a/packages/cli/src/commands/snapshot.ts +++ b/packages/cli/src/commands/snapshot.ts @@ -3,6 +3,7 @@ import { buildZeroDurationDiagnostic, evaluateHfDiagnostic, extractMediaMetadata, + isHdrColorSpace, sdrCanvasColourFilters, } from "@hyperframes/engine"; import { parseTimeoutMs } from "../utils/timeoutArg.js"; @@ -57,8 +58,11 @@ import { type ZoomTarget, } from "../capture/captureCompositionFrame.js"; import { + firstFrameColourArgs, + hdrToSdrToneMapFilter, isClipVisibleAt, isInClipWindow, + parseFirstFrameColour, readElementRateSpec, sourceTimeAt, timeAtSourceTime, @@ -459,6 +463,16 @@ function warnColourUnknownOnce(): void { ); } +let warnedNoHdrFilters = false; + +function warnNoHdrFiltersOnce(): void { + if (warnedNoHdrFilters) return; + warnedNoHdrFilters = true; + console.warn( + ` ${c.warn("⚠")} A video in this project is HDR, but this ffmpeg has no zscale/tonemap (libzimg), so the still shows it as the browser does and will not match an SDR export. Install an ffmpeg with libzimg.`, + ); +} + class FrameExtractTimeoutError extends Error {} export async function extractVideoFrameToBuffer( @@ -478,24 +492,50 @@ export async function extractVideoFrameToBuffer( holdLastFrame, ); if (seek === null) return null; - const args = ["-hide_banner", "-loglevel", "error"]; - if (useVp9AlphaDecoder) args.push("-c:v", "libvpx-vp9"); - if (seek.seek !== undefined) args.push("-ss", String(seek.seek)); - args.push("-i", videoPath, "-map", "0:v:0"); - const filters: string[] = []; - if (seek.select !== undefined) filters.push(`select=gte(t-start_t\\,${seek.select})`); const metadata = findFFprobe() ? await extractMediaMetadata(videoPath) : null; - if (metadata) filters.push(...sdrCanvasColourFilters(metadata)); - else warnColourUnknownOnce(); - // Shrink so the short side still covers the displayed box at any rotation or fit; never enlarge. - if (displayedPixels > 0) { - filters.push( - `scale='min(iw,${displayedPixels})':'min(ih,${displayedPixels})':force_original_aspect_ratio=increase`, - ); + if (!metadata) warnColourUnknownOnce(); + // HDR footage takes the tone map the SDR export uses, so a still matches the render. + const toneMap = + metadata && isHdrColorSpace(metadata.colorSpace) + ? hdrToSdrToneMapFilter( + metadata.colorSpace ?? {}, + parseFirstFrameColour( + ( + await runFfmpegOnce( + ffmpegPath, + firstFrameColourArgs(videoPath), + FFMPEG_EXTRACT_TIMEOUT_MS, + ) + ).stderr, + ), + ) + : null; + const frameArgs = (toneMapFilter: string | null): string[] => { + const args = ["-hide_banner", "-loglevel", "error"]; + if (useVp9AlphaDecoder) args.push("-c:v", "libvpx-vp9"); + if (seek.seek !== undefined) args.push("-ss", String(seek.seek)); + args.push("-i", videoPath, "-map", "0:v:0"); + const filters: string[] = []; + if (seek.select !== undefined) filters.push(`select=gte(t-start_t\\,${seek.select})`); + if (metadata) { + if (toneMapFilter) filters.push(toneMapFilter); + filters.push(...sdrCanvasColourFilters(metadata, toneMapFilter !== null)); + } + // Shrink so the short side still covers the displayed box at any rotation or fit; never enlarge. + if (displayedPixels > 0) { + filters.push( + `scale='min(iw,${displayedPixels})':'min(ih,${displayedPixels})':force_original_aspect_ratio=increase`, + ); + } + if (filters.length > 0) args.push("-vf", filters.join(",")); + args.push("-frames:v", "1", "-q:v", "2", "-y", outPath); + return args; + }; + let result = await runFfmpegOnce(ffmpegPath, frameArgs(toneMap), FFMPEG_EXTRACT_TIMEOUT_MS); + if (toneMap && result.code !== 0 && /no such filter/i.test(result.stderr)) { + warnNoHdrFiltersOnce(); + result = await runFfmpegOnce(ffmpegPath, frameArgs(null), FFMPEG_EXTRACT_TIMEOUT_MS); } - if (filters.length > 0) args.push("-vf", filters.join(",")); - args.push("-frames:v", "1", "-q:v", "2", "-y", outPath); - const result = await runFfmpegOnce(ffmpegPath, args, FFMPEG_EXTRACT_TIMEOUT_MS); if (result.timedOut) { throw new FrameExtractTimeoutError( `FFmpeg timed out after ${FFMPEG_EXTRACT_TIMEOUT_MS / 1000}s extracting the frame of ${videoPath} at ${timeSeconds.toFixed(2)}s`, diff --git a/packages/cli/src/commands/snapshotHdrFrame.test.ts b/packages/cli/src/commands/snapshotHdrFrame.test.ts new file mode 100644 index 0000000000..12e453cd67 --- /dev/null +++ b/packages/cli/src/commands/snapshotHdrFrame.test.ts @@ -0,0 +1,53 @@ +import { spawnSync } from "node:child_process"; +import { mkdtempSync, writeFileSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { describe, expect, it } from "vitest"; +import { extractVideoFrameToBuffer } from "./snapshot.js"; + +const hasZscale = + spawnSync("ffmpeg", ["-hide_banner", "-filters"], { encoding: "utf8" }).stdout?.includes( + "zscale", + ) === true; + +function hlgClip(dir: string): string { + const clip = join(dir, "clip.mp4"); + const made = spawnSync( + "ffmpeg", + ["-v", "error", "-y", "-f", "lavfi", "-i", "smptehdbars=size=320x180:rate=10:duration=1"] + .concat(["-pix_fmt", "yuv420p", "-c:v", "libx264", "-crf", "12"]) + .concat(["-color_primaries", "bt2020", "-color_trc", "arib-std-b67"]) + .concat(["-colorspace", "bt2020nc", "-color_range", "tv", clip]), + ); + expect(made.status).toBe(0); + return clip; +} + +describe.runIf(hasZscale)("a snapshot still of HDR footage", () => { + it("is tone-mapped to SDR like the export, not left tagged HLG for the browser to map", async () => { + const dir = mkdtempSync(join(tmpdir(), "hf-snapshot-hdr-")); + const buffer = await extractVideoFrameToBuffer(hlgClip(dir), 0.5); + expect(buffer).not.toBeNull(); + const png = join(dir, "still.png"); + writeFileSync(png, buffer!); + const tags = spawnSync( + "ffprobe", + ["-v", "error", "-show_entries", "stream=color_transfer", "-of", "csv=p=0", "--", png], + { encoding: "utf8" }, + ).stdout.trim(); + expect(tags).not.toBe("arib-std-b67"); + const rgb = spawnSync( + "ffmpeg", + ["-v", "error", "-i", png, "-vf", "crop=1:1:25:75,format=rgb24"].concat([ + "-frames:v", + "1", + "-f", + "rawvideo", + "-", + ]), + ).stdout; + // The 40% grey bar: the export's tone map gives about 102 (an untouched HLG decode gives the same + // number but stays tagged HLG, which the browser then maps darker). + expect(Math.abs(rgb[0]! - 102)).toBeLessThanOrEqual(8); + }, 60_000); +}); diff --git a/packages/studio-server/src/helpers/hdrFilters.ts b/packages/studio-server/src/helpers/hdrFilters.ts new file mode 100644 index 0000000000..e4b67a4e08 --- /dev/null +++ b/packages/studio-server/src/helpers/hdrFilters.ts @@ -0,0 +1,25 @@ +import { spawn } from "node:child_process"; + +const checks = new Map>(); + +/** Whether this ffmpeg has the zscale and tonemap filters (libzimg) that the HDR to SDR copy needs. */ +export function ffmpegHasHdrFilters(ffmpegPath: string): Promise { + const known = checks.get(ffmpegPath); + if (known) return known; + const check = new Promise((resolveCheck) => { + const proc = spawn(ffmpegPath, ["-hide_banner", "-filters"], { + stdio: ["ignore", "pipe", "pipe"], + windowsHide: true, + }); + let stdout = ""; + proc.stdout?.on("data", (chunk: Buffer) => { + stdout += chunk.toString(); + }); + proc.on("error", () => resolveCheck(false)); + proc.on("close", (code) => + resolveCheck(code === 0 && /\bzscale\b/.test(stdout) && /\btonemap\b/.test(stdout)), + ); + }); + checks.set(ffmpegPath, check); + return check; +} diff --git a/packages/studio-server/src/helpers/mediaCodecMap.test.ts b/packages/studio-server/src/helpers/mediaCodecMap.test.ts index 501339209d..93c0b09d84 100644 --- a/packages/studio-server/src/helpers/mediaCodecMap.test.ts +++ b/packages/studio-server/src/helpers/mediaCodecMap.test.ts @@ -1,4 +1,4 @@ -import { afterEach, beforeEach, describe, expect, it } from "vitest"; +import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; import { mkdirSync, mkdtempSync, rmSync, utimesSync, writeFileSync } from "node:fs"; import { join } from "node:path"; import { tmpdir } from "node:os"; @@ -32,6 +32,7 @@ function makeRunner( tags?: Record; size?: [number, number]; bitRate?: string; + colorTransfer?: string; } >, ): FfprobeRunner { @@ -49,6 +50,8 @@ function makeRunner( width: normalized.size?.[0], height: normalized.size?.[1], bit_rate: normalized.bitRate, + color_transfer: normalized.colorTransfer, + color_primaries: normalized.colorTransfer ? "bt2020" : undefined, }, ] : []; @@ -558,3 +561,57 @@ describe("scanProjectMediaCodecMap", () => { expect(cache.has(join(project, "clip-512.mp4"))).toBe(true); }); }); + +describe("HDR video previews through the SDR tone-mapped copy", () => { + const hlgClip = { codecName: "h264", pixFmt: "yuv420p", colorTransfer: "arib-std-b67" }; + + async function probe(entry: Parameters[0][string], hasFilters: boolean) { + vi.resetModules(); + vi.doMock("./hdrFilters.js", () => ({ ffmpegHasHdrFilters: async () => hasFilters })); + vi.doMock("@hyperframes/parsers/ff-binaries", () => ({ + findFfBinary: (name: string) => (name === "ffprobe" ? FAKE_FFPROBE_PATH : "/fake/ffmpeg"), + })); + process.env.HYPERFRAMES_FFPROBE_PATH = FAKE_FFPROBE_PATH; + const fresh = await import("./mediaCodecMap.js"); + const filePath = join(mkdtempSync(join(tmpdir(), "hf-hdr-clip-")), "clip.mp4"); + writeFileSync(filePath, "x"); + return fresh.probeAssetCodec( + filePath, + makeRunner({ [filePath]: entry }), + fresh.createMediaCodecProbeCache(), + ); + } + + afterEach(() => { + vi.doUnmock("./hdrFilters.js"); + vi.doUnmock("@hyperframes/parsers/ff-binaries"); + vi.restoreAllMocks(); + }); + + it("marks an HLG H.264 clip browser-hostile so it gets the same copy the export uses", async () => { + const facts = await probe(hlgClip, true); + expect(facts?.browserHostile).toBe(true); + expect(decideMediaProxyEligibility(facts)).toEqual({ eligible: true }); + }); + + it("treats an HDR clip as not playable whatever its codec, so a browser that decodes HEVC still swaps it", async () => { + const facts = await probe({ ...hlgClip, codecName: "hevc" }, true); + expect(facts?.browserHostile).toBe(true); + expect(facts?.representativeMime).toBeNull(); + const vp9 = await probe({ ...hlgClip, codecName: "vp9" }, true); + expect(vp9?.representativeMime).toBeNull(); + }); + + it("leaves an SDR H.264 clip native", async () => { + const facts = await probe({ codecName: "h264", pixFmt: "yuv420p" }, true); + expect(facts?.browserHostile).toBe(false); + }); + + it("plays the clip natively and says why once when ffmpeg has no zscale/tonemap", async () => { + const warn = vi.spyOn(console, "warn").mockImplementation(() => undefined); + const facts = await probe(hlgClip, false); + expect(facts?.browserHostile).toBe(false); + expect(warn).toHaveBeenCalledOnce(); + expect(warn.mock.calls[0]![0]).toContain("libzimg"); + }); +}); diff --git a/packages/studio-server/src/helpers/mediaCodecMap.ts b/packages/studio-server/src/helpers/mediaCodecMap.ts index 32507da35c..e5553b9072 100644 --- a/packages/studio-server/src/helpers/mediaCodecMap.ts +++ b/packages/studio-server/src/helpers/mediaCodecMap.ts @@ -2,6 +2,8 @@ import { scanHtmlOpeningTags, decodeAuthoredAttribute } from "@hyperframes/parse import { statSync } from "node:fs"; import { realpath } from "./safePath.js"; import { sep } from "node:path"; +import { findFfBinary } from "@hyperframes/parsers/ff-binaries"; +import { ffmpegHasHdrFilters } from "./hdrFilters.js"; import { rewriteAssetPath } from "@hyperframes/parsers/asset-paths"; import { cleanAssetUrl, @@ -203,12 +205,14 @@ function codecFactsFor( codecName: string, hasAlpha: boolean, source?: PreviewSourceFacts, + toneMapsToSdr = false, ): AssetCodecFacts { const hostile = hostileCodecEntry(codecName); return { codecName, - browserHostile: hostile !== undefined, - representativeMime: hostile?.representativeMime ?? null, + browserHostile: hostile !== undefined || toneMapsToSdr, + // An HDR clip is never 'playable' for preview, whatever its codec: it takes the SDR copy the export uses. + representativeMime: toneMapsToSdr ? null : (hostile?.representativeMime ?? null), hasAlpha, ...(source && { source }), }; @@ -226,7 +230,25 @@ async function probeCodecFacts( if (metadata.kind !== "video") return null; const codecName = metadata.color.codecName; if (!codecName) return null; - return codecFactsFor(codecName, metadata.hasAlpha === true, metadata.source); + const hasAlpha = metadata.hasAlpha === true; + const toneMaps = !hasAlpha && (await toneMapsHdr(metadata.color.hdrTransfer)); + return codecFactsFor(codecName, hasAlpha, metadata.source, toneMaps); +} + +let warnedNoHdrFilters = false; + +/** An HDR video previews through the same tone-mapped copy the SDR export uses, so the two match. */ +async function toneMapsHdr(hdrTransfer: string | null): Promise { + if (hdrTransfer !== "pq" && hdrTransfer !== "hlg") return false; + const ffmpegPath = findFfBinary("ffmpeg"); + if (ffmpegPath && (await ffmpegHasHdrFilters(ffmpegPath))) return true; + if (!warnedNoHdrFilters) { + warnedNoHdrFilters = true; + console.warn( + `[hyperframes] A video in this project is HDR, but this ffmpeg has no zscale/tonemap (libzimg), so the preview plays it as the browser shows it and will not match an SDR export. Install an ffmpeg with libzimg.`, + ); + } + return false; } interface CachedAssetProbe { diff --git a/packages/studio-server/src/helpers/proxyTranscoder.ts b/packages/studio-server/src/helpers/proxyTranscoder.ts index 7022107e6a..24c0690d28 100644 --- a/packages/studio-server/src/helpers/proxyTranscoder.ts +++ b/packages/studio-server/src/helpers/proxyTranscoder.ts @@ -14,6 +14,7 @@ import { basename, dirname, extname, isAbsolute, join, relative, resolve, sep } import { hdrToSdrToneMapFilter, type PreviewProxyBox } from "@hyperframes/core"; import { findFfBinary } from "@hyperframes/parsers/ff-binaries"; import { alphaDecoderFor } from "@hyperframes/parsers/media-alpha"; +import { ffmpegHasHdrFilters } from "./hdrFilters.js"; import { probeFirstFrameColour, probeMediaMetadata } from "./mediaMetadata.js"; import { cleanupProxyCache } from "./proxyCache.js"; import { @@ -352,30 +353,8 @@ interface RememberedFailure { const failedTranscodes = new Map(); -let hdrFilterCheck: { ffmpegPath: string; promise: Promise } | undefined; - -function ensureHdrFilters(ffmpegPath: string): Promise { - if (hdrFilterCheck?.ffmpegPath === ffmpegPath) return hdrFilterCheck.promise; - const promise = new Promise((resolveCheck, rejectCheck) => { - const proc = spawn(ffmpegPath, ["-hide_banner", "-filters"], { - stdio: ["ignore", "pipe", "pipe"], - windowsHide: true, - }); - let stdout = ""; - proc.stdout?.on("data", (chunk: Buffer) => { - stdout += chunk.toString(); - }); - proc.on("error", () => rejectCheck(new FfmpegMissingFilterError())); - proc.on("close", (code) => { - if (code !== 0 || !/\bzscale\b/.test(stdout) || !/\btonemap\b/.test(stdout)) { - rejectCheck(new FfmpegMissingFilterError()); - } else { - resolveCheck(); - } - }); - }); - hdrFilterCheck = { ffmpegPath, promise }; - return promise; +async function ensureHdrFilters(ffmpegPath: string): Promise { + if (!(await ffmpegHasHdrFilters(ffmpegPath))) throw new FfmpegMissingFilterError(); } function rememberFailure(cachePath: string, error: unknown): void {