diff --git a/packages/cli/src/commands/measure/probe.ts b/packages/cli/src/commands/measure/probe.ts index f331af2eeb..c40927be74 100644 --- a/packages/cli/src/commands/measure/probe.ts +++ b/packages/cli/src/commands/measure/probe.ts @@ -9,6 +9,7 @@ import { runFfmpeg, type VideoInfo, } from "../../measure/media.js"; +import { toneMapReferenceNits } from "@hyperframes/core"; import { execFileSync } from "node:child_process"; import { dirname, resolve } from "node:path"; import { makeOutputFolder } from "../../utils/outputFolder.js"; @@ -54,7 +55,7 @@ function normalize(info: VideoInfo, out: string): { path: string; fps: string } "install an ffmpeg built with libzimg, e.g. brew install ffmpeg", ); filters.push( - "zscale=t=linear:npl=100,format=gbrpf32le,zscale=p=bt709,tonemap=hable:desat=0,zscale=t=bt709:m=bt709:r=tv", + `zscale=t=linear:npl=${toneMapReferenceNits(info.transfer ? [info.transfer] : [])},format=gbrpf32le,zscale=p=bt709,tonemap=hable:desat=0,zscale=t=bt709:m=bt709:r=tv`, ); } makeOutputFolder(dirname(out)); diff --git a/packages/cli/src/commands/snapshot.ts b/packages/cli/src/commands/snapshot.ts index 84220342c4..d1195e4a70 100644 --- a/packages/cli/src/commands/snapshot.ts +++ b/packages/cli/src/commands/snapshot.ts @@ -71,10 +71,14 @@ import { import { firstFrameColourArgs, hdrToSdrToneMapFilter, + keyframeTransferArgs, + parseKeyframeTransfers, + type ToneMapFrameColour, isClipVisibleAt, isInClipWindow, MEDIA_RENDER_ID_ATTR, parseFirstFrameColour, + readsKeyframeTransfers, readElementRateSpec, sourceTimeAt, timeAtSourceTime, @@ -620,20 +624,26 @@ async function videoColour(ffmpegPath: string, videoPath: string) { metadata && isHdrColorSpace(metadata.colorSpace) ? hdrToSdrToneMapFilter( metadata.colorSpace ?? {}, - parseFirstFrameColour( - ( - await runFfmpegOnce( - ffmpegPath, - firstFrameColourArgs(videoPath), - FFMPEG_EXTRACT_TIMEOUT_MS, - ) - ).stderr, - ), + await frameColour(ffmpegPath, videoPath, metadata.videoCodec), ) : null; return { metadata, toneMap }; } +async function frameColour( + ffmpegPath: string, + videoPath: string, + codec: string, +): Promise { + const read = (args: string[]) => runFfmpegOnce(ffmpegPath, args, FFMPEG_EXTRACT_TIMEOUT_MS); + const probe = await read(firstFrameColourArgs(videoPath)); + const first = parseFirstFrameColour(probe.stderr); + if (probe.code !== 0 || !readsKeyframeTransfers(first)) return first; + const keyframes = await read(keyframeTransferArgs(videoPath, codec)); + const keyframeTransfers = keyframes.code === 0 ? parseKeyframeTransfers(keyframes.stderr) : []; + return { ...first, keyframeTransfers }; +} + function frameFilters( select: number | undefined, displayedPixels: number, diff --git a/packages/cli/src/measure/media.ts b/packages/cli/src/measure/media.ts index 4c8038d340..b950a4541e 100644 --- a/packages/cli/src/measure/media.ts +++ b/packages/cli/src/measure/media.ts @@ -72,6 +72,7 @@ export interface VideoInfo { /** The container's frame rate and the stream's average differ: frames are not evenly spaced. */ vfr: boolean; hdr: boolean; + transfer?: string; codec: string; } @@ -160,6 +161,7 @@ export function probeVideo(path: string): VideoInfo { avgFps: avg || fps, vfr: avg > 0 && Math.abs(avg - fps) / fps > 0.01, hdr: /smpte2084|arib-std-b67/.test(video.color_transfer ?? ""), + transfer: video.color_transfer, codec: video.codec_name ?? "", }; } diff --git a/packages/core/src/hdrToneMap.test.ts b/packages/core/src/hdrToneMap.test.ts index 7edea21386..081d27f0e6 100644 --- a/packages/core/src/hdrToneMap.test.ts +++ b/packages/core/src/hdrToneMap.test.ts @@ -1,13 +1,19 @@ +import { spawnSync } from "node:child_process"; import { describe, expect, it } from "vitest"; -import { hdrToSdrToneMapFilter } from "./hdrToneMap"; +import { + hdrToSdrToneMapFilter, + keyframeTransferArgs, + parseKeyframeTransfers, + toneMapReferenceNits, +} from "./hdrToneMap"; const PQ = { colorSpace: "bt2020nc", colorPrimaries: "bt2020", colorTransfer: "smpte2084" }; -const TONE_MAP = - "zscale=t=linear:npl=100,tonemap=hable:desat=0,zscale=p=bt709:t=bt709:m=bt709:r=tv"; +const toneMap = (npl: number) => + `zscale=t=linear:npl=${npl},tonemap=hable:desat=0,zscale=p=bt709:t=bt709:m=bt709:r=tv`; describe("hdrToSdrToneMapFilter", () => { it("leaves every frame its own tags when the first frame carries them all", () => { - expect(hdrToSdrToneMapFilter(PQ, PQ)).toBe(TONE_MAP); + expect(hdrToSdrToneMapFilter(PQ, PQ)).toBe(toneMap(100)); }); it("resizes in the first zscale, before the tone map, when it is given a size", () => { @@ -18,10 +24,10 @@ describe("hdrToSdrToneMapFilter", () => { it("sets the probed tags the first frame lacks", () => { expect(hdrToSdrToneMapFilter(PQ, {})).toBe( - `setparams=colorspace=bt2020nc:color_primaries=bt2020:color_trc=smpte2084,${TONE_MAP}`, + `setparams=colorspace=bt2020nc:color_primaries=bt2020:color_trc=smpte2084,${toneMap(100)}`, ); expect(hdrToSdrToneMapFilter(PQ, { colorTransfer: "arib-std-b67" })).toBe( - `setparams=colorspace=bt2020nc:color_primaries=bt2020,${TONE_MAP}`, + `setparams=colorspace=bt2020nc:color_primaries=bt2020,${toneMap(100)}`, ); }); @@ -31,9 +37,82 @@ describe("hdrToSdrToneMapFilter", () => { { colorSpace: "unknown", colorTransfer: "arib-std-b67" }, { colorSpace: "unknown", colorPrimaries: "unknown", colorTransfer: "arib-std-b67" }, ), - ).toBe(`setparams=colorspace=bt2020nc:color_primaries=bt2020,${TONE_MAP}`); + ).toBe(`setparams=colorspace=bt2020nc:color_primaries=bt2020,${toneMap(100)}`); expect(hdrToSdrToneMapFilter({ colorPrimaries: "reserved" }, {})).toBe( - `setparams=colorspace=bt2020nc:color_primaries=bt2020,${TONE_MAP}`, + `setparams=colorspace=bt2020nc:color_primaries=bt2020,${toneMap(100)}`, ); }); + + it("maps a stream at BT.2408's 203-nit HLG reference white only when every keyframe is HLG", () => { + const hlg = { ...PQ, colorTransfer: "arib-std-b67" }; + const keyframes = (...keyframeTransfers: string[]) => ({ ...hlg, keyframeTransfers }); + expect(hdrToSdrToneMapFilter(hlg, keyframes("arib-std-b67", "arib-std-b67"))).toBe( + toneMap(203), + ); + expect(hdrToSdrToneMapFilter(hlg, keyframes("arib-std-b67", "smpte2084"))).toBe(toneMap(100)); + expect(hdrToSdrToneMapFilter(hlg, keyframes("arib-std-b67", "bt709"))).toBe(toneMap(100)); + expect(hdrToSdrToneMapFilter(hlg, keyframes())).toBe(toneMap(100)); + const untagged = (keyframeTransfers: string[]) => ({ + colorTransfer: "unknown", + keyframeTransfers, + }); + expect(hdrToSdrToneMapFilter(hlg, untagged(["unspecified", "arib-std-b67"]))).toContain( + "npl=203", + ); + expect(hdrToSdrToneMapFilter(PQ, untagged(["unspecified"]))).toContain("npl=100"); + expect(hdrToSdrToneMapFilter(PQ, { ...PQ, keyframeTransfers: ["smpte2084"] })).toBe( + toneMap(100), + ); + expect(toneMapReferenceNits(undefined)).toBe(100); + }); + + it("names each SPS transfer trace_headers prints, and reads only keyframes' headers", () => { + const line = (code: number) => + `[trace_headers @ 0x1] 287 transfer_characteristics 00010010 = ${code}`; + expect(parseKeyframeTransfers([18, 16, 1, 2, 4].map(line).join("\n"))).toEqual([ + "arib-std-b67", + "smpte2084", + "bt709", + "unspecified", + "code 4", + ]); + expect(keyframeTransferArgs("clip.mov", "hevc")).toContain( + "filter_units=pass_types=33,trace_headers", + ); + expect(keyframeTransferArgs("clip.mov")).toEqual(expect.arrayContaining(["-discard", "nokey"])); + }); +}); + +const FFMPEG_MAJOR = Number( + /ffmpeg version n?(\d+)/.exec(spawnSync("ffmpeg", ["-version"]).stdout?.toString() ?? "")?.[1] ?? + 0, +); +const HAS_ZSCALE = /\szscale\s/.test( + spawnSync("ffmpeg", ["-hide_banner", "-filters"]).stdout?.toString() ?? "", +); + +/** The 8-bit full-range grey the chain makes of one 10-bit HLG code value. */ +function toneMappedGrey(code: number): number { + const hlg = { colorSpace: "bt2020nc", colorPrimaries: "bt2020", colorTransfer: "arib-std-b67" }; + const tags = + "setparams=color_primaries=bt2020:color_trc=arib-std-b67:colorspace=bt2020nc:range=tv"; + const run = spawnSync("ffmpeg", [ + ...["-v", "error", "-f", "lavfi", "-i", "color=c=black:s=16x16:d=0.04"], + ...[ + "-vf", + `format=yuv420p10le,geq=lum=${code}:cb=512:cr=512,${tags},${hdrToSdrToneMapFilter(hlg, { ...hlg, keyframeTransfers: ["arib-std-b67"] })}`, + ], + ...["-frames:v", "1", "-pix_fmt", "gray", "-f", "rawvideo", "pipe:1"], + ]); + const luma = run.stdout[0] ?? Number.NaN; + return Math.round(((luma - 16) * 255) / 219); +} + +// ffmpeg before 8 bundles an older zimg that maps HLG white near 174, outside the tolerance. +describe.skipIf(!HAS_ZSCALE || FFMPEG_MAJOR < 8)("hdrToSdrToneMapFilter on HLG grey", () => { + // Apple's own SDR conversion (avconvert) of these 10-bit steps gives 105 and 161; at 100 nits ours gave 142 and 216. + it("puts mid grey and reference white where Apple's SDR conversion does", () => { + expect(Math.abs(toneMappedGrey(551) - 105)).toBeLessThanOrEqual(8); + expect(Math.abs(toneMappedGrey(745) - 161)).toBeLessThanOrEqual(8); + }); }); diff --git a/packages/core/src/hdrToneMap.ts b/packages/core/src/hdrToneMap.ts index 6dac52002e..d4a4d794de 100644 --- a/packages/core/src/hdrToneMap.ts +++ b/packages/core/src/hdrToneMap.ts @@ -4,6 +4,22 @@ export interface ToneMapSourceColour { colorTransfer?: string; } +export interface ToneMapFrameColour extends ToneMapSourceColour { + keyframeTransfers?: readonly string[]; +} + +const BT2408_HLG_REFERENCE_WHITE_NITS = 203; +const SDR_NOMINAL_PEAK_NITS = 100; + +/** One zscale serves a stream: only an all-HLG one takes HLG's white, so a mixed one's PQ stays as is. */ +export const toneMapReferenceNits = (keyframeTransfers: readonly string[] | undefined): number => + keyframeTransfers?.length && keyframeTransfers.every((transfer) => transfer === "arib-std-b67") + ? BT2408_HLG_REFERENCE_WHITE_NITS + : SDR_NOMINAL_PEAK_NITS; + +export const readsKeyframeTransfers = ({ colorTransfer }: ToneMapSourceColour): boolean => + colorTransfer === "arib-std-b67" || !known(colorTransfer); + function known(value: string | undefined): string | undefined { return value && value !== "unknown" && value !== "reserved" ? value : undefined; } @@ -14,7 +30,7 @@ function known(value: string | undefined): string | undefined { */ export function hdrToSdrToneMapFilter( probed: ToneMapSourceColour, - firstFrame: ToneMapSourceColour, + firstFrame: ToneMapFrameColour, size?: { width: string; height: string }, ): string { const fill = (key: keyof ToneMapSourceColour, fallback?: string) => @@ -26,7 +42,13 @@ export function hdrToSdrToneMapFilter( ].filter(([, value]) => value); const setTags = tags.length ? `setparams=${tags.map((tag) => tag.join("=")).join(":")},` : ""; const resizeBeforeToneMap = size ? `w=${size.width}:h=${size.height}:` : ""; - return `${setTags}zscale=${resizeBeforeToneMap}t=linear:npl=100,tonemap=hable:desat=0,zscale=p=bt709:t=bt709:m=bt709:r=tv`; + const stream = known(firstFrame.colorTransfer) ?? known(probed.colorTransfer); + const npl = toneMapReferenceNits( + firstFrame.keyframeTransfers?.map((transfer) => + transfer === "unspecified" ? (stream ?? transfer) : transfer, + ), + ); + return `${setTags}zscale=${resizeBeforeToneMap}t=linear:npl=${npl},tonemap=hable:desat=0,zscale=p=bt709:t=bt709:m=bt709:r=tv`; } /** ffmpeg args printing the first shown frame's tags; ffprobe's packet-bounded reads miss edit-list pre-roll. */ @@ -37,6 +59,29 @@ export function firstFrameColourArgs(videoPath: string): string[] { ]; } +const SPS_UNIT_TYPES: Record = { h264: "7", hevc: "33" }; +const TRANSFER_NAMES: Record = { + "1": "bt709", + "2": "unspecified", + "16": "smpte2084", + "18": "arib-std-b67", +}; + +/** ffmpeg args tracing the transfer of each SPS, in the header and in keyframes, without decoding a picture. */ +export function keyframeTransferArgs(videoPath: string, codec?: string): string[] { + const sps = codec ? SPS_UNIT_TYPES[codec] : undefined; + const trace = sps ? `filter_units=pass_types=${sps},trace_headers` : "trace_headers"; + return [ + ...["-hide_banner", "-nostats", "-discard", "nokey", "-i", videoPath, "-map", "0:v:0"], + ...["-c", "copy", "-bsf:v", trace, "-f", "null", "-"], + ]; +} + +export const parseKeyframeTransfers = (stderr: string): string[] => + [...stderr.matchAll(/transfer_characteristics +[01]+ = (\d+)/g)].map( + ([, code = ""]) => TRANSFER_NAMES[code] ?? `code ${code}`, + ); + export function parseFirstFrameColour(stderr: string): ToneMapSourceColour { const tags = / color_space:(\S+) color_primaries:(\S+) color_trc:(\S+)/.exec(stderr); return tags ? { colorSpace: tags[1], colorPrimaries: tags[2], colorTransfer: tags[3] } : {}; diff --git a/packages/core/src/index.ts b/packages/core/src/index.ts index f6f6f1f76b..d7a43d2bfb 100644 --- a/packages/core/src/index.ts +++ b/packages/core/src/index.ts @@ -280,7 +280,12 @@ export { bakeHfColorGradingCube, type BakeHfColorGradingCubeOptions } from "./co export { firstFrameColourArgs, hdrToSdrToneMapFilter, + keyframeTransferArgs, + parseKeyframeTransfers, + readsKeyframeTransfers, + toneMapReferenceNits, parseFirstFrameColour, + type ToneMapFrameColour, type ToneMapSourceColour, } from "./hdrToneMap"; export { diff --git a/packages/engine/src/services/videoFrameExtractor.concurrency.test.ts b/packages/engine/src/services/videoFrameExtractor.concurrency.test.ts index 9cf03655ee..83ec550fd4 100644 --- a/packages/engine/src/services/videoFrameExtractor.concurrency.test.ts +++ b/packages/engine/src/services/videoFrameExtractor.concurrency.test.ts @@ -146,7 +146,7 @@ describe.skipIf(process.platform === "win32")("extractAllVideoFrames ffmpeg conc delete process.env.HF_TEST_SHIM_DIR; } const alive = readFileSync(join(run, "alive"), "utf8").trim().split(/\s+/).map(Number); - expect(alive).toHaveLength(8 + 2); + expect(alive).toHaveLength(8 + 2 * 2); expect(Math.max(...alive)).toBe(1); }, 60_000, diff --git a/packages/engine/src/services/videoFrameExtractor.test.ts b/packages/engine/src/services/videoFrameExtractor.test.ts index 41f7dce3fd..153e76c729 100644 --- a/packages/engine/src/services/videoFrameExtractor.test.ts +++ b/packages/engine/src/services/videoFrameExtractor.test.ts @@ -3858,6 +3858,26 @@ const FFPROBE_READS_CONTAINER_TRANSFER = (FFPROBE_RELEASE !== undefined && (FFPROBE_RELEASE[0]! > 8 || (FFPROBE_RELEASE[0] === 8 && FFPROBE_RELEASE[1]! >= 1))); +/** The colour probes run so far: first-frame reads, and SPS transfer reads of an HLG source. */ +function colourProbeRuns() { + const runs = vi + .mocked(runFfmpeg) + .mock.calls.map(([args]) => args) + .filter((args) => args.some((arg) => /showinfo|trace_headers/.test(arg))); + return { + firstFrame: runs.filter((args) => args.includes("-frames:v")).length, + keyframes: runs.filter((args) => args.includes("nokey")).length, + }; +} + +function colourProbesSince(before: ReturnType) { + const now = colourProbeRuns(); + return { + firstFrame: now.firstFrame - before.firstFrame, + keyframes: now.keyframes - before.keyframes, + }; +} + describe.skipIf(!HAS_ZSCALE)("forced-SDR HDR extraction", () => { let fixtureDir = ""; @@ -3977,8 +3997,7 @@ describe.skipIf(!HAS_ZSCALE)("forced-SDR HDR extraction", () => { it("probes each source's first frame once, however many cuts of it extract together", async () => { const sources = ["a", "b"].map((name) => join(fixtureDir, `cuts-${name}.mp4`)); for (const source of sources) await synthesizeHlgClip(source); - const probes = () => - vi.mocked(runFfmpeg).mock.calls.filter(([args]) => args.includes("showinfo")).length; + const probes = colourProbeRuns; const before = probes(); await Promise.all( Array.from({ length: 12 }, (_, cut) => @@ -3990,14 +4009,13 @@ describe.skipIf(!HAS_ZSCALE)("forced-SDR HDR extraction", () => { }), ), ); - expect(probes() - before).toBe(2); + expect(colourProbesSince(before)).toEqual({ firstFrame: 2, keyframes: 2 }); }, 60_000); it("probes a source again after its first-frame probe failed", async () => { const source = join(fixtureDir, "probe-retry.mp4"); await synthesizeHlgClip(source); - const probes = () => - vi.mocked(runFfmpeg).mock.calls.filter(([args]) => args.includes("showinfo")).length; + const probes = colourProbeRuns; const { runFfmpeg: real } = await vi.importActual("../utils/runFfmpeg.js"); vi.mocked(runFfmpeg).mockImplementation((args, opts) => @@ -4025,7 +4043,40 @@ describe.skipIf(!HAS_ZSCALE)("forced-SDR HDR extraction", () => { vi.mocked(runFfmpeg).mockImplementation(real); } await extract("retry-again"); - expect(probes() - before).toBe(2); + expect(colourProbesSince(before)).toEqual({ firstFrame: 2, keyframes: 1 }); + }, 60_000); + + it("reads an HLG source's keyframes again after that read failed", async () => { + const source = join(fixtureDir, "keyframe-retry.mp4"); + await synthesizeHlgClip(source); + const { runFfmpeg: real } = + await vi.importActual("../utils/runFfmpeg.js"); + vi.mocked(runFfmpeg).mockImplementation((args, opts) => + args.includes("nokey") + ? Promise.resolve({ + success: false, + exitCode: 1, + stderr: "", + durationMs: 0, + terminationReason: "exit", + }) + : real(args, opts), + ); + const extract = (id: string) => + extractVideoFramesRange(source, id, 0, 1, { + fps: 1, + outputDir: join(fixtureDir, "keyframe-retry"), + format: "png", + toneMapHdrToSdr: true, + }); + const before = colourProbeRuns(); + try { + await extract("keyframes-failed"); + } finally { + vi.mocked(runFfmpeg).mockImplementation(real); + } + await extract("keyframes-again"); + expect(colourProbesSince(before)).toEqual({ firstFrame: 2, keyframes: 2 }); }, 60_000); // One stream with HLG frames, then PQ frames: each is tone-mapped with its own transfer. @@ -4148,7 +4199,7 @@ describe.skipIf(!HAS_ZSCALE)("forced-SDR HDR extraction", () => { "-t", "1", "-vf", - "fps=1,zscale=t=linear:npl=100,tonemap=hable:desat=0,zscale=p=bt709:t=bt709:m=bt709:r=tv", + "fps=1,zscale=t=linear:npl=203,tonemap=hable:desat=0,zscale=p=bt709:t=bt709:m=bt709:r=tv", "-q:v", "0", "-compression_level", diff --git a/packages/engine/src/services/videoFrameExtractor.ts b/packages/engine/src/services/videoFrameExtractor.ts index 6e420ea167..14c13ebc03 100644 --- a/packages/engine/src/services/videoFrameExtractor.ts +++ b/packages/engine/src/services/videoFrameExtractor.ts @@ -38,8 +38,12 @@ import { type Fps, type FpsInput, firstFrameColourArgs, + keyframeTransferArgs, + parseKeyframeTransfers, + type ToneMapFrameColour, hdrToSdrToneMapFilter, parseFirstFrameColour, + readsKeyframeTransfers, } from "@hyperframes/core"; import { resolveReferencedStart, type RefResolverEl } from "./referenceResolver.js"; import { isKnownInactiveTimelineWindow } from "./mediaTimelineWindow.js"; @@ -948,25 +952,35 @@ export function parseImageElements(html: string): ImageElement[] { return images; } -const firstFrameColourCache = new Map< - string, - MediaProbeCacheEntry> ->(); +const firstFrameColourCache = new Map>(); + +/** A source's SPS transfers, so a PQ or SDR keyframe keeps an HLG one at 100 nits; null when the read failed. */ +function readKeyframeTransfers(videoPath: string, codec: string) { + return inExtractionSlot(() => + runFfmpeg([...urlInputArgs(videoPath), ...keyframeTransferArgs(videoPath, codec)]), + ).then((result) => (result.success ? parseKeyframeTransfers(result.stderr) : null)); +} /** The first shown frame's colour tags (see {@link firstFrameColourArgs}), probed once per source file. */ -function readFirstFrameColour(videoPath: string) { +function readFirstFrameColour(videoPath: string, codec: string) { const identity = mediaFileIdentity(videoPath); const cached = readMetadataCache(firstFrameColourCache, videoPath, identity); if (cached) return cached; // In an extraction slot, so it never adds a decoder; unsignalled, so one cancel cannot cut it short for others. const probe = inExtractionSlot(() => runFfmpeg([...urlInputArgs(videoPath), ...firstFrameColourArgs(videoPath)]), - ).then((result) => { + ).then(async (result): Promise => { + const forget = () => { + if (firstFrameColourCache.get(videoPath)?.promise === probe) + firstFrameColourCache.delete(videoPath); + }; // A failed probe answers this cut only; the next one probes again. - if (!result.success && firstFrameColourCache.get(videoPath)?.promise === probe) { - firstFrameColourCache.delete(videoPath); - } - return parseFirstFrameColour(result.stderr); + if (!result.success) forget(); + const first = parseFirstFrameColour(result.stderr); + if (!result.success || !readsKeyframeTransfers(first)) return first; + const keyframeTransfers = await readKeyframeTransfers(videoPath, codec); + if (!keyframeTransfers) forget(); + return { ...first, keyframeTransfers: keyframeTransfers ?? [] }; }); writeMetadataCache(firstFrameColourCache, videoPath, identity, probe); return probe; @@ -1159,7 +1173,7 @@ interface VideoFilterChainInput { sampleAtOutputFps: boolean; keepsHdrForChrome: boolean; /** The first frame's colour tags; set exactly when HDR is tone-mapped to SDR. */ - toneMapFrom?: Awaited>; + toneMapFrom?: ToneMapFrameColour; isHdr: boolean; sdrToHdrTransfer?: ExtractionOptions["sdrToHdrTransfer"]; shownBox?: ExtractionOptions["shownBox"]; @@ -1371,7 +1385,9 @@ async function extractVideoFramesRangeUnredacted( ); } - const frameColour = toneMappedToSdr ? await readFirstFrameColour(videoPath) : undefined; + const frameColour = toneMappedToSdr + ? await readFirstFrameColour(videoPath, metadata.videoCodec) + : undefined; const filtersFor = videoFilterChain({ metadata, format, diff --git a/packages/studio-server/src/helpers/mediaMetadata.ts b/packages/studio-server/src/helpers/mediaMetadata.ts index 0ece211446..a093fe9f27 100644 --- a/packages/studio-server/src/helpers/mediaMetadata.ts +++ b/packages/studio-server/src/helpers/mediaMetadata.ts @@ -2,9 +2,12 @@ import { execFile } from "node:child_process"; import { extname } from "node:path"; import { firstFrameColourArgs, + keyframeTransferArgs, parseFirstFrameColour, + readsKeyframeTransfers, + parseKeyframeTransfers, type PreviewSourceFacts, - type ToneMapSourceColour, + type ToneMapFrameColour, } from "@hyperframes/core"; import { findFfBinary } from "@hyperframes/parsers/ff-binaries"; import { videoStreamHasAlpha } from "@hyperframes/parsers/media-alpha"; @@ -188,8 +191,9 @@ export function classifyMediaColor(stream: FfprobeStream | null | undefined): Me /** The first shown frame's colour tags, which zscale reads per frame. Empty when unread. */ export async function probeFirstFrameColour( filePath: string, + codec?: string, runner: FfprobeRunner = execFileRunner, -): Promise { +): Promise { const ffmpegPath = findFfBinary("ffmpeg", { configuredMustExist: true }) ?? (runner === execFileRunner ? undefined : "ffmpeg"); @@ -198,7 +202,15 @@ export async function probeFirstFrameColour( timeout: 15_000, maxBuffer: 1024 * 1024, }); - return result.status === 0 ? parseFirstFrameColour(String(result.stderr)) : {}; + const first = result.status === 0 ? parseFirstFrameColour(String(result.stderr)) : {}; + if (result.status !== 0 || !readsKeyframeTransfers(first)) return first; + const keyframes = await runner(ffmpegPath, keyframeTransferArgs(filePath, codec), { + timeout: 30_000, + maxBuffer: 8 * 1024 * 1024, + }); + const keyframeTransfers = + keyframes.status === 0 ? parseKeyframeTransfers(String(keyframes.stderr)) : []; + return { ...first, keyframeTransfers }; } function bitDepthOf(stream: FfprobeStream): number { diff --git a/packages/studio-server/src/helpers/proxyTranscoder.test.ts b/packages/studio-server/src/helpers/proxyTranscoder.test.ts index fbbfb55e93..c72571bf3d 100644 --- a/packages/studio-server/src/helpers/proxyTranscoder.test.ts +++ b/packages/studio-server/src/helpers/proxyTranscoder.test.ts @@ -841,11 +841,11 @@ describe("resolveProxy", () => { ]); const proxyPath = await resolveProxy(projectDir, sourcePath); - // Reference: the render tone map's RGB, converted with BT.709. Mean luma is ~53 there; - // skipping the tone map gives ~60, a BT.601 conversion ~56. + // Reference: the render tone map's RGB, converted with BT.709. Skipping the tone map or + // converting with BT.601 each moves mean luma by several levels. const reference = meanLuma( sourcePath, - `${hdrToSdrToneMapFilter({ colorTransfer: "arib-std-b67" }, { colorTransfer: "arib-std-b67" })},format=gbrp,scale=in_range=pc:out_color_matrix=bt709:out_range=tv,format=yuv420p,`, + `${hdrToSdrToneMapFilter({ colorTransfer: "arib-std-b67" }, { colorTransfer: "arib-std-b67", keyframeTransfers: ["arib-std-b67"] })},format=gbrp,scale=in_range=pc:out_color_matrix=bt709:out_range=tv,format=yuv420p,`, ); expect(Math.abs(meanLuma(proxyPath) - reference)).toBeLessThan(1); }, diff --git a/packages/studio-server/src/helpers/proxyTranscoder.ts b/packages/studio-server/src/helpers/proxyTranscoder.ts index 2f50ddae50..559bcc2219 100644 --- a/packages/studio-server/src/helpers/proxyTranscoder.ts +++ b/packages/studio-server/src/helpers/proxyTranscoder.ts @@ -704,7 +704,9 @@ async function buildVideoFilter( const { hdrTransfer } = metadata.color; const toneMap = (hdrTransfer === "pq" || hdrTransfer === "hlg") && !keepsAlpha; if (toneMap) await ensureHdrFilters(ffmpegPath); - const firstFrame = toneMap ? await probeFirstFrameColour(sourcePath) : {}; + const firstFrame = toneMap + ? await probeFirstFrameColour(sourcePath, metadata.color.codecName) + : {}; const evenScale = proxyScaleFilter(box); const pixelFormat = keepsAlpha ? "yuva420p" : "yuv420p"; // The tone map ends in RGB; older ffmpeg (seen on 5.1) converts it with BT.601 unless the matrix is named.