mirror of
https://github.com/heygen-com/hyperframes.git
synced 2026-11-18 07:50:13 +00:00
fix(core): hlg footage tone-maps to sdr at the bt.2408 reference white (#7234)
* fix(core): hlg footage tone-maps to sdr at the bt.2408 reference white * fix(core): an HLG clip whose stream leaves its transfer unset keeps HLG's white An SPS with an unspecified transfer takes the stream's tag, as its frames do; a failed keyframe read is not cached; the grey test needs ffmpeg 8.
This commit is contained in:
@@ -9,6 +9,7 @@ import {
|
||||
runFfmpeg,
|
||||
type VideoInfo,
|
||||
} from "../../measure/media.js";
|
||||
import { toneMapReferenceNits } from "@hyperframes/core";
|
||||
import { execFileSync } from "node:child_process";
|
||||
import { dirname, resolve } from "node:path";
|
||||
import { makeOutputFolder } from "../../utils/outputFolder.js";
|
||||
@@ -54,7 +55,7 @@ function normalize(info: VideoInfo, out: string): { path: string; fps: string }
|
||||
"install an ffmpeg built with libzimg, e.g. brew install ffmpeg",
|
||||
);
|
||||
filters.push(
|
||||
"zscale=t=linear:npl=100,format=gbrpf32le,zscale=p=bt709,tonemap=hable:desat=0,zscale=t=bt709:m=bt709:r=tv",
|
||||
`zscale=t=linear:npl=${toneMapReferenceNits(info.transfer ? [info.transfer] : [])},format=gbrpf32le,zscale=p=bt709,tonemap=hable:desat=0,zscale=t=bt709:m=bt709:r=tv`,
|
||||
);
|
||||
}
|
||||
makeOutputFolder(dirname(out));
|
||||
|
||||
@@ -71,10 +71,14 @@ import {
|
||||
import {
|
||||
firstFrameColourArgs,
|
||||
hdrToSdrToneMapFilter,
|
||||
keyframeTransferArgs,
|
||||
parseKeyframeTransfers,
|
||||
type ToneMapFrameColour,
|
||||
isClipVisibleAt,
|
||||
isInClipWindow,
|
||||
MEDIA_RENDER_ID_ATTR,
|
||||
parseFirstFrameColour,
|
||||
readsKeyframeTransfers,
|
||||
readElementRateSpec,
|
||||
sourceTimeAt,
|
||||
timeAtSourceTime,
|
||||
@@ -620,20 +624,26 @@ async function videoColour(ffmpegPath: string, videoPath: string) {
|
||||
metadata && isHdrColorSpace(metadata.colorSpace)
|
||||
? hdrToSdrToneMapFilter(
|
||||
metadata.colorSpace ?? {},
|
||||
parseFirstFrameColour(
|
||||
(
|
||||
await runFfmpegOnce(
|
||||
ffmpegPath,
|
||||
firstFrameColourArgs(videoPath),
|
||||
FFMPEG_EXTRACT_TIMEOUT_MS,
|
||||
)
|
||||
).stderr,
|
||||
),
|
||||
await frameColour(ffmpegPath, videoPath, metadata.videoCodec),
|
||||
)
|
||||
: null;
|
||||
return { metadata, toneMap };
|
||||
}
|
||||
|
||||
async function frameColour(
|
||||
ffmpegPath: string,
|
||||
videoPath: string,
|
||||
codec: string,
|
||||
): Promise<ToneMapFrameColour> {
|
||||
const read = (args: string[]) => runFfmpegOnce(ffmpegPath, args, FFMPEG_EXTRACT_TIMEOUT_MS);
|
||||
const probe = await read(firstFrameColourArgs(videoPath));
|
||||
const first = parseFirstFrameColour(probe.stderr);
|
||||
if (probe.code !== 0 || !readsKeyframeTransfers(first)) return first;
|
||||
const keyframes = await read(keyframeTransferArgs(videoPath, codec));
|
||||
const keyframeTransfers = keyframes.code === 0 ? parseKeyframeTransfers(keyframes.stderr) : [];
|
||||
return { ...first, keyframeTransfers };
|
||||
}
|
||||
|
||||
function frameFilters(
|
||||
select: number | undefined,
|
||||
displayedPixels: number,
|
||||
|
||||
@@ -72,6 +72,7 @@ export interface VideoInfo {
|
||||
/** The container's frame rate and the stream's average differ: frames are not evenly spaced. */
|
||||
vfr: boolean;
|
||||
hdr: boolean;
|
||||
transfer?: string;
|
||||
codec: string;
|
||||
}
|
||||
|
||||
@@ -160,6 +161,7 @@ export function probeVideo(path: string): VideoInfo {
|
||||
avgFps: avg || fps,
|
||||
vfr: avg > 0 && Math.abs(avg - fps) / fps > 0.01,
|
||||
hdr: /smpte2084|arib-std-b67/.test(video.color_transfer ?? ""),
|
||||
transfer: video.color_transfer,
|
||||
codec: video.codec_name ?? "",
|
||||
};
|
||||
}
|
||||
|
||||
@@ -1,13 +1,19 @@
|
||||
import { spawnSync } from "node:child_process";
|
||||
import { describe, expect, it } from "vitest";
|
||||
import { hdrToSdrToneMapFilter } from "./hdrToneMap";
|
||||
import {
|
||||
hdrToSdrToneMapFilter,
|
||||
keyframeTransferArgs,
|
||||
parseKeyframeTransfers,
|
||||
toneMapReferenceNits,
|
||||
} from "./hdrToneMap";
|
||||
|
||||
const PQ = { colorSpace: "bt2020nc", colorPrimaries: "bt2020", colorTransfer: "smpte2084" };
|
||||
const TONE_MAP =
|
||||
"zscale=t=linear:npl=100,tonemap=hable:desat=0,zscale=p=bt709:t=bt709:m=bt709:r=tv";
|
||||
const toneMap = (npl: number) =>
|
||||
`zscale=t=linear:npl=${npl},tonemap=hable:desat=0,zscale=p=bt709:t=bt709:m=bt709:r=tv`;
|
||||
|
||||
describe("hdrToSdrToneMapFilter", () => {
|
||||
it("leaves every frame its own tags when the first frame carries them all", () => {
|
||||
expect(hdrToSdrToneMapFilter(PQ, PQ)).toBe(TONE_MAP);
|
||||
expect(hdrToSdrToneMapFilter(PQ, PQ)).toBe(toneMap(100));
|
||||
});
|
||||
|
||||
it("resizes in the first zscale, before the tone map, when it is given a size", () => {
|
||||
@@ -18,10 +24,10 @@ describe("hdrToSdrToneMapFilter", () => {
|
||||
|
||||
it("sets the probed tags the first frame lacks", () => {
|
||||
expect(hdrToSdrToneMapFilter(PQ, {})).toBe(
|
||||
`setparams=colorspace=bt2020nc:color_primaries=bt2020:color_trc=smpte2084,${TONE_MAP}`,
|
||||
`setparams=colorspace=bt2020nc:color_primaries=bt2020:color_trc=smpte2084,${toneMap(100)}`,
|
||||
);
|
||||
expect(hdrToSdrToneMapFilter(PQ, { colorTransfer: "arib-std-b67" })).toBe(
|
||||
`setparams=colorspace=bt2020nc:color_primaries=bt2020,${TONE_MAP}`,
|
||||
`setparams=colorspace=bt2020nc:color_primaries=bt2020,${toneMap(100)}`,
|
||||
);
|
||||
});
|
||||
|
||||
@@ -31,9 +37,82 @@ describe("hdrToSdrToneMapFilter", () => {
|
||||
{ colorSpace: "unknown", colorTransfer: "arib-std-b67" },
|
||||
{ colorSpace: "unknown", colorPrimaries: "unknown", colorTransfer: "arib-std-b67" },
|
||||
),
|
||||
).toBe(`setparams=colorspace=bt2020nc:color_primaries=bt2020,${TONE_MAP}`);
|
||||
).toBe(`setparams=colorspace=bt2020nc:color_primaries=bt2020,${toneMap(100)}`);
|
||||
expect(hdrToSdrToneMapFilter({ colorPrimaries: "reserved" }, {})).toBe(
|
||||
`setparams=colorspace=bt2020nc:color_primaries=bt2020,${TONE_MAP}`,
|
||||
`setparams=colorspace=bt2020nc:color_primaries=bt2020,${toneMap(100)}`,
|
||||
);
|
||||
});
|
||||
|
||||
it("maps a stream at BT.2408's 203-nit HLG reference white only when every keyframe is HLG", () => {
|
||||
const hlg = { ...PQ, colorTransfer: "arib-std-b67" };
|
||||
const keyframes = (...keyframeTransfers: string[]) => ({ ...hlg, keyframeTransfers });
|
||||
expect(hdrToSdrToneMapFilter(hlg, keyframes("arib-std-b67", "arib-std-b67"))).toBe(
|
||||
toneMap(203),
|
||||
);
|
||||
expect(hdrToSdrToneMapFilter(hlg, keyframes("arib-std-b67", "smpte2084"))).toBe(toneMap(100));
|
||||
expect(hdrToSdrToneMapFilter(hlg, keyframes("arib-std-b67", "bt709"))).toBe(toneMap(100));
|
||||
expect(hdrToSdrToneMapFilter(hlg, keyframes())).toBe(toneMap(100));
|
||||
const untagged = (keyframeTransfers: string[]) => ({
|
||||
colorTransfer: "unknown",
|
||||
keyframeTransfers,
|
||||
});
|
||||
expect(hdrToSdrToneMapFilter(hlg, untagged(["unspecified", "arib-std-b67"]))).toContain(
|
||||
"npl=203",
|
||||
);
|
||||
expect(hdrToSdrToneMapFilter(PQ, untagged(["unspecified"]))).toContain("npl=100");
|
||||
expect(hdrToSdrToneMapFilter(PQ, { ...PQ, keyframeTransfers: ["smpte2084"] })).toBe(
|
||||
toneMap(100),
|
||||
);
|
||||
expect(toneMapReferenceNits(undefined)).toBe(100);
|
||||
});
|
||||
|
||||
it("names each SPS transfer trace_headers prints, and reads only keyframes' headers", () => {
|
||||
const line = (code: number) =>
|
||||
`[trace_headers @ 0x1] 287 transfer_characteristics 00010010 = ${code}`;
|
||||
expect(parseKeyframeTransfers([18, 16, 1, 2, 4].map(line).join("\n"))).toEqual([
|
||||
"arib-std-b67",
|
||||
"smpte2084",
|
||||
"bt709",
|
||||
"unspecified",
|
||||
"code 4",
|
||||
]);
|
||||
expect(keyframeTransferArgs("clip.mov", "hevc")).toContain(
|
||||
"filter_units=pass_types=33,trace_headers",
|
||||
);
|
||||
expect(keyframeTransferArgs("clip.mov")).toEqual(expect.arrayContaining(["-discard", "nokey"]));
|
||||
});
|
||||
});
|
||||
|
||||
const FFMPEG_MAJOR = Number(
|
||||
/ffmpeg version n?(\d+)/.exec(spawnSync("ffmpeg", ["-version"]).stdout?.toString() ?? "")?.[1] ??
|
||||
0,
|
||||
);
|
||||
const HAS_ZSCALE = /\szscale\s/.test(
|
||||
spawnSync("ffmpeg", ["-hide_banner", "-filters"]).stdout?.toString() ?? "",
|
||||
);
|
||||
|
||||
/** The 8-bit full-range grey the chain makes of one 10-bit HLG code value. */
|
||||
function toneMappedGrey(code: number): number {
|
||||
const hlg = { colorSpace: "bt2020nc", colorPrimaries: "bt2020", colorTransfer: "arib-std-b67" };
|
||||
const tags =
|
||||
"setparams=color_primaries=bt2020:color_trc=arib-std-b67:colorspace=bt2020nc:range=tv";
|
||||
const run = spawnSync("ffmpeg", [
|
||||
...["-v", "error", "-f", "lavfi", "-i", "color=c=black:s=16x16:d=0.04"],
|
||||
...[
|
||||
"-vf",
|
||||
`format=yuv420p10le,geq=lum=${code}:cb=512:cr=512,${tags},${hdrToSdrToneMapFilter(hlg, { ...hlg, keyframeTransfers: ["arib-std-b67"] })}`,
|
||||
],
|
||||
...["-frames:v", "1", "-pix_fmt", "gray", "-f", "rawvideo", "pipe:1"],
|
||||
]);
|
||||
const luma = run.stdout[0] ?? Number.NaN;
|
||||
return Math.round(((luma - 16) * 255) / 219);
|
||||
}
|
||||
|
||||
// ffmpeg before 8 bundles an older zimg that maps HLG white near 174, outside the tolerance.
|
||||
describe.skipIf(!HAS_ZSCALE || FFMPEG_MAJOR < 8)("hdrToSdrToneMapFilter on HLG grey", () => {
|
||||
// Apple's own SDR conversion (avconvert) of these 10-bit steps gives 105 and 161; at 100 nits ours gave 142 and 216.
|
||||
it("puts mid grey and reference white where Apple's SDR conversion does", () => {
|
||||
expect(Math.abs(toneMappedGrey(551) - 105)).toBeLessThanOrEqual(8);
|
||||
expect(Math.abs(toneMappedGrey(745) - 161)).toBeLessThanOrEqual(8);
|
||||
});
|
||||
});
|
||||
|
||||
@@ -4,6 +4,22 @@ export interface ToneMapSourceColour {
|
||||
colorTransfer?: string;
|
||||
}
|
||||
|
||||
export interface ToneMapFrameColour extends ToneMapSourceColour {
|
||||
keyframeTransfers?: readonly string[];
|
||||
}
|
||||
|
||||
const BT2408_HLG_REFERENCE_WHITE_NITS = 203;
|
||||
const SDR_NOMINAL_PEAK_NITS = 100;
|
||||
|
||||
/** One zscale serves a stream: only an all-HLG one takes HLG's white, so a mixed one's PQ stays as is. */
|
||||
export const toneMapReferenceNits = (keyframeTransfers: readonly string[] | undefined): number =>
|
||||
keyframeTransfers?.length && keyframeTransfers.every((transfer) => transfer === "arib-std-b67")
|
||||
? BT2408_HLG_REFERENCE_WHITE_NITS
|
||||
: SDR_NOMINAL_PEAK_NITS;
|
||||
|
||||
export const readsKeyframeTransfers = ({ colorTransfer }: ToneMapSourceColour): boolean =>
|
||||
colorTransfer === "arib-std-b67" || !known(colorTransfer);
|
||||
|
||||
function known(value: string | undefined): string | undefined {
|
||||
return value && value !== "unknown" && value !== "reserved" ? value : undefined;
|
||||
}
|
||||
@@ -14,7 +30,7 @@ function known(value: string | undefined): string | undefined {
|
||||
*/
|
||||
export function hdrToSdrToneMapFilter(
|
||||
probed: ToneMapSourceColour,
|
||||
firstFrame: ToneMapSourceColour,
|
||||
firstFrame: ToneMapFrameColour,
|
||||
size?: { width: string; height: string },
|
||||
): string {
|
||||
const fill = (key: keyof ToneMapSourceColour, fallback?: string) =>
|
||||
@@ -26,7 +42,13 @@ export function hdrToSdrToneMapFilter(
|
||||
].filter(([, value]) => value);
|
||||
const setTags = tags.length ? `setparams=${tags.map((tag) => tag.join("=")).join(":")},` : "";
|
||||
const resizeBeforeToneMap = size ? `w=${size.width}:h=${size.height}:` : "";
|
||||
return `${setTags}zscale=${resizeBeforeToneMap}t=linear:npl=100,tonemap=hable:desat=0,zscale=p=bt709:t=bt709:m=bt709:r=tv`;
|
||||
const stream = known(firstFrame.colorTransfer) ?? known(probed.colorTransfer);
|
||||
const npl = toneMapReferenceNits(
|
||||
firstFrame.keyframeTransfers?.map((transfer) =>
|
||||
transfer === "unspecified" ? (stream ?? transfer) : transfer,
|
||||
),
|
||||
);
|
||||
return `${setTags}zscale=${resizeBeforeToneMap}t=linear:npl=${npl},tonemap=hable:desat=0,zscale=p=bt709:t=bt709:m=bt709:r=tv`;
|
||||
}
|
||||
|
||||
/** ffmpeg args printing the first shown frame's tags; ffprobe's packet-bounded reads miss edit-list pre-roll. */
|
||||
@@ -37,6 +59,29 @@ export function firstFrameColourArgs(videoPath: string): string[] {
|
||||
];
|
||||
}
|
||||
|
||||
const SPS_UNIT_TYPES: Record<string, string> = { h264: "7", hevc: "33" };
|
||||
const TRANSFER_NAMES: Record<string, string> = {
|
||||
"1": "bt709",
|
||||
"2": "unspecified",
|
||||
"16": "smpte2084",
|
||||
"18": "arib-std-b67",
|
||||
};
|
||||
|
||||
/** ffmpeg args tracing the transfer of each SPS, in the header and in keyframes, without decoding a picture. */
|
||||
export function keyframeTransferArgs(videoPath: string, codec?: string): string[] {
|
||||
const sps = codec ? SPS_UNIT_TYPES[codec] : undefined;
|
||||
const trace = sps ? `filter_units=pass_types=${sps},trace_headers` : "trace_headers";
|
||||
return [
|
||||
...["-hide_banner", "-nostats", "-discard", "nokey", "-i", videoPath, "-map", "0:v:0"],
|
||||
...["-c", "copy", "-bsf:v", trace, "-f", "null", "-"],
|
||||
];
|
||||
}
|
||||
|
||||
export const parseKeyframeTransfers = (stderr: string): string[] =>
|
||||
[...stderr.matchAll(/transfer_characteristics +[01]+ = (\d+)/g)].map(
|
||||
([, code = ""]) => TRANSFER_NAMES[code] ?? `code ${code}`,
|
||||
);
|
||||
|
||||
export function parseFirstFrameColour(stderr: string): ToneMapSourceColour {
|
||||
const tags = / color_space:(\S+) color_primaries:(\S+) color_trc:(\S+)/.exec(stderr);
|
||||
return tags ? { colorSpace: tags[1], colorPrimaries: tags[2], colorTransfer: tags[3] } : {};
|
||||
|
||||
@@ -280,7 +280,12 @@ export { bakeHfColorGradingCube, type BakeHfColorGradingCubeOptions } from "./co
|
||||
export {
|
||||
firstFrameColourArgs,
|
||||
hdrToSdrToneMapFilter,
|
||||
keyframeTransferArgs,
|
||||
parseKeyframeTransfers,
|
||||
readsKeyframeTransfers,
|
||||
toneMapReferenceNits,
|
||||
parseFirstFrameColour,
|
||||
type ToneMapFrameColour,
|
||||
type ToneMapSourceColour,
|
||||
} from "./hdrToneMap";
|
||||
export {
|
||||
|
||||
@@ -146,7 +146,7 @@ describe.skipIf(process.platform === "win32")("extractAllVideoFrames ffmpeg conc
|
||||
delete process.env.HF_TEST_SHIM_DIR;
|
||||
}
|
||||
const alive = readFileSync(join(run, "alive"), "utf8").trim().split(/\s+/).map(Number);
|
||||
expect(alive).toHaveLength(8 + 2);
|
||||
expect(alive).toHaveLength(8 + 2 * 2);
|
||||
expect(Math.max(...alive)).toBe(1);
|
||||
},
|
||||
60_000,
|
||||
|
||||
@@ -3858,6 +3858,26 @@ const FFPROBE_READS_CONTAINER_TRANSFER =
|
||||
(FFPROBE_RELEASE !== undefined &&
|
||||
(FFPROBE_RELEASE[0]! > 8 || (FFPROBE_RELEASE[0] === 8 && FFPROBE_RELEASE[1]! >= 1)));
|
||||
|
||||
/** The colour probes run so far: first-frame reads, and SPS transfer reads of an HLG source. */
|
||||
function colourProbeRuns() {
|
||||
const runs = vi
|
||||
.mocked(runFfmpeg)
|
||||
.mock.calls.map(([args]) => args)
|
||||
.filter((args) => args.some((arg) => /showinfo|trace_headers/.test(arg)));
|
||||
return {
|
||||
firstFrame: runs.filter((args) => args.includes("-frames:v")).length,
|
||||
keyframes: runs.filter((args) => args.includes("nokey")).length,
|
||||
};
|
||||
}
|
||||
|
||||
function colourProbesSince(before: ReturnType<typeof colourProbeRuns>) {
|
||||
const now = colourProbeRuns();
|
||||
return {
|
||||
firstFrame: now.firstFrame - before.firstFrame,
|
||||
keyframes: now.keyframes - before.keyframes,
|
||||
};
|
||||
}
|
||||
|
||||
describe.skipIf(!HAS_ZSCALE)("forced-SDR HDR extraction", () => {
|
||||
let fixtureDir = "";
|
||||
|
||||
@@ -3977,8 +3997,7 @@ describe.skipIf(!HAS_ZSCALE)("forced-SDR HDR extraction", () => {
|
||||
it("probes each source's first frame once, however many cuts of it extract together", async () => {
|
||||
const sources = ["a", "b"].map((name) => join(fixtureDir, `cuts-${name}.mp4`));
|
||||
for (const source of sources) await synthesizeHlgClip(source);
|
||||
const probes = () =>
|
||||
vi.mocked(runFfmpeg).mock.calls.filter(([args]) => args.includes("showinfo")).length;
|
||||
const probes = colourProbeRuns;
|
||||
const before = probes();
|
||||
await Promise.all(
|
||||
Array.from({ length: 12 }, (_, cut) =>
|
||||
@@ -3990,14 +4009,13 @@ describe.skipIf(!HAS_ZSCALE)("forced-SDR HDR extraction", () => {
|
||||
}),
|
||||
),
|
||||
);
|
||||
expect(probes() - before).toBe(2);
|
||||
expect(colourProbesSince(before)).toEqual({ firstFrame: 2, keyframes: 2 });
|
||||
}, 60_000);
|
||||
|
||||
it("probes a source again after its first-frame probe failed", async () => {
|
||||
const source = join(fixtureDir, "probe-retry.mp4");
|
||||
await synthesizeHlgClip(source);
|
||||
const probes = () =>
|
||||
vi.mocked(runFfmpeg).mock.calls.filter(([args]) => args.includes("showinfo")).length;
|
||||
const probes = colourProbeRuns;
|
||||
const { runFfmpeg: real } =
|
||||
await vi.importActual<typeof import("../utils/runFfmpeg.js")>("../utils/runFfmpeg.js");
|
||||
vi.mocked(runFfmpeg).mockImplementation((args, opts) =>
|
||||
@@ -4025,7 +4043,40 @@ describe.skipIf(!HAS_ZSCALE)("forced-SDR HDR extraction", () => {
|
||||
vi.mocked(runFfmpeg).mockImplementation(real);
|
||||
}
|
||||
await extract("retry-again");
|
||||
expect(probes() - before).toBe(2);
|
||||
expect(colourProbesSince(before)).toEqual({ firstFrame: 2, keyframes: 1 });
|
||||
}, 60_000);
|
||||
|
||||
it("reads an HLG source's keyframes again after that read failed", async () => {
|
||||
const source = join(fixtureDir, "keyframe-retry.mp4");
|
||||
await synthesizeHlgClip(source);
|
||||
const { runFfmpeg: real } =
|
||||
await vi.importActual<typeof import("../utils/runFfmpeg.js")>("../utils/runFfmpeg.js");
|
||||
vi.mocked(runFfmpeg).mockImplementation((args, opts) =>
|
||||
args.includes("nokey")
|
||||
? Promise.resolve({
|
||||
success: false,
|
||||
exitCode: 1,
|
||||
stderr: "",
|
||||
durationMs: 0,
|
||||
terminationReason: "exit",
|
||||
})
|
||||
: real(args, opts),
|
||||
);
|
||||
const extract = (id: string) =>
|
||||
extractVideoFramesRange(source, id, 0, 1, {
|
||||
fps: 1,
|
||||
outputDir: join(fixtureDir, "keyframe-retry"),
|
||||
format: "png",
|
||||
toneMapHdrToSdr: true,
|
||||
});
|
||||
const before = colourProbeRuns();
|
||||
try {
|
||||
await extract("keyframes-failed");
|
||||
} finally {
|
||||
vi.mocked(runFfmpeg).mockImplementation(real);
|
||||
}
|
||||
await extract("keyframes-again");
|
||||
expect(colourProbesSince(before)).toEqual({ firstFrame: 2, keyframes: 2 });
|
||||
}, 60_000);
|
||||
|
||||
// One stream with HLG frames, then PQ frames: each is tone-mapped with its own transfer.
|
||||
@@ -4148,7 +4199,7 @@ describe.skipIf(!HAS_ZSCALE)("forced-SDR HDR extraction", () => {
|
||||
"-t",
|
||||
"1",
|
||||
"-vf",
|
||||
"fps=1,zscale=t=linear:npl=100,tonemap=hable:desat=0,zscale=p=bt709:t=bt709:m=bt709:r=tv",
|
||||
"fps=1,zscale=t=linear:npl=203,tonemap=hable:desat=0,zscale=p=bt709:t=bt709:m=bt709:r=tv",
|
||||
"-q:v",
|
||||
"0",
|
||||
"-compression_level",
|
||||
|
||||
@@ -38,8 +38,12 @@ import {
|
||||
type Fps,
|
||||
type FpsInput,
|
||||
firstFrameColourArgs,
|
||||
keyframeTransferArgs,
|
||||
parseKeyframeTransfers,
|
||||
type ToneMapFrameColour,
|
||||
hdrToSdrToneMapFilter,
|
||||
parseFirstFrameColour,
|
||||
readsKeyframeTransfers,
|
||||
} from "@hyperframes/core";
|
||||
import { resolveReferencedStart, type RefResolverEl } from "./referenceResolver.js";
|
||||
import { isKnownInactiveTimelineWindow } from "./mediaTimelineWindow.js";
|
||||
@@ -948,25 +952,35 @@ export function parseImageElements(html: string): ImageElement[] {
|
||||
return images;
|
||||
}
|
||||
|
||||
const firstFrameColourCache = new Map<
|
||||
string,
|
||||
MediaProbeCacheEntry<ReturnType<typeof parseFirstFrameColour>>
|
||||
>();
|
||||
const firstFrameColourCache = new Map<string, MediaProbeCacheEntry<ToneMapFrameColour>>();
|
||||
|
||||
/** A source's SPS transfers, so a PQ or SDR keyframe keeps an HLG one at 100 nits; null when the read failed. */
|
||||
function readKeyframeTransfers(videoPath: string, codec: string) {
|
||||
return inExtractionSlot(() =>
|
||||
runFfmpeg([...urlInputArgs(videoPath), ...keyframeTransferArgs(videoPath, codec)]),
|
||||
).then((result) => (result.success ? parseKeyframeTransfers(result.stderr) : null));
|
||||
}
|
||||
|
||||
/** The first shown frame's colour tags (see {@link firstFrameColourArgs}), probed once per source file. */
|
||||
function readFirstFrameColour(videoPath: string) {
|
||||
function readFirstFrameColour(videoPath: string, codec: string) {
|
||||
const identity = mediaFileIdentity(videoPath);
|
||||
const cached = readMetadataCache(firstFrameColourCache, videoPath, identity);
|
||||
if (cached) return cached;
|
||||
// In an extraction slot, so it never adds a decoder; unsignalled, so one cancel cannot cut it short for others.
|
||||
const probe = inExtractionSlot(() =>
|
||||
runFfmpeg([...urlInputArgs(videoPath), ...firstFrameColourArgs(videoPath)]),
|
||||
).then((result) => {
|
||||
).then(async (result): Promise<ToneMapFrameColour> => {
|
||||
const forget = () => {
|
||||
if (firstFrameColourCache.get(videoPath)?.promise === probe)
|
||||
firstFrameColourCache.delete(videoPath);
|
||||
};
|
||||
// A failed probe answers this cut only; the next one probes again.
|
||||
if (!result.success && firstFrameColourCache.get(videoPath)?.promise === probe) {
|
||||
firstFrameColourCache.delete(videoPath);
|
||||
}
|
||||
return parseFirstFrameColour(result.stderr);
|
||||
if (!result.success) forget();
|
||||
const first = parseFirstFrameColour(result.stderr);
|
||||
if (!result.success || !readsKeyframeTransfers(first)) return first;
|
||||
const keyframeTransfers = await readKeyframeTransfers(videoPath, codec);
|
||||
if (!keyframeTransfers) forget();
|
||||
return { ...first, keyframeTransfers: keyframeTransfers ?? [] };
|
||||
});
|
||||
writeMetadataCache(firstFrameColourCache, videoPath, identity, probe);
|
||||
return probe;
|
||||
@@ -1159,7 +1173,7 @@ interface VideoFilterChainInput {
|
||||
sampleAtOutputFps: boolean;
|
||||
keepsHdrForChrome: boolean;
|
||||
/** The first frame's colour tags; set exactly when HDR is tone-mapped to SDR. */
|
||||
toneMapFrom?: Awaited<ReturnType<typeof readFirstFrameColour>>;
|
||||
toneMapFrom?: ToneMapFrameColour;
|
||||
isHdr: boolean;
|
||||
sdrToHdrTransfer?: ExtractionOptions["sdrToHdrTransfer"];
|
||||
shownBox?: ExtractionOptions["shownBox"];
|
||||
@@ -1371,7 +1385,9 @@ async function extractVideoFramesRangeUnredacted(
|
||||
);
|
||||
}
|
||||
|
||||
const frameColour = toneMappedToSdr ? await readFirstFrameColour(videoPath) : undefined;
|
||||
const frameColour = toneMappedToSdr
|
||||
? await readFirstFrameColour(videoPath, metadata.videoCodec)
|
||||
: undefined;
|
||||
const filtersFor = videoFilterChain({
|
||||
metadata,
|
||||
format,
|
||||
|
||||
@@ -2,9 +2,12 @@ import { execFile } from "node:child_process";
|
||||
import { extname } from "node:path";
|
||||
import {
|
||||
firstFrameColourArgs,
|
||||
keyframeTransferArgs,
|
||||
parseFirstFrameColour,
|
||||
readsKeyframeTransfers,
|
||||
parseKeyframeTransfers,
|
||||
type PreviewSourceFacts,
|
||||
type ToneMapSourceColour,
|
||||
type ToneMapFrameColour,
|
||||
} from "@hyperframes/core";
|
||||
import { findFfBinary } from "@hyperframes/parsers/ff-binaries";
|
||||
import { videoStreamHasAlpha } from "@hyperframes/parsers/media-alpha";
|
||||
@@ -188,8 +191,9 @@ export function classifyMediaColor(stream: FfprobeStream | null | undefined): Me
|
||||
/** The first shown frame's colour tags, which zscale reads per frame. Empty when unread. */
|
||||
export async function probeFirstFrameColour(
|
||||
filePath: string,
|
||||
codec?: string,
|
||||
runner: FfprobeRunner = execFileRunner,
|
||||
): Promise<ToneMapSourceColour> {
|
||||
): Promise<ToneMapFrameColour> {
|
||||
const ffmpegPath =
|
||||
findFfBinary("ffmpeg", { configuredMustExist: true }) ??
|
||||
(runner === execFileRunner ? undefined : "ffmpeg");
|
||||
@@ -198,7 +202,15 @@ export async function probeFirstFrameColour(
|
||||
timeout: 15_000,
|
||||
maxBuffer: 1024 * 1024,
|
||||
});
|
||||
return result.status === 0 ? parseFirstFrameColour(String(result.stderr)) : {};
|
||||
const first = result.status === 0 ? parseFirstFrameColour(String(result.stderr)) : {};
|
||||
if (result.status !== 0 || !readsKeyframeTransfers(first)) return first;
|
||||
const keyframes = await runner(ffmpegPath, keyframeTransferArgs(filePath, codec), {
|
||||
timeout: 30_000,
|
||||
maxBuffer: 8 * 1024 * 1024,
|
||||
});
|
||||
const keyframeTransfers =
|
||||
keyframes.status === 0 ? parseKeyframeTransfers(String(keyframes.stderr)) : [];
|
||||
return { ...first, keyframeTransfers };
|
||||
}
|
||||
|
||||
function bitDepthOf(stream: FfprobeStream): number {
|
||||
|
||||
@@ -841,11 +841,11 @@ describe("resolveProxy", () => {
|
||||
]);
|
||||
|
||||
const proxyPath = await resolveProxy(projectDir, sourcePath);
|
||||
// Reference: the render tone map's RGB, converted with BT.709. Mean luma is ~53 there;
|
||||
// skipping the tone map gives ~60, a BT.601 conversion ~56.
|
||||
// Reference: the render tone map's RGB, converted with BT.709. Skipping the tone map or
|
||||
// converting with BT.601 each moves mean luma by several levels.
|
||||
const reference = meanLuma(
|
||||
sourcePath,
|
||||
`${hdrToSdrToneMapFilter({ colorTransfer: "arib-std-b67" }, { colorTransfer: "arib-std-b67" })},format=gbrp,scale=in_range=pc:out_color_matrix=bt709:out_range=tv,format=yuv420p,`,
|
||||
`${hdrToSdrToneMapFilter({ colorTransfer: "arib-std-b67" }, { colorTransfer: "arib-std-b67", keyframeTransfers: ["arib-std-b67"] })},format=gbrp,scale=in_range=pc:out_color_matrix=bt709:out_range=tv,format=yuv420p,`,
|
||||
);
|
||||
expect(Math.abs(meanLuma(proxyPath) - reference)).toBeLessThan(1);
|
||||
},
|
||||
|
||||
@@ -704,7 +704,9 @@ async function buildVideoFilter(
|
||||
const { hdrTransfer } = metadata.color;
|
||||
const toneMap = (hdrTransfer === "pq" || hdrTransfer === "hlg") && !keepsAlpha;
|
||||
if (toneMap) await ensureHdrFilters(ffmpegPath);
|
||||
const firstFrame = toneMap ? await probeFirstFrameColour(sourcePath) : {};
|
||||
const firstFrame = toneMap
|
||||
? await probeFirstFrameColour(sourcePath, metadata.color.codecName)
|
||||
: {};
|
||||
const evenScale = proxyScaleFilter(box);
|
||||
const pixelFormat = keepsAlpha ? "yuva420p" : "yuv420p";
|
||||
// The tone map ends in RGB; older ffmpeg (seen on 5.1) converts it with BT.601 unless the matrix is named.
|
||||
|
||||
Reference in New Issue
Block a user