feat(cli): expose agent-native color grading

This commit is contained in:
ukimsanov
2026-07-26 01:25:42 -07:00
parent e446de6023
commit 6d5961b802
11 changed files with 821 additions and 179 deletions
@@ -0,0 +1,66 @@
export interface GradeSignalFrame {
ptsTime?: number;
YMIN?: number;
YLOW?: number;
YAVG?: number;
YHIGH?: number;
YMAX?: number;
UAVG?: number;
VAVG?: number;
SATAVG?: number;
}
export interface GradeMediaProbe {
duration: number | null;
colorSpace: string;
transfer: string;
primaries: string;
pixelFormat: string;
}
export interface MediaTreatmentAnalysis {
adjust: Record<string, number>;
measured: {
frames: number;
yMin: number;
yLow: number;
yAvg: number;
yHigh: number;
yMax: number;
uAvg: number;
vAvg: number;
satAvg: number;
shadowClipRisk: number;
highlightClipRisk: number;
};
source: {
colorSpace: string;
transfer: string;
primaries: string;
pixelFormat: string;
hdr: boolean;
log: "unknown";
};
diagnosis: string[];
warnings: string[];
}
export function parseMediaTreatmentSignalStats(raw: string): GradeSignalFrame[];
export function statsToAdjust(
stats: Record<string, number>,
): Pick<MediaTreatmentAnalysis, "adjust" | "measured">;
export function summarizeMediaTreatmentAnalysis(
probe: GradeMediaProbe,
frames: readonly GradeSignalFrame[],
): MediaTreatmentAnalysis;
export function analyzeMediaGrade(
mediaPath: string,
options?: {
ffmpegPath?: string;
ffprobePath?: string;
},
): MediaTreatmentAnalysis;
export function formatMeasuredNote(
mediaPath: string,
measured: MediaTreatmentAnalysis["measured"],
): string;
+151 -63
View File
@@ -3,21 +3,15 @@ import { basename, extname } from "node:path";
const IMAGE_EXT = new Set([".jpg", ".jpeg", ".png", ".webp", ".gif", ".bmp", ".tif", ".tiff"]);
const SAMPLE_FRAMES = 5;
// A long HD clip on slow storage can exceed the default 15s signalstats window;
// override without a code change via HYPERFRAMES_ANALYZE_TIMEOUT_MS.
const SIGNALSTATS_TIMEOUT_MS = Number(process.env.HYPERFRAMES_ANALYZE_TIMEOUT_MS) || 15000;
const DEFAULT_TIMEOUT_MS = 15_000;
const ADJUST_LIMITS = {
exposure: { min: -2, max: 2 },
contrast: { min: -1, max: 1 },
highlights: { min: -1, max: 1 },
shadows: { min: -1, max: 1 },
whites: { min: -1, max: 1 },
blacks: { min: -1, max: 1 },
temperature: { min: -1, max: 1 },
tint: { min: -1, max: 1 },
vibrance: { min: -1, max: 1 },
saturation: { min: -1, max: 1 },
};
function clamp(value, key) {
@@ -27,91 +21,124 @@ function clamp(value, key) {
}
function round(value) {
return Math.round(value * 1000) / 1000;
const rounded = Math.round(value * 1000) / 1000;
return Object.is(rounded, -0) ? 0 : rounded;
}
function avg(values) {
if (values.length === 0) return 0;
return values.reduce((sum, value) => sum + value, 0) / values.length;
function average(values) {
return values.reduce((sum, value) => sum + value, 0) / Math.max(1, values.length);
}
function probeDuration(mediaPath) {
function probeMedia(mediaPath, ffprobePath) {
try {
const raw = execFileSync(
"ffprobe",
["-v", "quiet", "-print_format", "json", "-show_format", mediaPath],
{ encoding: "utf8", timeout: 5000 },
ffprobePath,
[
"-v",
"error",
"-select_streams",
"v:0",
"-show_entries",
"stream=color_space,color_transfer,color_primaries,pix_fmt,duration:format=duration",
"-of",
"json",
mediaPath,
],
{ encoding: "utf8", timeout: 5_000, stdio: ["ignore", "pipe", "pipe"] },
);
const parsed = JSON.parse(raw);
const duration = Number(parsed.format?.duration);
return Number.isFinite(duration) && duration > 0 ? duration : null;
const stream = Array.isArray(parsed.streams) ? (parsed.streams[0] ?? {}) : {};
const duration = Number(stream.duration ?? parsed.format?.duration);
const text = (key) =>
typeof stream[key] === "string" && stream[key] ? stream[key] : "unknown";
return {
duration: Number.isFinite(duration) && duration > 0 ? duration : null,
colorSpace: text("color_space"),
transfer: text("color_transfer"),
primaries: text("color_primaries"),
pixelFormat: text("pix_fmt"),
};
} catch {
return null;
return {
duration: null,
colorSpace: "unknown",
transfer: "unknown",
primaries: "unknown",
pixelFormat: "unknown",
};
}
}
function filterFor(mediaPath) {
const ext = extname(mediaPath).toLowerCase();
if (IMAGE_EXT.has(ext)) return "signalstats,metadata=print:file=-";
const duration = probeDuration(mediaPath);
if (!duration || duration <= 1) return "signalstats,metadata=print:file=-";
const fps = Math.max(0.1, Math.min(2, SAMPLE_FRAMES / duration));
return `fps=${fps.toFixed(4)},signalstats,metadata=print:file=-`;
}
function parseSignalStats(raw) {
export function parseMediaTreatmentSignalStats(raw) {
const frames = [];
let current = null;
for (const line of String(raw).split(/\r?\n/)) {
const frameMatch = line.match(/^frame:/);
if (frameMatch) {
const frame = line.match(/^frame:\d+.*pts_time:([+-]?(?:\d+(?:\.\d+)?|\.\d+))/);
if (frame) {
if (current) frames.push(current);
current = {};
current = { ptsTime: Number(frame[1]) };
continue;
}
const match = line.match(/lavfi\.signalstats\.([A-Z]+)=([+-]?(?:\d+(?:\.\d+)?|\.\d+))/);
if (!match) continue;
if (!current) current = {};
current[match[1]] = Number(match[2]);
const stat = line.match(/lavfi\.signalstats\.([A-Z]+)=([+-]?(?:\d+(?:\.\d+)?|\.\d+))/);
if (!stat) continue;
current ??= {};
current[stat[1]] = Number(stat[2]);
}
if (current) frames.push(current);
const complete = frames.filter(
return frames.filter(
(frame) =>
Number.isFinite(frame.YMIN) &&
Number.isFinite(frame.YMAX) &&
Number.isFinite(frame.YLOW) &&
Number.isFinite(frame.YAVG) &&
Number.isFinite(frame.YHIGH) &&
Number.isFinite(frame.YMAX) &&
Number.isFinite(frame.UAVG) &&
Number.isFinite(frame.VAVG),
);
if (complete.length === 0) {
throw new Error("no signalstats frames found");
}
}
function summarizeFrames(frames) {
if (frames.length === 0) throw new Error("FFmpeg returned no analyzable video frames");
return {
frames: complete.length,
yMin: Math.min(...complete.map((frame) => frame.YMIN)),
yMax: Math.max(...complete.map((frame) => frame.YMAX)),
yAvg: avg(complete.map((frame) => frame.YAVG)),
uAvg: avg(complete.map((frame) => frame.UAVG)),
vAvg: avg(complete.map((frame) => frame.VAVG)),
frames: frames.length,
yMin: Math.min(...frames.map((frame) => frame.YMIN)),
yLow: average(frames.map((frame) => frame.YLOW)),
yAvg: average(frames.map((frame) => frame.YAVG)),
yHigh: average(frames.map((frame) => frame.YHIGH)),
yMax: Math.max(...frames.map((frame) => frame.YMAX)),
uAvg: average(frames.map((frame) => frame.UAVG)),
vAvg: average(frames.map((frame) => frame.VAVG)),
satAvg: average(frames.map((frame) => frame.SATAVG ?? 0)),
shadowClipRisk: average(frames.map((frame) => (frame.YLOW <= 16 ? 1 : 0))),
highlightClipRisk: average(frames.map((frame) => (frame.YHIGH >= 235 ? 1 : 0))),
};
}
export function statsToAdjust(stats) {
const yMin = Number(stats.yMin);
const yLow = Number(stats.yLow ?? stats.yMin);
const yMax = Number(stats.yMax);
const yHigh = Number(stats.yHigh ?? stats.yMax);
const yAvg = Number(stats.yAvg);
const uAvg = Number(stats.uAvg);
const vAvg = Number(stats.vAvg);
const spread = (yMax - yMin) / 255;
const shadowClipRisk = Number(stats.shadowClipRisk ?? (yLow <= 16 ? 1 : 0));
const highlightClipRisk = Number(stats.highlightClipRisk ?? (yHigh >= 235 ? 1 : 0));
const percentileSpread = (yHigh - yLow) / 255;
const normalizedAvg = yAvg / 255;
const exposure = clamp((0.45 - normalizedAvg) * 1.8, "exposure");
const contrast = clamp((0.42 - spread) * 0.9, "contrast");
const whites =
yMax > 230 ? clamp(-((yMax - 230) / 40 + Math.max(0, normalizedAvg - 0.74)), "whites") : 0;
const blacks = yMin < 12 ? clamp((12 - yMin) / 80, "blacks") : 0;
const exposure =
normalizedAvg < 0.28 && yHigh / 255 < 0.65
? clamp((0.32 - normalizedAvg) * 1.2, "exposure")
: normalizedAvg > 0.72 && yLow / 255 > 0.3
? clamp((0.68 - normalizedAvg) * 1.2, "exposure")
: 0;
const contrast = percentileSpread < 0.35 ? clamp((0.35 - percentileSpread) * 0.4, "contrast") : 0;
const whites = clamp(-highlightClipRisk * 0.08, "whites");
const blacks = clamp(shadowClipRisk * 0.08, "blacks");
const chromaWarmth = (vAvg - 128 + (128 - uAvg)) / 128;
const temperature = clamp(-chromaWarmth * 0.7, "temperature");
const tint = clamp(-(uAvg - 128 + (vAvg - 128)) / 256, "tint");
const temperature =
Math.abs(chromaWarmth) >= 0.08 ? clamp(-chromaWarmth * 0.25, "temperature") : 0;
const tint = Math.abs(uAvg + vAvg - 256) >= 10 ? clamp(-(uAvg + vAvg - 256) / 512, "tint") : 0;
return {
adjust: {
@@ -125,18 +152,75 @@ export function statsToAdjust(stats) {
measured: {
frames: Number(stats.frames ?? 1),
yMin: round(yMin),
yMax: round(yMax),
yLow: round(yLow),
yAvg: round(yAvg),
yHigh: round(yHigh),
yMax: round(yMax),
uAvg: round(uAvg),
vAvg: round(vAvg),
satAvg: round(Number(stats.satAvg ?? 0)),
shadowClipRisk: round(shadowClipRisk),
highlightClipRisk: round(highlightClipRisk),
},
};
}
export function analyzeMediaGrade(mediaPath) {
export function summarizeMediaTreatmentAnalysis(probe, frames) {
const result = statsToAdjust(summarizeFrames(frames));
const hdr = ["smpte2084", "arib-std-b67"].includes(probe.transfer);
const diagnosis = [];
if (result.measured.shadowClipRisk > 0) {
diagnosis.push("sampled frames contain deep or clipped shadows");
}
if (result.measured.highlightClipRisk > 0) {
diagnosis.push("sampled frames contain bright or clipped highlights");
}
if (diagnosis.length === 0) diagnosis.push("no obvious technical imbalance in sampled frames");
const warnings = [];
if (hdr) warnings.push("HDR transfer detected; the realtime treatment path is SDR/Rec.709.");
if (probe.colorSpace === "unknown" || probe.transfer === "unknown") {
warnings.push(
"Source color metadata is incomplete; camera LOG cannot be identified reliably from container metadata alone.",
);
}
return {
...result,
source: {
colorSpace: probe.colorSpace,
transfer: probe.transfer,
primaries: probe.primaries,
pixelFormat: probe.pixelFormat,
hdr,
log: "unknown",
},
diagnosis,
warnings,
};
}
export function analyzeMediaGrade(
mediaPath,
{ ffmpegPath = "ffmpeg", ffprobePath = "ffprobe" } = {},
) {
try {
const probe = probeMedia(mediaPath, ffprobePath);
const isImage = IMAGE_EXT.has(extname(mediaPath).toLowerCase());
const fps =
!isImage && probe.duration
? Math.max(0.1, Math.min(2, SAMPLE_FRAMES / probe.duration))
: null;
const filters = [
fps ? `fps=${fps.toFixed(4)}` : null,
"format=yuv444p",
"signalstats",
"metadata=print:file=-",
]
.filter(Boolean)
.join(",");
const raw = execFileSync(
"ffmpeg",
ffmpegPath,
[
"-hide_banner",
"-nostdin",
@@ -145,21 +229,25 @@ export function analyzeMediaGrade(mediaPath) {
"-i",
mediaPath,
"-vf",
filterFor(mediaPath),
filters,
"-frames:v",
String(SAMPLE_FRAMES),
"-f",
"null",
"-",
],
{ encoding: "utf8", timeout: SIGNALSTATS_TIMEOUT_MS, stdio: ["ignore", "pipe", "pipe"] },
{
encoding: "utf8",
timeout: Number(process.env.HYPERFRAMES_ANALYZE_TIMEOUT_MS) || DEFAULT_TIMEOUT_MS,
stdio: ["ignore", "pipe", "pipe"],
},
);
return statsToAdjust(parseSignalStats(raw));
} catch (err) {
throw new Error(`grade analysis failed for ${mediaPath}: ${err.message}`);
return summarizeMediaTreatmentAnalysis(probe, parseMediaTreatmentSignalStats(raw));
} catch (error) {
throw new Error(`grade analysis failed for ${mediaPath}: ${error.message}`);
}
}
export function formatMeasuredNote(mediaPath, measured) {
return `media-use: measured ${basename(mediaPath)}: frames=${measured.frames}, YMIN=${measured.yMin}, YMAX=${measured.yMax}, YAVG=${measured.yAvg}, UAVG=${measured.uAvg}, VAVG=${measured.vAvg}; adjust is a starting suggestion`;
return `media-use: measured ${basename(mediaPath)}: frames=${measured.frames}, YMIN=${measured.yMin}, YLOW=${measured.yLow}, YAVG=${measured.yAvg}, YHIGH=${measured.yHigh}, YMAX=${measured.yMax}, UAVG=${measured.uAvg}, VAVG=${measured.vAvg}; adjust is a starting suggestion`;
}
@@ -4,7 +4,32 @@ import { existsSync, mkdtempSync, rmSync, unlinkSync } from "node:fs";
import { join } from "node:path";
import { tmpdir } from "node:os";
import { test } from "node:test";
import { analyzeMediaGrade, formatMeasuredNote, statsToAdjust } from "./grade-analyzer.mjs";
import {
analyzeMediaGrade,
formatMeasuredNote,
parseMediaTreatmentSignalStats,
statsToAdjust,
summarizeMediaTreatmentAnalysis,
} from "./grade-analyzer.mjs";
const SIGNALSTATS = `frame:0 pts:0 pts_time:0
lavfi.signalstats.YMIN=0
lavfi.signalstats.YLOW=8
lavfi.signalstats.YAVG=100
lavfi.signalstats.YHIGH=240
lavfi.signalstats.YMAX=255
lavfi.signalstats.UAVG=120
lavfi.signalstats.VAVG=140
lavfi.signalstats.SATAVG=40
frame:1 pts:2000 pts_time:2
lavfi.signalstats.YMIN=10
lavfi.signalstats.YLOW=20
lavfi.signalstats.YAVG=130
lavfi.signalstats.YHIGH=220
lavfi.signalstats.YMAX=245
lavfi.signalstats.UAVG=130
lavfi.signalstats.VAVG=125
lavfi.signalstats.SATAVG=60`;
// The "Test: skills" CI job runs bare `node --test` with no ffmpeg on PATH (by
// design — skills tests are meant to be node-builtin-only). Tests that shell to
@@ -70,18 +95,22 @@ test("under-exposed synthetic frame suggests positive exposure", { skip: FFMPEG_
}
});
test("over-exposed synthetic frame pulls exposure and whites down", { skip: FFMPEG_SKIP }, () => {
const dir = mkdtempSync(join(tmpdir(), "mu-grade-over-"));
try {
const file = makeFrame(dir, "over.png", "white");
const { adjust } = analyzeMediaGrade(file);
assert.ok(adjust.exposure < 0, `expected negative exposure, got ${adjust.exposure}`);
assert.ok(adjust.whites < 0, `expected negative whites, got ${adjust.whites}`);
assertWithinLimits(adjust);
} finally {
rmSync(dir, { recursive: true, force: true });
}
});
test(
"over-exposed synthetic frame pulls exposure down without inventing clipping",
{ skip: FFMPEG_SKIP },
() => {
const dir = mkdtempSync(join(tmpdir(), "mu-grade-over-"));
try {
const file = makeFrame(dir, "over.png", "white");
const { adjust } = analyzeMediaGrade(file);
assert.ok(adjust.exposure < 0, `expected negative exposure, got ${adjust.exposure}`);
assert.ok(adjust.whites <= 0, `expected non-positive whites, got ${adjust.whites}`);
assertWithinLimits(adjust);
} finally {
rmSync(dir, { recursive: true, force: true });
}
},
);
test(
"warm-cast synthetic frame suggests negative temperature correction",
@@ -150,6 +179,107 @@ test("measured note is a stderr-safe single-line summary", () => {
vAvg: 140,
});
assert.match(note, /^media-use: measured /);
assert.match(note, /YMIN=10/);
assert.match(note, /YMAX=240/);
assert.match(note, /YAVG=80/);
assert.equal(note.includes("\n"), false);
});
test("parses deterministic FFmpeg signalstats frames", () => {
const frames = parseMediaTreatmentSignalStats(SIGNALSTATS);
assert.equal(frames.length, 2);
assert.deepEqual(
{
ptsTime: frames[0]?.ptsTime,
YLOW: frames[0]?.YLOW,
YAVG: frames[0]?.YAVG,
YHIGH: frames[0]?.YHIGH,
SATAVG: frames[0]?.SATAVG,
},
{ ptsTime: 0, YLOW: 8, YAVG: 100, YHIGH: 240, SATAVG: 40 },
);
});
test("keeps adjust output while adding HDR metadata and warnings", () => {
const result = summarizeMediaTreatmentAnalysis(
{
duration: 4,
colorSpace: "bt2020nc",
transfer: "smpte2084",
primaries: "bt2020",
pixelFormat: "yuv420p10le",
},
parseMediaTreatmentSignalStats(SIGNALSTATS),
);
assert.deepEqual(
{
blacks: result.adjust.blacks,
whites: result.adjust.whites,
frames: result.measured.frames,
yLow: result.measured.yLow,
yHigh: result.measured.yHigh,
satAvg: result.measured.satAvg,
shadowClipRisk: result.measured.shadowClipRisk,
highlightClipRisk: result.measured.highlightClipRisk,
},
{
blacks: 0.04,
whites: -0.04,
frames: 2,
yLow: 14,
yHigh: 230,
satAvg: 50,
shadowClipRisk: 0.5,
highlightClipRisk: 0.5,
},
);
assert.equal(result.source.hdr, true);
assert.equal(result.source.log, "unknown");
assert.match(result.warnings[0] ?? "", /HDR/);
});
test("does not force middle-gray exposure onto healthy intentional contrast", () => {
const result = summarizeMediaTreatmentAnalysis(
{
duration: 4,
colorSpace: "bt709",
transfer: "bt709",
primaries: "bt709",
pixelFormat: "yuv420p",
},
[
{
YMIN: 20,
YLOW: 42,
YAVG: 122,
YHIGH: 190,
YMAX: 220,
UAVG: 128,
VAVG: 128,
SATAVG: 40,
},
{
YMIN: 18,
YLOW: 38,
YAVG: 118,
YHIGH: 196,
YMAX: 225,
UAVG: 128,
VAVG: 128,
SATAVG: 42,
},
],
);
assert.deepEqual(
{
exposure: result.adjust.exposure,
blacks: result.adjust.blacks,
whites: result.adjust.whites,
temperature: result.adjust.temperature,
tint: result.adjust.tint,
},
{ exposure: 0, blacks: 0, whites: 0, temperature: 0, tint: 0 },
);
});