mirror of
https://github.com/heygen-com/hyperframes.git
synced 2026-09-11 14:50:02 +00:00
feat(cli): expose agent-native color grading
This commit is contained in:
ukimsanov
@@ -3,21 +3,15 @@ import { basename, extname } from "node:path";
|
||||
|
||||
const IMAGE_EXT = new Set([".jpg", ".jpeg", ".png", ".webp", ".gif", ".bmp", ".tif", ".tiff"]);
|
||||
const SAMPLE_FRAMES = 5;
|
||||
// A long HD clip on slow storage can exceed the default 15s signalstats window;
|
||||
// override without a code change via HYPERFRAMES_ANALYZE_TIMEOUT_MS.
|
||||
const SIGNALSTATS_TIMEOUT_MS = Number(process.env.HYPERFRAMES_ANALYZE_TIMEOUT_MS) || 15000;
|
||||
const DEFAULT_TIMEOUT_MS = 15_000;
|
||||
|
||||
const ADJUST_LIMITS = {
|
||||
exposure: { min: -2, max: 2 },
|
||||
contrast: { min: -1, max: 1 },
|
||||
highlights: { min: -1, max: 1 },
|
||||
shadows: { min: -1, max: 1 },
|
||||
whites: { min: -1, max: 1 },
|
||||
blacks: { min: -1, max: 1 },
|
||||
temperature: { min: -1, max: 1 },
|
||||
tint: { min: -1, max: 1 },
|
||||
vibrance: { min: -1, max: 1 },
|
||||
saturation: { min: -1, max: 1 },
|
||||
};
|
||||
|
||||
function clamp(value, key) {
|
||||
@@ -27,91 +21,124 @@ function clamp(value, key) {
|
||||
}
|
||||
|
||||
function round(value) {
|
||||
return Math.round(value * 1000) / 1000;
|
||||
const rounded = Math.round(value * 1000) / 1000;
|
||||
return Object.is(rounded, -0) ? 0 : rounded;
|
||||
}
|
||||
|
||||
function avg(values) {
|
||||
if (values.length === 0) return 0;
|
||||
return values.reduce((sum, value) => sum + value, 0) / values.length;
|
||||
function average(values) {
|
||||
return values.reduce((sum, value) => sum + value, 0) / Math.max(1, values.length);
|
||||
}
|
||||
|
||||
function probeDuration(mediaPath) {
|
||||
function probeMedia(mediaPath, ffprobePath) {
|
||||
try {
|
||||
const raw = execFileSync(
|
||||
"ffprobe",
|
||||
["-v", "quiet", "-print_format", "json", "-show_format", mediaPath],
|
||||
{ encoding: "utf8", timeout: 5000 },
|
||||
ffprobePath,
|
||||
[
|
||||
"-v",
|
||||
"error",
|
||||
"-select_streams",
|
||||
"v:0",
|
||||
"-show_entries",
|
||||
"stream=color_space,color_transfer,color_primaries,pix_fmt,duration:format=duration",
|
||||
"-of",
|
||||
"json",
|
||||
mediaPath,
|
||||
],
|
||||
{ encoding: "utf8", timeout: 5_000, stdio: ["ignore", "pipe", "pipe"] },
|
||||
);
|
||||
const parsed = JSON.parse(raw);
|
||||
const duration = Number(parsed.format?.duration);
|
||||
return Number.isFinite(duration) && duration > 0 ? duration : null;
|
||||
const stream = Array.isArray(parsed.streams) ? (parsed.streams[0] ?? {}) : {};
|
||||
const duration = Number(stream.duration ?? parsed.format?.duration);
|
||||
const text = (key) =>
|
||||
typeof stream[key] === "string" && stream[key] ? stream[key] : "unknown";
|
||||
return {
|
||||
duration: Number.isFinite(duration) && duration > 0 ? duration : null,
|
||||
colorSpace: text("color_space"),
|
||||
transfer: text("color_transfer"),
|
||||
primaries: text("color_primaries"),
|
||||
pixelFormat: text("pix_fmt"),
|
||||
};
|
||||
} catch {
|
||||
return null;
|
||||
return {
|
||||
duration: null,
|
||||
colorSpace: "unknown",
|
||||
transfer: "unknown",
|
||||
primaries: "unknown",
|
||||
pixelFormat: "unknown",
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
function filterFor(mediaPath) {
|
||||
const ext = extname(mediaPath).toLowerCase();
|
||||
if (IMAGE_EXT.has(ext)) return "signalstats,metadata=print:file=-";
|
||||
const duration = probeDuration(mediaPath);
|
||||
if (!duration || duration <= 1) return "signalstats,metadata=print:file=-";
|
||||
const fps = Math.max(0.1, Math.min(2, SAMPLE_FRAMES / duration));
|
||||
return `fps=${fps.toFixed(4)},signalstats,metadata=print:file=-`;
|
||||
}
|
||||
|
||||
function parseSignalStats(raw) {
|
||||
export function parseMediaTreatmentSignalStats(raw) {
|
||||
const frames = [];
|
||||
let current = null;
|
||||
for (const line of String(raw).split(/\r?\n/)) {
|
||||
const frameMatch = line.match(/^frame:/);
|
||||
if (frameMatch) {
|
||||
const frame = line.match(/^frame:\d+.*pts_time:([+-]?(?:\d+(?:\.\d+)?|\.\d+))/);
|
||||
if (frame) {
|
||||
if (current) frames.push(current);
|
||||
current = {};
|
||||
current = { ptsTime: Number(frame[1]) };
|
||||
continue;
|
||||
}
|
||||
const match = line.match(/lavfi\.signalstats\.([A-Z]+)=([+-]?(?:\d+(?:\.\d+)?|\.\d+))/);
|
||||
if (!match) continue;
|
||||
if (!current) current = {};
|
||||
current[match[1]] = Number(match[2]);
|
||||
const stat = line.match(/lavfi\.signalstats\.([A-Z]+)=([+-]?(?:\d+(?:\.\d+)?|\.\d+))/);
|
||||
if (!stat) continue;
|
||||
current ??= {};
|
||||
current[stat[1]] = Number(stat[2]);
|
||||
}
|
||||
if (current) frames.push(current);
|
||||
const complete = frames.filter(
|
||||
return frames.filter(
|
||||
(frame) =>
|
||||
Number.isFinite(frame.YMIN) &&
|
||||
Number.isFinite(frame.YMAX) &&
|
||||
Number.isFinite(frame.YLOW) &&
|
||||
Number.isFinite(frame.YAVG) &&
|
||||
Number.isFinite(frame.YHIGH) &&
|
||||
Number.isFinite(frame.YMAX) &&
|
||||
Number.isFinite(frame.UAVG) &&
|
||||
Number.isFinite(frame.VAVG),
|
||||
);
|
||||
if (complete.length === 0) {
|
||||
throw new Error("no signalstats frames found");
|
||||
}
|
||||
}
|
||||
|
||||
function summarizeFrames(frames) {
|
||||
if (frames.length === 0) throw new Error("FFmpeg returned no analyzable video frames");
|
||||
return {
|
||||
frames: complete.length,
|
||||
yMin: Math.min(...complete.map((frame) => frame.YMIN)),
|
||||
yMax: Math.max(...complete.map((frame) => frame.YMAX)),
|
||||
yAvg: avg(complete.map((frame) => frame.YAVG)),
|
||||
uAvg: avg(complete.map((frame) => frame.UAVG)),
|
||||
vAvg: avg(complete.map((frame) => frame.VAVG)),
|
||||
frames: frames.length,
|
||||
yMin: Math.min(...frames.map((frame) => frame.YMIN)),
|
||||
yLow: average(frames.map((frame) => frame.YLOW)),
|
||||
yAvg: average(frames.map((frame) => frame.YAVG)),
|
||||
yHigh: average(frames.map((frame) => frame.YHIGH)),
|
||||
yMax: Math.max(...frames.map((frame) => frame.YMAX)),
|
||||
uAvg: average(frames.map((frame) => frame.UAVG)),
|
||||
vAvg: average(frames.map((frame) => frame.VAVG)),
|
||||
satAvg: average(frames.map((frame) => frame.SATAVG ?? 0)),
|
||||
shadowClipRisk: average(frames.map((frame) => (frame.YLOW <= 16 ? 1 : 0))),
|
||||
highlightClipRisk: average(frames.map((frame) => (frame.YHIGH >= 235 ? 1 : 0))),
|
||||
};
|
||||
}
|
||||
|
||||
export function statsToAdjust(stats) {
|
||||
const yMin = Number(stats.yMin);
|
||||
const yLow = Number(stats.yLow ?? stats.yMin);
|
||||
const yMax = Number(stats.yMax);
|
||||
const yHigh = Number(stats.yHigh ?? stats.yMax);
|
||||
const yAvg = Number(stats.yAvg);
|
||||
const uAvg = Number(stats.uAvg);
|
||||
const vAvg = Number(stats.vAvg);
|
||||
const spread = (yMax - yMin) / 255;
|
||||
const shadowClipRisk = Number(stats.shadowClipRisk ?? (yLow <= 16 ? 1 : 0));
|
||||
const highlightClipRisk = Number(stats.highlightClipRisk ?? (yHigh >= 235 ? 1 : 0));
|
||||
const percentileSpread = (yHigh - yLow) / 255;
|
||||
const normalizedAvg = yAvg / 255;
|
||||
const exposure = clamp((0.45 - normalizedAvg) * 1.8, "exposure");
|
||||
const contrast = clamp((0.42 - spread) * 0.9, "contrast");
|
||||
const whites =
|
||||
yMax > 230 ? clamp(-((yMax - 230) / 40 + Math.max(0, normalizedAvg - 0.74)), "whites") : 0;
|
||||
const blacks = yMin < 12 ? clamp((12 - yMin) / 80, "blacks") : 0;
|
||||
const exposure =
|
||||
normalizedAvg < 0.28 && yHigh / 255 < 0.65
|
||||
? clamp((0.32 - normalizedAvg) * 1.2, "exposure")
|
||||
: normalizedAvg > 0.72 && yLow / 255 > 0.3
|
||||
? clamp((0.68 - normalizedAvg) * 1.2, "exposure")
|
||||
: 0;
|
||||
const contrast = percentileSpread < 0.35 ? clamp((0.35 - percentileSpread) * 0.4, "contrast") : 0;
|
||||
const whites = clamp(-highlightClipRisk * 0.08, "whites");
|
||||
const blacks = clamp(shadowClipRisk * 0.08, "blacks");
|
||||
const chromaWarmth = (vAvg - 128 + (128 - uAvg)) / 128;
|
||||
const temperature = clamp(-chromaWarmth * 0.7, "temperature");
|
||||
const tint = clamp(-(uAvg - 128 + (vAvg - 128)) / 256, "tint");
|
||||
const temperature =
|
||||
Math.abs(chromaWarmth) >= 0.08 ? clamp(-chromaWarmth * 0.25, "temperature") : 0;
|
||||
const tint = Math.abs(uAvg + vAvg - 256) >= 10 ? clamp(-(uAvg + vAvg - 256) / 512, "tint") : 0;
|
||||
|
||||
return {
|
||||
adjust: {
|
||||
@@ -125,18 +152,75 @@ export function statsToAdjust(stats) {
|
||||
measured: {
|
||||
frames: Number(stats.frames ?? 1),
|
||||
yMin: round(yMin),
|
||||
yMax: round(yMax),
|
||||
yLow: round(yLow),
|
||||
yAvg: round(yAvg),
|
||||
yHigh: round(yHigh),
|
||||
yMax: round(yMax),
|
||||
uAvg: round(uAvg),
|
||||
vAvg: round(vAvg),
|
||||
satAvg: round(Number(stats.satAvg ?? 0)),
|
||||
shadowClipRisk: round(shadowClipRisk),
|
||||
highlightClipRisk: round(highlightClipRisk),
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
export function analyzeMediaGrade(mediaPath) {
|
||||
export function summarizeMediaTreatmentAnalysis(probe, frames) {
|
||||
const result = statsToAdjust(summarizeFrames(frames));
|
||||
const hdr = ["smpte2084", "arib-std-b67"].includes(probe.transfer);
|
||||
const diagnosis = [];
|
||||
if (result.measured.shadowClipRisk > 0) {
|
||||
diagnosis.push("sampled frames contain deep or clipped shadows");
|
||||
}
|
||||
if (result.measured.highlightClipRisk > 0) {
|
||||
diagnosis.push("sampled frames contain bright or clipped highlights");
|
||||
}
|
||||
if (diagnosis.length === 0) diagnosis.push("no obvious technical imbalance in sampled frames");
|
||||
|
||||
const warnings = [];
|
||||
if (hdr) warnings.push("HDR transfer detected; the realtime treatment path is SDR/Rec.709.");
|
||||
if (probe.colorSpace === "unknown" || probe.transfer === "unknown") {
|
||||
warnings.push(
|
||||
"Source color metadata is incomplete; camera LOG cannot be identified reliably from container metadata alone.",
|
||||
);
|
||||
}
|
||||
|
||||
return {
|
||||
...result,
|
||||
source: {
|
||||
colorSpace: probe.colorSpace,
|
||||
transfer: probe.transfer,
|
||||
primaries: probe.primaries,
|
||||
pixelFormat: probe.pixelFormat,
|
||||
hdr,
|
||||
log: "unknown",
|
||||
},
|
||||
diagnosis,
|
||||
warnings,
|
||||
};
|
||||
}
|
||||
|
||||
export function analyzeMediaGrade(
|
||||
mediaPath,
|
||||
{ ffmpegPath = "ffmpeg", ffprobePath = "ffprobe" } = {},
|
||||
) {
|
||||
try {
|
||||
const probe = probeMedia(mediaPath, ffprobePath);
|
||||
const isImage = IMAGE_EXT.has(extname(mediaPath).toLowerCase());
|
||||
const fps =
|
||||
!isImage && probe.duration
|
||||
? Math.max(0.1, Math.min(2, SAMPLE_FRAMES / probe.duration))
|
||||
: null;
|
||||
const filters = [
|
||||
fps ? `fps=${fps.toFixed(4)}` : null,
|
||||
"format=yuv444p",
|
||||
"signalstats",
|
||||
"metadata=print:file=-",
|
||||
]
|
||||
.filter(Boolean)
|
||||
.join(",");
|
||||
const raw = execFileSync(
|
||||
"ffmpeg",
|
||||
ffmpegPath,
|
||||
[
|
||||
"-hide_banner",
|
||||
"-nostdin",
|
||||
@@ -145,21 +229,25 @@ export function analyzeMediaGrade(mediaPath) {
|
||||
"-i",
|
||||
mediaPath,
|
||||
"-vf",
|
||||
filterFor(mediaPath),
|
||||
filters,
|
||||
"-frames:v",
|
||||
String(SAMPLE_FRAMES),
|
||||
"-f",
|
||||
"null",
|
||||
"-",
|
||||
],
|
||||
{ encoding: "utf8", timeout: SIGNALSTATS_TIMEOUT_MS, stdio: ["ignore", "pipe", "pipe"] },
|
||||
{
|
||||
encoding: "utf8",
|
||||
timeout: Number(process.env.HYPERFRAMES_ANALYZE_TIMEOUT_MS) || DEFAULT_TIMEOUT_MS,
|
||||
stdio: ["ignore", "pipe", "pipe"],
|
||||
},
|
||||
);
|
||||
return statsToAdjust(parseSignalStats(raw));
|
||||
} catch (err) {
|
||||
throw new Error(`grade analysis failed for ${mediaPath}: ${err.message}`);
|
||||
return summarizeMediaTreatmentAnalysis(probe, parseMediaTreatmentSignalStats(raw));
|
||||
} catch (error) {
|
||||
throw new Error(`grade analysis failed for ${mediaPath}: ${error.message}`);
|
||||
}
|
||||
}
|
||||
|
||||
export function formatMeasuredNote(mediaPath, measured) {
|
||||
return `media-use: measured ${basename(mediaPath)}: frames=${measured.frames}, YMIN=${measured.yMin}, YMAX=${measured.yMax}, YAVG=${measured.yAvg}, UAVG=${measured.uAvg}, VAVG=${measured.vAvg}; adjust is a starting suggestion`;
|
||||
return `media-use: measured ${basename(mediaPath)}: frames=${measured.frames}, YMIN=${measured.yMin}, YLOW=${measured.yLow}, YAVG=${measured.yAvg}, YHIGH=${measured.yHigh}, YMAX=${measured.yMax}, UAVG=${measured.uAvg}, VAVG=${measured.vAvg}; adjust is a starting suggestion`;
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user