mirror of
https://github.com/heygen-com/hyperframes.git
synced 2026-09-01 19:42:03 +00:00
fix(cli): report the level-shift bias and correct the compare --against floor
The documented caveat was wrong. Probing which reference frame the live seek lands on shows it lands exactly on the right one (peak SSIM at frame 210 for t=7, falling off on both sides), so there is no seek drift to work around. The real floor is the decode gap: a replica is a live browser paint, a reference is a decoded compressed video. Flat graphics self-compare at 0.998-0.999; photographic video sits near 0.93 at high quality and 0.89 at draft, and most of that is a uniform level shift rather than a structural error. Adds meanSignedDiff so that shift is a number instead of an eyeball call on the overlay, printed as 'diff X% (bias +Y%)', and replaces the caveat with the measured per-content floors.
This commit is contained in:
Miguel Angel Simon Sierra
+11
-3
@@ -748,9 +748,17 @@ npx hyperframes compare . --against reference.mp4 --at 0,4,10,21 --fail-under 0.
|
||||
|
||||
Each run writes a reference-over-replica contact sheet, a red/cyan deviation
|
||||
overlay per sampled time (agreement grey, reference-only ink red, replica-only
|
||||
ink cyan), and per-time numbers: `ssim`, `meanAbsDiff`, and ink bounding-box
|
||||
deltas `dw` / `dh` / `dcx` / `dcy` / `scale`. There is no default threshold:
|
||||
measure a known-good state first, then gate slightly below it.
|
||||
ink cyan), and per-time numbers: `ssim`, `meanAbsDiff`, `meanSignedDiff`, and
|
||||
ink bounding-box deltas `dw` / `dh` / `dcx` / `dcy` / `scale`. When
|
||||
`meanSignedDiff` is close to `meanAbsDiff` the replica is uniformly lighter or
|
||||
darker, which is a level shift from encoding or colour conversion rather than a
|
||||
defect in the composition.
|
||||
|
||||
There is no default threshold, because the floor depends on the content. A
|
||||
composition compared against its own render scores 0.998 to 0.999 for flat
|
||||
graphics and type, but around 0.93 (0.89 at draft quality) once photographic
|
||||
video is on screen, where encode loss and browser-versus-FFmpeg colour
|
||||
conversion dominate. Measure that floor first, then gate just below it.
|
||||
|
||||
`grade-compare` does the same for colour: candidate grades or LUTs applied to
|
||||
one reference frame.
|
||||
|
||||
@@ -24,6 +24,7 @@ import {
|
||||
boundsDeviation,
|
||||
inkBounds,
|
||||
meanAbsDiff,
|
||||
meanSignedDiff,
|
||||
parseSsimAll,
|
||||
redCyanOverlayRaw,
|
||||
ssimFfmpegArgs,
|
||||
@@ -60,6 +61,8 @@ export interface ReferenceSample {
|
||||
/** Full-frame SSIM (1 = identical); null when ffmpeg could not measure it. */
|
||||
ssim: number | null;
|
||||
meanAbsDiff: number;
|
||||
/** Signed counterpart of meanAbsDiff; close to it means a uniform level shift. */
|
||||
meanSignedDiff: number;
|
||||
deviation: BoundsDeviation;
|
||||
overlay: string;
|
||||
}
|
||||
@@ -230,6 +233,7 @@ export async function compareAgainstReference(
|
||||
time,
|
||||
ssim: await frameSsim(ffmpegPath, referenceFrame, normalized),
|
||||
meanAbsDiff: meanAbsDiff(referenceGray, replicaGray),
|
||||
meanSignedDiff: meanSignedDiff(referenceGray, replicaGray),
|
||||
deviation: boundsDeviation(
|
||||
inkBounds(referenceGray, width, height),
|
||||
inkBounds(replicaGray, width, height),
|
||||
|
||||
@@ -440,6 +440,7 @@ interface ReferenceComparePayload {
|
||||
time: number;
|
||||
ssim: number | null;
|
||||
meanAbsDiff: number;
|
||||
meanSignedDiff: number;
|
||||
deviation: BoundsDeviation;
|
||||
overlay: string;
|
||||
}[];
|
||||
@@ -481,6 +482,7 @@ async function runReferenceCompare(
|
||||
time: sample.time,
|
||||
ssim: sample.ssim,
|
||||
meanAbsDiff: Number(sample.meanAbsDiff.toFixed(4)),
|
||||
meanSignedDiff: Number(sample.meanSignedDiff.toFixed(4)),
|
||||
deviation: sample.deviation,
|
||||
overlay: sample.overlay,
|
||||
})),
|
||||
@@ -496,8 +498,9 @@ function printReferenceReport(payload: ReferenceComparePayload): void {
|
||||
for (const sample of payload.samples) {
|
||||
const { dw, dh, dcx, dcy, scale } = sample.deviation;
|
||||
const ssim = sample.ssim === null ? "n/a" : sample.ssim.toFixed(4);
|
||||
const signed = sample.meanSignedDiff * 100;
|
||||
console.log(
|
||||
` t=${sample.time}s SSIM ${ssim} diff ${(sample.meanAbsDiff * 100).toFixed(1)}%`,
|
||||
` t=${sample.time}s SSIM ${ssim} diff ${(sample.meanAbsDiff * 100).toFixed(1)}% (bias ${signed >= 0 ? "+" : ""}${signed.toFixed(1)}%)`,
|
||||
);
|
||||
console.log(
|
||||
c.dim(
|
||||
|
||||
@@ -3,6 +3,7 @@ import {
|
||||
boundsDeviation,
|
||||
inkBounds,
|
||||
meanAbsDiff,
|
||||
meanSignedDiff,
|
||||
parseSsimAll,
|
||||
redCyanOverlayRaw,
|
||||
ssimFfmpegArgs,
|
||||
@@ -62,6 +63,23 @@ describe("meanAbsDiff", () => {
|
||||
});
|
||||
});
|
||||
|
||||
describe("meanSignedDiff", () => {
|
||||
it("matches meanAbsDiff when the replica is uniformly brighter", () => {
|
||||
const reference = new Uint8Array(64).fill(100);
|
||||
const replica = new Uint8Array(64).fill(110);
|
||||
expect(meanSignedDiff(reference, replica)).toBeCloseTo(meanAbsDiff(reference, replica), 6);
|
||||
});
|
||||
|
||||
it("cancels to ~0 when the deviation is localized in both directions", () => {
|
||||
const reference = new Uint8Array(64).fill(100);
|
||||
const replica = new Uint8Array(64).fill(100);
|
||||
replica[0] = 200;
|
||||
replica[1] = 0;
|
||||
expect(meanSignedDiff(reference, replica)).toBeCloseTo(0, 6);
|
||||
expect(meanAbsDiff(reference, replica)).toBeGreaterThan(0);
|
||||
});
|
||||
});
|
||||
|
||||
describe("redCyanOverlayRaw", () => {
|
||||
it("puts the reference in red and the replica in green+blue", () => {
|
||||
const overlay = redCyanOverlayRaw(new Uint8Array([200, 0]), new Uint8Array([0, 100]), 2, 1);
|
||||
|
||||
@@ -117,6 +117,23 @@ export function meanAbsDiff(reference: Uint8Array, replica: Uint8Array): number
|
||||
return total / (length * 255);
|
||||
}
|
||||
|
||||
/**
|
||||
* Mean *signed* luma difference (replica minus reference), normalized to -1..1.
|
||||
*
|
||||
* Separates the two things `meanAbsDiff` sums together. A comparison whose
|
||||
* signed value is close to its absolute value is uniformly lighter or darker,
|
||||
* which is a level shift from encoding or colour conversion, not a structural
|
||||
* error. Signed near zero with a large absolute value means the deviation is
|
||||
* real and localized.
|
||||
*/
|
||||
export function meanSignedDiff(reference: Uint8Array, replica: Uint8Array): number {
|
||||
const length = Math.min(reference.length, replica.length);
|
||||
if (length === 0) return 0;
|
||||
let total = 0;
|
||||
for (let i = 0; i < length; i++) total += replica[i]! - reference[i]!;
|
||||
return total / (length * 255);
|
||||
}
|
||||
|
||||
/**
|
||||
* Interleaved RGB where the reference drives red and the replica drives
|
||||
* green+blue: agreement reads neutral grey, reference-only ink glows red,
|
||||
|
||||
@@ -26,7 +26,7 @@
|
||||
"files": 121
|
||||
},
|
||||
"hyperframes-cli": {
|
||||
"hash": "f2bac1c4d74b9853",
|
||||
"hash": "0934aa3b30945c81",
|
||||
"files": 11
|
||||
},
|
||||
"hyperframes-core": {
|
||||
|
||||
@@ -50,17 +50,29 @@ Exactly one composition path is allowed with `--against`. The reference may be a
|
||||
|
||||
Each run produces three instruments:
|
||||
|
||||
| Artifact | Where | Reads as |
|
||||
| ---------------------------- | ---------------------- | ------------------------------------------------------------------------------- |
|
||||
| Reference-over-replica sheet | `--out` path | Row 1 reference, row 2 replica, one column per sampled time |
|
||||
| Red/cyan deviation overlay | `<out>-overlay-NN.png` | Agreement grey, reference-only ink red, replica-only ink cyan |
|
||||
| Numbers | stdout and `--json` | `ssim`, `meanAbsDiff`, and ink-box deltas `dw` / `dh` / `dcx` / `dcy` / `scale` |
|
||||
| Artifact | Where | Reads as |
|
||||
| ---------------------------- | ---------------------- | --------------------------------------------------------------------------------------------- |
|
||||
| Reference-over-replica sheet | `--out` path | Row 1 reference, row 2 replica, one column per sampled time |
|
||||
| Red/cyan deviation overlay | `<out>-overlay-NN.png` | Agreement grey, reference-only ink red, replica-only ink cyan |
|
||||
| Numbers | stdout and `--json` | `ssim`, `meanAbsDiff`, `meanSignedDiff`, ink-box deltas `dw` / `dh` / `dcx` / `dcy` / `scale` |
|
||||
|
||||
How to read them:
|
||||
|
||||
- **`ssim`** is full-frame structural similarity, 1.0 = identical. A graphics-only composition measured against its own render lands at 0.998–0.999, so treat anything below ~0.99 on a self-comparison as a real difference, not measurement noise.
|
||||
- **`ssim`** is full-frame structural similarity, 1.0 = identical. What counts as good depends on the content, so read it against the floor below rather than against 1.0.
|
||||
- **`meanAbsDiff` vs `meanSignedDiff`** separates two things a single number confuses. `meanSignedDiff` is the same average without the absolute value, so when the two are close the replica is uniformly lighter or darker, which is a level shift from encoding or colour conversion and not a mistake you can fix in the composition. Signed near zero with a large absolute value means the deviation is real and localized. The CLI prints it as `diff X% (bias +Y%)`.
|
||||
- **Ink-box deltas** answer "is my title the right size and in the right place": `dw`/`dh` are the replica's ink bounding box minus the reference's in reference pixels, `dcx`/`dcy` the centre offset, `scale` the width ratio. They are meaningful for type and graphic frames; a full-bleed photograph makes every pixel ink and the box degenerates to the whole canvas.
|
||||
- **The overlay** localizes the deviation the numbers only total up. A uniform tint across the whole frame is a global level shift (encode quality, grade); localized red/cyan ghosting is a position, size or timing error.
|
||||
- **The overlay** localizes the deviation the numbers only total up. A flat tint across the whole frame is the level shift `meanSignedDiff` already quantified; localized red/cyan ghosting is a position, size or timing error.
|
||||
|
||||
### The floor is set by content, not by your composition
|
||||
|
||||
A replica is a live browser paint; a reference is a decoded compressed video. The gap between those two decode paths is a floor no correction can go below, and it depends entirely on what is on screen. Measured against their own renders:
|
||||
|
||||
| Composition content | Self-comparison SSIM | Why |
|
||||
| ------------------------- | ------------------------------------------- | ------------------------------------------------------------------ |
|
||||
| Flat graphics and type | 0.998–0.999 | Encodes near-losslessly; a real defect shows immediately |
|
||||
| Photographic video layers | ~0.93 at `--quality high`, ~0.89 at `draft` | Encode loss plus browser/FFmpeg colour conversion, visible as bias |
|
||||
|
||||
So a 0.93 on a video-backed composition can be a perfect rebuild, and a 0.98 on a typographic one is a real defect. Establish the floor before you read any number: compare the composition against its own render first, then treat that value as your zero.
|
||||
|
||||
Gate on it with `--fail-under <ssim>`, which exits non-zero when the worst sampled SSIM falls below the threshold:
|
||||
|
||||
@@ -68,9 +80,7 @@ Gate on it with `--fail-under <ssim>`, which exits non-zero when the worst sampl
|
||||
npx hyperframes compare . --against reference.mp4 --at 0,4,10,21 --fail-under 0.95
|
||||
```
|
||||
|
||||
There is no default threshold, deliberately: pick one from a measured baseline (compare a known-good state first, then gate slightly below it). A threshold guessed before measuring passes everything and gates nothing.
|
||||
|
||||
Caveat for video-backed compositions: a composition whose scenes play `<video>` will not reproduce its own render exactly, because live seek and the render pipeline do not always land the same source frame. Measure those against the composition's own rendered output to size the floor before choosing `--fail-under`, or sample times away from video-heavy scenes.
|
||||
There is no default threshold, deliberately: pick one just below the floor you measured above. A threshold guessed before measuring passes everything and gates nothing, and one copied from a graphics-only project will fail every video-backed build for no reason.
|
||||
|
||||
`--against` needs FFmpeg on PATH.
|
||||
|
||||
|
||||
Reference in New Issue
Block a user