Merge pull request #2507 from heygen-com/via/parity-telemetry-gate

feat(producer): parity telemetry gate for per-clip frame-count invariant
This commit is contained in:
Vance Ingalls
2026-07-15 17:22:13 -07:00
committed by GitHub
4 changed files with 586 additions and 1 deletions
@@ -109,6 +109,28 @@ export interface RenderExtractionObservability {
vfrPreflightCount?: number;
cacheHits?: number;
cacheMisses?: number;
/**
* Per-clip captured-vs-expected-frame gauges. Emitted by the parity gate
* at extract finalization (see `videoFrameCoverage.ts`). Undefined when
* the render has no source videos to cover.
*
* • `minVideoFrameCoverageRatio` — worst clip's `captured / expected`
* ratio (0 when a clip was never extracted; a strong "later-injected
* clip silently dropped" signal per field ts=1784139267).
* • `coverageShortfallClipCount` — clips whose ratio fell below the
* configured threshold (`HF_VIDEO_COVERAGE_THRESHOLD`, default 0.95).
* Non-zero only ever accompanies a `VideoFrameCoverageError` throw.
*/
minVideoFrameCoverageRatio?: number;
coverageShortfallClipCount?: number;
/**
* Count of authored `[data-start]` clip windows in the compiled HTML —
* a coarse proxy for the ts=1784144554 field signal shape (147-clip
* composition, 130 word-level caption divs authored-clip-count-scaled
* failure). Static scan; dynamic script-inserted timed clips land in
* the probe-stage's `hasRuntimeInsertedMedia` path (PR #2474).
*/
authoredTimedClipCount?: number;
}
export interface RenderInitObservability {
@@ -0,0 +1,301 @@
import type { ExtractedFrames, VideoElement, VideoMetadata } from "@hyperframes/engine";
import { describe, expect, it } from "vitest";
import {
assertVideoFrameCoverage,
computeVideoFrameCoverage,
countAuthoredTimedClips,
expectedFramesForClip,
isVideoFrameCoverageError,
resolveVideoCoverageThreshold,
VideoFrameCoverageError,
} from "./videoFrameCoverage.js";
function makeVideo(overrides: Partial<VideoElement> & { id: string }): VideoElement {
return {
id: overrides.id,
src: overrides.src ?? `${overrides.id}.mp4`,
start: overrides.start ?? 0,
end: overrides.end ?? 1,
mediaStart: overrides.mediaStart ?? 0,
loop: overrides.loop ?? false,
hasAudio: overrides.hasAudio ?? false,
};
}
function makeExtracted(videoId: string, delivered: number, fps = 30): ExtractedFrames {
const framePaths = new Map<number, string>();
for (let i = 0; i < delivered; i += 1) framePaths.set(i, `/tmp/${videoId}/${i}.jpg`);
const metadata: VideoMetadata = {
durationSeconds: delivered / fps,
videoStreamDurationSeconds: delivered / fps,
width: 1280,
height: 720,
fps,
videoCodec: "h264",
hasAudio: false,
isVFR: false,
hasAlpha: false,
colorSpace: null,
};
return {
videoId,
srcPath: `/tmp/${videoId}.mp4`,
outputDir: `/tmp/${videoId}`,
framePattern: `${videoId}-%06d.jpg`,
fps,
totalFrames: delivered,
metadata,
framePaths,
};
}
describe("expectedFramesForClip", () => {
it("returns 0 for invalid inputs", () => {
expect(expectedFramesForClip(Number.NaN, 1, 30)).toBe(0);
expect(expectedFramesForClip(0, 1, 0)).toBe(0);
expect(expectedFramesForClip(0, 1, -1)).toBe(0);
expect(expectedFramesForClip(2, 1, 30)).toBe(0); // negative window collapses to 0
});
it("ceils fractional-fps windows so a 29.97fps 1s clip demands 30 frames", () => {
expect(expectedFramesForClip(0, 1, 29.97)).toBe(30);
expect(expectedFramesForClip(0, 5, 30)).toBe(150);
});
});
describe("resolveVideoCoverageThreshold", () => {
it("defaults to 0.95 when env is unset or non-numeric", () => {
expect(resolveVideoCoverageThreshold(undefined)).toBe(0.95);
expect(resolveVideoCoverageThreshold("garbage")).toBe(0.95);
});
it("returns null when env is 0 or negative — gate disabled", () => {
expect(resolveVideoCoverageThreshold("0")).toBeNull();
expect(resolveVideoCoverageThreshold("-1")).toBeNull();
});
it("clamps values above 1 to 1", () => {
expect(resolveVideoCoverageThreshold("2")).toBe(1);
});
it("passes through in-range values", () => {
expect(resolveVideoCoverageThreshold("0.8")).toBe(0.8);
expect(resolveVideoCoverageThreshold("0.99")).toBe(0.99);
expect(resolveVideoCoverageThreshold("1")).toBe(1);
});
});
describe("computeVideoFrameCoverage", () => {
it("reports 1.0 ratio when every video delivered its authored window", () => {
const videos = [
makeVideo({ id: "a", start: 0, end: 1 }),
makeVideo({ id: "b", start: 1, end: 3 }),
];
const extracted = [makeExtracted("a", 30), makeExtracted("b", 60)];
const reports = computeVideoFrameCoverage(videos, extracted, 30);
expect(reports).toHaveLength(2);
expect(reports[0]).toMatchObject({
videoId: "a",
expectedFrames: 30,
capturedFrames: 30,
ratio: 1,
});
expect(reports[1]).toMatchObject({
videoId: "b",
expectedFrames: 60,
capturedFrames: 60,
ratio: 1,
});
});
it("reports 0 capturedFrames when a video was never extracted (injection failure)", () => {
// Field signal ts=1784139267: later-injected video clips silently drop
// out of the extractor under injection-count saturation.
const videos = [makeVideo({ id: "later-injection", start: 0, end: 5 })];
const reports = computeVideoFrameCoverage(videos, [], 30);
expect(reports).toHaveLength(1);
expect(reports[0]).toMatchObject({
videoId: "later-injection",
expectedFrames: 150,
capturedFrames: 0,
ratio: 0,
});
});
it("uses delivered framePaths.size, not the possibly-stale totalFrames field", () => {
const videos = [makeVideo({ id: "a", start: 0, end: 1 })];
// Simulate an extractor whose totalFrames reports 30 but only 5 frames
// landed in framePaths (mid-extraction crash, partial cache read, …).
const partial = makeExtracted("a", 5);
partial.totalFrames = 30;
const reports = computeVideoFrameCoverage(videos, [partial], 30);
expect(reports[0]!.capturedFrames).toBe(5);
expect(reports[0]!.ratio).toBeCloseTo(5 / 30, 5);
});
});
describe("assertVideoFrameCoverage", () => {
it("does not throw when every clip has full frames", () => {
const reports = [
{ videoId: "a", clipStart: 0, clipEnd: 1, expectedFrames: 30, capturedFrames: 30, ratio: 1 },
{ videoId: "b", clipStart: 1, clipEnd: 2, expectedFrames: 30, capturedFrames: 30, ratio: 1 },
];
expect(() => assertVideoFrameCoverage(reports, 0.95)).not.toThrow();
});
it("fails loudly with VideoFrameCoverageError when any clip has zero frames", () => {
const reports = [
{
videoId: "good",
clipStart: 0,
clipEnd: 1,
expectedFrames: 30,
capturedFrames: 30,
ratio: 1,
},
{
videoId: "blank",
clipStart: 5,
clipEnd: 10,
expectedFrames: 150,
capturedFrames: 0,
ratio: 0,
},
];
let caught: unknown = null;
try {
assertVideoFrameCoverage(reports, 0.95);
} catch (err) {
caught = err;
}
expect(caught).toBeInstanceOf(VideoFrameCoverageError);
expect(isVideoFrameCoverageError(caught)).toBe(true);
const err = caught as VideoFrameCoverageError;
expect(err.worst.videoId).toBe("blank");
expect(err.threshold).toBe(0.95);
expect(err.message).toContain("blank");
expect(err.message).toContain("check/snapshot");
expect(err.message).toContain("HF_VIDEO_COVERAGE_THRESHOLD=0");
});
it("fails loudly when a clip is below the threshold (partial coverage)", () => {
// 80% coverage: 24/30 — below the 0.95 default, above zero.
const reports = [
{
videoId: "partial",
clipStart: 0,
clipEnd: 1,
expectedFrames: 30,
capturedFrames: 24,
ratio: 0.8,
},
];
expect(() => assertVideoFrameCoverage(reports, 0.95)).toThrow(VideoFrameCoverageError);
});
it("respects a threshold override — 0.5 passes 60% coverage", () => {
const reports = [
{
videoId: "partial",
clipStart: 0,
clipEnd: 1,
expectedFrames: 30,
capturedFrames: 18,
ratio: 0.6,
},
];
expect(() => assertVideoFrameCoverage(reports, 0.5)).not.toThrow();
});
it("is a no-op when the threshold is null (env opt-out)", () => {
const reports = [
{
videoId: "blank",
clipStart: 0,
clipEnd: 1,
expectedFrames: 30,
capturedFrames: 0,
ratio: 0,
},
];
expect(() => assertVideoFrameCoverage(reports, null)).not.toThrow();
});
it("ignores 0-expected-frame windows so the gate never fires on a degenerate clip", () => {
const reports = [
{
videoId: "zero-duration",
clipStart: 3,
clipEnd: 3,
expectedFrames: 0,
capturedFrames: 0,
ratio: 1,
},
];
expect(() => assertVideoFrameCoverage(reports, 1)).not.toThrow();
});
it("cites the worst-ratio clip in the error, not the first-found", () => {
// Field-signal shape: 15 injected videos, several later ones blank. Cite
// the deepest failure so operators fix root cause first, not the closest.
const reports = [
{
videoId: "slightly-low",
clipStart: 0,
clipEnd: 1,
expectedFrames: 30,
capturedFrames: 27,
ratio: 0.9,
},
{
videoId: "totally-blank",
clipStart: 1,
clipEnd: 2,
expectedFrames: 30,
capturedFrames: 0,
ratio: 0,
},
{
videoId: "moderately-low",
clipStart: 2,
clipEnd: 3,
expectedFrames: 30,
capturedFrames: 15,
ratio: 0.5,
},
];
try {
assertVideoFrameCoverage(reports, 0.95);
throw new Error("expected coverage assertion to throw");
} catch (err) {
expect(isVideoFrameCoverageError(err)).toBe(true);
const cov = err as VideoFrameCoverageError;
expect(cov.worst.videoId).toBe("totally-blank");
expect(cov.failedReports.map((r) => r.videoId)).toEqual([
"totally-blank",
"moderately-low",
"slightly-low",
]);
expect(cov.message).toContain("+2 more clip(s) below threshold");
}
});
});
describe("countAuthoredTimedClips", () => {
it("counts every [data-start] element in the compiled HTML", () => {
// Field signal ts=1784144554: 147-clip composition with 130 word-level
// caption divs. Static scan gives a coarse proxy — enough to make a
// 147-clip render distinguishable in telemetry from a 3-clip render.
const html = `<html><body>
<div data-start="0" data-duration="1">a</div>
<div data-start="1" data-duration="1">b</div>
<video data-start="2" data-duration="3" src="v.mp4"></video>
<div>not timed</div>
</body></html>`;
expect(countAuthoredTimedClips(html)).toBe(3);
});
it("returns 0 when no timed clips exist", () => {
expect(countAuthoredTimedClips("<html><body><div>hi</div></body></html>")).toBe(0);
});
});
@@ -0,0 +1,206 @@
/**
* Per-clip render-time frame-coverage accounting + threshold fail-loud gate.
*
* Sibling to #2474's `hasRuntimeInsertedMedia` probe: that PR guarantees the
* DISCOVERY of runtime-inserted media (so the browser probe launches and
* reconciles element identity). This module owns the DELIVERY side —
* for each authored/discovered video clip on the timeline, did the
* extractor actually produce enough source-video frames to composite the
* clip's authored `[data-start,data-end]` window? Any clip whose
* `capturedFrames / expectedFrames` ratio falls below a configurable
* threshold aborts the render with a `VideoFrameCoverageError` at extract
* finalization, BEFORE encode produces an MP4 that silently drops the
* clip's pixels to black.
*
* Two field signals defined the failure surface this exists to close
* (both `#hyperframes-cli-feedback`, both `check`/`snapshot` pass /
* final MP4 wrong):
*
* • ts=1784139267 · win32/x64 CLI 0.7.58 156s render, 15 injected
* videos: several later-injected video clips render BLANK in the
* encoded MP4. Injection-count-scaled — 12 injections fail, 4
* succeed (workaround: pre-compose into one base timeline). Points
* at injector scheduling / worker-seek saturation / extractor
* concurrency. Directly covered here — the per-video capture
* shortfall lands with `capturedFrames << expectedFrames`.
*
* • ts=1784144554 · darwin/arm64 CLI 0.7.59 3/10 147-clip
* composition (130 word-level caption divs): producer left a
* subset of authored `[data-start]` div clips permanently visible
* in the rendered MP4 while preview/snapshot showed correct
* visibility. Authored-clip-count-scaled — 147 clips fails, 3
* succeeds. This case runs through `syncTimedElementVisibility`
* at runtime and is NOT source-video-frame-shaped; the coverage
* gate cannot directly observe it. We surface an
* `authoredTimedClipCount` gauge so a 147-clip composition is
* visible in telemetry, and leave the per-tick visibility parity
* check as follow-up work (a separate runtime observability
* channel is required — the extractor doesn't see div visibility).
*
* Threshold: default 0.95 (a 5% capture-frame drop is loud); override
* via `HF_VIDEO_COVERAGE_THRESHOLD` env; disable entirely by setting
* the env to `0` or a negative number. A threshold of `1` requires
* exact coverage (no slack for the ffmpeg ±1-frame boundary rounding
* that legitimately happens on a 29.97 fps timeline).
*/
import { parseHTML } from "linkedom";
import type { ExtractedFrames, VideoElement } from "@hyperframes/engine";
export interface VideoFrameCoverageReport {
videoId: string;
clipStart: number;
clipEnd: number;
expectedFrames: number;
capturedFrames: number;
/** `capturedFrames / expectedFrames`; `1` when `expectedFrames === 0` (nothing to cover). */
ratio: number;
}
/**
* Discriminant-based error so callers cross-module (producer server,
* distributed worker) can identify a coverage-gate failure without
* `instanceof` (which is fragile across duplicated module instances,
* see `DrawElementVerificationError` for the same problem).
*/
export interface VideoFrameCoverageErrorDetails {
readonly hyperframesVideoFrameCoverageError: true;
readonly threshold: number;
readonly worst: VideoFrameCoverageReport;
readonly failedReports: VideoFrameCoverageReport[];
}
export class VideoFrameCoverageError extends Error {
readonly hyperframesVideoFrameCoverageError = true as const;
readonly threshold: number;
readonly worst: VideoFrameCoverageReport;
readonly failedReports: VideoFrameCoverageReport[];
constructor(
message: string,
details: Omit<VideoFrameCoverageErrorDetails, "hyperframesVideoFrameCoverageError">,
) {
super(message);
this.name = "VideoFrameCoverageError";
this.threshold = details.threshold;
this.worst = details.worst;
this.failedReports = details.failedReports;
}
}
export function isVideoFrameCoverageError(err: unknown): err is VideoFrameCoverageError {
return (
typeof err === "object" &&
err !== null &&
(err as { hyperframesVideoFrameCoverageError?: unknown }).hyperframesVideoFrameCoverageError ===
true
);
}
/**
* Resolve the coverage threshold from `HF_VIDEO_COVERAGE_THRESHOLD` env.
*
* Defaults to `0.95`. Values outside `(0, 1]` disable the gate: `0` or
* negative → off (return `null`); `>1` clamps to `1`. Non-numeric env
* values fall back to the default (with a caller-side warning if a log
* is available).
*/
export function resolveVideoCoverageThreshold(
envValue: string | undefined = process.env.HF_VIDEO_COVERAGE_THRESHOLD,
): number | null {
if (envValue === undefined) return 0.95;
const parsed = Number(envValue);
if (!Number.isFinite(parsed)) return 0.95;
if (parsed <= 0) return null;
if (parsed > 1) return 1;
return parsed;
}
/**
* Ceil the clip's authored `[start,end)` window at `fps` — the number of
* captured render frames whose center-time falls inside the window. Kept
* separate so a caller (or a test) can override it if a composition uses
* a non-integer fps whose sampling makes the naive count off by one.
*/
export function expectedFramesForClip(start: number, end: number, fps: number): number {
if (!Number.isFinite(start) || !Number.isFinite(end) || !Number.isFinite(fps)) return 0;
if (fps <= 0) return 0;
const duration = Math.max(0, end - start);
return Math.ceil(duration * fps);
}
export function computeVideoFrameCoverage(
videos: readonly VideoElement[],
extracted: readonly ExtractedFrames[],
fps: number,
): VideoFrameCoverageReport[] {
const byId = new Map<string, ExtractedFrames>();
for (const entry of extracted) byId.set(entry.videoId, entry);
const reports: VideoFrameCoverageReport[] = [];
for (const video of videos) {
const entry = byId.get(video.id);
const expectedFrames = expectedFramesForClip(video.start, video.end, fps);
// framePaths is a Map — `size` is the number of distinct captured frames
// delivered to the runtime injector, which is the load-bearing count
// (some extractors report a total that includes cache-hit-skipped frames
// via a stale `totalFrames`, so we trust the delivered-path count).
const capturedFrames = entry ? entry.framePaths.size : 0;
const ratio = expectedFrames === 0 ? 1 : capturedFrames / expectedFrames;
reports.push({
videoId: video.id,
clipStart: video.start,
clipEnd: video.end,
expectedFrames,
capturedFrames,
ratio,
});
}
return reports;
}
/**
* Throws `VideoFrameCoverageError` when any per-clip ratio is below
* `threshold`. A `null` threshold disables the gate (env opt-out).
* A clip with `expectedFrames === 0` (0-duration or non-authored) is
* unconditionally passing so the gate never fires on a degenerate window.
*/
export function assertVideoFrameCoverage(
reports: readonly VideoFrameCoverageReport[],
threshold: number | null,
): void {
if (threshold === null) return;
const failed = reports.filter((report) => report.expectedFrames > 0 && report.ratio < threshold);
if (failed.length === 0) return;
// Sort ascending by ratio so the "worst" is first — that's what we cite
// in the message and pin on the error details for telemetry.
const sorted = [...failed].sort((a, b) => a.ratio - b.ratio);
const worst = sorted[0]!;
const pct = (worst.ratio * 100).toFixed(1);
const thresholdPct = (threshold * 100).toFixed(1);
const suffix = sorted.length > 1 ? ` (+${sorted.length - 1} more clip(s) below threshold)` : "";
throw new VideoFrameCoverageError(
`Video "${worst.videoId}" captured ${worst.capturedFrames} of expected ${worst.expectedFrames} frames ` +
`(coverage ${pct}%, threshold ${thresholdPct}%). ` +
`check/snapshot may pass while the encoded MP4 renders this clip blank — aborting render ` +
`to prevent shipping a wrong MP4.${suffix} ` +
`Set HF_VIDEO_COVERAGE_THRESHOLD=0 to disable this gate.`,
{ threshold, worst, failedReports: sorted },
);
}
/**
* Count authored `[data-start]` clip windows in the compiled HTML.
*
* Not a fail-loud gate — a raw counter that lands in
* `RenderExtractionObservability.authoredTimedClipCount` so a 147-clip
* composition is queryable in telemetry (the ts=1784144554 field signal
* shape). Runtime `syncTimedElementVisibility` iterates the same set at
* render time; counting statically here is a coarse proxy — dynamic
* script-inserted `[data-start]` divs land in `hasRuntimeInsertedMedia`'s
* probe path (PR #2474), not this static scan.
*/
export function countAuthoredTimedClips(html: string): number {
const { document } = parseHTML(html);
return document.querySelectorAll("[data-start]").length;
}
@@ -45,7 +45,13 @@ import {
} from "fs";
import { tmpdir } from "node:os";
import { parseHTML } from "linkedom";
import { type CanvasResolution, type Fps, type FpsInput, toFps } from "@hyperframes/core";
import {
type CanvasResolution,
type Fps,
type FpsInput,
fpsToNumber,
toFps,
} from "@hyperframes/core";
import {
type EngineConfig,
resolveConfig,
@@ -127,6 +133,13 @@ import {
type RenderObservabilitySummary,
} from "./render/observability.js";
import { type HdrPerfCollector, type HdrPerfSummary } from "./render/hdrPerf.js";
import {
assertVideoFrameCoverage,
computeVideoFrameCoverage,
countAuthoredTimedClips,
resolveVideoCoverageThreshold,
type VideoFrameCoverageReport,
} from "./render/videoFrameCoverage.js";
import { runCompileStage } from "./render/stages/compileStage.js";
import { runProbeStage } from "./render/stages/probeStage.js";
import {
@@ -177,11 +190,25 @@ function sampleDirectoryBytes(dir: string): number {
function summarizeExtractionObservability(
extractionResult: ExtractionResult | null,
videoCount: number,
coverageReports?: readonly VideoFrameCoverageReport[],
authoredTimedClipCount?: number,
): RenderExtractionObservability {
const extracted = extractionResult?.extracted ?? [];
const totalFramesExtracted = extractionResult?.totalFramesExtracted ?? 0;
const maxFramesPerVideo = extracted.reduce((max, item) => Math.max(max, item.totalFrames), 0);
const phaseBreakdown = extractionResult?.phaseBreakdown;
// Only surface the coverage gauges when we actually ran the gate — a
// no-video render must not emit a spurious `minVideoFrameCoverageRatio`
// that dashboards interpret as "coverage measured, was 0/0=1".
const coverageGauges =
coverageReports && coverageReports.length > 0
? {
minVideoFrameCoverageRatio: coverageReports.reduce(
(min, r) => Math.min(min, r.ratio),
Number.POSITIVE_INFINITY,
),
}
: {};
return {
videoCount,
extractedVideoCount: extracted.length,
@@ -194,6 +221,8 @@ function summarizeExtractionObservability(
vfrPreflightCount: phaseBreakdown?.vfrPreflightCount,
cacheHits: phaseBreakdown?.cacheHits,
cacheMisses: phaseBreakdown?.cacheMisses,
...coverageGauges,
authoredTimedClipCount,
};
}
@@ -1920,9 +1949,28 @@ export async function executeRenderJob(
imageColorSpaces,
} = extractResult;
perfStages.videoExtractMs = extractResult.videoExtractMs;
// ── Parity gate: per-clip captured-vs-expected-frame coverage ───────
// Fail loudly BEFORE encode if any clip's delivered frames fall below
// the threshold — check/snapshot passes on individual frames while the
// encoded MP4 silently renders the clip blank (field signal
// ts=1784139267: 15-injection later-clip drop; see videoFrameCoverage.ts).
// Also count authored `[data-start]` clip windows as a coarse proxy
// for the ts=1784144554 authored-clip-count-scaled failure shape.
const coverageReports: VideoFrameCoverageReport[] = extractionResult
? computeVideoFrameCoverage(
composition.videos,
extractionResult.extracted,
fpsToNumber(job.config.fps),
)
: [];
const coverageThreshold = resolveVideoCoverageThreshold();
const authoredTimedClipCount = countAuthoredTimedClips(compiled.html);
extractionObservability = summarizeExtractionObservability(
extractionResult,
composition.videos.length,
coverageReports,
authoredTimedClipCount,
);
observability.checkpoint("video_extract", "frames resolved", {
videoCount: extractionObservability.videoCount,
@@ -1934,7 +1982,15 @@ export async function executeRenderJob(
vfrPreflightMs: extractionObservability.vfrPreflightMs ?? null,
cacheHits: extractionObservability.cacheHits ?? null,
cacheMisses: extractionObservability.cacheMisses ?? null,
minVideoFrameCoverageRatio: extractionObservability.minVideoFrameCoverageRatio ?? null,
authoredTimedClipCount: extractionObservability.authoredTimedClipCount ?? null,
});
// Gate AFTER the checkpoint so a coverage-failed render still emits
// the observability row (partial telemetry is still worth having).
// `assertVideoFrameCoverage` no-ops on an empty report list AND on a
// null threshold, so the gate is inert for no-video + opted-out
// renders alike.
assertVideoFrameCoverage(coverageReports, coverageThreshold);
// ── HDR auto-detection ──────────────────────────────────────────────
const effectiveHdr = resolveEffectiveHdrMode({