mirror of
https://github.com/heygen-com/hyperframes.git
synced 2026-11-18 07:50:13 +00:00
fix(producer): refuse a too-slow render only when its typical frame says it cannot finish (#7479)
* fix(producer): refuse a too-slow render only when its typical frame says it cannot finish * fix(producer): keep the calibration estimate fields self-describing
This commit is contained in:
@@ -35,6 +35,7 @@ export interface CaptureCostEstimate {
|
||||
multiplier: number;
|
||||
reasons: string[];
|
||||
p95Ms?: number;
|
||||
medianMs?: number;
|
||||
}
|
||||
|
||||
export interface CaptureCalibrationSample {
|
||||
@@ -100,12 +101,14 @@ function combineCaptureCostEstimates(
|
||||
multiplier: staticCost.multiplier,
|
||||
reasons: [...staticCost.reasons, ...measuredCost.reasons],
|
||||
p95Ms: measuredCost.p95Ms,
|
||||
medianMs: measuredCost.medianMs,
|
||||
};
|
||||
}
|
||||
return {
|
||||
multiplier: measuredCost.multiplier,
|
||||
reasons: [...measuredCost.reasons, ...staticCost.reasons],
|
||||
p95Ms: measuredCost.p95Ms,
|
||||
medianMs: measuredCost.medianMs,
|
||||
};
|
||||
}
|
||||
|
||||
@@ -228,6 +231,12 @@ export function estimateMeasuredCaptureCostMultiplier(
|
||||
return { multiplier: 1, reasons: [] };
|
||||
}
|
||||
const p95Ms = Math.round(p95Sample.captureTimeMs);
|
||||
const mid = sorted.length / 2;
|
||||
const medianMs = Math.round(
|
||||
sorted.length % 2 === 1
|
||||
? sorted[Math.floor(mid)]!.captureTimeMs
|
||||
: (sorted[mid - 1]!.captureTimeMs + sorted[mid]!.captureTimeMs) / 2,
|
||||
);
|
||||
const multiplier = Math.min(
|
||||
MAX_MEASURED_CAPTURE_COST_MULTIPLIER,
|
||||
Math.max(1, Math.round((p95Ms / CAPTURE_CALIBRATION_TARGET_MS) * 100) / 100),
|
||||
@@ -237,6 +246,7 @@ export function estimateMeasuredCaptureCostMultiplier(
|
||||
multiplier,
|
||||
reasons: multiplier > 1 ? [`calibration-p95=${p95Ms}ms`] : [],
|
||||
p95Ms,
|
||||
medianMs,
|
||||
};
|
||||
}
|
||||
|
||||
|
||||
@@ -1,17 +1,22 @@
|
||||
import { afterEach, beforeEach, describe, expect, it, vi } from "vitest";
|
||||
import { RenderTooSlowError, checkRenderEta, startRenderEtaGuard } from "./renderEtaGuard.js";
|
||||
import {
|
||||
RenderTooSlowError,
|
||||
checkCalibratedRenderEta,
|
||||
checkRenderEta,
|
||||
startRenderEtaGuard,
|
||||
} from "./renderEtaGuard.js";
|
||||
|
||||
const RENDER_TOO_SLOW = "RENDER_TOO_SLOW";
|
||||
|
||||
describe("checkRenderEta", () => {
|
||||
const base = { maxRenderMs: 100_000, elapsedMs: 0, framesCompleted: 0, totalFrames: 100 };
|
||||
|
||||
it("passes at the 1.2x margin and fails just over it", () => {
|
||||
expect(checkRenderEta("capture", { ...base, msPerFrame: 1_200 })).toBeUndefined();
|
||||
const error = checkRenderEta("capture", { ...base, msPerFrame: 1_201 });
|
||||
it("passes at the 2x margin and fails just over it", () => {
|
||||
expect(checkRenderEta("capture", { ...base, msPerFrame: 2_000 })).toBeUndefined();
|
||||
const error = checkRenderEta("capture", { ...base, msPerFrame: 2_001 });
|
||||
expect(error).toBeInstanceOf(RenderTooSlowError);
|
||||
expect(error?.message).toMatch(/^render too slow:/);
|
||||
expect(error?.publicMetadata).toMatchObject({ stage: "capture", etaMs: 120_100 });
|
||||
expect(error?.publicMetadata).toMatchObject({ stage: "capture", etaMs: 200_100 });
|
||||
// The producer server reads these off the error to build the wire envelope.
|
||||
expect(error?.code).toBe(RENDER_TOO_SLOW);
|
||||
expect(error?.owner).toBe("user");
|
||||
@@ -20,7 +25,7 @@ describe("checkRenderEta", () => {
|
||||
|
||||
it("counts time already spent", () => {
|
||||
expect(
|
||||
checkRenderEta("capture", { ...base, elapsedMs: 30_000, msPerFrame: 1_000 }),
|
||||
checkRenderEta("capture", { ...base, elapsedMs: 110_000, msPerFrame: 1_000 }),
|
||||
).toBeInstanceOf(RenderTooSlowError);
|
||||
});
|
||||
|
||||
@@ -44,7 +49,7 @@ describe("checkRenderEta", () => {
|
||||
msPerFrame: 784 / 4,
|
||||
}),
|
||||
).toBeUndefined();
|
||||
// 570 frames, 1 worker, p95 15.8 s: 150 min, caught before capture.
|
||||
// 570 frames, 1 worker, 15.8 s a frame: 160 min, so it runs to the attempt limit as before.
|
||||
expect(
|
||||
checkRenderEta("capture_calibration", {
|
||||
maxRenderMs: budget,
|
||||
@@ -53,10 +58,62 @@ describe("checkRenderEta", () => {
|
||||
totalFrames: 570,
|
||||
msPerFrame: 15_800 / 1,
|
||||
}),
|
||||
).toBeUndefined();
|
||||
// Twice as many frames at that cost: over 5 h, caught before capture.
|
||||
expect(
|
||||
checkRenderEta("capture_calibration", {
|
||||
maxRenderMs: budget,
|
||||
elapsedMs: 600_000,
|
||||
framesCompleted: 0,
|
||||
totalFrames: 1_140,
|
||||
msPerFrame: 15_800 / 1,
|
||||
}),
|
||||
).toBeInstanceOf(RenderTooSlowError);
|
||||
});
|
||||
});
|
||||
|
||||
describe("checkCalibratedRenderEta", () => {
|
||||
const budget = { maxRenderMs: 5_100_000, elapsedMs: 15_000 };
|
||||
|
||||
it("lets a render through whose slowest calibration frame alone would refuse it", () => {
|
||||
// One slow calibration frame among fast ones: the slowest predicts about 3 h, the typical one under 1 h.
|
||||
const input = { maxRenderMs: 5_100_000, elapsedMs: 33_000, totalFrames: 1_800, workerCount: 1 };
|
||||
expect(checkCalibratedRenderEta({ ...input, estimate: { medianMs: 1_421 } })).toBeUndefined();
|
||||
expect(
|
||||
checkRenderEta("capture_calibration", { ...input, framesCompleted: 0, msPerFrame: 5_772 }),
|
||||
).toBeInstanceOf(RenderTooSlowError);
|
||||
});
|
||||
|
||||
it("lets through a long many-worker render whose typical frame fits the budget", () => {
|
||||
const input = {
|
||||
maxRenderMs: 5_100_000,
|
||||
elapsedMs: 438_000,
|
||||
totalFrames: 61_092,
|
||||
workerCount: 6,
|
||||
};
|
||||
expect(checkCalibratedRenderEta({ ...input, estimate: { medianMs: 613 } })).toBeUndefined();
|
||||
});
|
||||
|
||||
it("still refuses a render that would take hours on its typical frame", () => {
|
||||
const tooSlow = checkCalibratedRenderEta({
|
||||
...budget,
|
||||
totalFrames: 7_890,
|
||||
workerCount: 1,
|
||||
estimate: { medianMs: 1_905 },
|
||||
});
|
||||
expect(tooSlow?.publicMetadata).toMatchObject({
|
||||
stage: "capture_calibration",
|
||||
totalFrames: 7_890,
|
||||
});
|
||||
});
|
||||
|
||||
it("skips the check without a calibration estimate", () => {
|
||||
expect(
|
||||
checkCalibratedRenderEta({ ...budget, totalFrames: 2_145, workerCount: 1 }),
|
||||
).toBeUndefined();
|
||||
});
|
||||
});
|
||||
|
||||
describe("startRenderEtaGuard", () => {
|
||||
beforeEach(() => vi.useFakeTimers());
|
||||
afterEach(() => vi.useRealTimers());
|
||||
|
||||
@@ -1,7 +1,7 @@
|
||||
const RENDER_TOO_SLOW = "RENDER_TOO_SLOW" as const;
|
||||
|
||||
/** Fail only when the ETA clears the budget by this factor, so borderline renders survive. */
|
||||
const ETA_MARGIN = 1.2;
|
||||
/** Fail only at twice the budget: a film's per-frame cost varies, so early ETAs overshoot. */
|
||||
const ETA_MARGIN = 2;
|
||||
/** The measured rate needs this much capture time and frames; the first frame's warm-up is excluded. */
|
||||
const MIN_SAMPLE_MS = 60_000;
|
||||
const MIN_SAMPLE_FRAMES = 3;
|
||||
@@ -50,6 +50,23 @@ export function checkRenderEta(
|
||||
return new RenderTooSlowError(stage, { ...input, maxRenderMs, etaMs });
|
||||
}
|
||||
|
||||
/** The pre-capture check. The typical calibration frame predicts the render; the slowest one over-refuses. */
|
||||
export function checkCalibratedRenderEta(input: {
|
||||
maxRenderMs?: number;
|
||||
elapsedMs: number;
|
||||
totalFrames: number;
|
||||
estimate?: { medianMs?: number };
|
||||
workerCount: number;
|
||||
}): RenderTooSlowError | undefined {
|
||||
return checkRenderEta("capture_calibration", {
|
||||
maxRenderMs: input.maxRenderMs,
|
||||
elapsedMs: input.elapsedMs,
|
||||
framesCompleted: 0,
|
||||
totalFrames: input.totalFrames,
|
||||
msPerFrame: (input.estimate?.medianMs ?? 0) / input.workerCount,
|
||||
});
|
||||
}
|
||||
|
||||
interface RenderEtaGuardOptions {
|
||||
maxRenderMs?: number;
|
||||
job: { framesRendered?: number };
|
||||
|
||||
@@ -1681,6 +1681,18 @@ describe("estimateMeasuredCaptureCostMultiplier", () => {
|
||||
expect(estimate.reasons).toEqual(["calibration-p95=2400ms"]);
|
||||
});
|
||||
|
||||
it("records the typical calibration frame next to the slowest", () => {
|
||||
const estimate = estimateMeasuredCaptureCostMultiplier(
|
||||
[92, 1243, 1183, 983, 717].map((captureTimeMs, i) => ({
|
||||
frameIndex: i * 100,
|
||||
captureTimeMs,
|
||||
})),
|
||||
);
|
||||
|
||||
expect(estimate.p95Ms).toBe(1243);
|
||||
expect(estimate.medianMs).toBe(983);
|
||||
});
|
||||
|
||||
it("keeps fast calibration samples at baseline cost", () => {
|
||||
const estimate = estimateMeasuredCaptureCostMultiplier([
|
||||
{ frameIndex: 0, captureTimeMs: 120 },
|
||||
|
||||
@@ -170,7 +170,7 @@ import {
|
||||
import { runCaptureSegmentedStage } from "./render/stages/captureSegmentedStage.js";
|
||||
import {
|
||||
RenderTooSlowError,
|
||||
checkRenderEta,
|
||||
checkCalibratedRenderEta,
|
||||
startRenderEtaGuard,
|
||||
} from "./render/renderEtaGuard.js";
|
||||
import { checkPageNeverReady } from "./render/pageNeverReady.js";
|
||||
@@ -4257,6 +4257,7 @@ async function executeRenderPipeline(input: {
|
||||
forceScreenshot: captureForceScreenshot,
|
||||
multiplier: outcome.calibration?.estimate.multiplier ?? null,
|
||||
p95Ms: outcome.calibration?.estimate.p95Ms ?? null,
|
||||
medianMs: outcome.calibration?.estimate.medianMs ?? null,
|
||||
});
|
||||
} else {
|
||||
observability.checkpoint("capture_calibration", "skipped", {
|
||||
@@ -4393,12 +4394,12 @@ async function executeRenderPipeline(input: {
|
||||
|
||||
// No calibration estimate (explicit workers, short or forced-screenshot renders) skips the
|
||||
// pre-capture check; the measured-rate guard below still covers capture.
|
||||
const calibratedTooSlow = checkRenderEta("capture_calibration", {
|
||||
const calibratedTooSlow = checkCalibratedRenderEta({
|
||||
maxRenderMs: job.config.maxRenderMs,
|
||||
elapsedMs: Date.now() - pipelineStart,
|
||||
framesCompleted: 0,
|
||||
totalFrames,
|
||||
msPerFrame: (captureCalibration?.estimate.p95Ms ?? 0) / workerCount,
|
||||
estimate: captureCalibration?.estimate,
|
||||
workerCount,
|
||||
});
|
||||
if (calibratedTooSlow) throw calibratedTooSlow;
|
||||
execution.defer(
|
||||
|
||||
Reference in New Issue
Block a user