fix(producer): refuse a too-slow render only when its typical frame says it cannot finish (#7479)

* fix(producer): refuse a too-slow render only when its typical frame says it cannot finish

* fix(producer): keep the calibration estimate fields self-describing
This commit is contained in:
Miguel Ángel
2026-11-10 04:16:10 +00:00
committed by GitHub
parent 1a758fd60f
commit 88db83092a
5 changed files with 110 additions and 13 deletions
@@ -35,6 +35,7 @@ export interface CaptureCostEstimate {
multiplier: number;
reasons: string[];
p95Ms?: number;
medianMs?: number;
}
export interface CaptureCalibrationSample {
@@ -100,12 +101,14 @@ function combineCaptureCostEstimates(
multiplier: staticCost.multiplier,
reasons: [...staticCost.reasons, ...measuredCost.reasons],
p95Ms: measuredCost.p95Ms,
medianMs: measuredCost.medianMs,
};
}
return {
multiplier: measuredCost.multiplier,
reasons: [...measuredCost.reasons, ...staticCost.reasons],
p95Ms: measuredCost.p95Ms,
medianMs: measuredCost.medianMs,
};
}
@@ -228,6 +231,12 @@ export function estimateMeasuredCaptureCostMultiplier(
return { multiplier: 1, reasons: [] };
}
const p95Ms = Math.round(p95Sample.captureTimeMs);
const mid = sorted.length / 2;
const medianMs = Math.round(
sorted.length % 2 === 1
? sorted[Math.floor(mid)]!.captureTimeMs
: (sorted[mid - 1]!.captureTimeMs + sorted[mid]!.captureTimeMs) / 2,
);
const multiplier = Math.min(
MAX_MEASURED_CAPTURE_COST_MULTIPLIER,
Math.max(1, Math.round((p95Ms / CAPTURE_CALIBRATION_TARGET_MS) * 100) / 100),
@@ -237,6 +246,7 @@ export function estimateMeasuredCaptureCostMultiplier(
multiplier,
reasons: multiplier > 1 ? [`calibration-p95=${p95Ms}ms`] : [],
p95Ms,
medianMs,
};
}
@@ -1,17 +1,22 @@
import { afterEach, beforeEach, describe, expect, it, vi } from "vitest";
import { RenderTooSlowError, checkRenderEta, startRenderEtaGuard } from "./renderEtaGuard.js";
import {
RenderTooSlowError,
checkCalibratedRenderEta,
checkRenderEta,
startRenderEtaGuard,
} from "./renderEtaGuard.js";
const RENDER_TOO_SLOW = "RENDER_TOO_SLOW";
describe("checkRenderEta", () => {
const base = { maxRenderMs: 100_000, elapsedMs: 0, framesCompleted: 0, totalFrames: 100 };
it("passes at the 1.2x margin and fails just over it", () => {
expect(checkRenderEta("capture", { ...base, msPerFrame: 1_200 })).toBeUndefined();
const error = checkRenderEta("capture", { ...base, msPerFrame: 1_201 });
it("passes at the 2x margin and fails just over it", () => {
expect(checkRenderEta("capture", { ...base, msPerFrame: 2_000 })).toBeUndefined();
const error = checkRenderEta("capture", { ...base, msPerFrame: 2_001 });
expect(error).toBeInstanceOf(RenderTooSlowError);
expect(error?.message).toMatch(/^render too slow:/);
expect(error?.publicMetadata).toMatchObject({ stage: "capture", etaMs: 120_100 });
expect(error?.publicMetadata).toMatchObject({ stage: "capture", etaMs: 200_100 });
// The producer server reads these off the error to build the wire envelope.
expect(error?.code).toBe(RENDER_TOO_SLOW);
expect(error?.owner).toBe("user");
@@ -20,7 +25,7 @@ describe("checkRenderEta", () => {
it("counts time already spent", () => {
expect(
checkRenderEta("capture", { ...base, elapsedMs: 30_000, msPerFrame: 1_000 }),
checkRenderEta("capture", { ...base, elapsedMs: 110_000, msPerFrame: 1_000 }),
).toBeInstanceOf(RenderTooSlowError);
});
@@ -44,7 +49,7 @@ describe("checkRenderEta", () => {
msPerFrame: 784 / 4,
}),
).toBeUndefined();
// 570 frames, 1 worker, p95 15.8 s: 150 min, caught before capture.
// 570 frames, 1 worker, 15.8 s a frame: 160 min, so it runs to the attempt limit as before.
expect(
checkRenderEta("capture_calibration", {
maxRenderMs: budget,
@@ -53,10 +58,62 @@ describe("checkRenderEta", () => {
totalFrames: 570,
msPerFrame: 15_800 / 1,
}),
).toBeUndefined();
// Twice as many frames at that cost: over 5 h, caught before capture.
expect(
checkRenderEta("capture_calibration", {
maxRenderMs: budget,
elapsedMs: 600_000,
framesCompleted: 0,
totalFrames: 1_140,
msPerFrame: 15_800 / 1,
}),
).toBeInstanceOf(RenderTooSlowError);
});
});
describe("checkCalibratedRenderEta", () => {
const budget = { maxRenderMs: 5_100_000, elapsedMs: 15_000 };
it("lets a render through whose slowest calibration frame alone would refuse it", () => {
// One slow calibration frame among fast ones: the slowest predicts about 3 h, the typical one under 1 h.
const input = { maxRenderMs: 5_100_000, elapsedMs: 33_000, totalFrames: 1_800, workerCount: 1 };
expect(checkCalibratedRenderEta({ ...input, estimate: { medianMs: 1_421 } })).toBeUndefined();
expect(
checkRenderEta("capture_calibration", { ...input, framesCompleted: 0, msPerFrame: 5_772 }),
).toBeInstanceOf(RenderTooSlowError);
});
it("lets through a long many-worker render whose typical frame fits the budget", () => {
const input = {
maxRenderMs: 5_100_000,
elapsedMs: 438_000,
totalFrames: 61_092,
workerCount: 6,
};
expect(checkCalibratedRenderEta({ ...input, estimate: { medianMs: 613 } })).toBeUndefined();
});
it("still refuses a render that would take hours on its typical frame", () => {
const tooSlow = checkCalibratedRenderEta({
...budget,
totalFrames: 7_890,
workerCount: 1,
estimate: { medianMs: 1_905 },
});
expect(tooSlow?.publicMetadata).toMatchObject({
stage: "capture_calibration",
totalFrames: 7_890,
});
});
it("skips the check without a calibration estimate", () => {
expect(
checkCalibratedRenderEta({ ...budget, totalFrames: 2_145, workerCount: 1 }),
).toBeUndefined();
});
});
describe("startRenderEtaGuard", () => {
beforeEach(() => vi.useFakeTimers());
afterEach(() => vi.useRealTimers());
@@ -1,7 +1,7 @@
const RENDER_TOO_SLOW = "RENDER_TOO_SLOW" as const;
/** Fail only when the ETA clears the budget by this factor, so borderline renders survive. */
const ETA_MARGIN = 1.2;
/** Fail only at twice the budget: a film's per-frame cost varies, so early ETAs overshoot. */
const ETA_MARGIN = 2;
/** The measured rate needs this much capture time and frames; the first frame's warm-up is excluded. */
const MIN_SAMPLE_MS = 60_000;
const MIN_SAMPLE_FRAMES = 3;
@@ -50,6 +50,23 @@ export function checkRenderEta(
return new RenderTooSlowError(stage, { ...input, maxRenderMs, etaMs });
}
/** The pre-capture check. The typical calibration frame predicts the render; the slowest one over-refuses. */
export function checkCalibratedRenderEta(input: {
maxRenderMs?: number;
elapsedMs: number;
totalFrames: number;
estimate?: { medianMs?: number };
workerCount: number;
}): RenderTooSlowError | undefined {
return checkRenderEta("capture_calibration", {
maxRenderMs: input.maxRenderMs,
elapsedMs: input.elapsedMs,
framesCompleted: 0,
totalFrames: input.totalFrames,
msPerFrame: (input.estimate?.medianMs ?? 0) / input.workerCount,
});
}
interface RenderEtaGuardOptions {
maxRenderMs?: number;
job: { framesRendered?: number };
@@ -1681,6 +1681,18 @@ describe("estimateMeasuredCaptureCostMultiplier", () => {
expect(estimate.reasons).toEqual(["calibration-p95=2400ms"]);
});
it("records the typical calibration frame next to the slowest", () => {
const estimate = estimateMeasuredCaptureCostMultiplier(
[92, 1243, 1183, 983, 717].map((captureTimeMs, i) => ({
frameIndex: i * 100,
captureTimeMs,
})),
);
expect(estimate.p95Ms).toBe(1243);
expect(estimate.medianMs).toBe(983);
});
it("keeps fast calibration samples at baseline cost", () => {
const estimate = estimateMeasuredCaptureCostMultiplier([
{ frameIndex: 0, captureTimeMs: 120 },
@@ -170,7 +170,7 @@ import {
import { runCaptureSegmentedStage } from "./render/stages/captureSegmentedStage.js";
import {
RenderTooSlowError,
checkRenderEta,
checkCalibratedRenderEta,
startRenderEtaGuard,
} from "./render/renderEtaGuard.js";
import { checkPageNeverReady } from "./render/pageNeverReady.js";
@@ -4257,6 +4257,7 @@ async function executeRenderPipeline(input: {
forceScreenshot: captureForceScreenshot,
multiplier: outcome.calibration?.estimate.multiplier ?? null,
p95Ms: outcome.calibration?.estimate.p95Ms ?? null,
medianMs: outcome.calibration?.estimate.medianMs ?? null,
});
} else {
observability.checkpoint("capture_calibration", "skipped", {
@@ -4393,12 +4394,12 @@ async function executeRenderPipeline(input: {
// No calibration estimate (explicit workers, short or forced-screenshot renders) skips the
// pre-capture check; the measured-rate guard below still covers capture.
const calibratedTooSlow = checkRenderEta("capture_calibration", {
const calibratedTooSlow = checkCalibratedRenderEta({
maxRenderMs: job.config.maxRenderMs,
elapsedMs: Date.now() - pipelineStart,
framesCompleted: 0,
totalFrames,
msPerFrame: (captureCalibration?.estimate.p95Ms ?? 0) / workerCount,
estimate: captureCalibration?.estimate,
workerCount,
});
if (calibratedTooSlow) throw calibratedTooSlow;
execution.defer(