From 88db83092a635b9e3ef2b7f90e2f50261e8e5cdb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Miguel=20=C3=81ngel?= Date: Tue, 10 Nov 2026 04:16:10 +0000 Subject: [PATCH] fix(producer): refuse a too-slow render only when its typical frame says it cannot finish (#7479) * fix(producer): refuse a too-slow render only when its typical frame says it cannot finish * fix(producer): keep the calibration estimate fields self-describing --- .../src/services/render/captureCost.ts | 10 +++ .../services/render/renderEtaGuard.test.ts | 71 +++++++++++++++++-- .../src/services/render/renderEtaGuard.ts | 21 +++++- .../src/services/renderOrchestrator.test.ts | 12 ++++ .../src/services/renderOrchestrator.ts | 9 +-- 5 files changed, 110 insertions(+), 13 deletions(-) diff --git a/packages/producer/src/services/render/captureCost.ts b/packages/producer/src/services/render/captureCost.ts index 4098cfe3a6..689b9b50c2 100644 --- a/packages/producer/src/services/render/captureCost.ts +++ b/packages/producer/src/services/render/captureCost.ts @@ -35,6 +35,7 @@ export interface CaptureCostEstimate { multiplier: number; reasons: string[]; p95Ms?: number; + medianMs?: number; } export interface CaptureCalibrationSample { @@ -100,12 +101,14 @@ function combineCaptureCostEstimates( multiplier: staticCost.multiplier, reasons: [...staticCost.reasons, ...measuredCost.reasons], p95Ms: measuredCost.p95Ms, + medianMs: measuredCost.medianMs, }; } return { multiplier: measuredCost.multiplier, reasons: [...measuredCost.reasons, ...staticCost.reasons], p95Ms: measuredCost.p95Ms, + medianMs: measuredCost.medianMs, }; } @@ -228,6 +231,12 @@ export function estimateMeasuredCaptureCostMultiplier( return { multiplier: 1, reasons: [] }; } const p95Ms = Math.round(p95Sample.captureTimeMs); + const mid = sorted.length / 2; + const medianMs = Math.round( + sorted.length % 2 === 1 + ? sorted[Math.floor(mid)]!.captureTimeMs + : (sorted[mid - 1]!.captureTimeMs + sorted[mid]!.captureTimeMs) / 2, + ); const multiplier = Math.min( MAX_MEASURED_CAPTURE_COST_MULTIPLIER, Math.max(1, Math.round((p95Ms / CAPTURE_CALIBRATION_TARGET_MS) * 100) / 100), @@ -237,6 +246,7 @@ export function estimateMeasuredCaptureCostMultiplier( multiplier, reasons: multiplier > 1 ? [`calibration-p95=${p95Ms}ms`] : [], p95Ms, + medianMs, }; } diff --git a/packages/producer/src/services/render/renderEtaGuard.test.ts b/packages/producer/src/services/render/renderEtaGuard.test.ts index 3b3a49fe3b..45ff570cd4 100644 --- a/packages/producer/src/services/render/renderEtaGuard.test.ts +++ b/packages/producer/src/services/render/renderEtaGuard.test.ts @@ -1,17 +1,22 @@ import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; -import { RenderTooSlowError, checkRenderEta, startRenderEtaGuard } from "./renderEtaGuard.js"; +import { + RenderTooSlowError, + checkCalibratedRenderEta, + checkRenderEta, + startRenderEtaGuard, +} from "./renderEtaGuard.js"; const RENDER_TOO_SLOW = "RENDER_TOO_SLOW"; describe("checkRenderEta", () => { const base = { maxRenderMs: 100_000, elapsedMs: 0, framesCompleted: 0, totalFrames: 100 }; - it("passes at the 1.2x margin and fails just over it", () => { - expect(checkRenderEta("capture", { ...base, msPerFrame: 1_200 })).toBeUndefined(); - const error = checkRenderEta("capture", { ...base, msPerFrame: 1_201 }); + it("passes at the 2x margin and fails just over it", () => { + expect(checkRenderEta("capture", { ...base, msPerFrame: 2_000 })).toBeUndefined(); + const error = checkRenderEta("capture", { ...base, msPerFrame: 2_001 }); expect(error).toBeInstanceOf(RenderTooSlowError); expect(error?.message).toMatch(/^render too slow:/); - expect(error?.publicMetadata).toMatchObject({ stage: "capture", etaMs: 120_100 }); + expect(error?.publicMetadata).toMatchObject({ stage: "capture", etaMs: 200_100 }); // The producer server reads these off the error to build the wire envelope. expect(error?.code).toBe(RENDER_TOO_SLOW); expect(error?.owner).toBe("user"); @@ -20,7 +25,7 @@ describe("checkRenderEta", () => { it("counts time already spent", () => { expect( - checkRenderEta("capture", { ...base, elapsedMs: 30_000, msPerFrame: 1_000 }), + checkRenderEta("capture", { ...base, elapsedMs: 110_000, msPerFrame: 1_000 }), ).toBeInstanceOf(RenderTooSlowError); }); @@ -44,7 +49,7 @@ describe("checkRenderEta", () => { msPerFrame: 784 / 4, }), ).toBeUndefined(); - // 570 frames, 1 worker, p95 15.8 s: 150 min, caught before capture. + // 570 frames, 1 worker, 15.8 s a frame: 160 min, so it runs to the attempt limit as before. expect( checkRenderEta("capture_calibration", { maxRenderMs: budget, @@ -53,10 +58,62 @@ describe("checkRenderEta", () => { totalFrames: 570, msPerFrame: 15_800 / 1, }), + ).toBeUndefined(); + // Twice as many frames at that cost: over 5 h, caught before capture. + expect( + checkRenderEta("capture_calibration", { + maxRenderMs: budget, + elapsedMs: 600_000, + framesCompleted: 0, + totalFrames: 1_140, + msPerFrame: 15_800 / 1, + }), ).toBeInstanceOf(RenderTooSlowError); }); }); +describe("checkCalibratedRenderEta", () => { + const budget = { maxRenderMs: 5_100_000, elapsedMs: 15_000 }; + + it("lets a render through whose slowest calibration frame alone would refuse it", () => { + // One slow calibration frame among fast ones: the slowest predicts about 3 h, the typical one under 1 h. + const input = { maxRenderMs: 5_100_000, elapsedMs: 33_000, totalFrames: 1_800, workerCount: 1 }; + expect(checkCalibratedRenderEta({ ...input, estimate: { medianMs: 1_421 } })).toBeUndefined(); + expect( + checkRenderEta("capture_calibration", { ...input, framesCompleted: 0, msPerFrame: 5_772 }), + ).toBeInstanceOf(RenderTooSlowError); + }); + + it("lets through a long many-worker render whose typical frame fits the budget", () => { + const input = { + maxRenderMs: 5_100_000, + elapsedMs: 438_000, + totalFrames: 61_092, + workerCount: 6, + }; + expect(checkCalibratedRenderEta({ ...input, estimate: { medianMs: 613 } })).toBeUndefined(); + }); + + it("still refuses a render that would take hours on its typical frame", () => { + const tooSlow = checkCalibratedRenderEta({ + ...budget, + totalFrames: 7_890, + workerCount: 1, + estimate: { medianMs: 1_905 }, + }); + expect(tooSlow?.publicMetadata).toMatchObject({ + stage: "capture_calibration", + totalFrames: 7_890, + }); + }); + + it("skips the check without a calibration estimate", () => { + expect( + checkCalibratedRenderEta({ ...budget, totalFrames: 2_145, workerCount: 1 }), + ).toBeUndefined(); + }); +}); + describe("startRenderEtaGuard", () => { beforeEach(() => vi.useFakeTimers()); afterEach(() => vi.useRealTimers()); diff --git a/packages/producer/src/services/render/renderEtaGuard.ts b/packages/producer/src/services/render/renderEtaGuard.ts index efbc469051..dbab57f42e 100644 --- a/packages/producer/src/services/render/renderEtaGuard.ts +++ b/packages/producer/src/services/render/renderEtaGuard.ts @@ -1,7 +1,7 @@ const RENDER_TOO_SLOW = "RENDER_TOO_SLOW" as const; -/** Fail only when the ETA clears the budget by this factor, so borderline renders survive. */ -const ETA_MARGIN = 1.2; +/** Fail only at twice the budget: a film's per-frame cost varies, so early ETAs overshoot. */ +const ETA_MARGIN = 2; /** The measured rate needs this much capture time and frames; the first frame's warm-up is excluded. */ const MIN_SAMPLE_MS = 60_000; const MIN_SAMPLE_FRAMES = 3; @@ -50,6 +50,23 @@ export function checkRenderEta( return new RenderTooSlowError(stage, { ...input, maxRenderMs, etaMs }); } +/** The pre-capture check. The typical calibration frame predicts the render; the slowest one over-refuses. */ +export function checkCalibratedRenderEta(input: { + maxRenderMs?: number; + elapsedMs: number; + totalFrames: number; + estimate?: { medianMs?: number }; + workerCount: number; +}): RenderTooSlowError | undefined { + return checkRenderEta("capture_calibration", { + maxRenderMs: input.maxRenderMs, + elapsedMs: input.elapsedMs, + framesCompleted: 0, + totalFrames: input.totalFrames, + msPerFrame: (input.estimate?.medianMs ?? 0) / input.workerCount, + }); +} + interface RenderEtaGuardOptions { maxRenderMs?: number; job: { framesRendered?: number }; diff --git a/packages/producer/src/services/renderOrchestrator.test.ts b/packages/producer/src/services/renderOrchestrator.test.ts index de892d005f..dc22df0d20 100644 --- a/packages/producer/src/services/renderOrchestrator.test.ts +++ b/packages/producer/src/services/renderOrchestrator.test.ts @@ -1681,6 +1681,18 @@ describe("estimateMeasuredCaptureCostMultiplier", () => { expect(estimate.reasons).toEqual(["calibration-p95=2400ms"]); }); + it("records the typical calibration frame next to the slowest", () => { + const estimate = estimateMeasuredCaptureCostMultiplier( + [92, 1243, 1183, 983, 717].map((captureTimeMs, i) => ({ + frameIndex: i * 100, + captureTimeMs, + })), + ); + + expect(estimate.p95Ms).toBe(1243); + expect(estimate.medianMs).toBe(983); + }); + it("keeps fast calibration samples at baseline cost", () => { const estimate = estimateMeasuredCaptureCostMultiplier([ { frameIndex: 0, captureTimeMs: 120 }, diff --git a/packages/producer/src/services/renderOrchestrator.ts b/packages/producer/src/services/renderOrchestrator.ts index 9dfc80dc88..2badbe7288 100644 --- a/packages/producer/src/services/renderOrchestrator.ts +++ b/packages/producer/src/services/renderOrchestrator.ts @@ -170,7 +170,7 @@ import { import { runCaptureSegmentedStage } from "./render/stages/captureSegmentedStage.js"; import { RenderTooSlowError, - checkRenderEta, + checkCalibratedRenderEta, startRenderEtaGuard, } from "./render/renderEtaGuard.js"; import { checkPageNeverReady } from "./render/pageNeverReady.js"; @@ -4257,6 +4257,7 @@ async function executeRenderPipeline(input: { forceScreenshot: captureForceScreenshot, multiplier: outcome.calibration?.estimate.multiplier ?? null, p95Ms: outcome.calibration?.estimate.p95Ms ?? null, + medianMs: outcome.calibration?.estimate.medianMs ?? null, }); } else { observability.checkpoint("capture_calibration", "skipped", { @@ -4393,12 +4394,12 @@ async function executeRenderPipeline(input: { // No calibration estimate (explicit workers, short or forced-screenshot renders) skips the // pre-capture check; the measured-rate guard below still covers capture. - const calibratedTooSlow = checkRenderEta("capture_calibration", { + const calibratedTooSlow = checkCalibratedRenderEta({ maxRenderMs: job.config.maxRenderMs, elapsedMs: Date.now() - pipelineStart, - framesCompleted: 0, totalFrames, - msPerFrame: (captureCalibration?.estimate.p95Ms ?? 0) / workerCount, + estimate: captureCalibration?.estimate, + workerCount, }); if (calibratedTooSlow) throw calibratedTooSlow; execution.defer(