fix(producer): screenshot-retry recovery for disk-path drawElement verify failures

This commit is contained in:
Vance Ingalls
2026-07-23 15:00:26 -07:00
parent 060b6f8ae5
commit ec791e91d9
2 changed files with 96 additions and 25 deletions
@@ -124,6 +124,18 @@ function revertedRouting(routing: CaptureRouting): CaptureRouting {
/** Pure, exhaustive capture fallback transition. The input plan is never mutated. */
export function replanAfterFailure(plan: CapturePlan, failure: CapturePlanFailure): CapturePlan {
// Disk-path drawElement self-verification (parallel disk workers under the
// explicit fast-capture opt-in) can also trip — the retry stays on the disk
// path but forces the screenshot baseline.
if (plan.kind === "sdr_disk" && failure.kind === "draw_element_verification") {
return createCapturePlan({
...plan,
forceScreenshot: true,
useStreamingEncode: false,
useLayeredComposite: false,
forceParallelStream: false,
});
}
if (plan.kind !== "sdr_streaming") {
throw new Error(`Cannot apply ${failure.kind} to ${plan.kind} capture plan`);
}
@@ -109,6 +109,7 @@ import {
createCapturePlan,
replanAfterFailure,
type CapturePlan,
type SdrDiskCapturePlan,
type CaptureRouting,
} from "./render/capturePlan.js";
import { normalizeErrorMessage } from "../utils/errorMessage.js";
@@ -3144,34 +3145,92 @@ async function executeRenderPipeline(input: {
if (capturePlan.kind !== "sdr_disk") {
throw new Error(`Disk capture requires sdr_disk plan; got ${capturePlan.kind}`);
}
const diskPlan = capturePlan;
// ── Disk-based capture (original flow) ────────────────────────────
resetCaptureAttemptProgress(job);
const captureFrameStart = Date.now();
const captureRes = await observeRenderStage(
observability,
"capture_disk",
captureStageObservationData({ needsAlpha: diskPlan.needsAlpha }),
() =>
runCaptureStage({
fileServer: activeFileServer,
workDir,
framesDir,
job,
totalFrames,
cfg,
plan: diskPlan,
log,
probeSession,
captureAttempts,
dedupPerfs,
buildCaptureOptions,
createRenderVideoFrameInjector,
abortSignal: executionSignal,
assertNotAborted,
onProgress,
}),
);
const invokeDiskCapture = (diskPlan: SdrDiskCapturePlan) =>
observeRenderStage(
observability,
"capture_disk",
captureStageObservationData({ needsAlpha: diskPlan.needsAlpha }),
() =>
runCaptureStage({
fileServer: activeFileServer,
workDir,
framesDir,
job,
totalFrames,
cfg,
plan: diskPlan,
log,
probeSession,
captureAttempts,
dedupPerfs,
buildCaptureOptions,
createRenderVideoFrameInjector,
abortSignal: executionSignal,
assertNotAborted,
onProgress,
}),
);
let captureRes;
try {
captureRes = await invokeDiskCapture(capturePlan);
} catch (err) {
// Disk-path drawElement self-verification tripped (a parallel disk
// worker's sampled frame diverged from its pre-injection ground
// truth — reachable only under the explicit fast-capture opt-in).
// Same recovery contract as the streaming drain: re-render on the
// screenshot baseline. Anything else keeps its existing semantics.
if (
!isDrawElementVerificationError(err) ||
err instanceof RenderCancelledError ||
executionSignal?.aborted === true
) {
throw err;
}
const verifyDetails = getDrawElementVerificationDetails(err);
deSelfVerifyFallback = true;
deFallbackReason = verifyDetails?.kind ?? "psnr";
deFallbackFailedDb = roundDb(verifyDetails?.failedDb);
deFallbackFrameIndex = verifyDetails?.frameIndex;
deFallbackThresholdDb = roundDb(verifyDetails?.verifyThresholdDb);
log.warn(
"[Render] drawElement self-verification failed on the parallel disk path; " +
"re-rendering via screenshot",
{ error: err instanceof Error ? err.message : String(err) },
);
observability.checkpoint(
"capture_disk",
"drawElement self-verify failed; retrying with forceScreenshot",
);
// The failed attempt's frames are untrusted BUT satisfy the
// completeness check — wipe them so the retry re-captures everything
// instead of silently keeping damaged files.
rmSync(framesDir, { recursive: true, force: true });
mkdirSync(framesDir, { recursive: true });
resetCaptureAttemptProgress(job);
dedupPerfs.length = 0;
cfg.useDrawElement = false;
probeSession = null;
capturePlan = replanAfterFailure(capturePlan, { kind: "draw_element_verification" });
syncCapturePlan();
updateCaptureObservability({
forceScreenshot: capturePlan.forceScreenshot,
deSelfVerifyFallback,
deFallbackReason,
deFallbackFailedDb,
deFallbackFrameIndex,
deFallbackThresholdDb,
});
if (capturePlan.kind !== "sdr_disk") {
throw new Error(`Disk verify retry requires sdr_disk plan; got ${capturePlan.kind}`);
}
captureRes = await invokeDiskCapture(capturePlan);
// The first attempt's error marked the phase failed; the retry
// recovered it — don't brand the render as failed in telemetry.
observability.clearFailure("capture_disk");
}
const captureFrameMs = Date.now() - captureFrameStart;
workerCount = captureRes.workerCount;
updateCaptureObservability({ workerCount });