mirror of
https://github.com/heygen-com/hyperframes.git
synced 2026-09-11 06:30:03 +00:00
fix(producer): screenshot-retry recovery for disk-path drawElement verify failures
This commit is contained in:
@@ -124,6 +124,18 @@ function revertedRouting(routing: CaptureRouting): CaptureRouting {
|
||||
|
||||
/** Pure, exhaustive capture fallback transition. The input plan is never mutated. */
|
||||
export function replanAfterFailure(plan: CapturePlan, failure: CapturePlanFailure): CapturePlan {
|
||||
// Disk-path drawElement self-verification (parallel disk workers under the
|
||||
// explicit fast-capture opt-in) can also trip — the retry stays on the disk
|
||||
// path but forces the screenshot baseline.
|
||||
if (plan.kind === "sdr_disk" && failure.kind === "draw_element_verification") {
|
||||
return createCapturePlan({
|
||||
...plan,
|
||||
forceScreenshot: true,
|
||||
useStreamingEncode: false,
|
||||
useLayeredComposite: false,
|
||||
forceParallelStream: false,
|
||||
});
|
||||
}
|
||||
if (plan.kind !== "sdr_streaming") {
|
||||
throw new Error(`Cannot apply ${failure.kind} to ${plan.kind} capture plan`);
|
||||
}
|
||||
|
||||
@@ -109,6 +109,7 @@ import {
|
||||
createCapturePlan,
|
||||
replanAfterFailure,
|
||||
type CapturePlan,
|
||||
type SdrDiskCapturePlan,
|
||||
type CaptureRouting,
|
||||
} from "./render/capturePlan.js";
|
||||
import { normalizeErrorMessage } from "../utils/errorMessage.js";
|
||||
@@ -3144,34 +3145,92 @@ async function executeRenderPipeline(input: {
|
||||
if (capturePlan.kind !== "sdr_disk") {
|
||||
throw new Error(`Disk capture requires sdr_disk plan; got ${capturePlan.kind}`);
|
||||
}
|
||||
const diskPlan = capturePlan;
|
||||
// ── Disk-based capture (original flow) ────────────────────────────
|
||||
resetCaptureAttemptProgress(job);
|
||||
const captureFrameStart = Date.now();
|
||||
const captureRes = await observeRenderStage(
|
||||
observability,
|
||||
"capture_disk",
|
||||
captureStageObservationData({ needsAlpha: diskPlan.needsAlpha }),
|
||||
() =>
|
||||
runCaptureStage({
|
||||
fileServer: activeFileServer,
|
||||
workDir,
|
||||
framesDir,
|
||||
job,
|
||||
totalFrames,
|
||||
cfg,
|
||||
plan: diskPlan,
|
||||
log,
|
||||
probeSession,
|
||||
captureAttempts,
|
||||
dedupPerfs,
|
||||
buildCaptureOptions,
|
||||
createRenderVideoFrameInjector,
|
||||
abortSignal: executionSignal,
|
||||
assertNotAborted,
|
||||
onProgress,
|
||||
}),
|
||||
);
|
||||
const invokeDiskCapture = (diskPlan: SdrDiskCapturePlan) =>
|
||||
observeRenderStage(
|
||||
observability,
|
||||
"capture_disk",
|
||||
captureStageObservationData({ needsAlpha: diskPlan.needsAlpha }),
|
||||
() =>
|
||||
runCaptureStage({
|
||||
fileServer: activeFileServer,
|
||||
workDir,
|
||||
framesDir,
|
||||
job,
|
||||
totalFrames,
|
||||
cfg,
|
||||
plan: diskPlan,
|
||||
log,
|
||||
probeSession,
|
||||
captureAttempts,
|
||||
dedupPerfs,
|
||||
buildCaptureOptions,
|
||||
createRenderVideoFrameInjector,
|
||||
abortSignal: executionSignal,
|
||||
assertNotAborted,
|
||||
onProgress,
|
||||
}),
|
||||
);
|
||||
let captureRes;
|
||||
try {
|
||||
captureRes = await invokeDiskCapture(capturePlan);
|
||||
} catch (err) {
|
||||
// Disk-path drawElement self-verification tripped (a parallel disk
|
||||
// worker's sampled frame diverged from its pre-injection ground
|
||||
// truth — reachable only under the explicit fast-capture opt-in).
|
||||
// Same recovery contract as the streaming drain: re-render on the
|
||||
// screenshot baseline. Anything else keeps its existing semantics.
|
||||
if (
|
||||
!isDrawElementVerificationError(err) ||
|
||||
err instanceof RenderCancelledError ||
|
||||
executionSignal?.aborted === true
|
||||
) {
|
||||
throw err;
|
||||
}
|
||||
const verifyDetails = getDrawElementVerificationDetails(err);
|
||||
deSelfVerifyFallback = true;
|
||||
deFallbackReason = verifyDetails?.kind ?? "psnr";
|
||||
deFallbackFailedDb = roundDb(verifyDetails?.failedDb);
|
||||
deFallbackFrameIndex = verifyDetails?.frameIndex;
|
||||
deFallbackThresholdDb = roundDb(verifyDetails?.verifyThresholdDb);
|
||||
log.warn(
|
||||
"[Render] drawElement self-verification failed on the parallel disk path; " +
|
||||
"re-rendering via screenshot",
|
||||
{ error: err instanceof Error ? err.message : String(err) },
|
||||
);
|
||||
observability.checkpoint(
|
||||
"capture_disk",
|
||||
"drawElement self-verify failed; retrying with forceScreenshot",
|
||||
);
|
||||
// The failed attempt's frames are untrusted BUT satisfy the
|
||||
// completeness check — wipe them so the retry re-captures everything
|
||||
// instead of silently keeping damaged files.
|
||||
rmSync(framesDir, { recursive: true, force: true });
|
||||
mkdirSync(framesDir, { recursive: true });
|
||||
resetCaptureAttemptProgress(job);
|
||||
dedupPerfs.length = 0;
|
||||
cfg.useDrawElement = false;
|
||||
probeSession = null;
|
||||
capturePlan = replanAfterFailure(capturePlan, { kind: "draw_element_verification" });
|
||||
syncCapturePlan();
|
||||
updateCaptureObservability({
|
||||
forceScreenshot: capturePlan.forceScreenshot,
|
||||
deSelfVerifyFallback,
|
||||
deFallbackReason,
|
||||
deFallbackFailedDb,
|
||||
deFallbackFrameIndex,
|
||||
deFallbackThresholdDb,
|
||||
});
|
||||
if (capturePlan.kind !== "sdr_disk") {
|
||||
throw new Error(`Disk verify retry requires sdr_disk plan; got ${capturePlan.kind}`);
|
||||
}
|
||||
captureRes = await invokeDiskCapture(capturePlan);
|
||||
// The first attempt's error marked the phase failed; the retry
|
||||
// recovered it — don't brand the render as failed in telemetry.
|
||||
observability.clearFailure("capture_disk");
|
||||
}
|
||||
const captureFrameMs = Date.now() - captureFrameStart;
|
||||
workerCount = captureRes.workerCount;
|
||||
updateCaptureObservability({ workerCount });
|
||||
|
||||
Reference in New Issue
Block a user