mirror of
https://github.com/heygen-com/hyperframes.git
synced 2026-09-11 14:50:02 +00:00
fix(producer): screenshot-retry recovery for disk-path drawElement verify failures
This commit is contained in:
@@ -124,6 +124,18 @@ function revertedRouting(routing: CaptureRouting): CaptureRouting {
|
|||||||
|
|
||||||
/** Pure, exhaustive capture fallback transition. The input plan is never mutated. */
|
/** Pure, exhaustive capture fallback transition. The input plan is never mutated. */
|
||||||
export function replanAfterFailure(plan: CapturePlan, failure: CapturePlanFailure): CapturePlan {
|
export function replanAfterFailure(plan: CapturePlan, failure: CapturePlanFailure): CapturePlan {
|
||||||
|
// Disk-path drawElement self-verification (parallel disk workers under the
|
||||||
|
// explicit fast-capture opt-in) can also trip — the retry stays on the disk
|
||||||
|
// path but forces the screenshot baseline.
|
||||||
|
if (plan.kind === "sdr_disk" && failure.kind === "draw_element_verification") {
|
||||||
|
return createCapturePlan({
|
||||||
|
...plan,
|
||||||
|
forceScreenshot: true,
|
||||||
|
useStreamingEncode: false,
|
||||||
|
useLayeredComposite: false,
|
||||||
|
forceParallelStream: false,
|
||||||
|
});
|
||||||
|
}
|
||||||
if (plan.kind !== "sdr_streaming") {
|
if (plan.kind !== "sdr_streaming") {
|
||||||
throw new Error(`Cannot apply ${failure.kind} to ${plan.kind} capture plan`);
|
throw new Error(`Cannot apply ${failure.kind} to ${plan.kind} capture plan`);
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -109,6 +109,7 @@ import {
|
|||||||
createCapturePlan,
|
createCapturePlan,
|
||||||
replanAfterFailure,
|
replanAfterFailure,
|
||||||
type CapturePlan,
|
type CapturePlan,
|
||||||
|
type SdrDiskCapturePlan,
|
||||||
type CaptureRouting,
|
type CaptureRouting,
|
||||||
} from "./render/capturePlan.js";
|
} from "./render/capturePlan.js";
|
||||||
import { normalizeErrorMessage } from "../utils/errorMessage.js";
|
import { normalizeErrorMessage } from "../utils/errorMessage.js";
|
||||||
@@ -3144,11 +3145,11 @@ async function executeRenderPipeline(input: {
|
|||||||
if (capturePlan.kind !== "sdr_disk") {
|
if (capturePlan.kind !== "sdr_disk") {
|
||||||
throw new Error(`Disk capture requires sdr_disk plan; got ${capturePlan.kind}`);
|
throw new Error(`Disk capture requires sdr_disk plan; got ${capturePlan.kind}`);
|
||||||
}
|
}
|
||||||
const diskPlan = capturePlan;
|
|
||||||
// ── Disk-based capture (original flow) ────────────────────────────
|
// ── Disk-based capture (original flow) ────────────────────────────
|
||||||
resetCaptureAttemptProgress(job);
|
resetCaptureAttemptProgress(job);
|
||||||
const captureFrameStart = Date.now();
|
const captureFrameStart = Date.now();
|
||||||
const captureRes = await observeRenderStage(
|
const invokeDiskCapture = (diskPlan: SdrDiskCapturePlan) =>
|
||||||
|
observeRenderStage(
|
||||||
observability,
|
observability,
|
||||||
"capture_disk",
|
"capture_disk",
|
||||||
captureStageObservationData({ needsAlpha: diskPlan.needsAlpha }),
|
captureStageObservationData({ needsAlpha: diskPlan.needsAlpha }),
|
||||||
@@ -3172,6 +3173,64 @@ async function executeRenderPipeline(input: {
|
|||||||
onProgress,
|
onProgress,
|
||||||
}),
|
}),
|
||||||
);
|
);
|
||||||
|
let captureRes;
|
||||||
|
try {
|
||||||
|
captureRes = await invokeDiskCapture(capturePlan);
|
||||||
|
} catch (err) {
|
||||||
|
// Disk-path drawElement self-verification tripped (a parallel disk
|
||||||
|
// worker's sampled frame diverged from its pre-injection ground
|
||||||
|
// truth — reachable only under the explicit fast-capture opt-in).
|
||||||
|
// Same recovery contract as the streaming drain: re-render on the
|
||||||
|
// screenshot baseline. Anything else keeps its existing semantics.
|
||||||
|
if (
|
||||||
|
!isDrawElementVerificationError(err) ||
|
||||||
|
err instanceof RenderCancelledError ||
|
||||||
|
executionSignal?.aborted === true
|
||||||
|
) {
|
||||||
|
throw err;
|
||||||
|
}
|
||||||
|
const verifyDetails = getDrawElementVerificationDetails(err);
|
||||||
|
deSelfVerifyFallback = true;
|
||||||
|
deFallbackReason = verifyDetails?.kind ?? "psnr";
|
||||||
|
deFallbackFailedDb = roundDb(verifyDetails?.failedDb);
|
||||||
|
deFallbackFrameIndex = verifyDetails?.frameIndex;
|
||||||
|
deFallbackThresholdDb = roundDb(verifyDetails?.verifyThresholdDb);
|
||||||
|
log.warn(
|
||||||
|
"[Render] drawElement self-verification failed on the parallel disk path; " +
|
||||||
|
"re-rendering via screenshot",
|
||||||
|
{ error: err instanceof Error ? err.message : String(err) },
|
||||||
|
);
|
||||||
|
observability.checkpoint(
|
||||||
|
"capture_disk",
|
||||||
|
"drawElement self-verify failed; retrying with forceScreenshot",
|
||||||
|
);
|
||||||
|
// The failed attempt's frames are untrusted BUT satisfy the
|
||||||
|
// completeness check — wipe them so the retry re-captures everything
|
||||||
|
// instead of silently keeping damaged files.
|
||||||
|
rmSync(framesDir, { recursive: true, force: true });
|
||||||
|
mkdirSync(framesDir, { recursive: true });
|
||||||
|
resetCaptureAttemptProgress(job);
|
||||||
|
dedupPerfs.length = 0;
|
||||||
|
cfg.useDrawElement = false;
|
||||||
|
probeSession = null;
|
||||||
|
capturePlan = replanAfterFailure(capturePlan, { kind: "draw_element_verification" });
|
||||||
|
syncCapturePlan();
|
||||||
|
updateCaptureObservability({
|
||||||
|
forceScreenshot: capturePlan.forceScreenshot,
|
||||||
|
deSelfVerifyFallback,
|
||||||
|
deFallbackReason,
|
||||||
|
deFallbackFailedDb,
|
||||||
|
deFallbackFrameIndex,
|
||||||
|
deFallbackThresholdDb,
|
||||||
|
});
|
||||||
|
if (capturePlan.kind !== "sdr_disk") {
|
||||||
|
throw new Error(`Disk verify retry requires sdr_disk plan; got ${capturePlan.kind}`);
|
||||||
|
}
|
||||||
|
captureRes = await invokeDiskCapture(capturePlan);
|
||||||
|
// The first attempt's error marked the phase failed; the retry
|
||||||
|
// recovered it — don't brand the render as failed in telemetry.
|
||||||
|
observability.clearFailure("capture_disk");
|
||||||
|
}
|
||||||
const captureFrameMs = Date.now() - captureFrameStart;
|
const captureFrameMs = Date.now() - captureFrameStart;
|
||||||
workerCount = captureRes.workerCount;
|
workerCount = captureRes.workerCount;
|
||||||
updateCaptureObservability({ workerCount });
|
updateCaptureObservability({ workerCount });
|
||||||
|
|||||||
Reference in New Issue
Block a user