mirror of
https://github.com/heygen-com/hyperframes.git
synced 2026-11-17 23:47:33 +00:00
fix(producer,engine,cli): oom wrapping, cancellation, fallback-reason gaps
Three defects found by max-effort code review of this branch: 1. The Bun OOM exact-match regex was defeated by this codebase's own parallel-worker error wrapping. executeParallelCapture/formatWorkerFailure (parallelCoordinator.ts) always wrap a worker's error as "Worker N: <message>", optionally suffixed and joined with other workers' segments, all prefixed "[Parallel] Capture failed: ". That wrapping defeated the exact-message check for exactly the cohort (deParallelRouter routed, N separate Chrome processes) the OOM-drops-to-1 fix targets — a real OOM there would retry at the SAME worker count instead of dropping to 1. Added a second pattern that recovers the signal by requiring "out of memory" appear as the WHOLE content of a "Worker N: ..." segment (bounded by end-of-string/"; "), preserving the same exact-match property (no bare substring match) while surviving the wrapping. Verified against the real wrapping logic, not a hand-typed guess at its shape. 2. shouldRetryViaPinnedFallback didn't exclude cancellation, so aborting a render mid-capture on the pinned router/inversion cohort would detour through spawning a fresh encoder/capture session before the outer catch's RenderCancelledError branch ended the render — delaying "stop" with a pointless resource spin-up/tear-down. Added an isCancellation param (checked first, before isVerifyError) using the same `err instanceof RenderCancelledError || abortSignal?.aborted` check the outer catch already uses. 3. deFallbackReason (this PR's new "oom"/"capture_error" values) was set locally but never mirrored into RenderCaptureObservability alongside deSelfVerifyFallback, so a render that fails AFTER a fallback attempt (perfSummary never built) was indistinguishable in render_error telemetry from one that never attempted any fallback — undercutting the "how often does the OOM retry fire on a render that still ultimately fails" question this branch exists to answer. Threaded through RenderCaptureObservability → RenderObservabilityTelemetryPayload → renderObservabilityTelemetryPayload, mirroring the existing deSelfVerifyFallback plumbing. Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com>
This commit is contained in:
co-authored by
Claude Sonnet 5
parent
b3f244a7e9
commit
a355fb2f6b
@@ -75,6 +75,28 @@ describe("render telemetry events", () => {
|
||||
);
|
||||
});
|
||||
|
||||
it("carries de_fallback_reason on render_error so a render that fails AFTER an OOM-triggered fallback attempt is distinguishable from one that never attempted a fallback", () => {
|
||||
trackRenderError({
|
||||
fps: 30,
|
||||
quality: "standard",
|
||||
docker: false,
|
||||
errorMessage: "worker crashed again after fallback",
|
||||
captureDeParallelRouter: "reverted",
|
||||
captureDeSelfVerifyFallback: false,
|
||||
captureDeFallbackReason: "oom",
|
||||
});
|
||||
|
||||
expect(trackEvent).toHaveBeenCalledWith(
|
||||
"render_error",
|
||||
expect.objectContaining({
|
||||
de_parallel_router: "reverted",
|
||||
de_self_verify_fallback: false,
|
||||
de_fallback_reason: "oom",
|
||||
}),
|
||||
undefined,
|
||||
);
|
||||
});
|
||||
|
||||
it("prefers the explicit perfSummary-sourced de_worker_inversion over the capture-observability fallback on render_complete", () => {
|
||||
trackRenderComplete({
|
||||
durationMs: 1000,
|
||||
|
||||
@@ -46,6 +46,7 @@ export interface RenderObservabilityTelemetryPayload {
|
||||
captureDeParallelRouter?: string;
|
||||
captureDePreRouterWorkers?: number;
|
||||
captureDeSelfVerifyFallback?: boolean;
|
||||
captureDeFallbackReason?: string;
|
||||
observabilityExtractVideoCount?: number;
|
||||
observabilityExtractedVideoCount?: number;
|
||||
observabilityExtractTotalFrames?: number;
|
||||
@@ -96,6 +97,7 @@ function renderObservabilityEventProperties(props: RenderObservabilityTelemetryP
|
||||
de_parallel_router: props.captureDeParallelRouter,
|
||||
de_pre_router_workers: props.captureDePreRouterWorkers,
|
||||
de_self_verify_fallback: props.captureDeSelfVerifyFallback,
|
||||
de_fallback_reason: props.captureDeFallbackReason,
|
||||
observability_extract_video_count: props.observabilityExtractVideoCount,
|
||||
observability_extracted_video_count: props.observabilityExtractedVideoCount,
|
||||
observability_extract_total_frames: props.observabilityExtractTotalFrames,
|
||||
|
||||
@@ -65,4 +65,20 @@ describe("renderObservabilityTelemetryPayload — DE inversion/router cohort (fa
|
||||
);
|
||||
expect(payload.captureDeSelfVerifyFallback).toBe(true);
|
||||
});
|
||||
|
||||
it("carries deFallbackReason so a render that fails AFTER an OOM-triggered fallback attempt is distinguishable from one that never attempted a fallback", () => {
|
||||
const payload = renderObservabilityTelemetryPayload(
|
||||
makeSummary({
|
||||
deParallelRouter: "routed",
|
||||
deSelfVerifyFallback: false,
|
||||
deFallbackReason: "oom",
|
||||
}),
|
||||
);
|
||||
expect(payload.captureDeFallbackReason).toBe("oom");
|
||||
});
|
||||
|
||||
it("leaves deFallbackReason undefined when no fallback was ever attempted", () => {
|
||||
const payload = renderObservabilityTelemetryPayload(makeSummary({}));
|
||||
expect(payload.captureDeFallbackReason).toBeUndefined();
|
||||
});
|
||||
});
|
||||
|
||||
@@ -45,6 +45,7 @@ export function renderObservabilityTelemetryPayload(
|
||||
captureDeParallelRouter: capture.deParallelRouter,
|
||||
captureDePreRouterWorkers: capture.dePreRouterWorkers,
|
||||
captureDeSelfVerifyFallback: capture.deSelfVerifyFallback,
|
||||
captureDeFallbackReason: capture.deFallbackReason,
|
||||
observabilityExtractVideoCount: extraction?.videoCount,
|
||||
observabilityExtractedVideoCount: extraction?.extractedVideoCount,
|
||||
observabilityExtractTotalFrames: extraction?.totalFramesExtracted,
|
||||
|
||||
@@ -109,4 +109,39 @@ describe("isMemoryExhaustionError", () => {
|
||||
false,
|
||||
);
|
||||
});
|
||||
|
||||
// The parallel-DE capture path (the exact cohort the OOM-aware retry
|
||||
// targets) never delivers a bare message — executeParallelCapture /
|
||||
// formatWorkerFailure (parallelCoordinator.ts) wrap it as
|
||||
// "[Parallel] Capture failed: Worker N: <message>", optionally joined with
|
||||
// other workers' segments and/or suffixed "; diagnostics: ...". Confirmed
|
||||
// against the real wrapping logic (not a hand-typed guess at its shape).
|
||||
it("recognizes Bun's OOM message through this codebase's own parallel-worker error wrapping", () => {
|
||||
expect(
|
||||
isMemoryExhaustionError(new Error("[Parallel] Capture failed: Worker 2: Out of memory")),
|
||||
).toBe(true);
|
||||
expect(
|
||||
isMemoryExhaustionError(
|
||||
new Error("[Parallel] Capture failed: Worker 1: net::ERR_FAILED; Worker 2: Out of memory"),
|
||||
),
|
||||
).toBe(true);
|
||||
expect(
|
||||
isMemoryExhaustionError(
|
||||
new Error(
|
||||
"[Parallel] Capture failed: Worker 2: Out of memory; diagnostics: ERROR foo | bar",
|
||||
),
|
||||
),
|
||||
).toBe(true);
|
||||
});
|
||||
|
||||
// The wrapped-worker-message pattern must stay as exact-match-per-segment
|
||||
// as the bare-message one — a worker's own error text merely containing
|
||||
// "out of memory" (e.g. surfaced WebGL/GPU noise) must not misclassify.
|
||||
it("does not match 'out of memory' as a mere substring inside a wrapped worker segment", () => {
|
||||
expect(
|
||||
isMemoryExhaustionError(
|
||||
new Error("Worker 2: WebGL context lost, out of memory reported by driver"),
|
||||
),
|
||||
).toBe(false);
|
||||
});
|
||||
});
|
||||
|
||||
@@ -3304,8 +3304,26 @@ const MEMORY_EXHAUSTION_ERROR_PATTERNS = [
|
||||
// it — a compound message with other text around the phrase still misses.
|
||||
const BUN_MEMORY_EXHAUSTION_EXACT_MESSAGE = /^out of memory\.?$/i;
|
||||
|
||||
// The parallel-DE capture path — the exact cohort the OOM-aware retry in
|
||||
// renderOrchestrator.ts targets — never reaches isMemoryExhaustionError with
|
||||
// a bare message: `executeParallelCapture`/`formatWorkerFailure`
|
||||
// (parallelCoordinator.ts) always wrap a worker's error as
|
||||
// "Worker N: <message>", optionally suffixed "; diagnostics: ..." and joined
|
||||
// with other failed workers' segments via "; ", all prefixed
|
||||
// "[Parallel] Capture failed: ". The exact-match check above is defeated by
|
||||
// that wrapping entirely (verified) — this pattern recovers the Bun OOM
|
||||
// signal by requiring "out of memory" appear immediately after "Worker N: "
|
||||
// and immediately before end-of-string, ";", or ".", i.e. as the WHOLE
|
||||
// worker-segment content, not merely somewhere inside it. This preserves the
|
||||
// exact-match property (no bare "out of memory" substring inside otherwise-
|
||||
// unrelated worker text, e.g. "Worker 2: WebGL context lost, out of memory
|
||||
// reported by driver" does NOT match) while surviving this codebase's own
|
||||
// error-flattening.
|
||||
const BUN_MEMORY_EXHAUSTION_WRAPPED_WORKER_MESSAGE = /\bworker \d+: out of memory\.?(?:;|$)/i;
|
||||
|
||||
export function isMemoryExhaustionError(error: unknown): boolean {
|
||||
const message = error instanceof Error ? error.message : String(error);
|
||||
if (BUN_MEMORY_EXHAUSTION_EXACT_MESSAGE.test(message.trim())) return true;
|
||||
if (BUN_MEMORY_EXHAUSTION_WRAPPED_WORKER_MESSAGE.test(message)) return true;
|
||||
return MEMORY_EXHAUSTION_ERROR_PATTERNS.some((pattern) => pattern.test(message));
|
||||
}
|
||||
|
||||
@@ -42,6 +42,16 @@ export interface RenderCaptureObservability {
|
||||
browserGpuMode?: string;
|
||||
/** drawElement per-render self-verification tripped → whole render re-ran via screenshot. */
|
||||
deSelfVerifyFallback?: boolean;
|
||||
/**
|
||||
* Why the capture-stage retry (self-verify OR the pinned-worker-count
|
||||
* fallback) fired: "blank"/"psnr" for a real self-verify trip,
|
||||
* "oom"/"capture_error" for the widened generic-failure retry. Set
|
||||
* whenever a fallback is attempted, independent of whether that retry
|
||||
* itself later succeeds — so a render that fails AFTER a fallback attempt
|
||||
* (perfSummary never built) is still distinguishable in failure-path
|
||||
* telemetry from one that never attempted any fallback.
|
||||
*/
|
||||
deFallbackReason?: string;
|
||||
/** Auto-parallel inversion outcome: "inverted" (fired, held) | "reverted" (fired, self-verify retry rolled back). */
|
||||
deWorkerInversion?: "inverted" | "reverted";
|
||||
/** Worker count the resolver would have used absent the inversion; undefined if it never fired. */
|
||||
|
||||
@@ -1878,6 +1878,7 @@ describe("shouldRetryViaPinnedFallback (widen the self-verify retry to generic c
|
||||
expect(
|
||||
shouldRetryViaPinnedFallback({
|
||||
isVerifyError: true,
|
||||
isCancellation: false,
|
||||
deWorkerInversion: undefined,
|
||||
deParallelRouter: undefined,
|
||||
}),
|
||||
@@ -1888,6 +1889,7 @@ describe("shouldRetryViaPinnedFallback (widen the self-verify retry to generic c
|
||||
expect(
|
||||
shouldRetryViaPinnedFallback({
|
||||
isVerifyError: false,
|
||||
isCancellation: false,
|
||||
deWorkerInversion: undefined,
|
||||
deParallelRouter: "routed",
|
||||
}),
|
||||
@@ -1898,6 +1900,7 @@ describe("shouldRetryViaPinnedFallback (widen the self-verify retry to generic c
|
||||
expect(
|
||||
shouldRetryViaPinnedFallback({
|
||||
isVerifyError: false,
|
||||
isCancellation: false,
|
||||
deWorkerInversion: "inverted",
|
||||
deParallelRouter: undefined,
|
||||
}),
|
||||
@@ -1908,6 +1911,7 @@ describe("shouldRetryViaPinnedFallback (widen the self-verify retry to generic c
|
||||
expect(
|
||||
shouldRetryViaPinnedFallback({
|
||||
isVerifyError: false,
|
||||
isCancellation: false,
|
||||
deWorkerInversion: undefined,
|
||||
deParallelRouter: undefined,
|
||||
}),
|
||||
@@ -1918,6 +1922,7 @@ describe("shouldRetryViaPinnedFallback (widen the self-verify retry to generic c
|
||||
expect(
|
||||
shouldRetryViaPinnedFallback({
|
||||
isVerifyError: false,
|
||||
isCancellation: false,
|
||||
deWorkerInversion: undefined,
|
||||
deParallelRouter: "routed",
|
||||
}),
|
||||
@@ -1928,6 +1933,7 @@ describe("shouldRetryViaPinnedFallback (widen the self-verify retry to generic c
|
||||
expect(
|
||||
shouldRetryViaPinnedFallback({
|
||||
isVerifyError: false,
|
||||
isCancellation: false,
|
||||
deWorkerInversion: "inverted",
|
||||
deParallelRouter: undefined,
|
||||
}),
|
||||
@@ -1938,9 +1944,40 @@ describe("shouldRetryViaPinnedFallback (widen the self-verify retry to generic c
|
||||
expect(
|
||||
shouldRetryViaPinnedFallback({
|
||||
isVerifyError: false,
|
||||
isCancellation: false,
|
||||
deWorkerInversion: "reverted",
|
||||
deParallelRouter: undefined,
|
||||
}),
|
||||
).toBe(false);
|
||||
});
|
||||
|
||||
it("never retries a cancellation, even on a pinned cohort — must propagate immediately, not detour through a fresh encoder spin-up", () => {
|
||||
expect(
|
||||
shouldRetryViaPinnedFallback({
|
||||
isVerifyError: false,
|
||||
isCancellation: true,
|
||||
deWorkerInversion: "inverted",
|
||||
deParallelRouter: undefined,
|
||||
}),
|
||||
).toBe(false);
|
||||
expect(
|
||||
shouldRetryViaPinnedFallback({
|
||||
isVerifyError: false,
|
||||
isCancellation: true,
|
||||
deWorkerInversion: undefined,
|
||||
deParallelRouter: "routed",
|
||||
}),
|
||||
).toBe(false);
|
||||
});
|
||||
|
||||
it("cancellation wins even if the error also looks like a self-verify failure", () => {
|
||||
expect(
|
||||
shouldRetryViaPinnedFallback({
|
||||
isVerifyError: true,
|
||||
isCancellation: true,
|
||||
deWorkerInversion: undefined,
|
||||
deParallelRouter: undefined,
|
||||
}),
|
||||
).toBe(false);
|
||||
});
|
||||
});
|
||||
|
||||
@@ -1208,12 +1208,20 @@ export function resolveParallelRouterRetryPlan(args: {
|
||||
* the parallel-SS fallback uses the default pooled browser (one shared
|
||||
* process). Retrying at a possibly-higher worker count is still fewer total
|
||||
* Chrome processes than what just failed.
|
||||
*
|
||||
* Excludes cancellation (review): a user-initiated abort must propagate
|
||||
* immediately, not detour through spawning a fresh encoder/capture session
|
||||
* before the outer catch's `RenderCancelledError` branch ends the render —
|
||||
* that would delay honoring "stop" with a pointless resource spin-up/
|
||||
* tear-down cycle.
|
||||
*/
|
||||
export function shouldRetryViaPinnedFallback(args: {
|
||||
isVerifyError: boolean;
|
||||
isCancellation: boolean;
|
||||
deWorkerInversion: "inverted" | "reverted" | undefined;
|
||||
deParallelRouter: "routed" | "reverted" | undefined;
|
||||
}): boolean {
|
||||
if (args.isCancellation) return false;
|
||||
if (args.isVerifyError) return true;
|
||||
return args.deWorkerInversion === "inverted" || args.deParallelRouter === "routed";
|
||||
}
|
||||
@@ -2341,7 +2349,16 @@ export async function executeRenderJob(
|
||||
// spawns on retry. See shouldRetryViaPinnedFallback for exactly
|
||||
// which errors qualify.
|
||||
const isVerifyError = isDrawElementVerificationError(err);
|
||||
if (!shouldRetryViaPinnedFallback({ isVerifyError, deWorkerInversion, deParallelRouter }))
|
||||
const isCancellation =
|
||||
err instanceof RenderCancelledError || abortSignal?.aborted === true;
|
||||
if (
|
||||
!shouldRetryViaPinnedFallback({
|
||||
isVerifyError,
|
||||
isCancellation,
|
||||
deWorkerInversion,
|
||||
deParallelRouter,
|
||||
})
|
||||
)
|
||||
throw err;
|
||||
const isMemoryExhaustion = !isVerifyError && isMemoryExhaustionError(err);
|
||||
deSelfVerifyFallback = isVerifyError;
|
||||
@@ -2368,6 +2385,7 @@ export async function executeRenderJob(
|
||||
updateCaptureObservability({
|
||||
forceScreenshot: true,
|
||||
deSelfVerifyFallback,
|
||||
deFallbackReason,
|
||||
});
|
||||
probeSession = null;
|
||||
// Must clear BEFORE resolveParallelRouterRetryPlan recomputes
|
||||
|
||||
Reference in New Issue
Block a user