feat(producer,cli): render-reliability telemetry counters for capture hardening (#1850)

Follow-up to the render-reliability batch (#1841/#1842/#1843). Threads two capture-reliability counters through the existing observability → CLI-telemetry pipeline (no new PostHog wiring) so #1842's hardening is measurable on dashboard 1783183:

- transient-retry burn (CaptureAttemptSummary.reason gains "transient-retry"; counted into RenderCaptureObservability.transientRetries on BOTH the recovered and the still-failed paths via a shared helper).
- OOM classification (memoryExhaustionDetected set when describeMemoryExhaustion classifies the failure).

Surfaced as capture_transient_retries + capture_memory_exhaustion_detected render-event props. Tests cover the attempt tagging and the payload mapping.

Further follow-up (different subsystems): encoder-frame-0-exit signal, and P1-3 pre-flight-rejection / P1-4 cli_env_check counters.

Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>

packages/cli/src/telemetry/events.ts

@@ -28,6 +28,8 @@ export interface RenderObservabilityTelemetryPayload {
   captureProtocolTimeoutMs?: number;
   capturePageNavigationTimeoutMs?: number;
   capturePlayerReadyTimeoutMs?: number;
+  captureTransientRetries?: number;
+  captureMemoryExhaustionDetected?: boolean;
   observabilityExtractVideoCount?: number;
   observabilityExtractedVideoCount?: number;
   observabilityExtractTotalFrames?: number;
@@ -70,6 +72,8 @@ function renderObservabilityEventProperties(props: RenderObservabilityTelemetryP
     capture_protocol_timeout_ms: props.captureProtocolTimeoutMs,
     capture_page_navigation_timeout_ms: props.capturePageNavigationTimeoutMs,
     capture_player_ready_timeout_ms: props.capturePlayerReadyTimeoutMs,
+    capture_transient_retries: props.captureTransientRetries,
+    capture_memory_exhaustion_detected: props.captureMemoryExhaustionDetected,
     observability_extract_video_count: props.observabilityExtractVideoCount,
     observability_extracted_video_count: props.observabilityExtractedVideoCount,
     observability_extract_total_frames: props.observabilityExtractTotalFrames,

packages/cli/src/telemetry/renderObservability.test.ts

@@ -0,0 +1,40 @@
+import { describe, expect, it } from "vitest";
+import type { RenderObservabilitySummary } from "@hyperframes/producer";
+import { renderObservabilityTelemetryPayload } from "./renderObservability.js";
+
+function makeSummary(
+  capture: Partial<RenderObservabilitySummary["capture"]>,
+): RenderObservabilitySummary {
+  return {
+    events: [],
+    eventCount: 0,
+    browserDiagnostics: {
+      total: 0,
+      errors: 0,
+      pageErrors: 0,
+      requestFailed: 0,
+      httpErrors: 0,
+      navigationStarts: 0,
+      navigationFailures: 0,
+      consoleErrors: 0,
+      consoleWarnings: 0,
+    },
+    capture: { forceScreenshot: false, captureMode: "beginframe", ...capture },
+  };
+}
+
+describe("renderObservabilityTelemetryPayload — render-reliability counters", () => {
+  it("maps the transient-retry and OOM counters through to the telemetry payload", () => {
+    const payload = renderObservabilityTelemetryPayload(
+      makeSummary({ transientRetries: 2, memoryExhaustionDetected: true }),
+    );
+    expect(payload.captureTransientRetries).toBe(2);
+    expect(payload.captureMemoryExhaustionDetected).toBe(true);
+  });
+
+  it("leaves the counters undefined when the render didn't retry or OOM", () => {
+    const payload = renderObservabilityTelemetryPayload(makeSummary({}));
+    expect(payload.captureTransientRetries).toBeUndefined();
+    expect(payload.captureMemoryExhaustionDetected).toBeUndefined();
+  });
+});

packages/cli/src/telemetry/renderObservability.ts

@@ -38,6 +38,8 @@ export function renderObservabilityTelemetryPayload(
     captureProtocolTimeoutMs: capture.protocolTimeoutMs,
     capturePageNavigationTimeoutMs: capture.pageNavigationTimeoutMs,
     capturePlayerReadyTimeoutMs: capture.playerReadyTimeoutMs,
+    captureTransientRetries: capture.transientRetries,
+    captureMemoryExhaustionDetected: capture.memoryExhaustionDetected,
     observabilityExtractVideoCount: extraction?.videoCount,
     observabilityExtractedVideoCount: extraction?.extractedVideoCount,
     observabilityExtractTotalFrames: extraction?.totalFramesExtracted,

packages/producer/src/services/render/observability.ts

@@ -43,6 +43,15 @@ export interface RenderCaptureObservability {
   protocolTimeoutMs?: number;
   pageNavigationTimeoutMs?: number;
   playerReadyTimeoutMs?: number;
+  /**
+   * Render-reliability counters (see PostHog dashboard 1783183). Emitted so the
+   * capture-hardening in #1842 is measurable from a metric, not just logs:
+   * how often the bounded transient-tab-death retry fired on a render that
+   * ultimately succeeded, and whether the failure was classified as an
+   * out-of-memory exhaustion (`Set maximum size exceeded` and friends).
+   */
+  transientRetries?: number;
+  memoryExhaustionDetected?: boolean;
 }
 
 export interface RenderExtractionObservability {

packages/producer/src/services/renderOrchestrator.test.ts

@@ -223,6 +223,9 @@ describe("executeDiskCaptureWithAdaptiveRetry — transient Target-closed single
       expect(vi.mocked(executeParallelCapture)).toHaveBeenCalledTimes(2);
       // Both attempts ran at the same worker count (transient retry doesn't halve).
       expect(attempts.map((a) => a.workers)).toEqual([1, 1]);
+      // The retry attempt is tagged `transient-retry` (vs the worker-halving
+      // `retry`) so it's countable for telemetry (dashboard 1783183).
+      expect(attempts.map((a) => a.reason)).toEqual(["initial", "transient-retry"]);
       expect(log.warn).toHaveBeenCalledWith(
         expect.stringContaining("Transient browser failure"),
         expect.objectContaining({ transientRetriesUsed: 1 }),

packages/producer/src/services/renderOrchestrator.ts

@@ -367,7 +367,13 @@ export interface CaptureAttemptSummary {
   attempt: number;
   workers: number;
   frameCount: number;
-  reason: "initial" | "retry";
+  /**
+   * `"transient-retry"` is a same-worker-count retry after a transient browser
+   * death (Target closed / tab crash); `"retry"` is the worker-halving retry
+   * after a recoverable timeout. Distinguished so transient-retry burn is
+   * countable for telemetry (dashboard 1783183).
+   */
+  reason: "initial" | "retry" | "transient-retry";
 }
 
 export interface RenderJob {
@@ -665,6 +671,10 @@ export async function executeDiskCaptureWithAdaptiveRetry(options: {
   let missingRanges: FrameRange[] | null = null;
   let attempt = 0;
   let transientRetriesUsed = 0;
+  // Set when the *previous* iteration retried after a transient browser death,
+  // so the attempt it spawns is tagged `"transient-retry"` (vs the worker-halving
+  // `"retry"`) for telemetry. Reset after each attempt is recorded.
+  let pendingTransientRetry = false;
   const rangeStart = options.frameRangeStart ?? 0;
 
   while (true) {
@@ -673,8 +683,9 @@ export async function executeDiskCaptureWithAdaptiveRetry(options: {
       attempt,
       workers: currentWorkers,
       frameCount,
-      reason: attempt === 0 ? "initial" : "retry",
+      reason: attempt === 0 ? "initial" : pendingTransientRetry ? "transient-retry" : "retry",
     });
+    pendingTransientRetry = false;
 
     const attemptWorkDir = join(options.workDir, `capture-attempt-${attempt}`);
     const batches = missingRanges
@@ -807,6 +818,7 @@ export async function executeDiskCaptureWithAdaptiveRetry(options: {
         );
         missingRanges = remaining;
         attempt++;
+        pendingTransientRetry = true;
         continue;
       }
 
@@ -1025,6 +1037,14 @@ export async function executeRenderJob(
       ? "screenshot"
       : "beginframe";
   };
+  // Function-scoped (not inside the try) so both the success path AND the catch
+  // can read it — the catch records transient-retry burn on renders that still
+  // failed, which is the more actionable signal for tuning the retry cap.
+  const captureAttempts: CaptureAttemptSummary[] = [];
+  const recordTransientRetryObservability = (): void => {
+    const count = captureAttempts.filter((a) => a.reason === "transient-retry").length;
+    if (count > 0) updateCaptureObservability({ transientRetries: count });
+  };
   // Declared outside the try so `finally` can stop the interval, but
   // the sampler is created INSIDE the try so a synchronous throw
   // between declaration and the try-block (currently impossible, but
@@ -1538,9 +1558,9 @@ export async function executeRenderJob(
       maxDurationSeconds: cfg.streamingEncodeMaxDurationSeconds,
     });
 
-    const captureAttempts: CaptureAttemptSummary[] = [];
+    // `captureAttempts` is declared at function scope above (shared with the
-    // Static-dedup perf, appended per sequential session / per parallel worker
+    // catch block). Static-dedup perf, appended per sequential session / per
-    // by the capture stage, aggregated into the perf summary below.
+    // parallel worker by the capture stage, aggregated into the perf summary below.
     const dedupPerfs: CapturePerfSummary[] = [];
 
     // png-sequence is "no container" — outputPath is treated as a directory and
@@ -1909,6 +1929,9 @@ export async function executeRenderJob(
     const totalElapsed = Date.now() - pipelineStart;
 
     const tmpPeakBytes = existsSync(workDir) ? sampleDirectoryBytes(workDir) : 0;
+    // Record transient-tab-death retry burn (recovered case) so it's visible on
+    // dashboard 1783183, not just logs. The catch mirrors this for the failed case.
+    recordTransientRetryObservability();
     observability.checkpoint("pipeline", "completed", { totalElapsedMs: totalElapsed });
     const observabilitySummary = observability.summary({
       lastBrowserConsole,
@@ -1998,6 +2021,14 @@ export async function executeRenderJob(
       height: captureCompositionHeight,
       totalFrames: captureTotalFrames,
     });
+    // Flag OOM-classified failures so the "is OOM the dominant tail?" question is
+    // answerable from a metric (dashboard 1783183), not just the error string.
+    if (memoryGuidance) {
+      updateCaptureObservability({ memoryExhaustionDetected: true });
+    }
+    // Retry burn on a render that STILL failed — the actionable signal for tuning
+    // MAX_TRANSIENT_CAPTURE_RETRIES (mirrors the success-path record above).
+    recordTransientRetryObservability();
     const errorMessage = memoryGuidance ?? normalizeErrorMessage(error);
     const carriedBrowserConsole = getCaptureStageBrowserConsole(error);
     if (carriedBrowserConsole.length > 0) {