From b26c27576b71d9a86c57828f7e1712b14479edeb Mon Sep 17 00:00:00 2001
From: Vance Ingalls <vance@heygen.com>
Date: Mon, 6 Jul 2026 22:47:29 -0700
Subject: [PATCH] feat(engine,producer,cli): verify video comps via deferred DE
 init + capture p50 (#2015)
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

* feat(engine,producer,cli): verify video comps via deferred DE init + capture p50

Closes the two biggest gaps in the first day of v0.7.38 wild data: 88% of
drawElement renders (video comps initialized via probe sessions) ran with
self-verification unarmed, and speedup was measurable on only 3 of 76 renders.

- Deferred drawElement init: probe sessions initialize before video
  extraction, so they have no frame injector — ground-truth screenshots
  would capture black <video> boxes, and verification skipped the whole
  comp. DE init now stops after the gates for injector-less video comps
  (deInitDeferred; autoAlpha flag retracted in case no path completes it)
  and completeDeferredDrawElementInit finishes verification + canvas
  injection + worker-encode at capture time, once
  prepareCaptureSessionForReuse has attached the injector. Validated
  end-to-end: a probe-path video comp now arms 4 ground-truth frames with
  real video pixels (3x inf + 64.7dB) and renders drawElement verified.
- capture_p50_ms: per-frame capture durations are sampled
  (capturePerf.frameMs; batch frames get the batch mean) and the median
  ships as CapturePerfSummary.p50TotalMs -> RenderPerfSummary.captureP50Ms
  -> render_complete capture_p50_ms. Unlike capture_avg_ms it is immune
  to first-frame warmup and stage-setup amortization — smoke: avg 15ms vs
  p50 8ms on the same render, p50 matching the measured steady-state
  floor. Dashboard speedup tiles can drop their frame-count floor once
  this ships.
- video_count on render_complete: segments speedup by video-injection
  comps (whose per-frame gain is legitimately lower) vs pure-graphics.

Canary suite 7/7; engine suite 905 passed (1 pre-existing upstream
failure); tsc/oxlint/oxfmt clean.

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>

* fix(producer): complete deferred drawElement init on the disk capture path

Review (miga): a probe-initialized video comp falling back to the disk path
kept deInitDeferred and silently stayed in screenshot mode — a regression
for PRODUCER_EXPERIMENTAL_FAST_CAPTURE=true renders that previously ran
drawElement there. Complete the deferred init on the sequential disk path
under the same explicit-opt-in test the orchestrator clamp uses; default-on
renders stay on the screenshot baseline (this path has no drain-time
self-verification, per the #1998 confinement rule).

Validated: video comp + PRODUCER_ENABLE_STREAMING_ENCODE=false + explicit
opt-in logs "(deferred drawElement init)" completion on capture_disk and
renders correct video pixels.

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>

---------

Co-authored-by: Claude Fable 5 <noreply@anthropic.com>
---
 packages/cli/src/commands/render.ts           |   2 +
 packages/cli/src/telemetry/events.ts          |   6 +
 packages/engine/src/index.ts                  |   1 +
 packages/engine/src/services/frameCapture.ts  | 160 ++++++++++++++----
 packages/engine/src/types.ts                  |   6 +
 .../src/services/render/perfSummary.ts        |   7 +
 .../services/render/stages/captureStage.ts    |   8 +
 .../render/stages/captureStreamingStage.ts    |   5 +
 .../src/services/renderOrchestrator.ts        |   7 +
 9 files changed, 166 insertions(+), 36 deletions(-)

diff --git a/packages/cli/src/commands/render.ts b/packages/cli/src/commands/render.ts
index fdef0444e..ba2ee8374 100644
--- a/packages/cli/src/commands/render.ts
+++ b/packages/cli/src/commands/render.ts
@@ -1684,6 +1684,8 @@ function trackRenderMetrics(
     totalFrames: perf?.totalFrames,
     speedRatio,
     captureAvgMs: perf?.captureAvgMs,
+    captureP50Ms: perf?.captureP50Ms,
+    videoCount: perf?.videoCount,
     capturePeakMs: perf?.capturePeakMs,
     tmpPeakBytes: perf?.tmpPeakBytes,
     stageCompileMs: stages.compileMs,
diff --git a/packages/cli/src/telemetry/events.ts b/packages/cli/src/telemetry/events.ts
index fcf03cbb2..9fe7e40f8 100644
--- a/packages/cli/src/telemetry/events.ts
+++ b/packages/cli/src/telemetry/events.ts
@@ -143,6 +143,10 @@ export function trackRenderComplete(
     // Processing efficiency
     speedRatio?: number;
     captureAvgMs?: number;
+    /** Warmup-robust per-frame capture median (basis for speedup estimates). */
+    captureP50Ms?: number;
+    /** <video> element count (speedup segmentation: injection comps read lower). */
+    videoCount?: number;
     capturePeakMs?: number;
     // Resource usage
     peakMemoryMb?: number;
@@ -211,6 +215,8 @@ export function trackRenderComplete(
       total_frames: props.totalFrames,
       speed_ratio: props.speedRatio,
       capture_avg_ms: props.captureAvgMs,
+      capture_p50_ms: props.captureP50Ms,
+      video_count: props.videoCount,
       capture_peak_ms: props.capturePeakMs,
       peak_memory_mb: props.peakMemoryMb,
       memory_free_mb: props.memoryFreeMb,
diff --git a/packages/engine/src/index.ts b/packages/engine/src/index.ts
index 94173a9d8..659843165 100644
--- a/packages/engine/src/index.ts
+++ b/packages/engine/src/index.ts
@@ -87,6 +87,7 @@ export {
   DrawElementVerificationError,
   isDrawElementVerificationError,
   recaptureDrawElementFrameForVerify,
+  completeDeferredDrawElementInit,
   writeCapturedFrame,
   discardWarmupCapture,
   getCompositionDuration,
diff --git a/packages/engine/src/services/frameCapture.ts b/packages/engine/src/services/frameCapture.ts
index 36487cfc6..f6b8f0d86 100644
--- a/packages/engine/src/services/frameCapture.ts
+++ b/packages/engine/src/services/frameCapture.ts
@@ -104,6 +104,9 @@ export interface CaptureSession {
     beforeCaptureMs: number;
     screenshotMs: number;
     totalMs: number;
+    /** Per-frame capture durations (batch frames get the batch mean). Basis for
+     * the warmup-robust p50 in the perf summary. */
+    frameMs: number[];
   };
   captureMode: CaptureMode;
   /**
@@ -160,6 +163,15 @@ export interface CaptureSession {
   deVerifyInitMs?: number;
   /** Count of per-frame "No cached paint record" screenshot fallbacks (telemetry). */
   deNcprFallbacks?: number;
+  /**
+   * drawElement init passed every gate but stopped before verification +
+   * canvas injection: the session has no video-frame injector yet (probe
+   * sessions initialize before extraction) and the comp has <video> elements,
+   * so ground-truth screenshots would capture black video boxes. The capture
+   * stage completes the init via completeDeferredDrawElementInit once
+   * prepareCaptureSessionForReuse attaches the injector.
+   */
+  deInitDeferred?: boolean;
 }
 
 /**
@@ -629,48 +641,104 @@ async function initDrawElementOrTransparentBackground(
       // passed) and the DOM is still normal (canvas not yet injected, so the
       // verification screenshots are valid). drawElement-path only; see armStaticDedup.
       await armStaticDedup(session, page, logInitPhase);
-      // Self-verification ground truth: must ALSO run pre-injection — after the
-      // canvas wraps the root, a page screenshot shows the canvas's last-drawn
-      // bitmap, not the live DOM (see the Lim 6 boundary-screenshot note).
-      {
-        const verifyStart = Date.now();
-        await captureDeVerificationFrames(session, page, logInitPhase);
-        session.deVerifyInitMs = Date.now() - verifyStart;
-      }
-      await injectDrawElementCanvas(page, session.options.width, session.options.height);
-      if (transparent) {
-        await initTransparentBackground(session.page);
-      }
-      session.captureMode = "drawelement";
-      session.drawElementReady = true;
-      logInitPhase("drawElement canvas injected");
-      // Lim 6: clip-cut boundary frames — screenshot these instead of drawElement.
-      if (process.env.HF_FAST_CAPTURE_BOUNDARY_SS !== "false" && !forceDE) {
-        const fps = fpsToNumber(session.options.fps);
-        const boundaryFrames = await computeClipBoundaryFrames(page, fps);
-        if (boundaryFrames.size > 0) {
-          session.clipBoundaryFrames = boundaryFrames;
-          logInitPhase(`screenshot fallback: ${boundaryFrames.size} clip-boundary frame(s)`);
+      // Video comps on injector-less sessions (probe sessions initialize before
+      // video extraction) DEFER the rest of the drawElement init: ground-truth
+      // screenshots would capture black <video> boxes, and once the canvas is
+      // injected they can never be retaken. The capture stage completes the
+      // init after prepareCaptureSessionForReuse attaches the injector.
+      // Retract the autoAlpha rewrite flag while deferred — if no path ever
+      // completes the init (e.g. the disk path takes over), the session
+      // captures via screenshot, where the armed flag causes measured damage.
+      if (!session.onBeforeCapture && !forceDE) {
+        const hasVideos = await page.evaluate(() => document.querySelector("video") !== null);
+        if (hasVideos) {
+          session.deInitDeferred = true;
+          await retractAutoAlphaFlag();
+          logInitPhase("drawElement init deferred: video comp awaiting frame injector");
+          return;
         }
       }
-      // Worker-encode pipeline: macOS hardware GPU path only (syncToPaintEvent=true,
-      // beginFrameTimeTicks=0). Skip for BeginFrame (Linux/Docker) and transparent
-      // (PNG) output — those use the existing synchronous path unchanged.
-      const workerEncodeEnabled =
-        (session.config?.enableDrawElementWorkerEncode ?? false) &&
-        !transparent &&
-        session.beginFrameTimeTicks === 0;
-      if (workerEncodeEnabled) {
-        await initDrawElementWorkerEncode(page);
-        session.workerEncodeEnabled = true;
-        logInitPhase("drawElement worker encode initialized");
-      }
+      await finalizeDrawElementInit(session, page, logInitPhase, { transparent, forceDE });
     }
   } else if (session.options.format === "png") {
     await initTransparentBackground(session.page);
   }
 }
 
+/**
+ * The tail of drawElement init: self-verification ground truth (pre-injection),
+ * canvas injection, capture-mode flip, clip-boundary predictor, worker-encode.
+ * Runs inline when the session already has its video-frame injector (or the
+ * comp has no videos); runs deferred via completeDeferredDrawElementInit for
+ * probe-initialized video comps.
+ */
+async function finalizeDrawElementInit(
+  session: CaptureSession,
+  page: Page,
+  logInitPhase: (phase: string) => void,
+  opts: { transparent: boolean; forceDE: boolean },
+): Promise<void> {
+  const { transparent, forceDE } = opts;
+  // Self-verification ground truth: must run pre-injection — after the canvas
+  // wraps the root, a page screenshot shows the canvas's last-drawn bitmap,
+  // not the live DOM (see the Lim 6 boundary-screenshot note).
+  {
+    const verifyStart = Date.now();
+    await captureDeVerificationFrames(session, page, logInitPhase);
+    session.deVerifyInitMs = Date.now() - verifyStart;
+  }
+  await injectDrawElementCanvas(page, session.options.width, session.options.height);
+  if (transparent) {
+    await initTransparentBackground(session.page);
+  }
+  session.captureMode = "drawelement";
+  session.drawElementReady = true;
+  logInitPhase("drawElement canvas injected");
+  // Lim 6: clip-cut boundary frames — screenshot these instead of drawElement.
+  if (process.env.HF_FAST_CAPTURE_BOUNDARY_SS !== "false" && !forceDE) {
+    const fps = fpsToNumber(session.options.fps);
+    const boundaryFrames = await computeClipBoundaryFrames(page, fps);
+    if (boundaryFrames.size > 0) {
+      session.clipBoundaryFrames = boundaryFrames;
+      logInitPhase(`screenshot fallback: ${boundaryFrames.size} clip-boundary frame(s)`);
+    }
+  }
+  // Worker-encode pipeline: macOS hardware GPU path only (syncToPaintEvent=true,
+  // beginFrameTimeTicks=0). Skip for BeginFrame (Linux/Docker) and transparent
+  // (PNG) output — those use the existing synchronous path unchanged.
+  const workerEncodeEnabled =
+    (session.config?.enableDrawElementWorkerEncode ?? false) &&
+    !transparent &&
+    session.beginFrameTimeTicks === 0;
+  if (workerEncodeEnabled) {
+    await initDrawElementWorkerEncode(page);
+    session.workerEncodeEnabled = true;
+    logInitPhase("drawElement worker encode initialized");
+  }
+}
+
+/**
+ * Complete a deferred drawElement init (see CaptureSession.deInitDeferred).
+ * Call after prepareCaptureSessionForReuse has attached the video-frame
+ * injector; no-op when the session is not deferred or still has no injector.
+ * Re-asserts the autoAlpha rewrite flag retracted at deferral time.
+ */
+export async function completeDeferredDrawElementInit(session: CaptureSession): Promise<void> {
+  if (!session.deInitDeferred || !session.onBeforeCapture) return;
+  const page = session.page;
+  await page.evaluate(() => {
+    (window as Window & { __HF_FAST_CAPTURE_AUTOALPHA__?: boolean }).__HF_FAST_CAPTURE_AUTOALPHA__ =
+      true;
+  });
+  const logInitPhase = (phase: string) =>
+    console.log(`[initSession:${session.captureMode}] ${phase} (deferred drawElement init)`);
+  await finalizeDrawElementInit(session, page, logInitPhase, {
+    transparent: session.options.format === "png",
+    forceDE: process.env.HF_FORCE_DRAWELEMENT === "1",
+  });
+  session.deInitDeferred = false;
+}
+
 // fallow-ignore-next-line unit-size
 export async function createCaptureSession(
   serverUrl: string,
@@ -865,6 +933,7 @@ export async function createCaptureSession(
       beforeCaptureMs: 0,
       screenshotMs: 0,
       totalMs: 0,
+      frameMs: [],
     },
     captureMode,
     launchCaptureMode: captureMode,
@@ -2387,6 +2456,7 @@ async function captureFrameCore(
     session.capturePerf.beforeCaptureMs += beforeCaptureMs;
     session.capturePerf.screenshotMs += screenshotMs;
     session.capturePerf.totalMs += captureTimeMs;
+    session.capturePerf.frameMs.push(captureTimeMs);
 
     // Retain this freshly-captured buffer so the following static frames can reuse it.
     if (session.staticFrames) session.lastFrameBuffer = screenshotBuffer;
@@ -2510,7 +2580,11 @@ export async function captureFrameToBufferPipelined(
       session.capturePerf.frames += 1;
       session.capturePerf.seekMs += seekMs;
       session.capturePerf.beforeCaptureMs += beforeCaptureMs;
-      session.capturePerf.totalMs += Date.now() - startTime;
+      {
+        const boundaryMs = Date.now() - startTime;
+        session.capturePerf.totalMs += boundaryMs;
+        session.capturePerf.frameMs.push(boundaryMs);
+      }
       const boundaryResult = Promise.resolve(buffer);
       if (session.staticFrames) session.lastEncodeResult = boundaryResult;
       return { encodeResult: boundaryResult, captureTimeMs: Date.now() - startTime };
@@ -2536,6 +2610,7 @@ export async function captureFrameToBufferPipelined(
     // screenshotMs reflects produce time only (encode is async, not tracked here)
     session.capturePerf.screenshotMs += captureTimeMs - seekMs - beforeCaptureMs;
     session.capturePerf.totalMs += captureTimeMs;
+    session.capturePerf.frameMs.push(captureTimeMs);
 
     // Task B: retain this encode result so a following static frame can reuse it.
     if (session.staticFrames) session.lastEncodeResult = encodeResult;
@@ -2635,9 +2710,14 @@ export async function captureFramesBatchPipelined(
   const okCount = failedAt === null ? frameIndices.length : failedAt;
   const elapsed = Date.now() - startTime;
   session.capturePerf.frames += okCount;
-  // Round-trips are fused — attribute the whole batch to produce time.
+  // Round-trips are fused — attribute the whole batch to produce time; each
+  // frame gets the batch mean for the per-frame sample series.
   session.capturePerf.screenshotMs += elapsed;
   session.capturePerf.totalMs += elapsed;
+  if (okCount > 0) {
+    const perFrame = elapsed / okCount;
+    for (let s2 = 0; s2 < okCount; s2++) session.capturePerf.frameMs.push(perFrame);
+  }
 
   const results: Array<{ frameIndex: number; encodeResult: Promise<Buffer> }> = [];
   for (let i = 0; i < okCount; i++) {
@@ -2811,6 +2891,7 @@ export function prepareCaptureSessionForReuse(
     beforeCaptureMs: 0,
     screenshotMs: 0,
     totalMs: 0,
+    frameMs: [],
   };
   session.beginFrameHasDamageCount = 0;
   session.beginFrameNoDamageCount = 0;
@@ -2938,6 +3019,12 @@ async function captureDeVerificationFrames(
   );
 }
 
+function medianOf(samples: number[]): number {
+  if (samples.length === 0) return 0;
+  const sorted = [...samples].sort((a, b) => a - b);
+  return Math.round(sorted[Math.floor(sorted.length / 2)] ?? 0);
+}
+
 export function getCapturePerfSummary(session: CaptureSession): CapturePerfSummary {
   const frames = Math.max(1, session.capturePerf.frames);
   return {
@@ -2946,6 +3033,7 @@ export function getCapturePerfSummary(session: CaptureSession): CapturePerfSumma
     avgSeekMs: Math.round(session.capturePerf.seekMs / frames),
     avgBeforeCaptureMs: Math.round(session.capturePerf.beforeCaptureMs / frames),
     avgScreenshotMs: Math.round(session.capturePerf.screenshotMs / frames),
+    p50TotalMs: medianOf(session.capturePerf.frameMs),
     staticDedupReused: session.staticDedupCount ?? 0,
     staticDedupEnabled: session.staticDedupEnabled ?? false,
     // armed ⟺ a non-empty static set survived verification; predicted === its size.
diff --git a/packages/engine/src/types.ts b/packages/engine/src/types.ts
index 53c1aa146..f2faa0651 100644
--- a/packages/engine/src/types.ts
+++ b/packages/engine/src/types.ts
@@ -177,6 +177,12 @@ export interface CapturePerfSummary {
   avgSeekMs: number;
   avgBeforeCaptureMs: number;
   avgScreenshotMs: number;
+  /**
+   * Median per-frame capture time — warmup-robust, unlike `avgTotalMs`
+   * (first frames pay font/image decode + GC that swamps short renders'
+   * averages). Basis for in-the-wild speedup estimates. 0 when no frames.
+   */
+  p50TotalMs: number;
   /**
    * Frames served from the static-dedup cache instead of a real seek+screenshot
    * (opt-out HF_STATIC_DEDUP=false). 0 when dedup was off or never armed. NOT counted
diff --git a/packages/producer/src/services/render/perfSummary.ts b/packages/producer/src/services/render/perfSummary.ts
index ab400f426..498ed9890 100644
--- a/packages/producer/src/services/render/perfSummary.ts
+++ b/packages/producer/src/services/render/perfSummary.ts
@@ -192,6 +192,13 @@ export function buildRenderPerfSummary(input: {
               input.totalFrames,
           )
         : undefined,
+    captureP50Ms: (() => {
+      // Per-frame median from the engine's samples; when parallel workers
+      // report separately, take the busiest session's median.
+      const withSamples = input.dedupPerfs.filter((p) => (p.p50TotalMs ?? 0) > 0);
+      if (withSamples.length === 0) return undefined;
+      return withSamples.reduce((a, b) => (b.frames > a.frames ? b : a)).p50TotalMs;
+    })(),
     peakRssMb: Math.round(input.peakRssBytes / (1024 * 1024)),
     peakHeapUsedMb: Math.round(input.peakHeapUsedBytes / (1024 * 1024)),
     staticDedup: aggregateDedup(input.dedupPerfs),
diff --git a/packages/producer/src/services/render/stages/captureStage.ts b/packages/producer/src/services/render/stages/captureStage.ts
index b76cb1b97..f52cad23e 100644
--- a/packages/producer/src/services/render/stages/captureStage.ts
+++ b/packages/producer/src/services/render/stages/captureStage.ts
@@ -46,6 +46,7 @@ import {
   captureFrameToBufferPipelined,
   writeCapturedFrame,
   closeCaptureSession,
+  completeDeferredDrawElementInit,
   createCaptureSession,
   getCapturePerfSummary,
   initializeSession,
@@ -260,6 +261,13 @@ export async function runCaptureStage(input: CaptureStageInput): Promise<Capture
     try {
       if (!session.isInitialized) {
         await initializeSession(session);
+      } else if (process.env.PRODUCER_EXPERIMENTAL_FAST_CAPTURE === "true") {
+        // Deferred drawElement init (probe-initialized video comps). The disk
+        // path has no drain-time self-verification, so only an explicit opt-in
+        // completes it here — mirroring the orchestrator clamp that routes
+        // default-on drawElement renders to the screenshot baseline on this
+        // path. No-op unless the session is deferred with an injector attached.
+        await completeDeferredDrawElementInit(session);
       }
       assertNotAborted();
       lastBrowserConsole = session.browserConsoleBuffer;
diff --git a/packages/producer/src/services/render/stages/captureStreamingStage.ts b/packages/producer/src/services/render/stages/captureStreamingStage.ts
index 260e67642..721ef6688 100644
--- a/packages/producer/src/services/render/stages/captureStreamingStage.ts
+++ b/packages/producer/src/services/render/stages/captureStreamingStage.ts
@@ -65,6 +65,7 @@ import {
   getCapturePerfSummary,
   getFfmpegBinary,
   recaptureDrawElementFrameForVerify,
+  completeDeferredDrawElementInit,
   initializeSession,
   prepareCaptureSessionForReuse,
   spawnStreamingEncoder,
@@ -560,6 +561,10 @@ export async function runCaptureStreamingStage(
         if (!session.isInitialized) {
           await initializeSession(session);
         }
+        // Probe-initialized video comps defer verification + canvas injection
+        // until the injector exists (attached by prepareCaptureSessionForReuse
+        // above) — complete it now so drawElement runs VERIFIED on video comps.
+        await completeDeferredDrawElementInit(session);
         assertNotAborted();
         lastBrowserConsole = session.browserConsoleBuffer;
 
diff --git a/packages/producer/src/services/renderOrchestrator.ts b/packages/producer/src/services/renderOrchestrator.ts
index e483c76c6..824f4a625 100644
--- a/packages/producer/src/services/renderOrchestrator.ts
+++ b/packages/producer/src/services/renderOrchestrator.ts
@@ -309,6 +309,13 @@ export interface RenderPerfSummary {
    * without the split fall back to `stages.captureMs`.
    */
   captureAvgMs?: number;
+  /**
+   * Median per-frame capture time from the engine's per-frame samples —
+   * warmup-robust (first frames pay font/image decode) and free of stage
+   * setup amortization, unlike `captureAvgMs`. From the session that
+   * captured the most frames when parallel workers report separately.
+   */
+  captureP50Ms?: number;
   capturePeakMs?: number;
   captureCalibration?: {
     sampledFrames: number[];