From 0e541673e0e7d2ecdb1c78c48263cdea47a38338 Mon Sep 17 00:00:00 2001
From: James <james.russo@heygen.com>
Date: Mon, 4 May 2026 19:00:07 -0700
Subject: [PATCH] fix(engine): wait for first frame decode + drop B-frames so
 renders play in every player
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Three related render robustness fixes:

1. frameCapture.ts: bump videos-ready check from `readyState >= 1`
   (HAVE_METADATA — only dimensions known) to `>= 2` (HAVE_CURRENT_DATA —
   first frame is rasterized). Without this, when two `<video>` elements
   with different codecs (h264 mp4 + VP9 webm) decode at different rates,
   the faster one passes readiness while the slower one still hasn't
   painted, producing a black "first frame" for the slower clip.

2. chunkEncoder.ts (libx264 path) + streamingEncoder.ts: disable B-frames
   for h264 (`-bf 0`). Standard libx264 with B-frames produces negative
   DTS at stream start (the first B-frame's decode order is "before" the
   first I-frame's presentation time). VS Code preview, several browser
   <video> implementations, and some HW decoders freeze on the first
   frame and only audio plays. -bf 0 makes PTS == DTS at every frame,
   eliminating the issue at the source. Quality cost is ~5–10% larger
   files at the same CRF — worthwhile for "the file plays everywhere".

3. chunkEncoder.ts (encoder + mux paths): add `-avoid_negative_ts make_zero`
   as belt-and-suspenders against negative DTS sneaking back in via
   `-c:v copy` mux passes when audio/video PTS bases differ.

Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
---
 packages/engine/src/services/chunkEncoder.ts  | 17 +++++++++++++++
 packages/engine/src/services/frameCapture.ts  | 21 +++++++++++--------
 .../engine/src/services/streamingEncoder.ts   | 12 +++++++++++
 3 files changed, 41 insertions(+), 9 deletions(-)

diff --git a/packages/engine/src/services/chunkEncoder.ts b/packages/engine/src/services/chunkEncoder.ts
index 594cda66d..a5c7ffe3e 100644
--- a/packages/engine/src/services/chunkEncoder.ts
+++ b/packages/engine/src/services/chunkEncoder.ts
@@ -145,6 +145,18 @@ export function buildEncoderArgs(
       if (bitrate) args.push("-b:v", bitrate);
       else args.push("-crf", String(quality));
 
+      // Disable B-frames. Standard h264 with B-frames produces negative DTS
+      // at the start of the stream (the first B-frame's decode order is
+      // "before" the first I-frame's presentation time). VS Code's video
+      // preview, several browser <video> pipelines, and some HW decoders
+      // freeze on the first frame when DTS is negative, so audio plays alone.
+      // -bf 0 makes PTS == DTS at every frame, eliminating the issue at the
+      // source. Quality cost is ~5–10% larger files at the same CRF — a
+      // worthwhile trade for "the file plays everywhere".
+      if (codec === "h264") {
+        args.push("-bf", "0");
+      }
+
       // Encoder-specific params: anti-banding + color space tagging.
       // aq-mode=3 redistributes bits to dark flat areas (gradients).
       // For HDR x265 paths we additionally embed BT.2020 + transfer + HDR static
@@ -239,6 +251,8 @@ export function buildEncoderArgs(
     args.push("-pix_fmt", pixelFormat);
   }
 
+  args.push("-avoid_negative_ts", "make_zero");
+
   args.push("-y", outputPath);
   return args;
 }
@@ -510,6 +524,9 @@ export async function muxVideoWithAudio(
   } else {
     args.push("-c:a", "aac", "-b:a", "192k", "-movflags", "+faststart");
   }
+  // PTS bases can diverge during mux and reintroduce negative DTS. See
+  // buildEncoderArgs for the full reasoning on why that breaks playback.
+  args.push("-avoid_negative_ts", "make_zero");
   args.push("-shortest", "-y", outputPath);
 
   const processTimeout = config?.ffmpegProcessTimeout ?? DEFAULT_CONFIG.ffmpegProcessTimeout;
diff --git a/packages/engine/src/services/frameCapture.ts b/packages/engine/src/services/frameCapture.ts
index 9e4cff429..d19e8cf6f 100644
--- a/packages/engine/src/services/frameCapture.ts
+++ b/packages/engine/src/services/frameCapture.ts
@@ -388,8 +388,14 @@ export async function initializeSession(session: CaptureSession): Promise<void>
 
     await applyVideoMetadataHints(page, session.options.videoMetadataHints);
 
-    // Wait for all video elements to have loaded metadata (dimensions + duration)
-    // Without this, frame 0 captures videos at their 300x150 default size.
+    // Wait for all video elements to have decoded their CURRENT frame, not
+    // just metadata. readyState >= 2 (HAVE_CURRENT_DATA) means a frame is
+    // actually rasterized and ready to paint — at >= 1 (HAVE_METADATA) we
+    // only know the dimensions, and the first <video> screenshot can come
+    // back as a black/blank rectangle. This bites compositions with two
+    // <video> elements of different codecs (h264 mp4 + VP9 webm) where the
+    // faster decoder lets the readiness check pass while the slower one
+    // hasn't painted, producing a black "first frame" for the slower clip.
     // skipReadinessVideoIds excludes natively-extracted videos (e.g. HDR HEVC
     // sources) whose frames come from ffmpeg out-of-band. videoMetadataHints
     // supply intrinsic dimensions for skipped videos whose layout depends on
@@ -397,12 +403,12 @@ export async function initializeSession(session: CaptureSession): Promise<void>
     const skipIdsLiteral = JSON.stringify(session.options.skipReadinessVideoIds ?? []);
     const videosReady = await pollPageExpression(
       page,
-      `(() => { const skip = new Set(${skipIdsLiteral}); const vids = Array.from(document.querySelectorAll("video")).filter(v => !skip.has(v.id)); return vids.length === 0 || vids.every(v => v.readyState >= 1); })()`,
+      `(() => { const skip = new Set(${skipIdsLiteral}); const vids = Array.from(document.querySelectorAll("video")).filter(v => !skip.has(v.id)); return vids.length === 0 || vids.every(v => v.readyState >= 2); })()`,
       pageReadyTimeout,
     );
     if (!videosReady) {
       throw new Error(
-        `[FrameCapture] video metadata not ready after ${pageReadyTimeout}ms. Video elements must load metadata before capture starts.`,
+        `[FrameCapture] video first frame not decoded after ${pageReadyTimeout}ms. Video elements must reach readyState >= 2 (HAVE_CURRENT_DATA) before capture starts.`,
       );
     }
 
@@ -484,16 +490,13 @@ export async function initializeSession(session: CaptureSession): Promise<void>
 
   await applyVideoMetadataHints(page, session.options.videoMetadataHints);
 
-  // Wait for all video elements to have loaded metadata (dimensions + duration).
-  // Without this, frame 0 captures videos at their 300x150 default size.
-  // See screenshot-mode comment above for why skipReadinessVideoIds and
-  // videoMetadataHints are paired.
+  // Same readyState contract as the screenshot path above (>= 2 / HAVE_CURRENT_DATA).
   const beginframeSkipIdsLiteral = JSON.stringify(session.options.skipReadinessVideoIds ?? []);
   const videoDeadline =
     Date.now() + (session.config?.playerReadyTimeout ?? DEFAULT_CONFIG.playerReadyTimeout);
   while (Date.now() < videoDeadline) {
     const videosReady = await page.evaluate(
-      `(() => { const skip = new Set(${beginframeSkipIdsLiteral}); const vids = Array.from(document.querySelectorAll("video")).filter(v => !skip.has(v.id)); return vids.length === 0 || vids.every(v => v.readyState >= 1); })()`,
+      `(() => { const skip = new Set(${beginframeSkipIdsLiteral}); const vids = Array.from(document.querySelectorAll("video")).filter(v => !skip.has(v.id)); return vids.length === 0 || vids.every(v => v.readyState >= 2); })()`,
     );
     if (videosReady) break;
     await new Promise((r) => setTimeout(r, 100));
diff --git a/packages/engine/src/services/streamingEncoder.ts b/packages/engine/src/services/streamingEncoder.ts
index a39073b7d..a527df830 100644
--- a/packages/engine/src/services/streamingEncoder.ts
+++ b/packages/engine/src/services/streamingEncoder.ts
@@ -227,6 +227,14 @@ export function buildStreamingArgs(
       if (bitrate) args.push("-b:v", bitrate);
       else args.push("-crf", String(quality));
 
+      // Mirrors chunkEncoder: disable B-frames for h264 so PTS == DTS, no
+      // negative DTS at stream start. Without this, files freeze on the
+      // first frame in VS Code preview, several browsers, and some HW
+      // decoders. See chunkEncoder.buildEncoderArgs for the full reasoning.
+      if (codec === "h264") {
+        args.push("-bf", "0");
+      }
+
       // Encoder-specific params: anti-banding + color space tagging.
       // For HDR, getHdrEncoderColorParams also emits the SMPTE ST 2086
       // mastering-display and CTA-861.3 MaxCLL/MaxFALL SEI messages —
@@ -313,6 +321,10 @@ export function buildStreamingArgs(
     args.push("-pix_fmt", pixelFormat);
   }
 
+  // Belt-and-suspenders against negative DTS at stream start. See chunkEncoder
+  // for the full explanation; same playback compatibility class.
+  args.push("-avoid_negative_ts", "make_zero");
+
   args.push("-y", outputPath);
   return args;
 }