mirror of
https://github.com/heygen-com/hyperframes.git
synced 2026-09-03 04:38:33 +00:00
fix(engine): wait for first frame decode + drop B-frames so renders play in every player
Three related render robustness fixes: 1. frameCapture.ts: bump videos-ready check from `readyState >= 1` (HAVE_METADATA — only dimensions known) to `>= 2` (HAVE_CURRENT_DATA — first frame is rasterized). Without this, when two `<video>` elements with different codecs (h264 mp4 + VP9 webm) decode at different rates, the faster one passes readiness while the slower one still hasn't painted, producing a black "first frame" for the slower clip. 2. chunkEncoder.ts (libx264 path) + streamingEncoder.ts: disable B-frames for h264 (`-bf 0`). Standard libx264 with B-frames produces negative DTS at stream start (the first B-frame's decode order is "before" the first I-frame's presentation time). VS Code preview, several browser <video> implementations, and some HW decoders freeze on the first frame and only audio plays. -bf 0 makes PTS == DTS at every frame, eliminating the issue at the source. Quality cost is ~5–10% larger files at the same CRF — worthwhile for "the file plays everywhere". 3. chunkEncoder.ts (encoder + mux paths): add `-avoid_negative_ts make_zero` as belt-and-suspenders against negative DTS sneaking back in via `-c:v copy` mux passes when audio/video PTS bases differ. Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
This commit is contained in:
@@ -388,8 +388,14 @@ export async function initializeSession(session: CaptureSession): Promise<void>
|
||||
|
||||
await applyVideoMetadataHints(page, session.options.videoMetadataHints);
|
||||
|
||||
// Wait for all video elements to have loaded metadata (dimensions + duration)
|
||||
// Without this, frame 0 captures videos at their 300x150 default size.
|
||||
// Wait for all video elements to have decoded their CURRENT frame, not
|
||||
// just metadata. readyState >= 2 (HAVE_CURRENT_DATA) means a frame is
|
||||
// actually rasterized and ready to paint — at >= 1 (HAVE_METADATA) we
|
||||
// only know the dimensions, and the first <video> screenshot can come
|
||||
// back as a black/blank rectangle. This bites compositions with two
|
||||
// <video> elements of different codecs (h264 mp4 + VP9 webm) where the
|
||||
// faster decoder lets the readiness check pass while the slower one
|
||||
// hasn't painted, producing a black "first frame" for the slower clip.
|
||||
// skipReadinessVideoIds excludes natively-extracted videos (e.g. HDR HEVC
|
||||
// sources) whose frames come from ffmpeg out-of-band. videoMetadataHints
|
||||
// supply intrinsic dimensions for skipped videos whose layout depends on
|
||||
@@ -397,12 +403,12 @@ export async function initializeSession(session: CaptureSession): Promise<void>
|
||||
const skipIdsLiteral = JSON.stringify(session.options.skipReadinessVideoIds ?? []);
|
||||
const videosReady = await pollPageExpression(
|
||||
page,
|
||||
`(() => { const skip = new Set(${skipIdsLiteral}); const vids = Array.from(document.querySelectorAll("video")).filter(v => !skip.has(v.id)); return vids.length === 0 || vids.every(v => v.readyState >= 1); })()`,
|
||||
`(() => { const skip = new Set(${skipIdsLiteral}); const vids = Array.from(document.querySelectorAll("video")).filter(v => !skip.has(v.id)); return vids.length === 0 || vids.every(v => v.readyState >= 2); })()`,
|
||||
pageReadyTimeout,
|
||||
);
|
||||
if (!videosReady) {
|
||||
throw new Error(
|
||||
`[FrameCapture] video metadata not ready after ${pageReadyTimeout}ms. Video elements must load metadata before capture starts.`,
|
||||
`[FrameCapture] video first frame not decoded after ${pageReadyTimeout}ms. Video elements must reach readyState >= 2 (HAVE_CURRENT_DATA) before capture starts.`,
|
||||
);
|
||||
}
|
||||
|
||||
@@ -484,16 +490,13 @@ export async function initializeSession(session: CaptureSession): Promise<void>
|
||||
|
||||
await applyVideoMetadataHints(page, session.options.videoMetadataHints);
|
||||
|
||||
// Wait for all video elements to have loaded metadata (dimensions + duration).
|
||||
// Without this, frame 0 captures videos at their 300x150 default size.
|
||||
// See screenshot-mode comment above for why skipReadinessVideoIds and
|
||||
// videoMetadataHints are paired.
|
||||
// Same readyState contract as the screenshot path above (>= 2 / HAVE_CURRENT_DATA).
|
||||
const beginframeSkipIdsLiteral = JSON.stringify(session.options.skipReadinessVideoIds ?? []);
|
||||
const videoDeadline =
|
||||
Date.now() + (session.config?.playerReadyTimeout ?? DEFAULT_CONFIG.playerReadyTimeout);
|
||||
while (Date.now() < videoDeadline) {
|
||||
const videosReady = await page.evaluate(
|
||||
`(() => { const skip = new Set(${beginframeSkipIdsLiteral}); const vids = Array.from(document.querySelectorAll("video")).filter(v => !skip.has(v.id)); return vids.length === 0 || vids.every(v => v.readyState >= 1); })()`,
|
||||
`(() => { const skip = new Set(${beginframeSkipIdsLiteral}); const vids = Array.from(document.querySelectorAll("video")).filter(v => !skip.has(v.id)); return vids.length === 0 || vids.every(v => v.readyState >= 2); })()`,
|
||||
);
|
||||
if (videosReady) break;
|
||||
await new Promise((r) => setTimeout(r, 100));
|
||||
|
||||
Reference in New Issue
Block a user