From 35a01d9058fca2c4f5e9a4aa13a86d2eafc77c68 Mon Sep 17 00:00:00 2001
From: miga-heygen <miguel.sierra_miga@heygen.com>
Date: Sun, 28 Jun 2026 14:36:21 +0000
Subject: [PATCH] perf(engine): reduce init overhead in headless capture
 sessions (#1718)

Flush the GSAP proxy queue synchronously during capture session initialization and parallelize independent media/font/tailwind readiness waits.

Closes #1715.

Co-authored-by: Miguel Angel Simon Sierra <miguel.sierra_miga@heygen.com>
---
 packages/engine/src/services/frameCapture.ts  | 157 +++++++++---------
 .../src/generated/hf-early-stub-inline.ts     |   2 +-
 packages/producer/stubs/hf-early-stub.ts      |  11 ++
 3 files changed, 90 insertions(+), 80 deletions(-)

diff --git a/packages/engine/src/services/frameCapture.ts b/packages/engine/src/services/frameCapture.ts
index d9fe500f6..d6f81d0a2 100644
--- a/packages/engine/src/services/frameCapture.ts
+++ b/packages/engine/src/services/frameCapture.ts
@@ -990,6 +990,13 @@ export async function initializeSession(session: CaptureSession): Promise<void>
     await gotoEntryPage();
     logInitPhase("page.goto complete");
 
+    // Flush the GSAP proxy queue synchronously instead of waiting for
+    // rAF-based batch ticks (100 ops/tick at ~16ms). In headless mode there's
+    // no UI responsiveness concern, so draining instantly eliminates the
+    // largest init-time cost for tween-heavy compositions.
+    await page.evaluate(`window.__hfFlushSync?.()`);
+    logInitPhase("GSAP proxy flush complete");
+
     const pageReadyTimeout =
       session.config?.playerReadyTimeout ?? DEFAULT_CONFIG.playerReadyTimeout;
     await pollHfReady(page, pageReadyTimeout);
@@ -1001,24 +1008,37 @@ export async function initializeSession(session: CaptureSession): Promise<void>
     await applyVideoMetadataHints(page, session.options.videoMetadataHints);
     logInitPhase("applyVideoMetadataHints complete");
 
-    // Wait for all video elements to have decoded their CURRENT frame, not
-    // just metadata. readyState >= 2 (HAVE_CURRENT_DATA) means a frame is
-    // actually rasterized and ready to paint — at >= 1 (HAVE_METADATA) we
-    // only know the dimensions, and the first <video> screenshot can come
-    // back as a black/blank rectangle. This bites compositions with two
-    // <video> elements of different codecs (h264 mp4 + VP9 webm) where the
-    // faster decoder lets the readiness check pass while the slower one
-    // hasn't painted, producing a black "first frame" for the slower clip.
-    // skipReadinessVideoIds excludes natively-extracted videos (e.g. HDR HEVC
-    // sources) whose frames come from ffmpeg out-of-band. videoMetadataHints
-    // supply intrinsic dimensions for skipped videos whose layout depends on
-    // aspect ratio, while Chromium may still fail to decode/load metadata.
-    const videosReady = await pollVideosReady(
-      page,
-      session.options.skipReadinessVideoIds ?? [],
-      pageReadyTimeout,
-    );
-    logInitPhase("pollVideosReady complete");
+    // Run independent readiness checks in parallel — videos, images, fonts,
+    // and Tailwind don't depend on each other's completion.
+    const skipVideoIds = session.options.skipReadinessVideoIds ?? [];
+    const [videosReady] = await Promise.all([
+      pollVideosReady(page, skipVideoIds, pageReadyTimeout),
+      pollImagesReady(page, pageReadyTimeout).then(async (ready) => {
+        if (!ready) {
+          const failedImages = await page.evaluate(() => {
+            return Array.from(document.querySelectorAll("img"))
+              .filter((img) => {
+                const ie = img as HTMLImageElement;
+                const src = ie.getAttribute("src") || "";
+                if (!src || src.startsWith("data:")) return false;
+                return !(ie.complete && ie.naturalWidth > 0);
+              })
+              .map((img) => (img as HTMLImageElement).src || img.getAttribute("src") || "(no src)")
+              .join(", ");
+          });
+          console.warn(
+            `[FrameCapture] Some image elements did not load within ${pageReadyTimeout}ms: ${failedImages}. ` +
+              `Continuing render — affected images may appear blank/missing in early frames.`,
+          );
+        }
+        await decodeAllImages(page);
+        return ready;
+      }),
+      page.evaluate(`document.fonts?.ready`),
+      waitForOptionalTailwindReady(page, pageReadyTimeout),
+    ]);
+    logInitPhase("media + fonts + tailwind ready");
+
     if (!videosReady) {
       const failedVideos = await page.evaluate((skipIdList: readonly string[]) => {
         const skip = new Set(skipIdList);
@@ -1027,38 +1047,13 @@ export async function initializeSession(session: CaptureSession): Promise<void>
           .filter((v) => (v as HTMLVideoElement).readyState < 2 && !(v as HTMLVideoElement).error)
           .map((v) => (v as HTMLVideoElement).src || v.getAttribute("src") || "(no src)")
           .join(", ");
-      }, session.options.skipReadinessVideoIds ?? []);
+      }, skipVideoIds);
       console.warn(
         `[FrameCapture] Some video elements did not decode within ${pageReadyTimeout}ms: ${failedVideos}. ` +
           `Continuing render — affected videos will appear as blank/black frames.`,
       );
     }
 
-    const imagesReady = await pollImagesReady(page, pageReadyTimeout);
-    if (!imagesReady) {
-      const failedImages = await page.evaluate(() => {
-        return Array.from(document.querySelectorAll("img"))
-          .filter((img) => {
-            const ie = img as HTMLImageElement;
-            const src = ie.getAttribute("src") || "";
-            if (!src || src.startsWith("data:")) return false;
-            return !(ie.complete && ie.naturalWidth > 0);
-          })
-          .map((img) => (img as HTMLImageElement).src || img.getAttribute("src") || "(no src)")
-          .join(", ");
-      });
-      console.warn(
-        `[FrameCapture] Some image elements did not load within ${pageReadyTimeout}ms: ${failedImages}. ` +
-          `Continuing render — affected images may appear blank/missing in early frames.`,
-      );
-    }
-    await decodeAllImages(page);
-    logInitPhase("images ready + decoded");
-
-    await page.evaluate(`document.fonts?.ready`);
-    logInitPhase("fonts ready");
-    await waitForOptionalTailwindReady(page, pageReadyTimeout);
-    logInitPhase("tailwind ready");
     await recordSessionInitTelemetry(session, initStart);
 
     // For PNG captures, force the page background fully transparent so the
@@ -1131,6 +1126,13 @@ export async function initializeSession(session: CaptureSession): Promise<void>
   await gotoEntryPage();
   logInitPhase("page.goto complete");
 
+  // Flush the GSAP proxy queue synchronously. In BeginFrame mode the rAF-based
+  // batch drain runs on the warmup loop's 33ms ticks — for tween-heavy
+  // compositions this is the dominant init cost. Flushing synchronously
+  // eliminates the wait entirely.
+  await page.evaluate(`window.__hfFlushSync?.()`);
+  logInitPhase("GSAP proxy flush complete");
+
   // Poll for window.__hf readiness using manual evaluate loop (waitForFunction
   // uses rAF polling internally, which won't fire in beginFrame mode).
   const pageReadyTimeout = session.config?.playerReadyTimeout ?? DEFAULT_CONFIG.playerReadyTimeout;
@@ -1148,12 +1150,37 @@ export async function initializeSession(session: CaptureSession): Promise<void>
   await applyVideoMetadataHints(page, session.options.videoMetadataHints);
   logInitPhase("applyVideoMetadataHints complete");
 
-  // Same readyState contract as the screenshot path above (>= 2 / HAVE_CURRENT_DATA).
-  const bfVideosReady = await pollVideosReady(
-    page,
-    session.options.skipReadinessVideoIds ?? [],
-    session.config?.playerReadyTimeout ?? DEFAULT_CONFIG.playerReadyTimeout,
-  );
+  // Run independent readiness checks in parallel — videos, images, fonts,
+  // and Tailwind don't depend on each other's completion.
+  const bfSkipVideoIds = session.options.skipReadinessVideoIds ?? [];
+  const [bfVideosReady] = await Promise.all([
+    pollVideosReady(page, bfSkipVideoIds, pageReadyTimeout),
+    pollImagesReady(page, pageReadyTimeout).then(async (ready) => {
+      if (!ready) {
+        const failedImages = await page.evaluate(() => {
+          return Array.from(document.querySelectorAll("img"))
+            .filter((img) => {
+              const ie = img as HTMLImageElement;
+              const src = ie.getAttribute("src") || "";
+              if (!src || src.startsWith("data:")) return false;
+              return !(ie.complete && ie.naturalWidth > 0);
+            })
+            .map((img) => (img as HTMLImageElement).src || img.getAttribute("src") || "(no src)")
+            .join(", ");
+        });
+        console.warn(
+          `[FrameCapture] Some image elements did not load within ${pageReadyTimeout}ms: ${failedImages}. ` +
+            `Continuing render — affected images may appear blank/missing in early frames.`,
+        );
+      }
+      await decodeAllImages(page);
+      return ready;
+    }),
+    page.evaluate(`document.fonts?.ready`),
+    waitForOptionalTailwindReady(page, pageReadyTimeout),
+  ]);
+  logInitPhase("media + fonts + tailwind ready");
+
   if (!bfVideosReady) {
     const failedVideos = await page.evaluate((skipIdList: readonly string[]) => {
       const skip = new Set(skipIdList);
@@ -1162,41 +1189,13 @@ export async function initializeSession(session: CaptureSession): Promise<void>
         .filter((v) => (v as HTMLVideoElement).readyState < 2 && !(v as HTMLVideoElement).error)
         .map((v) => (v as HTMLVideoElement).src || v.getAttribute("src") || "(no src)")
         .join(", ");
-    }, session.options.skipReadinessVideoIds ?? []);
+    }, bfSkipVideoIds);
     console.warn(
       `[FrameCapture] Some video elements did not decode within ${pageReadyTimeout}ms: ${failedVideos}. ` +
         `Continuing render — affected videos will appear as blank/black frames.`,
     );
   }
-  logInitPhase("pollVideosReady complete");
 
-  // Image readiness — parity with pollVideosReady. Defense against remote
-  // <img> URLs that bypass the htmlCompiler localize step.
-  const bfImagesReady = await pollImagesReady(page, pageReadyTimeout);
-  if (!bfImagesReady) {
-    const failedImages = await page.evaluate(() => {
-      return Array.from(document.querySelectorAll("img"))
-        .filter((img) => {
-          const ie = img as HTMLImageElement;
-          const src = ie.getAttribute("src") || "";
-          if (!src || src.startsWith("data:")) return false;
-          return !(ie.complete && ie.naturalWidth > 0);
-        })
-        .map((img) => (img as HTMLImageElement).src || img.getAttribute("src") || "(no src)")
-        .join(", ");
-    });
-    console.warn(
-      `[FrameCapture] Some image elements did not load within ${pageReadyTimeout}ms: ${failedImages}. ` +
-        `Continuing render — affected images may appear blank/missing in early frames.`,
-    );
-  }
-  await decodeAllImages(page);
-  logInitPhase("images ready + decoded");
-
-  await page.evaluate(`document.fonts?.ready`);
-  logInitPhase("fonts ready");
-  await waitForOptionalTailwindReady(page, pageReadyTimeout);
-  logInitPhase("tailwind ready");
   await recordSessionInitTelemetry(session, initStart);
 
   // Stop warmup. Unlocked mode exits on this flag; locked mode keeps ticking
diff --git a/packages/producer/src/generated/hf-early-stub-inline.ts b/packages/producer/src/generated/hf-early-stub-inline.ts
index eb1c1290e..8a938fbe8 100644
--- a/packages/producer/src/generated/hf-early-stub-inline.ts
+++ b/packages/producer/src/generated/hf-early-stub-inline.ts
@@ -1,6 +1,6 @@
 // AUTO-GENERATED by scripts/build-hf-early-stub.ts — do not edit
 const HF_EARLY_STUB_IIFE: string =
-  '"use strict";(()=>{var T=100,_=[],u=[],l=!1,s=!1;function w(n){let i=window.__HF_VIRTUAL_TIME__?.originalRequestAnimationFrame;return typeof i=="function"?i(n):requestAnimationFrame(n)}function y(n){let i=window.__HF_VIRTUAL_TIME__?.originalSetTimeout;if(typeof i=="function"){i(n,0);return}setTimeout(n,0)}function g(n){return n!==null&&typeof n=="object"&&"__hfIsProxy"in n?n.__hfReal:n}function m(n){let i=n.proxy.__hfReal,e=i[n.method];if(typeof e=="function"){let o=n.method==="add"?n.args.map(g):n.args;e.call(i,...o)}}function r(n,i,e){let o={proxy:n,method:i,args:e};return n.__hfQueue.push(o),u.push(o),P(),n}function c(n){let i=n.proxy.__hfQueue.indexOf(n);i>=0&&n.proxy.__hfQueue.splice(i,1)}function t(){for(;u.length>0;){let n=u.shift();n&&(c(n),m(n))}x()}function d(){s=!1,window.__hfTimelinesBuilding=!1;try{window.dispatchEvent(new CustomEvent("hf-timelines-built"))}catch{}}function x(){s||(s=!0,y(()=>{u.length===0?d():s=!1}))}function k(){l=!1;let n=u.splice(0,T);for(let i of n)c(i),m(i);u.length>0?(l=!0,w(k)):d()}function P(){l||(l=!0,window.__hfTimelinesBuilding=!0,w(k))}var O=new Set(["to","from","fromTo","set","add"]);function b(n,i){let e=i;for(;e!==null&&e!==Object.prototype;){for(let o of Object.getOwnPropertyNames(e)){if(o==="constructor"||o==="then"||o in n||O.has(o)||o.charAt(0)==="_")continue;let a=Object.getOwnPropertyDescriptor(e,o);if(!a||typeof a.value!="function")continue;let p=a.value;n[o]=function(...h){t();let f=p.call(i,...h);return f===i?n:f}}e=Object.getPrototypeOf(e)}}function v(n){let i={__hfReal:n,__hfQueue:[],__hfIsProxy:!0,to(...e){return r(i,"to",e)},from(...e){return r(i,"from",e)},fromTo(...e){return r(i,"fromTo",e)},set(...e){return r(i,"set",e)},add(...e){return r(i,"add",e)},pause(...e){return t(),n.pause(...e),i},play(...e){return t(),n.play(...e),i},seek(...e){return t(),n.seek(...e),i},totalTime(...e){return t(),e.length>0?(n.totalTime(...e),i):n.totalTime()},time(...e){return t(),e.length>0?(n.time(...e),i):n.time()},duration(...e){return t(),e.length>0?(n.duration(...e),i):n.duration()},getChildren(...e){t();let o=n.getChildren(...e);return Array.isArray(o)?o:[]},paused(...e){return t(),e.length>0?(n.paused(...e),i):n.paused()},timeScale(...e){return t(),e.length>0?(n.timeScale(...e),i):n.timeScale()},kill(){t(),n.kill()}};return b(i,n),_.push(i),i}if(typeof window<"u"){window.__hf||(window.__hf={}),window.__hfTimelinesBuilding=!1;let n=null;try{Object.defineProperty(window,"gsap",{configurable:!0,enumerable:!0,get(){return n},set(i){if(n=i,!i||typeof i.timeline!="function")return;let e=i.timeline.bind(i);i.timeline=o=>v(e(o))}})}catch{}}})();\n';
+  '"use strict";(()=>{var T=100,_=[],u=[],l=!1,s=!1;function m(n){let i=window.__HF_VIRTUAL_TIME__?.originalRequestAnimationFrame;return typeof i=="function"?i(n):requestAnimationFrame(n)}function y(n){let i=window.__HF_VIRTUAL_TIME__?.originalSetTimeout;if(typeof i=="function"){i(n,0);return}setTimeout(n,0)}function g(n){return n!==null&&typeof n=="object"&&"__hfIsProxy"in n?n.__hfReal:n}function c(n){let i=n.proxy.__hfReal,e=i[n.method];if(typeof e=="function"){let o=n.method==="add"?n.args.map(g):n.args;e.call(i,...o)}}function r(n,i,e){let o={proxy:n,method:i,args:e};return n.__hfQueue.push(o),u.push(o),P(),n}function d(n){let i=n.proxy.__hfQueue.indexOf(n);i>=0&&n.proxy.__hfQueue.splice(i,1)}function t(){for(;u.length>0;){let n=u.shift();n&&(d(n),c(n))}x()}function f(){s=!1,window.__hfTimelinesBuilding=!1;try{window.dispatchEvent(new CustomEvent("hf-timelines-built"))}catch{}}function x(){s||(s=!0,y(()=>{u.length===0?f():s=!1}))}function k(){l=!1;let n=u.splice(0,T);for(let i of n)d(i),c(i);u.length>0?(l=!0,m(k)):f()}function P(){l||(l=!0,window.__hfTimelinesBuilding=!0,m(k))}var O=new Set(["to","from","fromTo","set","add"]);function b(n,i){let e=i;for(;e!==null&&e!==Object.prototype;){for(let o of Object.getOwnPropertyNames(e)){if(o==="constructor"||o==="then"||o in n||O.has(o)||o.charAt(0)==="_")continue;let a=Object.getOwnPropertyDescriptor(e,o);if(!a||typeof a.value!="function")continue;let h=a.value;n[o]=function(...p){t();let w=h.call(i,...p);return w===i?n:w}}e=Object.getPrototypeOf(e)}}function v(n){let i={__hfReal:n,__hfQueue:[],__hfIsProxy:!0,to(...e){return r(i,"to",e)},from(...e){return r(i,"from",e)},fromTo(...e){return r(i,"fromTo",e)},set(...e){return r(i,"set",e)},add(...e){return r(i,"add",e)},pause(...e){return t(),n.pause(...e),i},play(...e){return t(),n.play(...e),i},seek(...e){return t(),n.seek(...e),i},totalTime(...e){return t(),e.length>0?(n.totalTime(...e),i):n.totalTime()},time(...e){return t(),e.length>0?(n.time(...e),i):n.time()},duration(...e){return t(),e.length>0?(n.duration(...e),i):n.duration()},getChildren(...e){t();let o=n.getChildren(...e);return Array.isArray(o)?o:[]},paused(...e){return t(),e.length>0?(n.paused(...e),i):n.paused()},timeScale(...e){return t(),e.length>0?(n.timeScale(...e),i):n.timeScale()},kill(){t(),n.kill()}};return b(i,n),_.push(i),i}if(typeof window<"u"){window.__hf||(window.__hf={}),window.__hfTimelinesBuilding=!1,window.__hfFlushSync=()=>{t(),u.length===0&&window.__hfTimelinesBuilding&&f()};let n=null;try{Object.defineProperty(window,"gsap",{configurable:!0,enumerable:!0,get(){return n},set(i){if(n=i,!i||typeof i.timeline!="function")return;let e=i.timeline.bind(i);i.timeline=o=>v(e(o))}})}catch{}}})();\n';
 
 /**
  * Returns the pre-built HyperFrames early stub IIFE as a string constant.
diff --git a/packages/producer/stubs/hf-early-stub.ts b/packages/producer/stubs/hf-early-stub.ts
index aa2634bef..25b1cdcde 100644
--- a/packages/producer/stubs/hf-early-stub.ts
+++ b/packages/producer/stubs/hf-early-stub.ts
@@ -391,6 +391,17 @@ function wrapTimeline(real: GsapTimeline): TimelineProxy {
 if (typeof window !== "undefined") {
   if (!window.__hf) window.__hf = {};
   window.__hfTimelinesBuilding = false;
+  // Expose a synchronous flush so headless renderers can drain the queue
+  // instantly instead of waiting for rAF-based batch ticks. Also force-
+  // publishes the "timelines built" signal immediately (normally deferred
+  // via setTimeout(0)) — the caller guarantees page scripts have finished
+  // loading, so no more operations can arrive after the flush.
+  (window as Record<string, unknown>).__hfFlushSync = () => {
+    flushPendingOperations();
+    if (pendingOperations.length === 0 && window.__hfTimelinesBuilding) {
+      publishTimelinesBuilt();
+    }
+  };
 
   // Intercept window.gsap assignment via a property trap so we can wrap
   // `gsap.timeline()` before any user script calls it. GSAP is not yet