From e87a389f4229580a916072e3feb445a55974aafc Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Miguel=20=C3=81ngel?= <miguel.sierra@heygen.com>
Date: Sat, 18 Apr 2026 18:30:16 +0100
Subject: [PATCH] fix(core): preload image decodes to eliminate jerky preview
 (GH #317)
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Reported: compositions with many images stutter visibly during the
first ~second of preview playback, with the only workaround being to
render the composition to mp4 and watch the output file.

Root cause: every `<img>` that has never been painted before decodes
synchronously on the main thread the first time it enters the paint
tree. With a composition of several dozen images, that's several dozen
main-thread stalls stacked inside the first paint window — visible as
jank on every seek and the first few seconds of play.

Fix: a new `packages/core/src/runtime/imageDecode.ts` module that the
runtime binds alongside `bindMediaMetadataListeners`. For every `<img>`
in the document, it:

- Sets `decoding="async"` so the browser never falls back to sync-decode
  on subsequent paints.
- Calls `img.decode()` off the main thread as soon as the image has a
  `currentSrc` (either immediately if already loaded, or on the next
  `load` event otherwise).
- Tracks bound images in a Set so repeated polling from the runtime's
  observation loop is free after the first bind.

`img.decode()` is the canonical platform primitive for this — it's a
no-op for already-decoded images, runs on the decoder thread, and
resolves when the pixels are in the bitmap cache. Decode rejections
(CORS, corrupt file, 404) are intentionally swallowed because the
runtime's existing asset-error diagnostic pipeline already surfaces
those via the `error` event.

Lazy images (`loading="lazy"`) are intentionally NOT force-loaded — if
the author opted them out, we respect that; they'll decode when they're
needed.

The module is wired in three call sites in `init.ts`:
- Initial boot (after `runAdapters("discover")`)
- Post-load of external/inline compositions
- Every 10th poll tick (catches DOM changes from author scripts,
  sub-composition mounts, and dynamically-inserted content)

Tests: `imageDecode.test.ts` covers nine cases — async-flag set,
immediate vs. load-deferred decode, idempotence, torn-down no-op,
swallowed rejections, missing currentSrc, multiple imgs in one pass,
and dynamically-added imgs across bind calls. 9/9 pass.

Docs: new "Preloading and decoding" subsection in `core.mdx` under
the Runtime section, documents the invariant for future contributors.

Closes #317.
---
 docs/packages/core.mdx                        |  19 ++
 packages/core/src/runtime/imageDecode.test.ts | 163 ++++++++++++++++++
 packages/core/src/runtime/imageDecode.ts      | 114 ++++++++++++
 packages/core/src/runtime/init.ts             |  12 ++
 4 files changed, 308 insertions(+)
 create mode 100644 packages/core/src/runtime/imageDecode.test.ts
 create mode 100644 packages/core/src/runtime/imageDecode.ts

diff --git a/docs/packages/core.mdx b/docs/packages/core.mdx
index 896d899875..7073eeecb0 100644
--- a/docs/packages/core.mdx
+++ b/docs/packages/core.mdx
@@ -353,6 +353,25 @@ The pre-built runtime IIFE is available as a direct import:
 import runtime from '@hyperframes/core/runtime';
 ```
 
+### Preloading and decoding
+
+The runtime eagerly prepares every asset in the composition so the first
+paint and the first play call never stall the compositor.
+
+- **Media** — every `<video>` and `<audio>` has `preload="auto"` set and
+  `.load()` kicked on bind. The first `play()` lands on buffered data
+  instead of triggering a network fetch + decoder spin-up.
+- **Images** — every `<img>` gets `decoding="async"` and
+  `img.decode()` invoked off the main thread. Compositions with many
+  images (several dozen through several hundred) otherwise stutter
+  during the first ~second of preview because each image decodes
+  synchronously the first time it enters the paint tree (see
+  [GH #317](https://github.com/heygen-com/hyperframes/issues/317)).
+
+Both binders are **idempotent** — the runtime re-runs them periodically
+as sub-compositions mount and DOM changes land, and they skip elements
+they've already processed.
+
 ## Frame Adapters
 
 The core package defines the [Frame Adapter](/concepts/frame-adapters) interface and provides the built-in GSAP adapter:
diff --git a/packages/core/src/runtime/imageDecode.test.ts b/packages/core/src/runtime/imageDecode.test.ts
new file mode 100644
index 0000000000..e66c37e4ef
--- /dev/null
+++ b/packages/core/src/runtime/imageDecode.test.ts
@@ -0,0 +1,163 @@
+/**
+ * Tests for the eager image-decode binder.
+ *
+ * Regression coverage for GH #317 — compositions with many images
+ * stuttered during preview because each image decoded synchronously on
+ * first paint. The binder forces every `<img>` through `decode()` off
+ * the main thread so the first paint lands on pre-decoded bitmaps.
+ *
+ * We exercise the binder with a tiny hand-rolled DOM stub rather than
+ * spinning up jsdom — the surface is small enough (three method calls)
+ * that a stub keeps the test fast and the assertions legible.
+ */
+
+import { describe, expect, it, vi } from "vitest";
+
+import { createImageDecodeBinder } from "./imageDecode";
+
+interface StubImg {
+  decoding: string;
+  complete: boolean;
+  currentSrc: string;
+  isConnected: boolean;
+  decode: () => Promise<void>;
+  addEventListener: (type: string, handler: () => void, opts?: AddEventListenerOptions) => void;
+}
+
+function makeStubImg(overrides: Partial<StubImg> = {}): StubImg {
+  return {
+    decoding: "auto",
+    complete: true,
+    currentSrc: "https://example.com/a.png",
+    isConnected: true,
+    decode: vi.fn().mockResolvedValue(undefined),
+    addEventListener: vi.fn(),
+    ...overrides,
+  };
+}
+
+function makeRoot(imgs: StubImg[]) {
+  return {
+    querySelectorAll: vi.fn().mockReturnValue(imgs as unknown as NodeListOf<Element>),
+  };
+}
+
+describe("createImageDecodeBinder", () => {
+  it("sets decoding='async' on every img it finds", () => {
+    const img = makeStubImg({ decoding: "auto" });
+    const binder = createImageDecodeBinder({ root: makeRoot([img]) });
+    binder.bind();
+    expect(img.decoding).toBe("async");
+  });
+
+  it("calls decode() immediately on images that are already loaded", () => {
+    const img = makeStubImg({ complete: true, currentSrc: "https://x/y.png" });
+    const binder = createImageDecodeBinder({ root: makeRoot([img]) });
+    binder.bind();
+    expect(img.decode).toHaveBeenCalledTimes(1);
+    expect(img.addEventListener).not.toHaveBeenCalled();
+  });
+
+  it("waits for 'load' on images that haven't finished loading yet", () => {
+    const img = makeStubImg({ complete: false, currentSrc: "" });
+    const addListener = vi.fn((_el: StubImg, _type: "load", handler: () => void) => {
+      // Simulate the browser firing `load` after a network fetch.
+      setTimeout(() => {
+        img.complete = true;
+        img.currentSrc = "https://x/y.png";
+        handler();
+      }, 0);
+    });
+    const binder = createImageDecodeBinder({
+      root: makeRoot([img]),
+      addEventListener: addListener as unknown as NonNullable<
+        Parameters<typeof createImageDecodeBinder>[0]
+      >["addEventListener"],
+    });
+
+    binder.bind();
+    expect(addListener).toHaveBeenCalledTimes(1);
+    expect(addListener.mock.calls[0][1]).toBe("load");
+    // decode() deferred until after the load handler fires.
+    expect(img.decode).not.toHaveBeenCalled();
+  });
+
+  it("is idempotent — calling bind() twice does not re-bind the same img", () => {
+    const img = makeStubImg();
+    const binder = createImageDecodeBinder({ root: makeRoot([img]) });
+    binder.bind();
+    binder.bind();
+    expect(img.decode).toHaveBeenCalledTimes(1);
+    expect(binder.size()).toBe(1);
+  });
+
+  it("no-ops when tornDown predicate returns true", () => {
+    const img = makeStubImg();
+    const binder = createImageDecodeBinder({
+      root: makeRoot([img]),
+      tornDown: () => true,
+    });
+    binder.bind();
+    expect(img.decode).not.toHaveBeenCalled();
+    expect(img.addEventListener).not.toHaveBeenCalled();
+    expect(binder.size()).toBe(0);
+  });
+
+  it("swallows decode() rejections silently (real failures surface via the 'error' event)", async () => {
+    const img = makeStubImg({
+      decode: vi.fn().mockRejectedValue(new Error("CORS blocked")),
+    });
+    const binder = createImageDecodeBinder({ root: makeRoot([img]) });
+    binder.bind();
+    // Let the microtask for the rejected promise run. No throw should reach us.
+    await Promise.resolve();
+    await Promise.resolve();
+    expect(img.decode).toHaveBeenCalled();
+    // The fact that no unhandled rejection leaked is what we're asserting.
+  });
+
+  it("does not trigger decode() for images without a currentSrc", () => {
+    const img = makeStubImg({ complete: true, currentSrc: "" });
+    const binder = createImageDecodeBinder({ root: makeRoot([img]) });
+    binder.bind();
+    // complete=true but currentSrc="" means the browser has no bitmap;
+    // decoding it would reject. We wait for `load` instead.
+    expect(img.decode).not.toHaveBeenCalled();
+    expect(img.addEventListener).toHaveBeenCalled();
+  });
+
+  it("binds multiple images in a single pass", () => {
+    const a = makeStubImg();
+    const b = makeStubImg();
+    const c = makeStubImg();
+    const binder = createImageDecodeBinder({ root: makeRoot([a, b, c]) });
+    binder.bind();
+    expect(a.decode).toHaveBeenCalledTimes(1);
+    expect(b.decode).toHaveBeenCalledTimes(1);
+    expect(c.decode).toHaveBeenCalledTimes(1);
+    expect(binder.size()).toBe(3);
+  });
+
+  it("picks up newly-added images on subsequent bind() calls", () => {
+    const first = makeStubImg();
+    const imgs = [first];
+    const root = {
+      querySelectorAll: vi.fn(() => imgs as unknown as NodeListOf<Element>),
+    };
+    const binder = createImageDecodeBinder({ root });
+
+    binder.bind();
+    expect(binder.size()).toBe(1);
+    expect(first.decode).toHaveBeenCalledTimes(1);
+
+    // A composition adds an image dynamically (sub-composition mount,
+    // author-inserted content). Next bind() must pick it up.
+    const second = makeStubImg();
+    imgs.push(second);
+    binder.bind();
+    expect(binder.size()).toBe(2);
+    expect(second.decode).toHaveBeenCalledTimes(1);
+    // The already-bound image isn't re-decoded.
+    expect(first.decode).toHaveBeenCalledTimes(1);
+  });
+});
diff --git a/packages/core/src/runtime/imageDecode.ts b/packages/core/src/runtime/imageDecode.ts
new file mode 100644
index 0000000000..33dddb700c
--- /dev/null
+++ b/packages/core/src/runtime/imageDecode.ts
@@ -0,0 +1,114 @@
+/**
+ * Eager image-decode binding.
+ *
+ * Compositions with many `<img>` elements (reported in GH #317) stutter
+ * during the first ~second of preview because each image decodes
+ * synchronously the first time it enters the paint tree. Forcing every
+ * image through `HTMLImageElement.decode()` off the main thread ahead of
+ * paint eliminates that cost.
+ *
+ * `img.decode()` is the canonical platform primitive for this:
+ *   - resolves when the pixels are in the decoded bitmap cache
+ *   - is a fast no-op for already-decoded images
+ *   - runs on the decoder thread, never blocks the main thread
+ *
+ * We also set `decoding="async"` so the browser never falls back to
+ * sync-decode on subsequent paints. We deliberately do NOT flip
+ * `loading="lazy"` images — if the author asked for them to be deferred,
+ * we respect that; they'll decode when they're actually needed.
+ *
+ * Binding is idempotent: each img is tracked in a WeakSet so repeated
+ * polling calls from the runtime's observation loop are free after the
+ * first bind.
+ */
+
+export interface BindImageDecodeOptions {
+  /**
+   * Document root to scan. Defaults to the globally-available `document`.
+   * Injectable so unit tests can pass a constructed DOM.
+   */
+  root?: Pick<Document, "querySelectorAll">;
+  /**
+   * Poison pill — when true, the binder is a no-op. Wired to the
+   * runtime's `state.tornDown` flag so teardown races don't leak
+   * listeners into an already-disposed runtime.
+   */
+  tornDown?: () => boolean;
+  /**
+   * Listener registration. Defaults to `el.addEventListener`. Injected
+   * for tests so they can drive the `load` event synchronously.
+   */
+  addEventListener?: (
+    el: HTMLImageElement,
+    type: "load",
+    handler: () => void,
+    opts?: AddEventListenerOptions,
+  ) => void;
+}
+
+export interface ImageDecodeBinder {
+  /**
+   * Scan the document for new `<img>` elements and bind each one once.
+   * Safe to call repeatedly — bound images are tracked and skipped.
+   */
+  bind(): void;
+  /** Number of images currently tracked (for tests + observability). */
+  size(): number;
+}
+
+/**
+ * Create a binder. Separated from the runtime's init.ts so the behaviour
+ * is testable in isolation without constructing the full runtime.
+ */
+export function createImageDecodeBinder(options: BindImageDecodeOptions = {}): ImageDecodeBinder {
+  const root = options.root ?? (typeof document !== "undefined" ? document : null);
+  const isTornDown = options.tornDown ?? (() => false);
+  const addListener =
+    options.addEventListener ??
+    ((el, type, handler, opts) => el.addEventListener(type, handler, opts));
+
+  // Use Set (not WeakSet) so we can report size() for tests + diagnostics.
+  // In long-lived runtimes a WeakSet would be preferable to avoid retaining
+  // removed images, but document-lifetime retention is acceptable here:
+  // images that are removed from the DOM don't hold pixel memory once the
+  // browser's decoded-bitmap cache evicts them, and the set entries are
+  // pointer-sized.
+  const bound = new Set<HTMLImageElement>();
+
+  return {
+    bind(): void {
+      if (isTornDown()) return;
+      if (!root) return;
+      const imgs = root.querySelectorAll("img");
+      for (let i = 0; i < imgs.length; i++) {
+        const img = imgs[i] as HTMLImageElement;
+        if (bound.has(img)) continue;
+        bound.add(img);
+
+        if (img.decoding !== "async") img.decoding = "async";
+
+        const start = () => {
+          if (isTornDown()) return;
+          if (!img.isConnected) return;
+          if (!img.currentSrc) return;
+          // `decode()` rejects for decode failures (CORS, 404, corrupt).
+          // We intentionally swallow — actual load failures are reported
+          // through the runtime's asset-error diagnostic pipeline, which
+          // listens on the "error" event, not on decode rejections.
+          img.decode().catch(() => {
+            /* see comment above */
+          });
+        };
+
+        if (img.complete && img.currentSrc) {
+          start();
+        } else {
+          addListener(img, "load", start, { once: true });
+        }
+      }
+    },
+    size(): number {
+      return bound.size;
+    },
+  };
+}
diff --git a/packages/core/src/runtime/init.ts b/packages/core/src/runtime/init.ts
index 7a430a47de..48528e14c0 100644
--- a/packages/core/src/runtime/init.ts
+++ b/packages/core/src/runtime/init.ts
@@ -13,6 +13,7 @@ import { collectRuntimeTimelinePayload } from "./timeline";
 import { createRuntimeStartTimeResolver } from "./startResolver";
 import { loadExternalCompositions, loadInlineTemplateCompositions } from "./compositionLoader";
 import { applyCaptionOverrides } from "./captionOverrides";
+import { createImageDecodeBinder } from "./imageDecode";
 import type { RuntimeDeterministicAdapter, RuntimeJson, RuntimeTimelineLike } from "./types";
 import type { PlayerAPI } from "../core.types";
 
@@ -1180,6 +1181,14 @@ export function initSandboxRuntimeModular(): void {
     }
   };
 
+  // Eagerly decode every <img> off the main thread so the first paint that
+  // needs it doesn't stall the compositor. See `imageDecode.ts` for the
+  // full rationale — addresses GH #317 (jerky preview with many images).
+  const imageDecodeBinder = createImageDecodeBinder({
+    tornDown: () => state.tornDown === true,
+  });
+  const bindImageDecodeListeners = () => imageDecodeBinder.bind();
+
   const syncMediaForCurrentState = () => {
     const cache = refreshRuntimeMediaCache({
       resolveStartSeconds: (element) => resolveStartForElement(element, 0),
@@ -1333,6 +1342,7 @@ export function initSandboxRuntimeModular(): void {
         externalCompositionsReady = true;
         runAdapters("discover", state.currentTime);
         bindMediaMetadataListeners();
+        bindImageDecodeListeners();
         installAssetFailureDiagnostics();
         applyCaptionOverrides();
         postTimeline();
@@ -1486,6 +1496,7 @@ export function initSandboxRuntimeModular(): void {
   installRuntimeErrorDiagnostics();
   runAdapters("discover");
   bindMediaMetadataListeners();
+  bindImageDecodeListeners();
   if (state.timelinePollIntervalId) {
     clearInterval(state.timelinePollIntervalId);
   }
@@ -1516,6 +1527,7 @@ export function initSandboxRuntimeModular(): void {
     }
     if (timelinePollTick % 10 === 0) {
       bindMediaMetadataListeners();
+      bindImageDecodeListeners();
     }
     syncCurrentTimeFromTimeline();
     if (state.isPlaying && state.capturedTimeline) {