From 39b588cbd0e196d1d1db14e959f837dc90cf7788 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Miguel=20=C3=81ngel?= <miguel.sierra@heygen.com>
Date: Thu, 16 Jul 2026 23:01:39 -0400
Subject: [PATCH] feat(core): swap undecodable video to its proxy at runtime
 (#2592)

* feat(studio-server): serve H.264 proxies from the preview route

Wires the codec manifest and the transcoder into the preview surface: the route
negotiates a proxy via a query param and serves it through the existing range
and ETag machinery, composition HTML carries a codec map for the runtime, and
hostile assets pre-warm so a first play does not wait on a cold transcode.
Exposes the three subpath exports the CLI surfaces consume upstack.

Drops the TEMP fallow entry added with the transcoder: it has real importers now.

* fix(studio-server): publish media proxy exports

* fix(parsers): scan HTML comments linearly

* feat(cli): let projects opt out of automatic proxying

Adds media.autoProxy to hyperframes.json plus --proxy/--no-proxy flags, and
forwards the resolved value into the studio and preview servers and the vite
adapter. Lands before the runtime slice that turns auto-proxying on, so the
switch exists before there is any behavior to switch off.

* fix(cli): align media config schema

* feat(core): swap undecodable video to its proxy at runtime

Adds the browser-side half: before first load the runtime consults the injected
codec map and swaps a hostile source to its proxy, and if a video still reports
zero decodable width it rescues it reactively. An HEVC file carrying AAC fires
no error event, so zero videoWidth, not the error event, is the reliable signal.
Audio elements and alpha sources are never proxied, render mode never proxies,
and each swap evicts the element's stale sync state and reports once.

This completes the loop: auto-proxying is live for preview and studio from here.
The opt-out (media.autoProxy, --no-proxy) shipped in the previous slice.
---
 packages/core/src/runtime/init.ts            |  31 ++
 packages/core/src/runtime/media.ts           |  32 +-
 packages/core/src/runtime/mediaProxy.test.ts | 529 +++++++++++++++++++
 packages/core/src/runtime/mediaProxy.ts      | 368 +++++++++++++
 4 files changed, 956 insertions(+), 4 deletions(-)
 create mode 100644 packages/core/src/runtime/mediaProxy.test.ts
 create mode 100644 packages/core/src/runtime/mediaProxy.ts

diff --git a/packages/core/src/runtime/init.ts b/packages/core/src/runtime/init.ts
index c6f606012..bac909fd6 100644
--- a/packages/core/src/runtime/init.ts
+++ b/packages/core/src/runtime/init.ts
@@ -24,6 +24,7 @@ import {
   resolveRuntimeMediaClipDuration,
   syncRuntimeMedia,
 } from "./media";
+import { handleErrorForProxy, handleMetadataForProxy, maybeProxyProactively } from "./mediaProxy";
 import { probeAndCacheElementVolume, type VolumeKeyframe } from "./mediaVolumeEnvelope.js";
 import { createPickerModule } from "./picker";
 import { createRuntimePlayer, type RuntimePlayerTransport } from "./player";
@@ -1644,10 +1645,29 @@ export function initSandboxRuntimeModular(): void {
     }, METADATA_REBIND_DEBOUNCE_MS);
   };
 
+  // Reactive/tertiary undecodable-media triggers (see mediaProxy.ts). Wrapped
+  // as event listeners here — rather than exported directly — because
+  // `addEventListener` hands the listener an `Event`, not the element;
+  // `event.currentTarget` recovers it. Bound/unbound alongside the metadata
+  // listeners below, reusing `metadataBoundMedia` as the once-per-element
+  // dedupe (no separate tracking set needed).
+  const onMediaLoadedMetadataForProxy = (event: Event) => {
+    if (event.currentTarget instanceof HTMLMediaElement) {
+      handleMetadataForProxy(event.currentTarget);
+    }
+  };
+  const onMediaErrorForProxy = (event: Event) => {
+    if (event.currentTarget instanceof HTMLMediaElement) {
+      handleErrorForProxy(event.currentTarget);
+    }
+  };
+
   const unbindMediaMetadataListeners = () => {
     for (const mediaEl of metadataBoundMedia) {
       mediaEl.removeEventListener("loadedmetadata", scheduleMetadataDurationHydration);
       mediaEl.removeEventListener("durationchange", scheduleMetadataDurationHydration);
+      mediaEl.removeEventListener("loadedmetadata", onMediaLoadedMetadataForProxy);
+      mediaEl.removeEventListener("error", onMediaErrorForProxy);
     }
     metadataBoundMedia.clear();
   };
@@ -1664,6 +1684,17 @@ export function initSandboxRuntimeModular(): void {
       }
       mediaEl.addEventListener("loadedmetadata", scheduleMetadataDurationHydration);
       mediaEl.addEventListener("durationchange", scheduleMetadataDurationHydration);
+      // Reactive (zero-videoWidth) + tertiary (error event) proxy-fallback
+      // triggers. Inert in render mode / when the codec map is absent /
+      // for <audio> — all guarded inside mediaProxy.ts itself.
+      mediaEl.addEventListener("loadedmetadata", onMediaLoadedMetadataForProxy);
+      mediaEl.addEventListener("error", onMediaErrorForProxy);
+
+      // Proactive proxy-fallback trigger: consult the codec map and swap
+      // BEFORE the eager load() below, so a known-hostile asset never even
+      // attempts to load (and error-flash) the original. No-op in render
+      // mode, for <audio>, or when the codec map is absent.
+      maybeProxyProactively(mediaEl);
 
       // Eagerly preload media data so audio/video is buffered before the user
       // clicks play. Without this, the first play() call fires on un-fetched
diff --git a/packages/core/src/runtime/media.ts b/packages/core/src/runtime/media.ts
index c95cecf7c..d69ce7dd7 100644
--- a/packages/core/src/runtime/media.ts
+++ b/packages/core/src/runtime/media.ts
@@ -152,6 +152,33 @@ function clampVolume(volume: number): number {
   return Math.max(0, Math.min(1, volume));
 }
 
+/**
+ * Drop every per-source sync baseline tracked for `el` — offset drift
+ * samples, the seek-past-buffered-range retry latch, and the last
+ * runtime-applied volume — so the next `syncRuntimeMedia` tick treats it as
+ * a first tick (hard resync, fresh drift baseline) instead of comparing
+ * against state computed for a different file. Used both when a clip leaves
+ * its active window (below) and by the runtime's proxy-swap helper
+ * (mediaProxy.ts) right after an in-place `src` swap, which points the same
+ * element at a different file without ever leaving its active window.
+ */
+export function evictMediaSyncState(el: HTMLMediaElement): void {
+  lastOffset.delete(el);
+  strictDriftSamples.delete(el);
+  seekLoadRetried.delete(el);
+  lastRuntimeAppliedVolume.delete(el);
+}
+
+/** Test-only seam: whether any per-source sync state is still tracked for `el`. */
+export function hasMediaSyncStateForTest(el: HTMLMediaElement): boolean {
+  return (
+    lastOffset.has(el) ||
+    strictDriftSamples.has(el) ||
+    seekLoadRetried.has(el) ||
+    lastRuntimeAppliedVolume.has(el)
+  );
+}
+
 // fallow-ignore-next-line complexity
 export function syncRuntimeMedia(params: {
   clips: RuntimeMediaClip[];
@@ -400,10 +427,7 @@ export function syncRuntimeMedia(params: {
     }
     // Clip left its active window — drop the offset baseline so the next
     // activation (e.g. re-entering a sub-composition) gets a hard resync.
-    lastOffset.delete(el);
-    strictDriftSamples.delete(el);
-    seekLoadRetried.delete(el);
-    lastRuntimeAppliedVolume.delete(el);
+    evictMediaSyncState(el);
     if (!el.paused) el.pause();
   }
 }
diff --git a/packages/core/src/runtime/mediaProxy.test.ts b/packages/core/src/runtime/mediaProxy.test.ts
new file mode 100644
index 000000000..ad6a541ce
--- /dev/null
+++ b/packages/core/src/runtime/mediaProxy.test.ts
@@ -0,0 +1,529 @@
+import { afterEach, describe, expect, it, vi } from "vitest";
+import { hasMediaSyncStateForTest, syncRuntimeMedia } from "./media";
+import {
+  deriveCodecMapKey,
+  handleErrorForProxy,
+  handleMetadataForProxy,
+  maybeProxyProactively,
+  swapToProxy,
+  type MediaCodecMapEntry,
+} from "./mediaProxy";
+
+vi.mock("./bridge", () => ({
+  postRuntimeMessage: vi.fn(),
+}));
+
+import { postRuntimeMessage } from "./bridge";
+
+const postRuntimeMessageMock = vi.mocked(postRuntimeMessage);
+
+const HEVC_ENTRY: MediaCodecMapEntry = {
+  codecName: "hevc",
+  browserHostile: true,
+  representativeMime: 'video/mp4; codecs="hvc1.1.6.L120.B0"',
+};
+
+const H264_ENTRY: MediaCodecMapEntry = {
+  codecName: "h264",
+  browserHostile: false,
+  representativeMime: 'video/mp4; codecs="avc1.640028"',
+};
+
+function createVideo(src: string): HTMLVideoElement {
+  const el = document.createElement("video");
+  el.src = src;
+  el.load = vi.fn();
+  document.body.appendChild(el);
+  return el;
+}
+
+function createAudio(src: string): HTMLAudioElement {
+  const el = document.createElement("audio");
+  el.src = src;
+  el.load = vi.fn();
+  document.body.appendChild(el);
+  return el;
+}
+
+function stubCanPlayType(el: HTMLVideoElement, result: string): void {
+  el.canPlayType = vi.fn(() => result) as unknown as HTMLVideoElement["canPlayType"];
+}
+
+function isProxied(el: HTMLMediaElement): boolean {
+  return new URL(el.src, document.baseURI).searchParams.get("hf-proxy") === "h264";
+}
+
+afterEach(() => {
+  document.body.innerHTML = "";
+  document.head.innerHTML = "";
+  delete (window as { __HF_MEDIA_CODEC_MAP__?: unknown }).__HF_MEDIA_CODEC_MAP__;
+  delete (window as { __HF_EXPORT_RENDER_SEEK_CONFIG?: unknown }).__HF_EXPORT_RENDER_SEEK_CONFIG;
+  vi.restoreAllMocks();
+});
+
+describe("maybeProxyProactively", () => {
+  it("rewrites src and calls load() before first load for a hostile, undecodable asset", () => {
+    window.__HF_MEDIA_CODEC_MAP__ = { "/video.mp4": HEVC_ENTRY };
+    const el = createVideo("/video.mp4");
+    stubCanPlayType(el, "");
+
+    maybeProxyProactively(el);
+
+    expect(isProxied(el)).toBe(true);
+    expect(el.load).toHaveBeenCalledTimes(1);
+    expect(postRuntimeMessageMock).toHaveBeenCalledWith(
+      expect.objectContaining({ code: "runtime_media_proxy_fallback" }),
+    );
+  });
+
+  it("matches codec-map paths across case and Unicode normalization differences", () => {
+    window.__HF_MEDIA_CODEC_MAP__ = { "/assets/Caf\u00e9.MP4": HEVC_ENTRY };
+    const el = createVideo("/assets/cafe\u0301.mp4");
+    stubCanPlayType(el, "");
+
+    maybeProxyProactively(el);
+
+    expect(isProxied(el)).toBe(true);
+  });
+
+  it("does not guess when two codec-map paths collide after normalization", () => {
+    window.__HF_MEDIA_CODEC_MAP__ = {
+      "/assets/CLIP.mp4": HEVC_ENTRY,
+      "/assets/clip.MP4": H264_ENTRY,
+    };
+    const el = createVideo("/assets/Clip.mp4");
+    stubCanPlayType(el, "");
+
+    maybeProxyProactively(el);
+
+    expect(isProxied(el)).toBe(false);
+  });
+
+  it("does not swap when canPlayType reports probably/maybe despite a hostile map entry", () => {
+    window.__HF_MEDIA_CODEC_MAP__ = { "/video.mp4": HEVC_ENTRY };
+    const el = createVideo("/video.mp4");
+    stubCanPlayType(el, "probably");
+
+    maybeProxyProactively(el);
+
+    expect(isProxied(el)).toBe(false);
+    expect(el.load).not.toHaveBeenCalled();
+  });
+
+  it("swaps when hostile and representativeMime is null (no canPlayType check possible)", () => {
+    window.__HF_MEDIA_CODEC_MAP__ = {
+      "/video.mp4": {
+        codecName: "prores",
+        browserHostile: true,
+        representativeMime: null,
+      },
+    };
+    const el = createVideo("/video.mp4");
+    stubCanPlayType(el, "probably"); // should not even be consulted
+
+    maybeProxyProactively(el);
+
+    expect(isProxied(el)).toBe(true);
+  });
+
+  it("does not swap a non-hostile (browser-safe) map entry", () => {
+    window.__HF_MEDIA_CODEC_MAP__ = { "/video.mp4": H264_ENTRY };
+    const el = createVideo("/video.mp4");
+    stubCanPlayType(el, "");
+
+    maybeProxyProactively(el);
+
+    expect(isProxied(el)).toBe(false);
+    expect(el.load).not.toHaveBeenCalled();
+  });
+
+  it("is a no-op in render mode even for a hostile entry", () => {
+    window.__HF_EXPORT_RENDER_SEEK_CONFIG = { mode: "seek" };
+    window.__HF_MEDIA_CODEC_MAP__ = { "/video.mp4": HEVC_ENTRY };
+    const el = createVideo("/video.mp4");
+    stubCanPlayType(el, "");
+
+    maybeProxyProactively(el);
+
+    expect(isProxied(el)).toBe(false);
+    expect(el.load).not.toHaveBeenCalled();
+    expect(postRuntimeMessageMock).not.toHaveBeenCalled();
+  });
+
+  it("is a no-op when the codec map global is absent", () => {
+    const el = createVideo("/video.mp4");
+    stubCanPlayType(el, "");
+
+    maybeProxyProactively(el);
+
+    expect(isProxied(el)).toBe(false);
+  });
+
+  it("never swaps an <audio> element even with a hostile-container src", () => {
+    window.__HF_MEDIA_CODEC_MAP__ = { "/video.mp4": HEVC_ENTRY };
+    const el = createAudio("/video.mp4");
+
+    maybeProxyProactively(el);
+
+    expect(isProxied(el)).toBe(false);
+    expect(el.load).not.toHaveBeenCalled();
+    expect(postRuntimeMessageMock).not.toHaveBeenCalled();
+  });
+
+  it("never swaps an alpha-bearing hostile entry; emits the unavailable diagnostic instead", () => {
+    window.__HF_MEDIA_CODEC_MAP__ = {
+      "/video.mov": {
+        codecName: "prores",
+        browserHostile: true,
+        representativeMime: null,
+        hasAlpha: true,
+      },
+    };
+    const el = createVideo("/video.mov");
+    stubCanPlayType(el, "");
+
+    maybeProxyProactively(el);
+
+    expect(isProxied(el)).toBe(false);
+    expect(el.load).not.toHaveBeenCalled();
+    expect(postRuntimeMessageMock).toHaveBeenCalledWith(
+      expect.objectContaining({
+        code: "runtime_media_proxy_unavailable",
+        details: expect.objectContaining({ reason: "alpha_source" }),
+      }),
+    );
+  });
+
+  it("is a no-op when the render-frame sibling image signals render mode", () => {
+    window.__HF_MEDIA_CODEC_MAP__ = { "/video.mp4": HEVC_ENTRY };
+    const el = createVideo("/video.mp4");
+    el.id = "clip-1";
+    const injected = document.createElement("img");
+    injected.id = "__render_frame_clip-1__";
+    document.body.appendChild(injected);
+    stubCanPlayType(el, "");
+
+    maybeProxyProactively(el);
+
+    expect(isProxied(el)).toBe(false);
+    expect(el.load).not.toHaveBeenCalled();
+    expect(postRuntimeMessageMock).not.toHaveBeenCalled();
+  });
+});
+
+describe("handleMetadataForProxy (reactive trigger)", () => {
+  function markZeroWidth(el: HTMLVideoElement): void {
+    Object.defineProperty(el, "videoWidth", { value: 0, configurable: true });
+  }
+
+  it("swaps once when videoWidth is 0 for a same-origin local video absent from the (present) map — unlisted-asset rescue", () => {
+    window.__HF_MEDIA_CODEC_MAP__ = {};
+    const el = createVideo("/video.mp4");
+    markZeroWidth(el);
+
+    handleMetadataForProxy(el);
+
+    expect(isProxied(el)).toBe(true);
+    expect(el.load).toHaveBeenCalledTimes(1);
+  });
+
+  it("never swaps when the codec map global is absent (opt-out surface serves no proxies)", () => {
+    const el = createVideo("/video.mp4");
+    markZeroWidth(el);
+
+    handleMetadataForProxy(el);
+
+    expect(isProxied(el)).toBe(false);
+    expect(el.load).not.toHaveBeenCalled();
+    expect(postRuntimeMessageMock).not.toHaveBeenCalled();
+  });
+
+  it("skips a MAPPED entry with hasAlpha (alpha sources are never proxied) and diagnoses instead", () => {
+    window.__HF_MEDIA_CODEC_MAP__ = {
+      "/video.mov": {
+        codecName: "prores",
+        browserHostile: true,
+        representativeMime: null,
+        hasAlpha: true,
+      },
+    };
+    const el = createVideo("/video.mov");
+    markZeroWidth(el);
+
+    handleMetadataForProxy(el);
+
+    expect(isProxied(el)).toBe(false);
+    expect(el.load).not.toHaveBeenCalled();
+    expect(postRuntimeMessageMock).toHaveBeenCalledWith(
+      expect.objectContaining({
+        code: "runtime_media_proxy_unavailable",
+        details: expect.objectContaining({ reason: "alpha_source" }),
+      }),
+    );
+  });
+
+  it("does not proxy a mapped browser-safe codec on the metadata path", () => {
+    window.__HF_MEDIA_CODEC_MAP__ = { "/video.mp4": H264_ENTRY };
+    const el = createVideo("/video.mp4");
+    markZeroWidth(el);
+
+    handleMetadataForProxy(el);
+
+    expect(isProxied(el)).toBe(false);
+    expect(el.load).not.toHaveBeenCalled();
+    expect(postRuntimeMessageMock).toHaveBeenCalledWith(
+      expect.objectContaining({
+        code: "runtime_media_proxy_unavailable",
+        details: expect.objectContaining({ reason: "browser_safe_codec" }),
+      }),
+    );
+  });
+
+  it("a second zero-width metadata event does not loop (no second swap)", () => {
+    window.__HF_MEDIA_CODEC_MAP__ = {};
+    const el = createVideo("/video.mp4");
+    markZeroWidth(el);
+
+    handleMetadataForProxy(el);
+    handleMetadataForProxy(el);
+
+    expect(el.load).toHaveBeenCalledTimes(1);
+  });
+
+  it("does nothing when videoWidth is non-zero", () => {
+    const el = createVideo("/video.mp4");
+    Object.defineProperty(el, "videoWidth", { value: 640, configurable: true });
+
+    handleMetadataForProxy(el);
+
+    expect(isProxied(el)).toBe(false);
+    expect(el.load).not.toHaveBeenCalled();
+  });
+
+  it("is a no-op in render mode", () => {
+    window.__HF_EXPORT_RENDER_SEEK_CONFIG = { mode: "seek" };
+    const el = createVideo("/video.mp4");
+    markZeroWidth(el);
+
+    handleMetadataForProxy(el);
+
+    expect(isProxied(el)).toBe(false);
+  });
+
+  it("never swaps <audio>", () => {
+    const el = createAudio("/video.mp4");
+    Object.defineProperty(el, "videoWidth", { value: 0, configurable: true });
+
+    handleMetadataForProxy(el);
+
+    expect(isProxied(el)).toBe(false);
+    expect(postRuntimeMessageMock).not.toHaveBeenCalled();
+  });
+
+  it("cross-origin src with zero videoWidth: no swap attempted, diagnostic still emitted (with its console.info line)", () => {
+    const infoSpy = vi.spyOn(console, "info").mockImplementation(() => {});
+    window.__HF_MEDIA_CODEC_MAP__ = {};
+    const el = createVideo("https://cdn.example.com/video.mp4");
+    markZeroWidth(el);
+
+    handleMetadataForProxy(el);
+
+    expect(el.load).not.toHaveBeenCalled();
+    expect(postRuntimeMessageMock).toHaveBeenCalledWith(
+      expect.objectContaining({
+        code: "runtime_media_proxy_unavailable",
+        details: expect.objectContaining({ reason: "cross_origin" }),
+      }),
+    );
+    // The console line mirrors the fallback line's shape: stable code token +
+    // reason + src, so checkBrowser.ts's scraper can surface it.
+    expect(infoSpy).toHaveBeenCalledTimes(1);
+    const line = String(infoSpy.mock.calls[0]?.[0]);
+    expect(line).toContain("runtime_media_proxy_unavailable");
+    expect(line).toContain("cross_origin");
+    expect(line).toContain("https://cdn.example.com/video.mp4");
+  });
+
+  it("resolves a ../-traversing sub-composition src (already rewritten to an absolute, prefixed URL) via longest-suffix map matching", () => {
+    const base = document.createElement("base");
+    base.href = `${window.location.origin}/api/projects/proj1/preview/`;
+    document.head.appendChild(base);
+    window.__HF_MEDIA_CODEC_MAP__ = { "/assets/video.mp4": HEVC_ENTRY };
+
+    // Mirrors what compositionLoader.ts's rewriteRuntimeAssetPath produces for
+    // a `../assets/video.mp4` src authored from a nested sub-composition: an
+    // absolute URL resolved against the sub-composition's own (prefixed) URL.
+    const el = createVideo(`${window.location.origin}/api/projects/proj1/preview/assets/video.mp4`);
+    expect(deriveCodecMapKey(el)).toBe("/api/projects/proj1/preview/assets/video.mp4");
+    stubCanPlayType(el, "");
+
+    maybeProxyProactively(el);
+
+    expect(isProxied(el)).toBe(true);
+  });
+});
+
+describe("handleErrorForProxy (tertiary trigger)", () => {
+  it("swaps once on an error event for a zero-stream (video-only hostile) file unlisted in the (present) map", () => {
+    window.__HF_MEDIA_CODEC_MAP__ = {};
+    const el = createVideo("/video.mp4");
+
+    handleErrorForProxy(el);
+
+    expect(isProxied(el)).toBe(true);
+    expect(el.load).toHaveBeenCalledTimes(1);
+  });
+
+  it("never swaps when the codec map global is absent (opt-out surface serves no proxies)", () => {
+    const el = createVideo("/video.mp4");
+
+    handleErrorForProxy(el);
+
+    expect(isProxied(el)).toBe(false);
+    expect(el.load).not.toHaveBeenCalled();
+    expect(postRuntimeMessageMock).not.toHaveBeenCalled();
+  });
+
+  it("does not swap a mapped browser-SAFE entry that errors (corrupt file — proxying can't help); diagnoses instead", () => {
+    window.__HF_MEDIA_CODEC_MAP__ = { "/video.mp4": H264_ENTRY };
+    const el = createVideo("/video.mp4");
+
+    handleErrorForProxy(el);
+
+    expect(isProxied(el)).toBe(false);
+    expect(el.load).not.toHaveBeenCalled();
+    expect(postRuntimeMessageMock).toHaveBeenCalledWith(
+      expect.objectContaining({
+        code: "runtime_media_proxy_unavailable",
+        details: expect.objectContaining({ reason: "browser_safe_codec" }),
+      }),
+    );
+  });
+
+  it("never swaps <audio> on error", () => {
+    window.__HF_MEDIA_CODEC_MAP__ = {};
+    const el = createAudio("/video.mp4");
+
+    handleErrorForProxy(el);
+
+    expect(isProxied(el)).toBe(false);
+    expect(postRuntimeMessageMock).not.toHaveBeenCalled();
+  });
+
+  it("the proxy URL itself erroring: no second swap, diagnostic reports the failure instead", () => {
+    const el = createVideo("/video.mp4");
+    swapToProxy(el, HEVC_ENTRY, "proactive");
+    expect(el.load).toHaveBeenCalledTimes(1);
+    postRuntimeMessageMock.mockClear();
+
+    handleErrorForProxy(el);
+
+    expect(el.load).toHaveBeenCalledTimes(1); // no second load()/swap
+    expect(postRuntimeMessageMock).toHaveBeenCalledWith(
+      expect.objectContaining({
+        code: "runtime_media_proxy_unavailable",
+        details: expect.objectContaining({ reason: "proxy_playback_failed" }),
+      }),
+    );
+  });
+});
+
+describe("swapToProxy", () => {
+  it("does not poison swap state when the source URL is malformed", () => {
+    const el = createVideo("/video.mp4");
+    Object.defineProperty(el, "currentSrc", {
+      value: "http://[",
+      configurable: true,
+    });
+
+    swapToProxy(el, HEVC_ENTRY, "reactive");
+
+    expect(el.load).not.toHaveBeenCalled();
+    expect(postRuntimeMessageMock).toHaveBeenCalledWith(
+      expect.objectContaining({
+        code: "runtime_media_proxy_unavailable",
+        details: expect.objectContaining({ reason: "invalid_source_url" }),
+      }),
+    );
+  });
+  it("evicts per-source sync state so the swapped element is treated as a first tick", () => {
+    const el = createVideo("/video.mp4");
+    // Populate sync state as a real active-clip tick would.
+    syncRuntimeMedia({
+      clips: [
+        {
+          el,
+          start: 0,
+          mediaStart: 0,
+          duration: 10,
+          end: 10,
+          volume: null,
+          playbackRate: 1,
+          loop: false,
+          sourceDuration: null,
+        },
+      ],
+      timeSeconds: 1,
+      playing: false,
+      playbackRate: 1,
+    });
+    expect(hasMediaSyncStateForTest(el)).toBe(true);
+
+    swapToProxy(el, HEVC_ENTRY, "reactive");
+
+    expect(hasMediaSyncStateForTest(el)).toBe(false);
+  });
+
+  it("preserves existing query strings when appending hf-proxy", () => {
+    const el = createVideo("/video.mp4?v=2");
+
+    swapToProxy(el, HEVC_ENTRY, "proactive");
+
+    const url = new URL(el.src, document.baseURI);
+    expect(url.searchParams.get("v")).toBe("2");
+    expect(url.searchParams.get("hf-proxy")).toBe("h264");
+  });
+
+  it("emits the diagnostic and a single console.info line exactly once per element", () => {
+    const infoSpy = vi.spyOn(console, "info").mockImplementation(() => {});
+    const el = createVideo("/video.mp4");
+
+    swapToProxy(el, HEVC_ENTRY, "proactive");
+    swapToProxy(el, HEVC_ENTRY, "proactive"); // idempotent re-call, e.g. from another trigger
+
+    const fallbackCalls = postRuntimeMessageMock.mock.calls.filter(
+      ([msg]) => (msg as { code?: string }).code === "runtime_media_proxy_fallback",
+    );
+    expect(fallbackCalls).toHaveLength(1);
+    expect(infoSpy).toHaveBeenCalledTimes(1);
+  });
+
+  it("is idempotent: a second call for an already-swapped element does not re-swap", () => {
+    const el = createVideo("/video.mp4");
+
+    swapToProxy(el, HEVC_ENTRY, "proactive");
+    const srcAfterFirstSwap = el.src;
+    swapToProxy(el, HEVC_ENTRY, "reactive");
+
+    expect(el.src).toBe(srcAfterFirstSwap);
+    expect(el.load).toHaveBeenCalledTimes(1);
+  });
+});
+
+describe("deriveCodecMapKey", () => {
+  it("returns the decoded, query-stripped pathname for a same-origin src", () => {
+    const el = createVideo("/assets/my%20clip.mp4?foo=bar");
+    expect(deriveCodecMapKey(el)).toBe("/assets/my clip.mp4");
+  });
+
+  it("returns null for a cross-origin src", () => {
+    const el = createVideo("https://cdn.example.com/video.mp4");
+    expect(deriveCodecMapKey(el)).toBeNull();
+  });
+
+  it("returns null when there is no src", () => {
+    const el = document.createElement("video");
+    document.body.appendChild(el);
+    expect(deriveCodecMapKey(el)).toBeNull();
+  });
+});
diff --git a/packages/core/src/runtime/mediaProxy.ts b/packages/core/src/runtime/mediaProxy.ts
new file mode 100644
index 000000000..d2e9f66b0
--- /dev/null
+++ b/packages/core/src/runtime/mediaProxy.ts
@@ -0,0 +1,368 @@
+import { postRuntimeMessage } from "./bridge";
+import { swallow } from "./diagnostics";
+import { evictMediaSyncState } from "./media";
+import type { RuntimeJson } from "./types";
+
+/**
+ * One entry per project-root-relative asset pathname, injected by the
+ * server as `window.__HF_MEDIA_CODEC_MAP__` (see the plan's server-contract
+ * KTD). `representativeMime` is a coarse per-codec-family MIME string (e.g.
+ * `video/mp4; codecs="hvc1.1.6.L120.B0"` for hevc) fed to `canPlayType`;
+ * `null` when ffprobe couldn't produce one, in which case the browser check
+ * is skipped and `browserHostile` alone decides.
+ */
+export type MediaCodecMapEntry = {
+  codecName: string;
+  browserHostile: boolean;
+  representativeMime: string | null;
+  /** Source carries an alpha channel — never proxy it (H.264 would destroy
+   * the transparency, e.g. ProRes 4444 alpha). Optional so pre-alpha-aware
+   * maps stay assignable; absent means "no alpha detected". */
+  hasAlpha?: boolean;
+};
+
+declare global {
+  interface Window {
+    __HF_MEDIA_CODEC_MAP__?: Record<string, MediaCodecMapEntry>;
+  }
+}
+
+const PROXY_QUERY_PARAM = "hf-proxy";
+const PROXY_QUERY_VALUE = "h264";
+
+/** Fired whenever an element is swapped to its H.264 proxy (any trigger). */
+const DIAGNOSTIC_FALLBACK_CODE = "runtime_media_proxy_fallback";
+/** Fired when the runtime detects an undecodable video but cannot (or already
+ *  did) proxy it — a remote asset, or the proxy URL itself failing. */
+const DIAGNOSTIC_UNAVAILABLE_CODE = "runtime_media_proxy_unavailable";
+
+type ProxyTrigger = "proactive" | "reactive" | "tertiary";
+
+// Elements already swapped to their proxy src. Gates every trigger so a
+// second undecodable-video signal (another zero-width metadata tick, a
+// stray error event) never re-swaps or loops.
+const swappedElements = new WeakSet<HTMLMediaElement>();
+// Elements that already got the "can't help you" diagnostic (cross-origin,
+// or the proxy itself failing) — one-shot per element, independent of
+// `swappedElements` so the proxy-failed case (which fires AFTER a real swap)
+// still gets its own single diagnostic.
+const unavailableDiagnosedElements = new WeakSet<HTMLMediaElement>();
+
+function currentSrcValue(el: HTMLMediaElement): string {
+  return el.currentSrc || el.src;
+}
+
+/**
+ * Render mode never proxies: the codec map arrives by injection (not a
+ * fetch, so determinism holds) but render always decodes the original via
+ * FFmpeg frame extraction, never a browser-side element. Mirrors the two
+ * render-mode signals already used elsewhere in the runtime — the global
+ * export-seek config (init.ts) and the per-element `__render_frame_<id>__`
+ * sibling image the producer's frame-injection pipeline creates during
+ * render (the same check `syncRuntimeMedia`'s `skipForInjectedVideo` makes
+ * in media.ts).
+ */
+function isRenderMode(el: HTMLMediaElement): boolean {
+  if (window.__HF_EXPORT_RENDER_SEEK_CONFIG) return true;
+  return (
+    el instanceof HTMLVideoElement &&
+    !!el.id &&
+    !!document.getElementById(`__render_frame_${el.id}__`)
+  );
+}
+
+/**
+ * Resolve an element's codec-map key: the served-root-relative,
+ * percent-decoded, query-string-stripped URL pathname (per the server
+ * contract KTD). Returns null when there's no usable src, or the src is
+ * cross-origin — the server can't have scanned (or proxy) a file it doesn't
+ * host, so there is never a map entry to look up.
+ */
+export function deriveCodecMapKey(el: HTMLMediaElement): string | null {
+  const raw = currentSrcValue(el);
+  if (!raw) return null;
+  let url: URL;
+  try {
+    url = new URL(raw, document.baseURI);
+  } catch {
+    return null;
+  }
+  if (url.origin !== window.location.origin) return null;
+  try {
+    return decodeURIComponent(url.pathname);
+  } catch {
+    return url.pathname;
+  }
+}
+
+/**
+ * The composition is served under a route prefix on some surfaces —
+ * studio-server's preview route injects `<base href="/api/projects/:id/preview/">`
+ * (packages/studio-server/src/routes/preview.ts) so every relative asset
+ * resolves through that prefix — while the codec map (per the server
+ * contract) is keyed by project-root-relative pathnames with no prefix.
+ * `play` and the CLI's static project server serve from the root with no
+ * such prefix, where an exact match already succeeds. So: try exact first
+ * (covers unprefixed servers), then fall back to the longest map key that
+ * is a suffix of the pathname. No separate segment-boundary check is
+ * needed: every key is contractually root-relative and leading-slash
+ * (`"/assets/x.mp4"`, never `"assets/x.mp4"`), so a suffix match's boundary
+ * is always that leading `/` itself — `pathname.endsWith(key)` can't match
+ * a partial segment (e.g. key `"/foo.mp4"` can never match a pathname
+ * ending in `"/notfoo.mp4"`, since that would require the literal substring
+ * `"/foo.mp4"` to appear where the last `/` already fell a segment later).
+ */
+function lookupLongestExactSuffix(
+  pathname: string,
+  map: Record<string, MediaCodecMapEntry>,
+): MediaCodecMapEntry | null {
+  let bestKey: string | null = null;
+  for (const key of Object.keys(map)) {
+    if (!pathname.endsWith(key)) continue;
+    if (bestKey === null || key.length > bestKey.length) bestKey = key;
+  }
+  return bestKey ? (map[bestKey] ?? null) : null;
+}
+
+function lookupNormalizedSuffix(
+  pathname: string,
+  map: Record<string, MediaCodecMapEntry>,
+): MediaCodecMapEntry | null {
+  // Case-insensitive filesystems and Unicode-normalizing filesystems can
+  // serve an authored URL whose spelling differs from the canonical path
+  // ffprobe returned (Clip.mp4 vs clip.mp4, NFC vs NFD). Exact matching stays
+  // authoritative; this normalized fallback is used only when it finds one
+  // unambiguous longest suffix, so case-sensitive projects containing both
+  // spellings never select the wrong asset.
+  const normalizedPathname = pathname.normalize("NFC").toLowerCase();
+  let normalizedBestLength = -1;
+  let normalizedBest: MediaCodecMapEntry | null = null;
+  let ambiguous = false;
+  for (const [key, entry] of Object.entries(map)) {
+    const normalizedKey = key.normalize("NFC").toLowerCase();
+    if (!normalizedPathname.endsWith(normalizedKey)) continue;
+    if (normalizedKey.length > normalizedBestLength) {
+      normalizedBestLength = normalizedKey.length;
+      normalizedBest = entry;
+      ambiguous = false;
+    } else if (normalizedKey.length === normalizedBestLength) {
+      ambiguous = true;
+    }
+  }
+  return ambiguous ? null : normalizedBest;
+}
+
+function lookupCodecMapEntry(
+  pathname: string,
+  map: Record<string, MediaCodecMapEntry>,
+): MediaCodecMapEntry | null {
+  return (
+    map[pathname] ??
+    lookupLongestExactSuffix(pathname, map) ??
+    lookupNormalizedSuffix(pathname, map)
+  );
+}
+
+function appendProxyParam(src: string): string {
+  const url = new URL(src, document.baseURI);
+  url.searchParams.set(PROXY_QUERY_PARAM, PROXY_QUERY_VALUE);
+  return url.href;
+}
+
+type UnavailableReason =
+  | "cross_origin"
+  | "proxy_playback_failed"
+  | "browser_safe_codec"
+  | "alpha_source"
+  | "invalid_source_url";
+
+const UNAVAILABLE_NOTES: Record<UnavailableReason, string> = {
+  cross_origin:
+    "video reports zero decodable width but its source is cross-origin; no local proxy can be served for it",
+  proxy_playback_failed: "the H.264 proxy itself failed to decode; render output is unaffected",
+  browser_safe_codec:
+    "the file errored but its codec is browser-decodable; an H.264 proxy cannot help (the file itself is likely corrupt)",
+  alpha_source:
+    "the source carries an alpha channel; an H.264 proxy would destroy the transparency, so it is never proxied",
+  invalid_source_url: "the media source URL is malformed and cannot be proxied",
+};
+
+function emitUnavailableDiagnostic(
+  el: HTMLMediaElement,
+  reason: UnavailableReason,
+  asset: string,
+): void {
+  if (unavailableDiagnosedElements.has(el)) return;
+  unavailableDiagnosedElements.add(el);
+  const note = UNAVAILABLE_NOTES[reason];
+  const details: Record<string, RuntimeJson> = {
+    asset,
+    codecName: null,
+    reason,
+    note,
+  };
+  postRuntimeMessage({
+    source: "hf-preview",
+    type: "diagnostic",
+    code: DIAGNOSTIC_UNAVAILABLE_CODE,
+    details,
+  });
+  // Mirrors swapToProxy's fallback line: the stable diagnostic code is in the
+  // text so checkBrowser.ts's console scraper can match a token, not prose.
+  console.info(`[hyperframes] ${DIAGNOSTIC_UNAVAILABLE_CODE}: "${asset}" (${reason}): ${note}`);
+}
+
+/**
+ * Swap `el` to its H.264 proxy URL, evict stale per-source sync state, and
+ * emit the one-time diagnostic + console line. Safe to call from any of the
+ * three triggers (proactive/reactive/tertiary); a no-op if already swapped.
+ */
+export function swapToProxy(
+  el: HTMLMediaElement,
+  entry: MediaCodecMapEntry | null = null,
+  trigger: ProxyTrigger = "reactive",
+): void {
+  if (swappedElements.has(el)) return;
+  const originalSrc = currentSrcValue(el);
+  let proxiedSrc: string;
+  try {
+    proxiedSrc = appendProxyParam(originalSrc);
+  } catch (err) {
+    swallow("runtime.mediaProxy.swap", err);
+    emitUnavailableDiagnostic(el, "invalid_source_url", originalSrc);
+    return;
+  }
+  swappedElements.add(el);
+  // The swapped src points at a different file — sync state (drift offsets,
+  // seek-retry latches, volume tracking) computed against the original
+  // source must not carry over, or the next tick misreads a fresh file's
+  // buffering as drift. Evict before `load()` so the very next sync tick
+  // treats this element as a first tick.
+  evictMediaSyncState(el);
+  el.src = proxiedSrc;
+  el.load();
+  const codecName = entry?.codecName ?? null;
+  const details: Record<string, RuntimeJson> = {
+    asset: originalSrc,
+    codecName,
+    trigger,
+    note: "render output is unaffected; only this preview element was swapped to an H.264 proxy",
+  };
+  postRuntimeMessage({
+    source: "hf-preview",
+    type: "diagnostic",
+    code: DIAGNOSTIC_FALLBACK_CODE,
+    details,
+  });
+  // The diagnostic code doubles as the stable token check's console scraper
+  // matches on (packages/cli/src/utils/checkBrowser.ts); keep it in the text.
+  console.info(
+    `[hyperframes] ${DIAGNOSTIC_FALLBACK_CODE}: "${originalSrc}" uses a codec (${codecName ?? "unknown"}) this browser can't decode; ` +
+      "auto-swapped to an H.264 proxy for this preview only. Render output is unaffected.",
+  );
+}
+
+/**
+ * Proactive trigger: consult the codec map before an element's first
+ * `load()` and swap ahead of time when the browser is known-unlikely to
+ * decode it, avoiding an error flash. `<audio>` is never proxied (per the
+ * plan's KTD — an HEVC container's AAC track demuxes fine regardless of the
+ * browser's video codec support).
+ */
+export function maybeProxyProactively(el: HTMLMediaElement): void {
+  if (isRenderMode(el)) return;
+  if (!(el instanceof HTMLVideoElement)) return;
+  if (swappedElements.has(el)) return;
+  const map = window.__HF_MEDIA_CODEC_MAP__;
+  if (!map) return;
+  const key = deriveCodecMapKey(el);
+  if (key === null) return;
+  const entry = lookupCodecMapEntry(key, map);
+  if (!entry || !entry.browserHostile) return;
+  if (entry.hasAlpha) {
+    // Alpha sources are never proxied (transparency would be destroyed);
+    // say so instead of silently leaving a possibly-undecodable element.
+    emitUnavailableDiagnostic(el, "alpha_source", currentSrcValue(el));
+    return;
+  }
+  const canPlay = entry.representativeMime ? el.canPlayType(entry.representativeMime) : "";
+  if (canPlay === "probably" || canPlay === "maybe") return;
+  swapToProxy(el, entry, "proactive");
+}
+
+/**
+ * Reactive trigger (the primary catch-all per the plan's KTD): a `<video>`
+ * reporting `videoWidth === 0` at `loadedmetadata` has an undecodable (or
+ * absent) video track — the common case for an HEVC+AAC file, which fires
+ * no error event because the AAC track satisfies the demuxer. Swaps once
+ * per element; if the element is already on its proxy src, the proxy
+ * itself is the one failing, so this only emits the failure diagnostic.
+ *
+ * No map, no swaps: the codec map global is only injected on surfaces where
+ * auto-proxying is enabled and served, so its absence means a `?hf-proxy=`
+ * request would 404 — never swap there. When the map is present but has no
+ * entry for this key, swapping stays allowed (unlisted-asset rescue). A
+ * mapped entry with alpha is never proxied.
+ */
+export function handleMetadataForProxy(el: HTMLMediaElement): void {
+  if (isRenderMode(el)) return;
+  if (!(el instanceof HTMLVideoElement)) return;
+  if (el.videoWidth !== 0) return;
+  const src = currentSrcValue(el);
+  if (swappedElements.has(el)) {
+    emitUnavailableDiagnostic(el, "proxy_playback_failed", src);
+    return;
+  }
+  const map = window.__HF_MEDIA_CODEC_MAP__;
+  if (!map) return;
+  const key = deriveCodecMapKey(el);
+  if (key === null) {
+    emitUnavailableDiagnostic(el, "cross_origin", src);
+    return;
+  }
+  const entry = lookupCodecMapEntry(key, map);
+  if (entry && !entry.browserHostile) {
+    emitUnavailableDiagnostic(el, "browser_safe_codec", src);
+    return;
+  }
+  if (entry?.hasAlpha) {
+    emitUnavailableDiagnostic(el, "alpha_source", src);
+    return;
+  }
+  swapToProxy(el, entry, "reactive");
+}
+
+/**
+ * Tertiary trigger: the `error` event, for the rarer zero-decodable-stream
+ * file (video-only hostile codec) where the demuxer has nothing to satisfy
+ * it and `loadedmetadata` never fires. Same guards and once-per-element
+ * behavior as the reactive path, plus one extra skip: an entry the scan
+ * mapped as browser-SAFE that still errors is a corrupt-but-safe file — an
+ * H.264 proxy of a broken source can't help, so only diagnose.
+ */
+export function handleErrorForProxy(el: HTMLMediaElement): void {
+  if (isRenderMode(el)) return;
+  if (!(el instanceof HTMLVideoElement)) return;
+  const src = currentSrcValue(el);
+  if (swappedElements.has(el)) {
+    emitUnavailableDiagnostic(el, "proxy_playback_failed", src);
+    return;
+  }
+  const map = window.__HF_MEDIA_CODEC_MAP__;
+  if (!map) return;
+  const key = deriveCodecMapKey(el);
+  if (key === null) {
+    emitUnavailableDiagnostic(el, "cross_origin", src);
+    return;
+  }
+  const entry = lookupCodecMapEntry(key, map);
+  if (entry && !entry.browserHostile) {
+    emitUnavailableDiagnostic(el, "browser_safe_codec", src);
+    return;
+  }
+  if (entry?.hasAlpha) {
+    emitUnavailableDiagnostic(el, "alpha_source", src);
+    return;
+  }
+  swapToProxy(el, entry, "tertiary");
+}