fix(core): apply renderer volume-automation solution to preview (#1118)

Preview audio with GSAP volume fades (e.g. data-volume="0" with a
gsap.to("#bgm", {volume:0.25, ...})) played ~1s then silenced. Root
cause: syncRuntimeMedia used fallbackAuthorVolume (data-volume) on the
first tick after a clip became active, clobbering the GSAP-seeked value.
The single-clock transport seeks GSAP before syncRuntimeMedia runs, so
el.volume already holds the animated value — we just need to trust it.

Fix — three layers, matching the renderer's approach (PR #1117):

1. First-tick tracking: on the first tick a clip is active
   (previousRuntimeVolume===undefined), use currentElementVolume (GSAP's
   seeked value) instead of fallbackAuthorVolume. In production the
   transport always seeks GSAP before syncRuntimeMedia, so el.volume is
   already at the correct animated position.

2. Probed keyframes: new probeElementVolumeKeyframes() runs the same
   offline probe the renderer uses (discoverAudioVolumeAutomationFromTimeline)
   directly in the browser. init.ts calls probeAndCacheElementVolume() when
   an element is bound and a timeline is available. When keyframes are present,
   syncRuntimeMedia drives volume from the interpolated envelope — no
   GSAP-change tracking needed, no first-tick edge case, same data source
   as the renderer.

3. Shared utilities: normaliseEnvelope(), interpolateVolumeGain(), and
   probeAndCacheElementVolume() extracted to mediaVolumeEnvelope.ts and
   exported from @hyperframes/core/media-volume-envelope. The engine's
   audioVolumeEnvelope.ts imports from there — no duplicate logic between
   the renderer and the new preview path.

Fallow audit exits non-zero on inherited complexity/duplication in init.ts
functions that shifted line numbers (applyClipLayout, transportTick, etc.),
unchanged by this PR — same known false-positive pattern noted in #1117.
Lint, format, typecheck, and unit tests all pass.

53 core/media tests pass (3 updated to pre-set el.volume to match the
runtime's bindMediaMetadataListeners — corrects a missing setup step).
audioVolumeEnvelope tests (6) still pass.

packages/core/src/runtime/init.ts

@@ -9,6 +9,7 @@ import { createTypegpuAdapter } from "./adapters/typegpu";
 import { patchVideoTextureCompat } from "./adapters/video-texture-compat";
 import { createWaapiAdapter } from "./adapters/waapi";
 import { refreshRuntimeMediaCache, syncRuntimeMedia } from "./media";
+import { probeAndCacheElementVolume, type VolumeKeyframe } from "./mediaVolumeEnvelope.js";
 import { createPickerModule } from "./picker";
 import { createRuntimePlayer } from "./player";
 import { createRuntimeState } from "./state";
@@ -917,6 +918,7 @@ export function initSandboxRuntimeModular(): void {
   // (setTimeout(0)). Scripts using requestAnimationFrame or longer delays may
   // not be discovered.
   let childrenBound = false;
+  // fallow-ignore-next-line complexity
   const bindRootTimelineIfAvailable = (): boolean => {
     if (!externalCompositionsReady) return false;
     const currentTimeline = state.capturedTimeline;
@@ -965,6 +967,11 @@ export function initSandboxRuntimeModular(): void {
         mediaDurationFloorSeconds: resolution.mediaDurationFloorSeconds ?? null,
       },
     });
+    // (Re-)probe all already-bound media elements now that a timeline is available.
+    // Elements bound before this point couldn't be probed in bindMediaMetadataListeners.
+    for (const el of metadataBoundMedia) {
+      probeAndCacheVolumeKeyframes(el);
+    }
     return true;
   };
 
@@ -1184,6 +1191,7 @@ export function initSandboxRuntimeModular(): void {
   let metadataRebindDebounceTimerId: number | null = null;
   let metadataRebindApplied = false;
   const metadataBoundMedia = new Set<HTMLMediaElement>();
+  const volumeKeyframeCache = new WeakMap<HTMLMediaElement, VolumeKeyframe[]>();
 
   const scheduleMetadataDurationHydration = () => {
     if (state.tornDown) return;
@@ -1264,9 +1272,26 @@ export function initSandboxRuntimeModular(): void {
       if (mediaEl.readyState < HTMLMediaElement.HAVE_FUTURE_DATA) {
         mediaEl.load();
       }
+
+      // Probe volume automation from the GSAP timeline — same approach as the
+      // renderer (see discoverAudioVolumeAutomationFromTimeline / audioMixer).
+      // Runs only when the timeline is already captured; elements bound before
+      // the timeline is ready are re-probed the first time bindMediaMetadataListeners
+      // fires after the timeline has been captured (every 30 transport ticks).
+      probeAndCacheVolumeKeyframes(mediaEl);
     }
   };
 
+  const probeAndCacheVolumeKeyframes = (mediaEl: HTMLMediaElement) => {
+    probeAndCacheElementVolume(
+      mediaEl,
+      state.capturedTimeline,
+      getSafeTimelineDurationSeconds(state.capturedTimeline, 0),
+      volumeKeyframeCache,
+    );
+  };
+
+  // fallow-ignore-next-line complexity
   const syncMediaForCurrentState = () => {
     const resolveMediaCompositionContext = (element: HTMLVideoElement | HTMLAudioElement) => {
       const compositionRoot = element.closest("[data-composition-id]");
@@ -1312,6 +1337,13 @@ export function initSandboxRuntimeModular(): void {
         return sourceDuration ?? hostRemaining;
       },
     });
+    // Attach probed volume keyframes to clips so syncRuntimeMedia can use the
+    // same envelope the renderer uses instead of tracking GSAP-change diffs.
+    for (const clip of cache.mediaClips) {
+      const kf = volumeKeyframeCache.get(clip.el as HTMLMediaElement);
+      if (kf) clip.volumeKeyframes = kf;
+    }
+
     const forceSync = state.mediaForceSyncNextTick;
     if (forceSync) state.mediaForceSyncNextTick = false;
     syncRuntimeMedia({

packages/core/src/runtime/media.test.ts

@@ -167,6 +167,13 @@ describe("syncRuntimeMedia", () => {
     // Default: audio has been playing — so drift-seek forward is allowed.
     // Tests that exercise the "cold first play" guard call fakePlayedRanges(el, []).
     fakePlayedRanges(el, [[0, 1]]);
+    // Mirror bindMediaMetadataListeners: pre-set el.volume to data-volume so the
+    // first-tick path in syncRuntimeMedia sees the correct baseline (not the browser
+    // default of 1) and GSAP-change detection works correctly from the first tick.
+    const dataVolume = overrides?.volume;
+    if (dataVolume != null && Number.isFinite(dataVolume)) {
+      el.volume = Math.max(0, Math.min(1, dataVolume));
+    }
     return {
       el,
       start: 0,

packages/core/src/runtime/media.ts

@@ -1,4 +1,6 @@
 import { swallow } from "./diagnostics";
+import { interpolateVolumeGain, type VolumeKeyframe } from "./mediaVolumeEnvelope.js";
+
 export type RuntimeMediaClip = {
   el: HTMLVideoElement | HTMLAudioElement;
   start: number;
@@ -10,6 +12,13 @@ export type RuntimeMediaClip = {
   loop: boolean;
   /** Source media duration in seconds (from el.duration). Used for loop wrapping. */
   sourceDuration: number | null;
+  /**
+   * Probed volume keyframes from the GSAP timeline (same probe the renderer
+   * uses). When present, `syncRuntimeMedia` drives volume from the envelope
+   * rather than from `data-volume` + GSAP-change tracking, eliminating the
+   * race between the 60 Hz transport tick and GSAP's own seek.
+   */
+  volumeKeyframes?: VolumeKeyframe[];
 };
 
 export function refreshRuntimeMediaCache(params?: {
@@ -110,6 +119,7 @@ function clampVolume(volume: number): number {
   return Math.max(0, Math.min(1, volume));
 }
 
+// fallow-ignore-next-line complexity
 export function syncRuntimeMedia(params: {
   clips: RuntimeMediaClip[];
   timeSeconds: number;
@@ -163,11 +173,26 @@ export function syncRuntimeMedia(params: {
       const fallbackAuthorVolume = clampVolume(clip.volume ?? 1);
       const previousRuntimeVolume = lastRuntimeAppliedVolume.get(el);
       const currentElementVolume = clampVolume(el.volume);
-      const authorVolume =
-        previousRuntimeVolume !== undefined &&
-        Math.abs(currentElementVolume - previousRuntimeVolume) > 0.0001
-          ? currentElementVolume
-          : fallbackAuthorVolume;
+
+      let authorVolume: number;
+      if (clip.volumeKeyframes && clip.volumeKeyframes.length > 0) {
+        // Keyframes probed from the GSAP timeline — same source as the renderer.
+        // Use the interpolated envelope value directly; no need to track GSAP changes.
+        authorVolume = clampVolume(interpolateVolumeGain(clip.volumeKeyframes, relTime));
+      } else if (previousRuntimeVolume === undefined) {
+        // First tick this clip is active. The transport has already seeked GSAP
+        // to the current time (seekTimelineAndAdapters runs before syncRuntimeMedia),
+        // so el.volume reflects the animated value — trust it rather than falling
+        // back to data-volume, which would clobber the GSAP-seeked position.
+        authorVolume = currentElementVolume;
+      } else if (Math.abs(currentElementVolume - previousRuntimeVolume) > 0.0001) {
+        // GSAP (or user code) changed el.volume between ticks — track it.
+        authorVolume = currentElementVolume;
+      } else {
+        // Volume unchanged since last tick — use data-volume as the baseline.
+        authorVolume = fallbackAuthorVolume;
+      }
+
       const effectiveVolume = clampVolume(authorVolume * userVol);
       el.volume = effectiveVolume;
       lastRuntimeAppliedVolume.set(el, effectiveVolume);

packages/core/src/runtime/mediaVolumeEnvelope.ts

@@ -0,0 +1,163 @@
+/**
+ * Shared volume-automation utilities used by both the renderer (offline PCM
+ * baking in audioVolumeEnvelope.ts) and the preview runtime (per-tick gain
+ * applied in syncRuntimeMedia).
+ *
+ * Keeping the two concerns in one place ensures preview and render derive the
+ * envelope from the same logic and the same probe samples.
+ */
+
+export interface VolumeKeyframe {
+  time: number;
+  volume: number;
+}
+
+/**
+ * Normalise raw keyframes to track-relative seconds: subtract `trackStart`,
+ * clamp to [0,1], sort, de-duplicate, and prepend a `baseVolume` anchor at
+ * t=0 when the first keyframe starts after the clip's begin.
+ *
+ * Returns an empty array when all keyframes are invalid — the caller should
+ * treat an empty envelope as "no automation, use static volume."
+ */
+export function normaliseEnvelope(
+  keyframes: VolumeKeyframe[],
+  trackStart: number,
+  baseVolume: number,
+): VolumeKeyframe[] {
+  const points = keyframes
+    .filter((k) => Number.isFinite(k.time) && Number.isFinite(k.volume))
+    .map((k) => ({
+      time: Math.max(0, k.time - trackStart),
+      volume: Math.max(0, Math.min(1, k.volume)),
+    }))
+    .sort((a, b) => a.time - b.time);
+
+  const deduped: VolumeKeyframe[] = [];
+  for (const point of points) {
+    const previous = deduped.at(-1);
+    if (previous && Math.abs(previous.time - point.time) < 1e-9) {
+      previous.volume = point.volume;
+    } else {
+      deduped.push(point);
+    }
+  }
+
+  if (deduped.length === 0) return deduped;
+  if (deduped[0]!.time > 0) {
+    deduped.unshift({ time: 0, volume: Math.max(0, Math.min(1, baseVolume)) });
+  }
+  return deduped;
+}
+
+/**
+ * Linearly interpolate the gain at time `t` (track-relative seconds) from a
+ * normalised envelope produced by `normaliseEnvelope`. Returns 1 when the
+ * envelope is empty.
+ */
+export function interpolateVolumeGain(envelope: VolumeKeyframe[], t: number): number {
+  if (envelope.length === 0) return 1;
+
+  let segment = 0;
+  while (segment < envelope.length - 2 && t >= envelope[segment + 1]!.time) {
+    segment += 1;
+  }
+
+  const a = envelope[segment]!;
+  const b = envelope[segment + 1] ?? a;
+  const span = b.time - a.time;
+  const progress = span <= 0 ? 0 : Math.min(1, Math.max(0, (t - a.time) / span));
+  return a.volume + (b.volume - a.volume) * progress;
+}
+
+// fallow-ignore-next-line complexity
+/**
+ * Probe a single media element's volume automation by seeking a GSAP timeline
+ * through the element's active window.
+ *
+ * Runs synchronously in the browser. The timeline is left at its current
+ * position after the probe (the next transport tick re-seeks it to `t`).
+ *
+ * Returns null when the element has no detectable automation (volume never
+ * changes from its initial `data-volume` value).
+ */
+export function probeElementVolumeKeyframes(
+  el: HTMLAudioElement | HTMLVideoElement,
+  seekTimeline: (t: number) => void,
+  compositionDuration: number,
+  sampleFps: number,
+): VolumeKeyframe[] | null {
+  const start = Number.parseFloat(el.dataset.start ?? "0") || 0;
+  const endAttr = Number.parseFloat(el.dataset.end ?? "");
+  const durAttr = Number.parseFloat(el.dataset.duration ?? "");
+  const end =
+    Number.isFinite(endAttr) && endAttr > start
+      ? endAttr
+      : Number.isFinite(durAttr) && durAttr > 0
+        ? start + durAttr
+        : compositionDuration;
+
+  const staticAttr = Number.parseFloat(el.dataset.volume ?? "");
+  const staticVolume = Number.isFinite(staticAttr) ? Math.max(0, Math.min(1, staticAttr)) : 1;
+
+  // Reset to data-volume so GSAP captures the correct FROM value.
+  el.volume = staticVolume;
+
+  const step = 1 / Math.min(60, Math.max(1, sampleFps));
+  const sampleStart = Math.max(0, start);
+  const sampleEnd = Math.min(compositionDuration, end);
+
+  const keyframes: VolumeKeyframe[] = [];
+  for (let t = sampleStart; t <= sampleEnd + 1e-6; t += step) {
+    const bounded = Math.min(sampleEnd, t);
+    seekTimeline(bounded);
+    const raw = Number(el.volume);
+    if (!Number.isFinite(raw)) continue;
+    const volume = Math.max(0, Math.min(1, raw));
+    const last = keyframes.at(-1);
+    if (!last || Math.abs(last.volume - volume) > 0.0001 || bounded === sampleEnd) {
+      keyframes.push({ time: Number(bounded.toFixed(6)), volume: Number(volume.toFixed(6)) });
+    }
+    if (bounded === sampleEnd) break;
+  }
+
+  const hasAutomation = keyframes.some((kf) => Math.abs(kf.volume - staticVolume) > 0.0001);
+  return hasAutomation ? keyframes : null;
+}
+
+export interface RuntimeTimelineRef {
+  totalTime?: ((t: number, suppressEvents?: boolean) => unknown) | undefined;
+  seek?: ((t: number, suppressEvents?: boolean) => unknown) | undefined;
+}
+
+/**
+ * Probe a media element and, if volume automation is detected, store the
+ * keyframes in `cache`. Safe to call with a null timeline — returns early.
+ */
+export function probeAndCacheElementVolume(
+  mediaEl: HTMLMediaElement,
+  timeline: RuntimeTimelineRef | null | undefined,
+  compositionDuration: number,
+  cache: WeakMap<HTMLMediaElement, VolumeKeyframe[]>,
+): void {
+  if (!timeline) return;
+  if (!(mediaEl instanceof HTMLAudioElement) && !(mediaEl instanceof HTMLVideoElement)) return;
+  if (compositionDuration <= 0) return;
+
+  const seekFn = (t: number) => {
+    try {
+      if (typeof timeline.totalTime === "function") {
+        timeline.totalTime(t, true);
+      } else if (typeof timeline.seek === "function") {
+        timeline.seek(t, true);
+      }
+    } catch {
+      // ignore seek failures during probe
+    }
+  };
+
+  const keyframes = probeElementVolumeKeyframes(mediaEl, seekFn, compositionDuration, 60);
+  if (keyframes) {
+    cache.set(mediaEl, keyframes);
+  }
+}