fix: support animated audio volume

2026-09-03 23:03:09 +00:00 · 2026-05-24 16:39:04 -04:00
parent 7461f1df30
commit 167a222318
10 changed files with 324 additions and 8 deletions
@@ -1241,6 +1241,10 @@ export function initSandboxRuntimeModular(): void {
    for (const mediaEl of mediaEls) {
      if (metadataBoundMedia.has(mediaEl)) continue;
      metadataBoundMedia.add(mediaEl);
+      const parsedVolume = Number.parseFloat(mediaEl.dataset.volume ?? "");
+      if (Number.isFinite(parsedVolume)) {
+        mediaEl.volume = Math.max(0, Math.min(1, parsedVolume));
+      }
      mediaEl.addEventListener("loadedmetadata", scheduleMetadataDurationHydration);
      mediaEl.addEventListener("durationchange", scheduleMetadataDurationHydration);

@@ -1312,6 +1316,7 @@ export function initSandboxRuntimeModular(): void {
      userMuted: state.bridgeMuted,
      userVolume: state.bridgeVolume,
      forceSync,
+      onElementVolume: (el, volume) => webAudio.setElementVolume(el, volume),
      onAutoplayBlocked: () => {
        if (state.mediaAutoplayBlockedPosted) return;
        state.mediaAutoplayBlockedPosted = true;
@@ -1461,8 +1466,8 @@ export function initSandboxRuntimeModular(): void {
        externalCompositionsReady = true;
        bindRootTimelineIfAvailable();
        window.__renderReady = true;
-        runAdapters("discover", state.currentTime);
        bindMediaMetadataListeners();
+        runAdapters("discover", state.currentTime);
        installAssetFailureDiagnostics();
        applyCaptionOverrides();
        postTimeline();
@@ -1667,8 +1672,8 @@ export function initSandboxRuntimeModular(): void {
  ] as RuntimeDeterministicAdapter[];
  patchVideoTextureCompat();
  installRuntimeErrorDiagnostics();
-  runAdapters("discover");
  bindMediaMetadataListeners();
+  runAdapters("discover");
  // ── Single-clock transport ──
  //
  // TransportClock is the sole time authority. GSAP is always paused —
@@ -343,6 +343,41 @@ describe("syncRuntimeMedia", () => {
    expect(clip.el.volume).toBeCloseTo(0.3);
  });

+  it("preserves authored volume changes made between sync ticks", () => {
+    const clip = createMockClip({ start: 0, end: 10, volume: 0 });
+    syncRuntimeMedia({ clips: [clip], timeSeconds: 0, playing: false, playbackRate: 1 });
+    expect(clip.el.volume).toBe(0);
+
+    clip.el.volume = 0.5;
+    syncRuntimeMedia({ clips: [clip], timeSeconds: 0.5, playing: false, playbackRate: 1 });
+
+    expect(clip.el.volume).toBe(0.5);
+  });
+
+  it("reports the effective element volume to external audio transports", () => {
+    const clip = createMockClip({ start: 0, end: 10, volume: 0 });
+    const onElementVolume = vi.fn();
+    syncRuntimeMedia({
+      clips: [clip],
+      timeSeconds: 0,
+      playing: false,
+      playbackRate: 1,
+      onElementVolume,
+    });
+    clip.el.volume = 0.75;
+    syncRuntimeMedia({
+      clips: [clip],
+      timeSeconds: 1,
+      playing: false,
+      playbackRate: 1,
+      userVolume: 0.5,
+      onElementVolume,
+    });
+
+    expect(clip.el.volume).toBeCloseTo(0.375);
+    expect(onElementVolume).toHaveBeenLastCalledWith(clip.el, 0.375);
+  });
+
  it("hard-syncs on the first active tick (sub-composition activation, mediaStart offsets)", () => {
    const clip = createMockClip({ start: 0, end: 10, mediaStart: 0 });
    Object.defineProperty(clip.el, "currentTime", { value: 0, writable: true });
@@ -103,6 +103,13 @@ function markPlayRequested(el: HTMLMediaElement): void {
  el.addEventListener("error", clear, { once: true });
 }

+const lastRuntimeAppliedVolume = new WeakMap<HTMLMediaElement, number>();
+
+function clampVolume(volume: number): number {
+  if (!Number.isFinite(volume)) return 1;
+  return Math.max(0, Math.min(1, volume));
+}
+
 export function syncRuntimeMedia(params: {
  clips: RuntimeMediaClip[];
  timeSeconds: number;
@@ -132,6 +139,7 @@ export function syncRuntimeMedia(params: {
   * outbound message; further invocations are suppressed by the caller.
   */
  onAutoplayBlocked?: () => void;
+  onElementVolume?: (el: HTMLMediaElement, volume: number) => void;
  forceSync?: boolean;
 }): void {
  // Either flag silences output. Combined up front so the per-clip loop is
@@ -151,8 +159,19 @@ export function syncRuntimeMedia(params: {
          relTime = clip.mediaStart + ((relTime - clip.mediaStart) % loopLength);
        }
      }
-      const userVol = params.userVolume ?? 1;
-      el.volume = (clip.volume ?? 1) * userVol;
+      const userVol = clampVolume(params.userVolume ?? 1);
+      const fallbackAuthorVolume = clampVolume(clip.volume ?? 1);
+      const previousRuntimeVolume = lastRuntimeAppliedVolume.get(el);
+      const currentElementVolume = clampVolume(el.volume);
+      const authorVolume =
+        previousRuntimeVolume !== undefined &&
+        Math.abs(currentElementVolume - previousRuntimeVolume) > 0.0001
+          ? currentElementVolume
+          : fallbackAuthorVolume;
+      const effectiveVolume = clampVolume(authorVolume * userVol);
+      el.volume = effectiveVolume;
+      lastRuntimeAppliedVolume.set(el, effectiveVolume);
+      params.onElementVolume?.(el, effectiveVolume);
      if (shouldMute) el.muted = true;
      // Ensure full preload for every active media element. Streaming
      // formats (MP3) may arrive with preload="metadata", which only
@@ -283,6 +302,7 @@ export function syncRuntimeMedia(params: {
    lastOffset.delete(el);
    strictDriftSamples.delete(el);
    seekLoadRetried.delete(el);
+    lastRuntimeAppliedVolume.delete(el);
    if (!el.paused) el.pause();
  }
 }
@@ -191,6 +191,18 @@ export class WebAudioTransport {
    }
  }

+  setElementVolume(el: HTMLMediaElement, volume: number): void {
+    const safeVolume = Math.max(0, Math.min(1, volume));
+    for (const source of this._activeSources) {
+      if (source.el !== el) continue;
+      try {
+        source.gainNode.gain.value = safeVolume;
+      } catch (err) {
+        swallow("webAudioTransport.setElementVolume", err);
+      }
+    }
+  }
+
  setMuted(muted: boolean): void {
    if (this._masterGain) {
      this._masterGain.gain.value = muted ? 0 : 1;
@@ -135,7 +135,12 @@ export {
 export { createVideoFrameInjector } from "./services/videoFrameInjector.js";

 export { parseAudioElements, processCompositionAudio } from "./services/audioMixer.js";
-export type { AudioElement, AudioTrack, MixResult } from "./services/audioMixer.types.js";
+export type {
+  AudioElement,
+  AudioTrack,
+  AudioVolumeKeyframe,
+  MixResult,
+} from "./services/audioMixer.types.js";

 // ── Parallel rendering ─────────────────────────────────────────────────────────
 export {
@@ -64,4 +64,47 @@ describe("processCompositionAudio", () => {
    expect(filter).toContain("volume=0");
    expect(filter).toContain("[mixed]volume=1[out]");
  });
+
+  it("uses frame-evaluated volume automation when keyframes are present", async () => {
+    const baseDir = mkdtempSync(join(tmpdir(), "hf-audio-base-"));
+    const workDir = mkdtempSync(join(tmpdir(), "hf-audio-work-"));
+    tempDirs.push(baseDir, workDir);
+
+    writeFileSync(join(baseDir, "voice.wav"), "stub");
+
+    const result = await processCompositionAudio(
+      [
+        {
+          id: "voice",
+          src: "voice.wav",
+          start: 2,
+          end: 5,
+          mediaStart: 0,
+          layer: 0,
+          volume: 0,
+          volumeKeyframes: [
+            { time: 2, volume: 0 },
+            { time: 3, volume: 1 },
+            { time: 5, volume: 0.5 },
+          ],
+          type: "audio",
+        },
+      ],
+      baseDir,
+      workDir,
+      join(baseDir, "out.m4a"),
+      5,
+    );
+
+    expect(result.success).toBe(true);
+
+    const mixArgs = runFfmpegMock.mock.calls[1]?.[0];
+    const filterIndex = mixArgs.indexOf("-filter_complex");
+    const filter = mixArgs[filterIndex + 1];
+
+    expect(filter).toContain("volume=");
+    expect(filter).toContain(":eval=frame");
+    expect(filter).toContain("lt(t\\,1)");
+    expect(filter).toContain("adelay=2000|2000");
+  });
 });
@@ -15,7 +15,67 @@ import { unwrapTemplate } from "../utils/htmlTemplate.js";
 import { resolveProjectRelativeSrc } from "./videoFrameExtractor.js";
 import type { AudioElement, AudioTrack, MixResult } from "./audioMixer.types.js";

-export type { AudioElement, AudioTrack, MixResult } from "./audioMixer.types.js";
+export type { AudioElement, MixResult } from "./audioMixer.types.js";
+
+function clampVolume(volume: number): number {
+  if (!Number.isFinite(volume)) return 1;
+  return Math.max(0, Math.min(1, volume));
+}
+
+function formatFilterNumber(value: number): string {
+  return Number(value.toFixed(6)).toString();
+}
+
+function escapeExpressionCommas(expression: string): string {
+  return expression.replace(/,/g, "\\,");
+}
+
+function buildVolumeExpression(track: AudioTrack): string {
+  const trimDuration = track.end - track.start;
+  const staticVolume = clampVolume(track.volume);
+  const keyframes = (track.volumeKeyframes ?? [])
+    .filter((keyframe) => Number.isFinite(keyframe.time) && Number.isFinite(keyframe.volume))
+    .map((keyframe) => ({
+      time: Math.max(0, Math.min(trimDuration, keyframe.time - track.start)),
+      volume: clampVolume(keyframe.volume),
+    }))
+    .sort((a, b) => a.time - b.time);
+
+  if (keyframes.length === 0) return `volume=${formatFilterNumber(staticVolume)}`;
+
+  if (keyframes[0]!.time > 0) {
+    keyframes.unshift({ time: 0, volume: staticVolume });
+  }
+
+  const deduped: typeof keyframes = [];
+  for (const keyframe of keyframes) {
+    const previous = deduped.at(-1);
+    if (previous && Math.abs(previous.time - keyframe.time) < 0.000001) {
+      previous.volume = keyframe.volume;
+    } else {
+      deduped.push(keyframe);
+    }
+  }
+
+  if (deduped.length === 1) {
+    return `volume=${formatFilterNumber(deduped[0]!.volume)}`;
+  }
+
+  let expression = formatFilterNumber(deduped.at(-1)!.volume);
+  for (let i = deduped.length - 2; i >= 0; i -= 1) {
+    const current = deduped[i]!;
+    const next = deduped[i + 1]!;
+    const currentTime = formatFilterNumber(current.time);
+    const nextTime = formatFilterNumber(next.time);
+    const currentVolume = formatFilterNumber(current.volume);
+    const span = Math.max(0.000001, next.time - current.time);
+    const slope = formatFilterNumber((next.volume - current.volume) / span);
+    const segment = `${currentVolume}+(${slope})*(t-${currentTime})`;
+    expression = `if(lt(t,${nextTime}),${segment},${expression})`;
+  }
+
+  return `volume=${escapeExpressionCommas(expression)}:eval=frame`;
+}

 interface ExtractResult {
  success: boolean;
@@ -246,8 +306,9 @@ async function mixAudioTracks(
    inputs.push("-i", track.srcPath);
    const delayMs = Math.round(track.start * 1000);
    const trimDuration = track.end - track.start;
+    const volumeFilter = buildVolumeExpression(track);
    filterParts.push(
-      `[${i}:a]atrim=0:${trimDuration},volume=${track.volume},adelay=${delayMs}|${delayMs},apad=whole_dur=${totalDuration}[a${i}]`,
+      `[${i}:a]atrim=0:${trimDuration},${volumeFilter},adelay=${delayMs}|${delayMs},apad=whole_dur=${totalDuration}[a${i}]`,
    );
  });

@@ -399,6 +460,7 @@ export async function processCompositionAudio(
          mediaStart: element.mediaStart,
          duration: element.end - element.start,
          volume: element.volume ?? 1.0,
+          volumeKeyframes: element.volumeKeyframes,
        });
      } catch (err: unknown) {
        errors.push(`Error: ${element.id} — ${err instanceof Error ? err.message : String(err)}`);
@@ -1,3 +1,8 @@
+export interface AudioVolumeKeyframe {
+  time: number;
+  volume: number;
+}
+
 export interface AudioElement {
  id: string;
  src: string;
@@ -6,6 +11,7 @@ export interface AudioElement {
  mediaStart: number;
  layer: number;
  volume?: number;
+  volumeKeyframes?: AudioVolumeKeyframe[];
  type: "audio" | "video";
 }

@@ -17,6 +23,7 @@ export interface AudioTrack {
  mediaStart: number;
  duration: number;
  volume: number;
+  volumeKeyframes?: AudioVolumeKeyframe[];
 }

 export interface MixResult {
@@ -31,6 +31,7 @@ import {
  type ImageElement,
  parseAudioElements,
  type AudioElement,
+  type AudioVolumeKeyframe,
  analyzeKeyframeIntervals,
 } from "@hyperframes/engine";
 import { downloadToTemp, isHttpUrl } from "../utils/urlDownloader.js";
@@ -1069,6 +1070,11 @@ export interface BrowserMediaElement {
  volume: number;
 }

+export interface BrowserAudioVolumeAutomation {
+  id: string;
+  keyframes: AudioVolumeKeyframe[];
+}
+
 export async function discoverMediaFromBrowser(page: Page): Promise<BrowserMediaElement[]> {
  const elements = await page.evaluate(() => {
    const results: {
@@ -1119,6 +1125,97 @@ export async function discoverMediaFromBrowser(page: Page): Promise<BrowserMedia
  return elements as BrowserMediaElement[];
 }

+export async function discoverAudioVolumeAutomationFromTimeline(
+  page: Page,
+  audioIds: string[],
+  compositionDuration: number,
+  sampleFps: number,
+): Promise<BrowserAudioVolumeAutomation[]> {
+  if (audioIds.length === 0 || compositionDuration <= 0) return [];
+
+  const sampleStep = 1 / Math.min(60, Math.max(1, sampleFps));
+  return page.evaluate(
+    ({ ids, duration, step }) => {
+      const results: { id: string; keyframes: { time: number; volume: number }[] }[] = [];
+      const timelines = (window as unknown as { __timelines?: Record<string, unknown> })
+        .__timelines;
+      if (!timelines) return results;
+
+      const rootEl = document.querySelector("[data-composition-id]");
+      const compId = rootEl?.getAttribute("data-composition-id");
+      if (!compId) return results;
+
+      const tl = timelines[compId] as
+        | {
+            totalTime?: (t: number, suppressEvents?: boolean) => unknown;
+            seek?: (t: number, suppressEvents?: boolean) => unknown;
+          }
+        | undefined;
+      if (!tl) return results;
+
+      const seekTl = (t: number) => {
+        if (typeof tl.totalTime === "function") {
+          tl.totalTime(t, false);
+        } else if (typeof tl.seek === "function") {
+          tl.seek(t, false);
+        }
+      };
+
+      for (const id of ids) {
+        const el =
+          document.getElementById(id) ?? document.getElementById(id.replace(/-audio$/, ""));
+        if (!(el instanceof HTMLAudioElement) && !(el instanceof HTMLVideoElement)) continue;
+
+        const start = Number.parseFloat(el.dataset.start ?? "0") || 0;
+        const endAttr = Number.parseFloat(el.dataset.end ?? "");
+        const durationAttr = Number.parseFloat(el.dataset.duration ?? "");
+        const end =
+          Number.isFinite(endAttr) && endAttr > start
+            ? endAttr
+            : Number.isFinite(durationAttr) && durationAttr > 0
+              ? start + durationAttr
+              : duration;
+        const sampleStart = Math.max(0, start);
+        const sampleEnd = Math.min(duration, end);
+        const initialVolumeAttr = Number.parseFloat(el.dataset.volume ?? "");
+        if (Number.isFinite(initialVolumeAttr)) {
+          el.volume = Math.max(0, Math.min(1, initialVolumeAttr));
+        }
+
+        const keyframes: { time: number; volume: number }[] = [];
+        for (let t = sampleStart; t <= sampleEnd + 0.000001; t += step) {
+          const boundedTime = Math.min(sampleEnd, t);
+          seekTl(boundedTime);
+          const rawVolume = Number(el.volume);
+          if (!Number.isFinite(rawVolume)) continue;
+          const volume = Math.max(0, Math.min(1, rawVolume));
+          const last = keyframes.at(-1);
+          if (!last || Math.abs(last.volume - volume) > 0.0001 || boundedTime === sampleEnd) {
+            keyframes.push({
+              time: Number(boundedTime.toFixed(6)),
+              volume: Number(volume.toFixed(6)),
+            });
+          }
+          if (boundedTime === sampleEnd) break;
+        }
+
+        const staticAttr = Number.parseFloat(el.dataset.volume ?? "");
+        const staticVolume = Number.isFinite(staticAttr) ? Math.max(0, Math.min(1, staticAttr)) : 1;
+        const hasAutomation = keyframes.some(
+          (keyframe) => Math.abs(keyframe.volume - staticVolume) > 0.0001,
+        );
+        if (hasAutomation) {
+          results.push({ id, keyframes });
+        }
+      }
+
+      seekTl(0);
+      return results;
+    },
+    { ids: audioIds, duration: compositionDuration, step: sampleStep },
+  );
+}
+
 export interface VideoVisibilityWindow {
  videoId: string;
  visibleStart: number;
@@ -40,6 +40,7 @@ import { fpsToNumber } from "@hyperframes/core";
 import type { CompiledComposition } from "../../htmlCompiler.js";
 import {
  discoverMediaFromBrowser,
+  discoverAudioVolumeAutomationFromTimeline,
  discoverVideoVisibilityFromTimeline,
  recompileWithResolutions,
  resolveCompositionDurations,
@@ -108,14 +109,22 @@ export async function runProbeStage(input: ProbeStageInput): Promise<ProbeStageR

  const probeStart = Date.now();
  const hasAutoStartVideos = compiled.html.includes("data-hf-auto-start");
+  const hasScriptedAudio =
+    composition.audios.length > 0 &&
+    /<script\b/i.test(compiled.html) &&
+    /\b(?:volume|data-volume)\b/i.test(compiled.html);
  const needsBrowser =
-    composition.duration <= 0 || compiled.unresolvedCompositions.length > 0 || hasAutoStartVideos;
+    composition.duration <= 0 ||
+    compiled.unresolvedCompositions.length > 0 ||
+    hasAutoStartVideos ||
+    hasScriptedAudio;

  if (needsBrowser) {
    const reasons = [];
    if (composition.duration <= 0) reasons.push("root duration unknown");
    if (compiled.unresolvedCompositions.length > 0)
      reasons.push(`${compiled.unresolvedCompositions.length} unresolved composition(s)`);
+    if (hasScriptedAudio) reasons.push("scripted audio volume");

    fileServer = await createFileServer({
      projectDir,
@@ -293,6 +302,27 @@ export async function runProbeStage(input: ProbeStageInput): Promise<ProbeStageR
      }
    }

+    if (composition.audios.length > 0) {
+      const automation = await discoverAudioVolumeAutomationFromTimeline(
+        probeSession.page,
+        composition.audios.map((audio) => audio.id),
+        composition.duration,
+        fpsToNumber(job.config.fps),
+      );
+      assertNotAborted();
+      if (automation.length > 0) {
+        const byId = new Map(automation.map((entry) => [entry.id, entry.keyframes]));
+        for (const audio of composition.audios) {
+          const keyframes = byId.get(audio.id);
+          if (!keyframes || keyframes.length === 0) continue;
+          audio.volumeKeyframes = keyframes;
+          log.info(`[Probe] Runtime audio volume automation: ${audio.id}`, {
+            keyframeCount: keyframes.length,
+          });
+        }
+      }
+    }
+
    // Runtime video discovery: for videos with auto-injected timing (data-hf-auto-start),
    // seek the GSAP timeline to find actual scene visibility windows and override start/end.
    if (composition.videos.length > 0) {