From 04e0ccce429c0c7c579cd06db09e7bad8fb550f3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Miguel=20=C3=81ngel?= Date: Wed, 29 Jul 2026 20:51:33 +0200 Subject: [PATCH] fix: preserve plateaus in sampled audio automation (#2863) * fix: preserve audio automation plateaus * fix(audio): align automation probe windows --- .../src/runtime/mediaVolumeEnvelope.test.ts | 91 ++++++++++++++++++- .../core/src/runtime/mediaVolumeEnvelope.ts | 81 ++++++++++++----- .../src/services/htmlCompiler.test.ts | 53 +++++++++++ .../producer/src/services/htmlCompiler.ts | 37 +++++--- 4 files changed, 227 insertions(+), 35 deletions(-) diff --git a/packages/core/src/runtime/mediaVolumeEnvelope.test.ts b/packages/core/src/runtime/mediaVolumeEnvelope.test.ts index 87ea462c3..b55d9f71a 100644 --- a/packages/core/src/runtime/mediaVolumeEnvelope.test.ts +++ b/packages/core/src/runtime/mediaVolumeEnvelope.test.ts @@ -1,6 +1,95 @@ /** @vitest-environment jsdom */ import { describe, expect, it } from "vitest"; -import { probeAndCacheElementVolume } from "./mediaVolumeEnvelope"; +import { probeAndCacheElementVolume, probeElementVolumeKeyframes } from "./mediaVolumeEnvelope"; + +describe("probeElementVolumeKeyframes", () => { + it("retains the last plateau sample before a short volume change", () => { + const audio = document.createElement("audio"); + audio.dataset.start = "0"; + audio.dataset.duration = "2"; + audio.dataset.volume = "0.8"; + + const keyframes = probeElementVolumeKeyframes( + audio, + (time) => { + audio.volume = time < 1.05 ? 0.8 : 0.2; + }, + 2, + 10, + ); + + expect(keyframes).toContainEqual({ time: 1, volume: 0.8 }); + expect(keyframes).toContainEqual({ time: 1.1, volume: 0.2 }); + }); + + it("samples a short transition at a clip end between frame intervals", () => { + const audio = document.createElement("audio"); + audio.dataset.start = "0"; + audio.dataset.duration = "1.05"; + audio.dataset.volume = "0.7"; + + const keyframes = probeElementVolumeKeyframes( + audio, + (time) => { + audio.volume = time < 1.02 ? 0.7 : 0.1; + }, + 1.05, + 10, + ); + + expect(keyframes).toEqual([ + { time: 0, volume: 0.7 }, + { time: 1, volume: 0.7 }, + { time: 1.05, volume: 0.1 }, + ]); + }); + + it("preserves every sampled point of a continuous ramp", () => { + const audio = document.createElement("audio"); + audio.dataset.start = "0"; + audio.dataset.duration = "0.5"; + audio.dataset.volume = "0"; + + const keyframes = probeElementVolumeKeyframes( + audio, + (time) => { + audio.volume = time * 2; + }, + 0.5, + 10, + ); + + expect(keyframes).toEqual([ + { time: 0, volume: 0 }, + { time: 0.1, volume: 0.2 }, + { time: 0.2, volume: 0.4 }, + { time: 0.3, volume: 0.6 }, + { time: 0.4, volume: 0.8 }, + { time: 0.5, volume: 1 }, + ]); + }); + + it("prefers data-duration when a stale data-end is also present", () => { + const video = document.createElement("video"); + video.dataset.start = "0"; + video.dataset.end = "0.25"; + video.dataset.duration = "1"; + video.dataset.volume = "0"; + + const sampledTimes: number[] = []; + probeElementVolumeKeyframes( + video, + (time) => { + sampledTimes.push(time); + video.volume = time; + }, + 1, + 10, + ); + + expect(sampledTimes.at(-1)).toBe(1); + }); +}); describe("probeAndCacheElementVolume", () => { it("does not seek or cache when live timeline probing is disabled", () => { diff --git a/packages/core/src/runtime/mediaVolumeEnvelope.ts b/packages/core/src/runtime/mediaVolumeEnvelope.ts index f32c12549..1992c9826 100644 --- a/packages/core/src/runtime/mediaVolumeEnvelope.ts +++ b/packages/core/src/runtime/mediaVolumeEnvelope.ts @@ -61,6 +61,9 @@ export function interpolateVolumeGain(envelope: VolumeKeyframe[], t: number): nu if (envelope.length === 0) return 1; let segment = 0; + // The PCM baker intentionally inlines this lookup with a monotonic cursor + // because calling this preview-oriented helper per sample would be O(N×M). + // fallow-ignore-next-line code-duplication while (segment < envelope.length - 2 && t >= envelope[segment + 1]!.time) { segment += 1; } @@ -72,7 +75,49 @@ export function interpolateVolumeGain(envelope: VolumeKeyframe[], t: number): nu return a.volume + (b.volume - a.volume) * progress; } -// fallow-ignore-next-line complexity +function recordVolumeSample( + keyframes: VolumeKeyframe[], + previousSample: VolumeKeyframe | undefined, + sample: VolumeKeyframe, + isFinalSample: boolean, +): void { + const last = keyframes.at(-1); + if (!last || Math.abs(last.volume - sample.volume) > 0.0001) { + // Change-only compression must retain the preceding real sample so a + // flat run stays flat instead of being interpolated into the next value. + // During a continuous ramp, that sample is already the last keyframe. + if (last && previousSample && previousSample.time > last.time) { + keyframes.push(previousSample); + } + keyframes.push(sample); + } else if (isFinalSample && sample.time > last.time) { + keyframes.push(sample); + } +} + +function parseFiniteDatasetNumber(value: string | undefined): number | undefined { + const parsed = Number.parseFloat(value ?? ""); + return Number.isFinite(parsed) ? parsed : undefined; +} + +function resolveVolumeProbeWindow( + el: HTMLAudioElement | HTMLVideoElement, + compositionDuration: number, +): { start: number; end: number; staticVolume: number } { + const start = parseFiniteDatasetNumber(el.dataset.start) ?? 0; + const endAttr = parseFiniteDatasetNumber(el.dataset.end); + const durAttr = parseFiniteDatasetNumber(el.dataset.duration); + let end = compositionDuration; + if (durAttr !== undefined && durAttr > 0) { + end = start + durAttr; + } else if (endAttr !== undefined && endAttr > start) { + end = endAttr; + } + const staticAttr = parseFiniteDatasetNumber(el.dataset.volume) ?? 1; + const staticVolume = Math.max(0, Math.min(1, staticAttr)); + return { start, end, staticVolume }; +} + /** * Probe a single media element's volume automation by seeking a GSAP timeline * through the element's active window. @@ -89,18 +134,7 @@ export function probeElementVolumeKeyframes( compositionDuration: number, sampleFps: number, ): VolumeKeyframe[] | null { - const start = Number.parseFloat(el.dataset.start ?? "0") || 0; - const endAttr = Number.parseFloat(el.dataset.end ?? ""); - const durAttr = Number.parseFloat(el.dataset.duration ?? ""); - const end = - Number.isFinite(endAttr) && endAttr > start - ? endAttr - : Number.isFinite(durAttr) && durAttr > 0 - ? start + durAttr - : compositionDuration; - - const staticAttr = Number.parseFloat(el.dataset.volume ?? ""); - const staticVolume = Number.isFinite(staticAttr) ? Math.max(0, Math.min(1, staticAttr)) : 1; + const { start, end, staticVolume } = resolveVolumeProbeWindow(el, compositionDuration); // Reset to data-volume so GSAP captures the correct FROM value. el.volume = staticVolume; @@ -110,17 +144,20 @@ export function probeElementVolumeKeyframes( const sampleEnd = Math.min(compositionDuration, end); const keyframes: VolumeKeyframe[] = []; - for (let t = sampleStart; t <= sampleEnd + 1e-6; t += step) { - const bounded = Math.min(sampleEnd, t); - seekTimeline(bounded); + let previousSample: VolumeKeyframe | undefined; + for (let t = sampleStart; t <= sampleEnd + 1e-6; t = Math.min(sampleEnd, t + step)) { + seekTimeline(t); const raw = Number(el.volume); - if (!Number.isFinite(raw)) continue; - const volume = Math.max(0, Math.min(1, raw)); - const last = keyframes.at(-1); - if (!last || Math.abs(last.volume - volume) > 0.0001 || bounded === sampleEnd) { - keyframes.push({ time: Number(bounded.toFixed(6)), volume: Number(volume.toFixed(6)) }); + if (Number.isFinite(raw)) { + const volume = Math.max(0, Math.min(1, raw)); + const sample = { + time: Number(t.toFixed(6)), + volume: Number(volume.toFixed(6)), + }; + recordVolumeSample(keyframes, previousSample, sample, t === sampleEnd); + previousSample = sample; } - if (bounded === sampleEnd) break; + if (t === sampleEnd) break; } const hasAutomation = keyframes.some((kf) => Math.abs(kf.volume - staticVolume) > 0.0001); diff --git a/packages/producer/src/services/htmlCompiler.test.ts b/packages/producer/src/services/htmlCompiler.test.ts index a4d0a8427..443c509ce 100644 --- a/packages/producer/src/services/htmlCompiler.test.ts +++ b/packages/producer/src/services/htmlCompiler.test.ts @@ -4,6 +4,7 @@ import { mkdtempSync, writeFileSync, mkdirSync } from "node:fs"; import { tmpdir } from "node:os"; import { join } from "node:path"; import { parseHTML } from "linkedom"; +import { interpolateVolumeGain } from "@hyperframes/core/media-volume-envelope"; import { defaultLogger } from "../logger.js"; import { collectExternalAssets, @@ -1763,6 +1764,58 @@ h1 { font-size: 2rem; }`; }); describe("discoverAudioVolumeAutomationFromTimeline", () => { + it("emits plateau boundaries around a sampled volume change", async () => { + class TestAudioElement { + id = "music"; + dataset = { start: "0", duration: "3", volume: "0.8" }; + volume = 0.8; + } + class TestVideoElement {} + + const audio = new TestAudioElement(); + const previousWindow = globalThis.window; + const previousDocument = globalThis.document; + const previousAudioElement = globalThis.HTMLAudioElement; + const previousVideoElement = globalThis.HTMLVideoElement; + + globalThis.window = { + __timelines: { + root: { + totalTime: (time: number) => { + audio.volume = time < 1.05 ? 0.8 : 0.2; + }, + }, + }, + } as any; + globalThis.document = { + querySelector: (selector: string) => + selector === "[data-composition-id]" + ? { getAttribute: (name: string) => (name === "data-composition-id" ? "root" : null) } + : null, + getElementById: (id: string) => (id === "music" ? audio : null), + } as any; + globalThis.HTMLAudioElement = TestAudioElement as any; + globalThis.HTMLVideoElement = TestVideoElement as any; + + try { + const page = { + evaluate: async (fn: (arg: unknown) => unknown, arg: unknown) => fn(arg), + } as any; + + const [automation] = await discoverAudioVolumeAutomationFromTimeline(page, ["music"], 3, 10); + + expect(automation?.keyframes).toContainEqual({ time: 1, volume: 0.8 }); + expect(automation?.keyframes).toContainEqual({ time: 1.1, volume: 0.2 }); + expect(interpolateVolumeGain(automation?.keyframes ?? [], 0.5)).toBeCloseTo(0.8, 6); + expect(interpolateVolumeGain(automation?.keyframes ?? [], 1.5)).toBeCloseTo(0.2, 6); + } finally { + globalThis.window = previousWindow; + globalThis.document = previousDocument; + globalThis.HTMLAudioElement = previousAudioElement; + globalThis.HTMLVideoElement = previousVideoElement; + } + }); + it("prefers runtime duration over stale data-end while sampling video-derived audio", async () => { class TestAudioElement {} class TestVideoElement { diff --git a/packages/producer/src/services/htmlCompiler.ts b/packages/producer/src/services/htmlCompiler.ts index 0894809e5..cf9157806 100644 --- a/packages/producer/src/services/htmlCompiler.ts +++ b/packages/producer/src/services/htmlCompiler.ts @@ -2150,20 +2150,33 @@ export async function discoverAudioVolumeAutomationFromTimeline( } const keyframes: { time: number; volume: number }[] = []; - for (let t = sampleStart; t <= sampleEnd + 0.000001; t += step) { - const boundedTime = Math.min(sampleEnd, t); - seekTl(boundedTime); + let previousSample: { time: number; volume: number } | undefined; + for (let t = sampleStart; t <= sampleEnd + 0.000001; t = Math.min(sampleEnd, t + step)) { + seekTl(t); const rawVolume = Number(el.volume); - if (!Number.isFinite(rawVolume)) continue; - const volume = Math.max(0, Math.min(1, rawVolume)); - const last = keyframes.at(-1); - if (!last || Math.abs(last.volume - volume) > 0.0001 || boundedTime === sampleEnd) { - keyframes.push({ - time: Number(boundedTime.toFixed(6)), - volume: Number(volume.toFixed(6)), - }); + if (!Number.isFinite(rawVolume)) { + if (t === sampleEnd) break; + continue; } - if (boundedTime === sampleEnd) break; + const volume = Math.max(0, Math.min(1, rawVolume)); + const sample = { + time: Number(t.toFixed(6)), + volume: Number(volume.toFixed(6)), + }; + const last = keyframes.at(-1); + if (!last || Math.abs(last.volume - volume) > 0.0001) { + // Retain the preceding real sample when compression omitted a flat + // run. Continuous ramps already have that sample as their last + // keyframe, so their interpolation remains unchanged. + if (last && previousSample && previousSample.time > last.time) { + keyframes.push(previousSample); + } + keyframes.push(sample); + } else if (t === sampleEnd && sample.time > last.time) { + keyframes.push(sample); + } + previousSample = sample; + if (t === sampleEnd) break; } const staticAttr = Number.parseFloat(el.dataset.volume ?? "");