mirror of
https://github.com/heygen-com/hyperframes.git
synced 2026-09-11 23:00:03 +00:00
fix(core): apply renderer volume-automation solution to preview (#1118)
Preview audio with GSAP volume fades (e.g. data-volume="0" with a
gsap.to("#bgm", {volume:0.25, ...})) played ~1s then silenced. Root
cause: syncRuntimeMedia used fallbackAuthorVolume (data-volume) on the
first tick after a clip became active, clobbering the GSAP-seeked value.
The single-clock transport seeks GSAP before syncRuntimeMedia runs, so
el.volume already holds the animated value — we just need to trust it.
Fix — three layers, matching the renderer's approach (PR #1117):
1. First-tick tracking: on the first tick a clip is active
(previousRuntimeVolume===undefined), use currentElementVolume (GSAP's
seeked value) instead of fallbackAuthorVolume. In production the
transport always seeks GSAP before syncRuntimeMedia, so el.volume is
already at the correct animated position.
2. Probed keyframes: new probeElementVolumeKeyframes() runs the same
offline probe the renderer uses (discoverAudioVolumeAutomationFromTimeline)
directly in the browser. init.ts calls probeAndCacheElementVolume() when
an element is bound and a timeline is available. When keyframes are present,
syncRuntimeMedia drives volume from the interpolated envelope — no
GSAP-change tracking needed, no first-tick edge case, same data source
as the renderer.
3. Shared utilities: normaliseEnvelope(), interpolateVolumeGain(), and
probeAndCacheElementVolume() extracted to mediaVolumeEnvelope.ts and
exported from @hyperframes/core/media-volume-envelope. The engine's
audioVolumeEnvelope.ts imports from there — no duplicate logic between
the renderer and the new preview path.
Fallow audit exits non-zero on inherited complexity/duplication in init.ts
functions that shifted line numbers (applyClipLayout, transportTick, etc.),
unchanged by this PR — same known false-positive pattern noted in #1117.
Lint, format, typecheck, and unit tests all pass.
53 core/media tests pass (3 updated to pre-set el.volume to match the
runtime's bindMediaMetadataListeners — corrects a missing setup step).
audioVolumeEnvelope tests (6) still pass.
This commit is contained in:
@@ -0,0 +1,163 @@
|
||||
/**
|
||||
* Shared volume-automation utilities used by both the renderer (offline PCM
|
||||
* baking in audioVolumeEnvelope.ts) and the preview runtime (per-tick gain
|
||||
* applied in syncRuntimeMedia).
|
||||
*
|
||||
* Keeping the two concerns in one place ensures preview and render derive the
|
||||
* envelope from the same logic and the same probe samples.
|
||||
*/
|
||||
|
||||
export interface VolumeKeyframe {
|
||||
time: number;
|
||||
volume: number;
|
||||
}
|
||||
|
||||
/**
|
||||
* Normalise raw keyframes to track-relative seconds: subtract `trackStart`,
|
||||
* clamp to [0,1], sort, de-duplicate, and prepend a `baseVolume` anchor at
|
||||
* t=0 when the first keyframe starts after the clip's begin.
|
||||
*
|
||||
* Returns an empty array when all keyframes are invalid — the caller should
|
||||
* treat an empty envelope as "no automation, use static volume."
|
||||
*/
|
||||
export function normaliseEnvelope(
|
||||
keyframes: VolumeKeyframe[],
|
||||
trackStart: number,
|
||||
baseVolume: number,
|
||||
): VolumeKeyframe[] {
|
||||
const points = keyframes
|
||||
.filter((k) => Number.isFinite(k.time) && Number.isFinite(k.volume))
|
||||
.map((k) => ({
|
||||
time: Math.max(0, k.time - trackStart),
|
||||
volume: Math.max(0, Math.min(1, k.volume)),
|
||||
}))
|
||||
.sort((a, b) => a.time - b.time);
|
||||
|
||||
const deduped: VolumeKeyframe[] = [];
|
||||
for (const point of points) {
|
||||
const previous = deduped.at(-1);
|
||||
if (previous && Math.abs(previous.time - point.time) < 1e-9) {
|
||||
previous.volume = point.volume;
|
||||
} else {
|
||||
deduped.push(point);
|
||||
}
|
||||
}
|
||||
|
||||
if (deduped.length === 0) return deduped;
|
||||
if (deduped[0]!.time > 0) {
|
||||
deduped.unshift({ time: 0, volume: Math.max(0, Math.min(1, baseVolume)) });
|
||||
}
|
||||
return deduped;
|
||||
}
|
||||
|
||||
/**
|
||||
* Linearly interpolate the gain at time `t` (track-relative seconds) from a
|
||||
* normalised envelope produced by `normaliseEnvelope`. Returns 1 when the
|
||||
* envelope is empty.
|
||||
*/
|
||||
export function interpolateVolumeGain(envelope: VolumeKeyframe[], t: number): number {
|
||||
if (envelope.length === 0) return 1;
|
||||
|
||||
let segment = 0;
|
||||
while (segment < envelope.length - 2 && t >= envelope[segment + 1]!.time) {
|
||||
segment += 1;
|
||||
}
|
||||
|
||||
const a = envelope[segment]!;
|
||||
const b = envelope[segment + 1] ?? a;
|
||||
const span = b.time - a.time;
|
||||
const progress = span <= 0 ? 0 : Math.min(1, Math.max(0, (t - a.time) / span));
|
||||
return a.volume + (b.volume - a.volume) * progress;
|
||||
}
|
||||
|
||||
// fallow-ignore-next-line complexity
|
||||
/**
|
||||
* Probe a single media element's volume automation by seeking a GSAP timeline
|
||||
* through the element's active window.
|
||||
*
|
||||
* Runs synchronously in the browser. The timeline is left at its current
|
||||
* position after the probe (the next transport tick re-seeks it to `t`).
|
||||
*
|
||||
* Returns null when the element has no detectable automation (volume never
|
||||
* changes from its initial `data-volume` value).
|
||||
*/
|
||||
export function probeElementVolumeKeyframes(
|
||||
el: HTMLAudioElement | HTMLVideoElement,
|
||||
seekTimeline: (t: number) => void,
|
||||
compositionDuration: number,
|
||||
sampleFps: number,
|
||||
): VolumeKeyframe[] | null {
|
||||
const start = Number.parseFloat(el.dataset.start ?? "0") || 0;
|
||||
const endAttr = Number.parseFloat(el.dataset.end ?? "");
|
||||
const durAttr = Number.parseFloat(el.dataset.duration ?? "");
|
||||
const end =
|
||||
Number.isFinite(endAttr) && endAttr > start
|
||||
? endAttr
|
||||
: Number.isFinite(durAttr) && durAttr > 0
|
||||
? start + durAttr
|
||||
: compositionDuration;
|
||||
|
||||
const staticAttr = Number.parseFloat(el.dataset.volume ?? "");
|
||||
const staticVolume = Number.isFinite(staticAttr) ? Math.max(0, Math.min(1, staticAttr)) : 1;
|
||||
|
||||
// Reset to data-volume so GSAP captures the correct FROM value.
|
||||
el.volume = staticVolume;
|
||||
|
||||
const step = 1 / Math.min(60, Math.max(1, sampleFps));
|
||||
const sampleStart = Math.max(0, start);
|
||||
const sampleEnd = Math.min(compositionDuration, end);
|
||||
|
||||
const keyframes: VolumeKeyframe[] = [];
|
||||
for (let t = sampleStart; t <= sampleEnd + 1e-6; t += step) {
|
||||
const bounded = Math.min(sampleEnd, t);
|
||||
seekTimeline(bounded);
|
||||
const raw = Number(el.volume);
|
||||
if (!Number.isFinite(raw)) continue;
|
||||
const volume = Math.max(0, Math.min(1, raw));
|
||||
const last = keyframes.at(-1);
|
||||
if (!last || Math.abs(last.volume - volume) > 0.0001 || bounded === sampleEnd) {
|
||||
keyframes.push({ time: Number(bounded.toFixed(6)), volume: Number(volume.toFixed(6)) });
|
||||
}
|
||||
if (bounded === sampleEnd) break;
|
||||
}
|
||||
|
||||
const hasAutomation = keyframes.some((kf) => Math.abs(kf.volume - staticVolume) > 0.0001);
|
||||
return hasAutomation ? keyframes : null;
|
||||
}
|
||||
|
||||
export interface RuntimeTimelineRef {
|
||||
totalTime?: ((t: number, suppressEvents?: boolean) => unknown) | undefined;
|
||||
seek?: ((t: number, suppressEvents?: boolean) => unknown) | undefined;
|
||||
}
|
||||
|
||||
/**
|
||||
* Probe a media element and, if volume automation is detected, store the
|
||||
* keyframes in `cache`. Safe to call with a null timeline — returns early.
|
||||
*/
|
||||
export function probeAndCacheElementVolume(
|
||||
mediaEl: HTMLMediaElement,
|
||||
timeline: RuntimeTimelineRef | null | undefined,
|
||||
compositionDuration: number,
|
||||
cache: WeakMap<HTMLMediaElement, VolumeKeyframe[]>,
|
||||
): void {
|
||||
if (!timeline) return;
|
||||
if (!(mediaEl instanceof HTMLAudioElement) && !(mediaEl instanceof HTMLVideoElement)) return;
|
||||
if (compositionDuration <= 0) return;
|
||||
|
||||
const seekFn = (t: number) => {
|
||||
try {
|
||||
if (typeof timeline.totalTime === "function") {
|
||||
timeline.totalTime(t, true);
|
||||
} else if (typeof timeline.seek === "function") {
|
||||
timeline.seek(t, true);
|
||||
}
|
||||
} catch {
|
||||
// ignore seek failures during probe
|
||||
}
|
||||
};
|
||||
|
||||
const keyframes = probeElementVolumeKeyframes(mediaEl, seekFn, compositionDuration, 60);
|
||||
if (keyframes) {
|
||||
cache.set(mediaEl, keyframes);
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user