mirror of
https://github.com/heygen-com/hyperframes.git
synced 2026-09-03 04:38:33 +00:00
fix(core,player,studio): bound trimmed audio playback to the clip window (#1430)
* fix(player): bound the parent audio proxy to its clip window When iframe autoplay is blocked, audible playback is promoted to a parent-frame audio proxy. The proxy read the clip's data-start/data-duration once at adopt time and mirrorTime() only skipped (never paused) the element outside that window — so a trimmed/moved music clip kept playing the full source past its on-timeline end, even though the iframe element was correctly paused. Fix: the proxy keeps a reference to its source iframe element and re-reads data-start/data-duration each mirror tick (live trims/moves apply), pauses the proxy when the playhead leaves [start, start+duration), and resumes it when the playhead re-enters during parent-owned playback. Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com> Co-authored-by: Miguel Ángel <miguel07alm@protonmail.com> * fix(core,studio): bound trimmed audio playback to the clip window Trimmed audio played to the source file's natural end instead of stopping at the clip edge, on every audio path: - WebAudio (the audible path in Studio): schedulePlayback now passes the clip's data-duration as the third start() arg, so the decoded buffer stops at the trimmed edge instead of running to the file end. - Runtime element gating: the duration resolver caps each clip by its own data-duration (min of source length, host window, authored duration), so a trimmed <audio>/<video> element pauses at its edge. Studio trim UX: - Resize live-patches the media-start/playback-start offset, so a start-edge drag trims into the source instead of only repositioning the clip. - AudioWaveform windows the rendered peaks to the trimmed slice so the waveform tracks the clip edges. Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com> Co-authored-by: Miguel Ángel <miguel07alm@protonmail.com> * fix(player,core): gate proxy playback to the live clip window Review follow-ups on the parent-audio-proxy / WebAudio bound: - seekAll now re-reads live source bounds (_refreshEntryBounds) before gating, so a paused scrub right after a trim/move uses the current clip window instead of the adopt-time one. - playAll and clip adoption only start a proxy when the playhead is inside the clip's window (_playEntryIfActive), so bulk starts / promotion no longer blip audio for clips outside their window until the next tick. - The WebAudio buffer is now bounded by the host-composition window too (matching resolveDurationSeconds), so a sub-composition-nested clip stops at the same edge on the WebAudio and HTMLMedia paths. Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com> Co-authored-by: Miguel Ángel <miguel07alm@protonmail.com> * fix(core,player): reschedule bounded WebAudio on rate change; guard NaN bounds A bounded WebAudio source's wall-clock length is baked into start()'s duration arg (in buffer-sample seconds) at its scheduling rate. Mutating playbackRate in place on a later rate change does not rescale that bound, so a trimmed clip ends early (fast) or late (slow). setRate now reports whether the rate changed and exposes hasBoundedActiveSources(); the runtime stopAll()+reschedules active clips at the new rate when any bounded source is live. The per-clip schedule loop is extracted to a shared closure so play() and the rate path agree. Also guard _refreshEntryBounds against a non-numeric duration attribute parsing to NaN, which would make every window check false and let the proxy play past its clip end. Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com> Co-authored-by: Miguel Ángel <miguel07alm@protonmail.com> --------- Co-authored-by: Miguel Ángel <miguel07alm@protonmail.com>
This commit is contained in:
Vance Ingalls
co-authored by
Claude Opus 4.8
Miguel Ángel
Claude Opus 4.8
Miguel Ángel
parent
5c8b637369
commit
a95e49dbda
@@ -1469,10 +1469,18 @@ export function initSandboxRuntimeModular(): void {
|
||||
Number.isFinite(element.duration) && element.duration > mediaStart
|
||||
? Math.max(0, element.duration - mediaStart)
|
||||
: null;
|
||||
if (sourceDuration != null && hostRemaining != null) {
|
||||
return Math.min(sourceDuration, hostRemaining);
|
||||
}
|
||||
return sourceDuration ?? hostRemaining;
|
||||
// The element's own data-duration is an explicit clip-length trim
|
||||
// (the studio writes it when you drag the clip edge). It must bound
|
||||
// playback so a trimmed track stops at its edge instead of running on
|
||||
// to the source-file or host-composition end. Absent → no cap (an
|
||||
// untrimmed clip plays its natural source length).
|
||||
const ownDuration = Number.parseFloat(element.dataset.duration ?? "");
|
||||
const explicitDuration =
|
||||
Number.isFinite(ownDuration) && ownDuration > 0 ? ownDuration : null;
|
||||
const candidates = [sourceDuration, hostRemaining, explicitDuration].filter(
|
||||
(value): value is number => value != null,
|
||||
);
|
||||
return candidates.length > 0 ? Math.min(...candidates) : null;
|
||||
},
|
||||
});
|
||||
// Attach probed volume keyframes to clips so syncRuntimeMedia can use the
|
||||
@@ -1760,7 +1768,7 @@ export function initSandboxRuntimeModular(): void {
|
||||
onSetPlaybackRate: (rate) => {
|
||||
applyPlaybackRate(rate);
|
||||
if (state.transportClock) state.transportClock.setRate(state.playbackRate);
|
||||
webAudio.setRate(state.playbackRate);
|
||||
applyWebAudioRate();
|
||||
},
|
||||
onTick: () => {
|
||||
if (state.tornDown || !clock.isPlaying()) return;
|
||||
@@ -2133,6 +2141,67 @@ export function initSandboxRuntimeModular(): void {
|
||||
};
|
||||
|
||||
// Player methods route through the TransportClock.
|
||||
// Schedule WebAudio playback for every in-window audio clip, bounding each
|
||||
// buffer to its clip window (own data-duration AND the remaining host
|
||||
// composition window) so trimmed / sub-composition-nested clips stop at the
|
||||
// same edge as the HTMLMedia path. Reused by play() and by the rate-change
|
||||
// handler (a rate change can't rescale a bounded source in place).
|
||||
const scheduleWebAudioForActiveClips = () => {
|
||||
const gen = webAudio.startGeneration();
|
||||
const audioEls = document.querySelectorAll("audio[data-start]");
|
||||
for (const rawEl of audioEls) {
|
||||
if (!(rawEl instanceof HTMLMediaElement) || !rawEl.isConnected) continue;
|
||||
const compStart = Number.parseFloat(rawEl.dataset.start ?? "");
|
||||
if (!Number.isFinite(compStart)) continue;
|
||||
const mediaStart =
|
||||
Number.parseFloat(rawEl.dataset.playbackStart ?? rawEl.dataset.mediaStart ?? "0") || 0;
|
||||
const volumeAttr = Number.parseFloat(rawEl.dataset.volume ?? "");
|
||||
const vol = Number.isFinite(volumeAttr) ? volumeAttr : 1;
|
||||
const durationAttr = Number.parseFloat(rawEl.dataset.duration ?? "");
|
||||
let clipDuration =
|
||||
Number.isFinite(durationAttr) && durationAttr > 0 ? durationAttr : Number.POSITIVE_INFINITY;
|
||||
const compositionRoot = rawEl.closest("[data-composition-id]");
|
||||
if (compositionRoot) {
|
||||
const inheritedStart = resolveStartForElement(compositionRoot, 0);
|
||||
const inheritedDuration = resolveDurationForElement(compositionRoot, {
|
||||
includeAuthoredTimingAttrs: true,
|
||||
});
|
||||
if (inheritedDuration != null && inheritedDuration > 0) {
|
||||
clipDuration = Math.min(
|
||||
clipDuration,
|
||||
Math.max(0, inheritedStart + inheritedDuration - compStart),
|
||||
);
|
||||
}
|
||||
}
|
||||
void webAudio.decodeAudioElement(rawEl).then((buffer) => {
|
||||
if (!buffer || !clock.isPlaying()) return;
|
||||
void webAudio.schedulePlayback(
|
||||
rawEl,
|
||||
buffer,
|
||||
compStart,
|
||||
mediaStart,
|
||||
clock.now(),
|
||||
vol * state.bridgeVolume,
|
||||
gen,
|
||||
state.playbackRate,
|
||||
clipDuration,
|
||||
);
|
||||
});
|
||||
}
|
||||
};
|
||||
|
||||
// Apply a new playback rate to the WebAudio transport. Unbounded sources are
|
||||
// rescaled in place; but a bounded source's window was baked into start()'s
|
||||
// duration at its prior rate and can't be rescaled, so when one is active we
|
||||
// stopAll()+reschedule at the new rate to keep trimmed clips ending on time.
|
||||
const applyWebAudioRate = () => {
|
||||
const changed = webAudio.setRate(state.playbackRate);
|
||||
if (changed && webAudioReady && clock.isPlaying() && webAudio.hasBoundedActiveSources()) {
|
||||
webAudio.stopAll();
|
||||
scheduleWebAudioForActiveClips();
|
||||
}
|
||||
};
|
||||
|
||||
player.play = () => {
|
||||
const tl = state.capturedTimeline;
|
||||
if (clock.isPlaying()) return;
|
||||
@@ -2157,32 +2226,7 @@ export function initSandboxRuntimeModular(): void {
|
||||
// Schedule audio through WebAudio for sample-accurate timing.
|
||||
// Falls back to HTMLMediaElement playback if WebAudio isn't ready
|
||||
// or decoding fails (the syncRuntimeMedia path handles that).
|
||||
if (webAudioReady) {
|
||||
const gen = webAudio.startGeneration();
|
||||
const audioEls = document.querySelectorAll("audio[data-start]");
|
||||
for (const rawEl of audioEls) {
|
||||
if (!(rawEl instanceof HTMLMediaElement) || !rawEl.isConnected) continue;
|
||||
const compStart = Number.parseFloat(rawEl.dataset.start ?? "");
|
||||
if (!Number.isFinite(compStart)) continue;
|
||||
const mediaStart =
|
||||
Number.parseFloat(rawEl.dataset.playbackStart ?? rawEl.dataset.mediaStart ?? "0") || 0;
|
||||
const volumeAttr = Number.parseFloat(rawEl.dataset.volume ?? "");
|
||||
const vol = Number.isFinite(volumeAttr) ? volumeAttr : 1;
|
||||
void webAudio.decodeAudioElement(rawEl).then((buffer) => {
|
||||
if (!buffer || !clock.isPlaying()) return;
|
||||
void webAudio.schedulePlayback(
|
||||
rawEl,
|
||||
buffer,
|
||||
compStart,
|
||||
mediaStart,
|
||||
clock.now(),
|
||||
vol * state.bridgeVolume,
|
||||
gen,
|
||||
state.playbackRate,
|
||||
);
|
||||
});
|
||||
}
|
||||
}
|
||||
if (webAudioReady) scheduleWebAudioForActiveClips();
|
||||
runAdapters("play");
|
||||
syncMediaForCurrentState();
|
||||
postState(true);
|
||||
@@ -2249,7 +2293,7 @@ export function initSandboxRuntimeModular(): void {
|
||||
player.setPlaybackRate = (rate: number) => {
|
||||
applyPlaybackRate(rate);
|
||||
clock.setRate(state.playbackRate);
|
||||
webAudio.setRate(state.playbackRate);
|
||||
applyWebAudioRate();
|
||||
};
|
||||
|
||||
// Sync clock duration from any captured timeline
|
||||
|
||||
@@ -171,6 +171,43 @@ describe("WebAudioTransport", () => {
|
||||
});
|
||||
});
|
||||
|
||||
describe("clip duration bound (trim)", () => {
|
||||
it("bounds an in-progress clip to its remaining authored window", async () => {
|
||||
const { transport, mock, gen } = setupTransport(100);
|
||||
// compStart=5, mediaStart=0, compTime=8 → elapsed=3; clipDuration=10 → 7 left
|
||||
await transport.schedulePlayback(mockEl, mockBuffer, 5, 0, 8, 1, gen, 1, 10);
|
||||
expect(mock.startFn).toHaveBeenCalledWith(0, 3, 7);
|
||||
});
|
||||
|
||||
it("bounds a future clip to its full authored window", async () => {
|
||||
const { transport, mock, gen } = setupTransport(100);
|
||||
// compStart=10, mediaStart=1.5, compTime=2 → elapsed=-8 → delay 8; clipDuration=4
|
||||
await transport.schedulePlayback(mockEl, mockBuffer, 10, 1.5, 2, 1, gen, 1, 4);
|
||||
expect(mock.startFn).toHaveBeenCalledWith(108, 1.5, 4);
|
||||
});
|
||||
|
||||
it("does not schedule a clip whose window has already elapsed", async () => {
|
||||
const { transport, mock, gen } = setupTransport(100);
|
||||
// elapsed=15 > clipDuration=10 → nothing to play
|
||||
const result = await transport.schedulePlayback(mockEl, mockBuffer, 5, 0, 20, 1, gen, 1, 10);
|
||||
expect(result).toBeNull();
|
||||
expect(mock.startFn).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it("scales the bound by playback rate (buffer seconds)", async () => {
|
||||
const { transport, mock, gen } = setupTransport(100);
|
||||
// rate=2, clipDuration=10 → clipSourceLen=20; elapsed=3 → 17 buffer seconds left
|
||||
await transport.schedulePlayback(mockEl, mockBuffer, 5, 0, 8, 1, gen, 2, 10);
|
||||
expect(mock.startFn).toHaveBeenCalledWith(0, 3, 17);
|
||||
});
|
||||
|
||||
it("plays unbounded when clipDuration is omitted (legacy behavior)", async () => {
|
||||
const { transport, mock, gen } = setupTransport(100);
|
||||
await transport.schedulePlayback(mockEl, mockBuffer, 5, 0, 8, 1, gen);
|
||||
expect(mock.startFn).toHaveBeenCalledWith(0, 3);
|
||||
});
|
||||
});
|
||||
|
||||
describe("playback rate", () => {
|
||||
it("sets sourceNode.playbackRate.value when rate is provided", async () => {
|
||||
const { transport, mock, gen } = setupTransport(100);
|
||||
|
||||
@@ -5,6 +5,42 @@ function normalizeRate(rate: number): number {
|
||||
return rate;
|
||||
}
|
||||
|
||||
/**
|
||||
* Start a buffer source, bounding it to the clip's authored window
|
||||
* (`data-duration`) so a trimmed clip stops at its edge instead of running the
|
||||
* buffer to the source file's natural end. `clipSourceLen` is the clip span in
|
||||
* buffer seconds; the third `start()` arg is the portion to play from the
|
||||
* offset. An infinite `clipDuration` plays unbounded (legacy behavior).
|
||||
*
|
||||
* Returns false when the playhead is already past the clip end (nothing to
|
||||
* play); the caller should discard the source.
|
||||
*/
|
||||
function startBoundedSource(
|
||||
node: AudioBufferSourceNode,
|
||||
opts: {
|
||||
elapsed: number;
|
||||
mediaStart: number;
|
||||
scheduledAt: number;
|
||||
safeRate: number;
|
||||
clipDuration: number;
|
||||
},
|
||||
): boolean {
|
||||
const { elapsed, mediaStart, scheduledAt, safeRate, clipDuration } = opts;
|
||||
const hasBound = Number.isFinite(clipDuration) && clipDuration > 0;
|
||||
const clipSourceLen = clipDuration * safeRate;
|
||||
if (elapsed >= 0) {
|
||||
const remaining = clipSourceLen - elapsed;
|
||||
if (hasBound && remaining <= 0) return false;
|
||||
if (hasBound) node.start(0, elapsed + mediaStart, remaining);
|
||||
else node.start(0, elapsed + mediaStart);
|
||||
return true;
|
||||
}
|
||||
const delay = -elapsed / safeRate;
|
||||
if (hasBound) node.start(scheduledAt + delay, mediaStart, clipSourceLen);
|
||||
else node.start(scheduledAt + delay, mediaStart);
|
||||
return true;
|
||||
}
|
||||
|
||||
export type ScheduledSource = {
|
||||
el: HTMLMediaElement;
|
||||
sourceNode: AudioBufferSourceNode;
|
||||
@@ -13,6 +49,10 @@ export type ScheduledSource = {
|
||||
mediaStart: number;
|
||||
scheduledAt: number;
|
||||
priorMuted: boolean;
|
||||
// The clip had a finite window, so start() was given a fixed duration in
|
||||
// buffer-sample seconds. That bound can't be rescaled in place on a rate
|
||||
// change — callers must stopAll()+reschedule (see hasBoundedActiveSources).
|
||||
bounded: boolean;
|
||||
};
|
||||
|
||||
export class WebAudioTransport {
|
||||
@@ -92,6 +132,7 @@ export class WebAudioTransport {
|
||||
volume: number,
|
||||
generation: number,
|
||||
rate = 1,
|
||||
clipDuration = Number.POSITIVE_INFINITY,
|
||||
): Promise<ScheduledSource | null> {
|
||||
if (!this._ctx || !this._masterGain) return null;
|
||||
if (generation !== this._playGeneration) return null;
|
||||
@@ -119,11 +160,19 @@ export class WebAudioTransport {
|
||||
this._rateAnchorCtx = scheduledAt;
|
||||
this._rateAnchorComp = compositionTime;
|
||||
|
||||
if (elapsed >= 0) {
|
||||
sourceNode.start(0, elapsed + mediaStart);
|
||||
} else {
|
||||
const delay = -elapsed / safeRate;
|
||||
sourceNode.start(scheduledAt + delay, mediaStart);
|
||||
if (
|
||||
!startBoundedSource(sourceNode, {
|
||||
elapsed,
|
||||
mediaStart,
|
||||
scheduledAt,
|
||||
safeRate,
|
||||
clipDuration,
|
||||
})
|
||||
) {
|
||||
// Playhead already past the clip end — discard the nodes we built.
|
||||
sourceNode.disconnect();
|
||||
gainNode.disconnect();
|
||||
return null;
|
||||
}
|
||||
|
||||
const priorMuted = el.muted;
|
||||
@@ -137,6 +186,7 @@ export class WebAudioTransport {
|
||||
mediaStart,
|
||||
scheduledAt,
|
||||
priorMuted,
|
||||
bounded: Number.isFinite(clipDuration) && clipDuration > 0,
|
||||
};
|
||||
this._activeSources.push(scheduled);
|
||||
this._paused = false;
|
||||
@@ -163,9 +213,9 @@ export class WebAudioTransport {
|
||||
* start in the future keep their original wallclock start time — callers
|
||||
* that need rate-correct future starts should `stopAll()` and reschedule.
|
||||
*/
|
||||
setRate(rate: number): void {
|
||||
setRate(rate: number): boolean {
|
||||
const safeRate = normalizeRate(rate);
|
||||
if (safeRate === this._rate) return;
|
||||
if (safeRate === this._rate) return false;
|
||||
if (this._ctx && !this._paused) {
|
||||
this._rateAnchorComp = this.getTime();
|
||||
this._rateAnchorCtx = this._ctx.currentTime;
|
||||
@@ -178,6 +228,14 @@ export class WebAudioTransport {
|
||||
swallow("webAudioTransport.setRate", err);
|
||||
}
|
||||
}
|
||||
return true;
|
||||
}
|
||||
|
||||
// A bounded source's wall-clock duration was baked into start()'s duration
|
||||
// arg at its original rate; a later rate change can't rescale it in place, so
|
||||
// the caller must stopAll()+reschedule to keep trimmed clips ending on time.
|
||||
hasBoundedActiveSources(): boolean {
|
||||
return this._activeSources.some((s) => s.bounded);
|
||||
}
|
||||
|
||||
stopAll(): void {
|
||||
|
||||
Reference in New Issue
Block a user