mirror of
https://github.com/heygen-com/hyperframes.git
synced 2026-10-30 09:16:55 +00:00
fix(core): audio clips join on time and their volume lane follows the audio (#6419)
* fix(core): audio clips join on time and their volume lane follows the audio An incoming <audio> clip is now pre-seeked to its in-point like a video, and a playing audio element that drifts is steered by rate instead of seeked while it sounds. A routed clip's volume lane and fades are re-aimed at the element's own position whenever it starts, is seeked or changes rate. * fix(core): sounding audio is nudged at most 10%, and a larger lag seeks under a short dip Speech audibly changes tempo past a 10% rate nudge, and a bare seek clicks. Audio that holds a lag rate cannot close in about a second is seeked under a 15 ms dip on its routed graph; an unrouted element is seeked plainly. Video keeps its steps. * fix(core): a routed clip's automation has one owner, its element, looping included A rate change re-aimed a routed clip's lane on the context clock and again when the element took the new rate. Only the element re-aims it now. A looping element's wrapped position is placed on the pass the transport is in, so a looping bed's lane follows it too. * style(core): wrap a steering comment to the line limit * refactor(core): one drift policy for playing media, exported for the parent proxy The steering steps, the audio cap and limit, and the steering state move to runtime/mediaSteer.ts as steerMedia(el, offset, baseRate, settled) and forgetSteering(el), exported as @hyperframes/core/runtime/media-steer so the player's parent-frame proxy uses the same policy instead of a second one. * style(core): trim the drift policy's comments to the package's share * fix(core): the dipped seek lands on the frame-start media time Routes the one currentTime write this branch adds through mediaElementTime, so Chrome's truncated currentTime cannot land it a frame early. * test(core): expect the plain audio seek at the frame-start media time
This commit is contained in:
@@ -392,6 +392,12 @@
|
||||
"types": "./dist/runtime/mediaClock.d.ts",
|
||||
"environments": ["browser", "bun", "node"]
|
||||
},
|
||||
"./runtime/media-steer": {
|
||||
"source": "./src/runtime/mediaSteer.ts",
|
||||
"runtime": "./dist/runtime/mediaSteer.js",
|
||||
"types": "./dist/runtime/mediaSteer.d.ts",
|
||||
"environments": ["browser", "bun", "node"]
|
||||
},
|
||||
"./runtime/clip-reveal": {
|
||||
"source": "./src/runtime/clipReveal.ts",
|
||||
"runtime": "./dist/runtime/clipReveal.js",
|
||||
|
||||
@@ -399,6 +399,12 @@
|
||||
"import": "./src/runtime/mediaClock.ts",
|
||||
"types": "./src/runtime/mediaClock.ts"
|
||||
},
|
||||
"./runtime/media-steer": {
|
||||
"bun": "./src/runtime/mediaSteer.ts",
|
||||
"node": "./dist/runtime/mediaSteer.js",
|
||||
"import": "./src/runtime/mediaSteer.ts",
|
||||
"types": "./src/runtime/mediaSteer.ts"
|
||||
},
|
||||
"./runtime/clip-reveal": {
|
||||
"bun": "./src/runtime/clipReveal.ts",
|
||||
"node": "./dist/runtime/clipReveal.js",
|
||||
@@ -845,6 +851,10 @@
|
||||
"import": "./dist/runtime/mediaClock.js",
|
||||
"types": "./dist/runtime/mediaClock.d.ts"
|
||||
},
|
||||
"./runtime/media-steer": {
|
||||
"import": "./dist/runtime/mediaSteer.js",
|
||||
"types": "./dist/runtime/mediaSteer.d.ts"
|
||||
},
|
||||
"./runtime/clip-reveal": {
|
||||
"import": "./dist/runtime/clipReveal.js",
|
||||
"types": "./dist/runtime/clipReveal.d.ts"
|
||||
|
||||
@@ -656,7 +656,7 @@ describe("attachElementFxChain", () => {
|
||||
}
|
||||
});
|
||||
|
||||
it("clears a limiter's state when a persistent chain is re-anchored for a new pass", async () => {
|
||||
it("clears a limiter's state for a new pass, not when the same pass is re-timed", async () => {
|
||||
const messages: unknown[] = [];
|
||||
class RecordingWorkletStub extends WorkletNodeStub {
|
||||
override port = { postMessage: (...sent: unknown[]) => void messages.push(...sent) };
|
||||
@@ -679,6 +679,7 @@ describe("attachElementFxChain", () => {
|
||||
timing,
|
||||
);
|
||||
await settle();
|
||||
fx?.retime({ scheduledAt: 1, elapsed: 0.9, rate: 1.03 });
|
||||
expect(messages).not.toContainEqual({ __hfReset: true });
|
||||
fx?.reanchor(timing);
|
||||
expect(messages).toContainEqual({ __hfReset: true });
|
||||
|
||||
@@ -112,6 +112,8 @@ export interface ElementFxHandle {
|
||||
* stuck value, which for a fade-out is silence for the rest of the session.
|
||||
*/
|
||||
reanchor(timing: AutomationTiming): void;
|
||||
/** Re-aim every booked envelope at `timing` without clearing effect state: the same pass, re-measured. */
|
||||
retime(timing: AutomationTiming): void;
|
||||
/**
|
||||
* Re-aim every booked envelope at a new playback rate.
|
||||
*
|
||||
@@ -343,15 +345,21 @@ export function attachElementFxChain(
|
||||
});
|
||||
}
|
||||
|
||||
const retime = (next: AutomationTiming): void => {
|
||||
if (disposed) return;
|
||||
const at = timingNow();
|
||||
if (at) cancelParamLane(automated, at.scheduledAt);
|
||||
frame = { ...next };
|
||||
scheduleFor(readChain(el).chain, frame);
|
||||
};
|
||||
|
||||
return {
|
||||
reanchor: (next: AutomationTiming) => {
|
||||
if (disposed) return;
|
||||
const at = timingNow();
|
||||
if (at) cancelParamLane(automated, at.scheduledAt);
|
||||
frame = { ...next };
|
||||
handle?.reset();
|
||||
scheduleFor(readChain(el).chain, frame);
|
||||
retime(next);
|
||||
},
|
||||
retime,
|
||||
setRate: (rate: number) => {
|
||||
const at = timingNow();
|
||||
if (disposed || !at || !Number.isFinite(rate) || rate <= 0 || rate === at.rate) return;
|
||||
|
||||
@@ -90,7 +90,7 @@ import {
|
||||
resolveNaturalMediaTimelineDuration,
|
||||
resolveRuntimeMediaClipDuration,
|
||||
syncRuntimeMedia,
|
||||
VIDEO_PRESEEK_SECONDS,
|
||||
MEDIA_PRESEEK_SECONDS,
|
||||
type RuntimeMediaClip,
|
||||
} from "./media";
|
||||
import {
|
||||
@@ -3493,14 +3493,14 @@ export function initSandboxRuntimeModular(): void {
|
||||
visiting.add(clip.el);
|
||||
}
|
||||
if (state.isPlaying && !window.__HF_EXPORT_RENDER_SEEK_CONFIG) {
|
||||
// An endpoint-only pass reaches trimmed video too late: its first visible tick starts a seek.
|
||||
// An endpoint-only pass reaches a trimmed clip too late: its first active tick starts a seek.
|
||||
for (const clip of clipsWithEndpointBetween(
|
||||
index.byStart,
|
||||
(c) => c.start,
|
||||
toSeconds,
|
||||
toSeconds + VIDEO_PRESEEK_SECONDS * state.playbackRate,
|
||||
toSeconds + MEDIA_PRESEEK_SECONDS * state.playbackRate,
|
||||
)) {
|
||||
if (clip.el.tagName === "VIDEO") visiting.add(clip.el);
|
||||
visiting.add(clip.el);
|
||||
}
|
||||
}
|
||||
return [...visiting];
|
||||
@@ -3659,6 +3659,7 @@ export function initSandboxRuntimeModular(): void {
|
||||
webAudio.setElementVolume(el, authorVolume, clipGain, unfadedVolume),
|
||||
isWebAudioOwned: (el) => webAudio.ownsElement(el),
|
||||
isWebAudioRouted: (el) => webAudio.routesElement(el),
|
||||
seekUnderDip: (el, target) => webAudio.seekUnderDip(el, target),
|
||||
onAutoplayBlocked: () => {
|
||||
if (state.mediaAutoplayBlockedPosted) return;
|
||||
state.mediaAutoplayBlockedPosted = true;
|
||||
|
||||
@@ -3,56 +3,58 @@ import { mediaElementTime } from "./mediaClock";
|
||||
import { initSandboxRuntimeModular } from "./init";
|
||||
import { createMockTimeline, resetRuntimeFixtureDom } from "./runtimeSeekFixture.test-helpers";
|
||||
|
||||
it("prepares a hidden video when playback enters its lookahead, after metadata arrives", () => {
|
||||
resetRuntimeFixtureDom();
|
||||
let now = 0;
|
||||
let id = 0;
|
||||
const callbacks = new Map<number, FrameRequestCallback>();
|
||||
vi.spyOn(performance, "now").mockImplementation(() => now);
|
||||
vi.spyOn(window, "requestAnimationFrame").mockImplementation((callback) => {
|
||||
callbacks.set(++id, callback);
|
||||
return id;
|
||||
});
|
||||
vi.spyOn(window, "cancelAnimationFrame").mockImplementation((key) => {
|
||||
callbacks.delete(key);
|
||||
});
|
||||
const step = () => {
|
||||
now += 100;
|
||||
const pending = [...callbacks.values()];
|
||||
callbacks.clear();
|
||||
for (const callback of pending) callback(now);
|
||||
};
|
||||
document.body.innerHTML =
|
||||
'<div data-composition-id="main" data-root="true" data-duration="6"><video id="incoming" data-start="4" data-duration="2" data-media-start="8" src="/movie.mp4"></video></div>';
|
||||
const video = document.querySelector("video")!;
|
||||
Object.defineProperty(video, "duration", { value: 20, configurable: true });
|
||||
Object.defineProperty(video, "readyState", { value: 0, configurable: true });
|
||||
Object.defineProperty(video, "paused", { value: true, writable: true, configurable: true });
|
||||
const play = vi.fn(() => {
|
||||
Object.assign(video, { paused: false });
|
||||
return Promise.resolve();
|
||||
});
|
||||
video.play = play;
|
||||
video.pause = vi.fn(() => {
|
||||
Object.assign(video, { paused: true });
|
||||
});
|
||||
video.load = vi.fn();
|
||||
const seek = vi.spyOn(video, "currentTime", "set");
|
||||
window.__timelines = { main: createMockTimeline(6) };
|
||||
initSandboxRuntimeModular();
|
||||
window.__player!.play();
|
||||
for (let i = 0; i < 28; i++) step();
|
||||
expect(seek).not.toHaveBeenCalled();
|
||||
Object.defineProperty(video, "readyState", { value: 4, configurable: true });
|
||||
step();
|
||||
expect(video.style.visibility).toBe("hidden");
|
||||
expect(video.currentTime).toBe(mediaElementTime(8));
|
||||
expect(play).not.toHaveBeenCalled();
|
||||
for (let i = 0; i < 11; i++) step();
|
||||
expect(video.style.visibility).toBe("visible");
|
||||
expect(seek).toHaveBeenCalledTimes(1);
|
||||
expect(play).toHaveBeenCalledTimes(1);
|
||||
});
|
||||
it.each(["video", "audio"])(
|
||||
"prepares an upcoming %s when playback enters its lookahead, after metadata arrives",
|
||||
(tag) => {
|
||||
resetRuntimeFixtureDom();
|
||||
let now = 0;
|
||||
let id = 0;
|
||||
const callbacks = new Map<number, FrameRequestCallback>();
|
||||
vi.spyOn(performance, "now").mockImplementation(() => now);
|
||||
vi.spyOn(window, "requestAnimationFrame").mockImplementation((callback) => {
|
||||
callbacks.set(++id, callback);
|
||||
return id;
|
||||
});
|
||||
vi.spyOn(window, "cancelAnimationFrame").mockImplementation((key) => {
|
||||
callbacks.delete(key);
|
||||
});
|
||||
const step = () => {
|
||||
now += 100;
|
||||
const pending = [...callbacks.values()];
|
||||
callbacks.clear();
|
||||
for (const callback of pending) callback(now);
|
||||
};
|
||||
document.body.innerHTML = `<div data-composition-id="main" data-root="true" data-duration="6"><${tag} id="incoming" data-start="4" data-duration="2" data-media-start="8" src="/movie.mp4"></${tag}></div>`;
|
||||
const video = document.querySelector<HTMLMediaElement>(tag)!;
|
||||
Object.defineProperty(video, "duration", { value: 20, configurable: true });
|
||||
Object.defineProperty(video, "readyState", { value: 0, configurable: true });
|
||||
Object.defineProperty(video, "paused", { value: true, writable: true, configurable: true });
|
||||
const play = vi.fn(() => {
|
||||
Object.assign(video, { paused: false });
|
||||
return Promise.resolve();
|
||||
});
|
||||
video.play = play;
|
||||
video.pause = vi.fn(() => {
|
||||
Object.assign(video, { paused: true });
|
||||
});
|
||||
video.load = vi.fn();
|
||||
const seek = vi.spyOn(video, "currentTime", "set");
|
||||
window.__timelines = { main: createMockTimeline(6) };
|
||||
initSandboxRuntimeModular();
|
||||
window.__player!.play();
|
||||
for (let i = 0; i < 28; i++) step();
|
||||
expect(seek).not.toHaveBeenCalled();
|
||||
Object.defineProperty(video, "readyState", { value: 4, configurable: true });
|
||||
step();
|
||||
if (tag === "video") expect(video.style.visibility).toBe("hidden");
|
||||
expect(video.currentTime).toBe(mediaElementTime(8));
|
||||
expect(play).not.toHaveBeenCalled();
|
||||
for (let i = 0; i < 11; i++) step();
|
||||
if (tag === "video") expect(video.style.visibility).toBe("visible");
|
||||
expect(seek).toHaveBeenCalledTimes(1);
|
||||
expect(play).toHaveBeenCalledTimes(1);
|
||||
},
|
||||
);
|
||||
|
||||
afterEach(() => {
|
||||
window.__hfRuntimeTeardown?.();
|
||||
|
||||
@@ -38,19 +38,22 @@ afterEach(() => {
|
||||
vi.restoreAllMocks();
|
||||
});
|
||||
|
||||
describe("hidden incoming video preparation", () => {
|
||||
it("lands the in-point before the cut without playing or seeking it again at activation", () => {
|
||||
const { el, seek, tick } = fixture();
|
||||
tick(0);
|
||||
expect(seek).not.toHaveBeenCalled();
|
||||
tick(2.5);
|
||||
expect(el.currentTime).toBe(mediaElementTime(8));
|
||||
expect(el.play).not.toHaveBeenCalled();
|
||||
tick(3);
|
||||
tick(4);
|
||||
expect(seek).toHaveBeenCalledTimes(1);
|
||||
expect(el.play).toHaveBeenCalledTimes(1);
|
||||
});
|
||||
describe("incoming media preparation", () => {
|
||||
it.each(["video", "audio"])(
|
||||
"lands a %s clip's in-point before the cut without playing or seeking it again at activation",
|
||||
(tag) => {
|
||||
const { el, seek, tick } = fixture({}, tag);
|
||||
tick(0);
|
||||
expect(seek).not.toHaveBeenCalled();
|
||||
tick(2.5);
|
||||
expect(el.currentTime).toBe(mediaElementTime(8));
|
||||
expect(el.play).not.toHaveBeenCalled();
|
||||
tick(3);
|
||||
tick(4);
|
||||
expect(seek).toHaveBeenCalledTimes(1);
|
||||
expect(el.play).toHaveBeenCalledTimes(1);
|
||||
},
|
||||
);
|
||||
|
||||
it("retries after metadata arrives without interrupting an outstanding seek", () => {
|
||||
const { el, seek, tick } = fixture();
|
||||
@@ -84,10 +87,10 @@ describe("hidden incoming video preparation", () => {
|
||||
expect(el.currentTime).toBe(mediaElementTime(8));
|
||||
});
|
||||
|
||||
it.each(["audio", "paused", "past", "export", "injected", "playing", "disconnected"])(
|
||||
it.each(["paused", "past", "export", "injected", "playing", "disconnected"])(
|
||||
"leaves %s media untouched",
|
||||
(kind) => {
|
||||
const { el, tick, seek } = fixture({}, kind === "audio" ? "audio" : "video");
|
||||
const { el, tick, seek } = fixture();
|
||||
if (kind === "export") window.__HF_EXPORT_RENDER_SEEK_CONFIG = {};
|
||||
if (kind === "injected") {
|
||||
el.id = "incoming";
|
||||
|
||||
@@ -1984,9 +1984,13 @@ describe("syncRuntimeMedia", () => {
|
||||
expect(clip.el.currentTime).toBe(mediaElementTime(5));
|
||||
});
|
||||
|
||||
describe("playing video drift", () => {
|
||||
function playingVideoAt(currentTime: number, playbackRate = 1) {
|
||||
const clip = createMockClip({ start: 0, end: 20, duration: 20 });
|
||||
describe("playing media drift", () => {
|
||||
function playingVideoAt(
|
||||
currentTime: number,
|
||||
playbackRate = 1,
|
||||
tag: "audio" | "video" = "video",
|
||||
) {
|
||||
const clip = createMockClip({ start: 0, end: 20, duration: 20 }, tag);
|
||||
Object.defineProperty(clip.el, "paused", { value: false, writable: true });
|
||||
Object.defineProperty(clip.el, "currentTime", { value: currentTime, writable: true });
|
||||
clip.el.playbackRate = playbackRate;
|
||||
@@ -2001,13 +2005,15 @@ describe("syncRuntimeMedia", () => {
|
||||
getCompositionDuration: () => 20,
|
||||
});
|
||||
|
||||
it("runs a video lagging past the sync tolerance 3% fast instead of seeking it", () => {
|
||||
const clip = playingVideoAt(4.908);
|
||||
tick(clip, 5);
|
||||
tick(clip, 5);
|
||||
expect(clip.el.playbackRate).toBeCloseTo(1.03, 9);
|
||||
expect(clip.el.currentTime).toBe(4.908);
|
||||
});
|
||||
it.each(["video", "audio"] as const)(
|
||||
"runs a playing %s lagging past the sync tolerance 3% fast instead of seeking it",
|
||||
(tag) => {
|
||||
const clip = playingVideoAt(4.908, 1, tag);
|
||||
for (let i = 0; i < 4; i++) tick(clip, 5);
|
||||
expect(clip.el.playbackRate).toBeCloseTo(1.03, 9);
|
||||
expect(clip.el.currentTime).toBe(4.908);
|
||||
},
|
||||
);
|
||||
|
||||
it("runs a video that is ahead 3% slow", () => {
|
||||
const clip = playingVideoAt(5.06);
|
||||
@@ -2107,6 +2113,53 @@ describe("syncRuntimeMedia", () => {
|
||||
expect(clip.el.playbackRate).toBeCloseTo(1.03, 9);
|
||||
});
|
||||
|
||||
describe("sounding audio past a gentle nudge", () => {
|
||||
const tickDipping = (clip: RuntimeMediaClip, timeSeconds: number, dips: boolean) => {
|
||||
const seekUnderDip = vi.fn(() => dips);
|
||||
syncRuntimeMedia({
|
||||
clips: [clip],
|
||||
timeSeconds,
|
||||
playing: true,
|
||||
playbackRate: 1,
|
||||
getCompositionDuration: () => 20,
|
||||
seekUnderDip,
|
||||
});
|
||||
return seekUnderDip;
|
||||
};
|
||||
|
||||
it("never nudges audio past 10% while its lag still grows", () => {
|
||||
const clip = playingVideoAt(4.6, 1, "audio");
|
||||
tickDipping(clip, 5, true);
|
||||
const seekUnderDip = tickDipping(clip, 5.02, true);
|
||||
expect(clip.el.playbackRate).toBeCloseTo(1.1, 9);
|
||||
expect(seekUnderDip).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it("seeks audio that holds a lag past 100 ms under a dip, at its authored rate", () => {
|
||||
const clip = playingVideoAt(4.85, 1, "audio");
|
||||
tickDipping(clip, 5, true);
|
||||
const seekUnderDip = tickDipping(clip, 5, true);
|
||||
expect(seekUnderDip).toHaveBeenCalledWith(clip.el, 5);
|
||||
expect(clip.el.currentTime).toBe(4.85);
|
||||
expect(clip.el.playbackRate).toBe(1);
|
||||
});
|
||||
|
||||
it("seeks it plainly when it has no dip to hide the seek under", () => {
|
||||
const clip = playingVideoAt(4.85, 1, "audio");
|
||||
tickDipping(clip, 5, false);
|
||||
tickDipping(clip, 5, false);
|
||||
expect(clip.el.currentTime).toBe(mediaElementTime(5));
|
||||
});
|
||||
|
||||
it("leaves video on its steps", () => {
|
||||
const clip = playingVideoAt(4.85);
|
||||
tickDipping(clip, 5, true);
|
||||
const seekUnderDip = tickDipping(clip, 5, true);
|
||||
expect(seekUnderDip).not.toHaveBeenCalled();
|
||||
expect(clip.el.playbackRate).toBeCloseTo(1.1, 9);
|
||||
});
|
||||
});
|
||||
|
||||
it("steers a video that is far ahead slower, by the same steps", () => {
|
||||
const clip = playingVideoAt(5.2);
|
||||
tick(clip, 5);
|
||||
|
||||
@@ -13,6 +13,13 @@ import { findInjectedRenderFrame } from "./renderFrameSibling.js";
|
||||
import { isRenderFrameVideo } from "./preloadMedia.js";
|
||||
import { registerSeekCompletion } from "./adapters/seek-dispatch.js";
|
||||
import { cutToHostSlots, type HostSlot } from "../mediaTiming.js";
|
||||
import {
|
||||
forgetSteering,
|
||||
isSteering,
|
||||
MEDIA_SYNC_TOLERANCE_SECONDS,
|
||||
steerMedia,
|
||||
} from "./mediaSteer.js";
|
||||
export { MEDIA_SYNC_TOLERANCE_SECONDS } from "./mediaSteer.js";
|
||||
export {
|
||||
readElementPlaybackRate,
|
||||
readElementRateSpec,
|
||||
@@ -250,7 +257,7 @@ export function evictMediaSyncState(el: HTMLMediaElement): void {
|
||||
strictDriftSamples.delete(el);
|
||||
seekLoadRetried.delete(el);
|
||||
lastRuntimeAppliedVolume.delete(el);
|
||||
videoSteering.delete(el);
|
||||
forgetSteering(el);
|
||||
}
|
||||
|
||||
/** Test-only seam: whether any per-source sync state is still tracked for `el`. */
|
||||
@@ -261,12 +268,12 @@ export function hasMediaSyncStateForTest(el: HTMLMediaElement): boolean {
|
||||
strictDriftSamples.has(el) ||
|
||||
seekLoadRetried.has(el) ||
|
||||
lastRuntimeAppliedVolume.has(el) ||
|
||||
videoSteering.has(el)
|
||||
isSteering(el)
|
||||
);
|
||||
}
|
||||
|
||||
export const MEDIA_HARD_SYNC_SECONDS = 0.5;
|
||||
export const VIDEO_PRESEEK_SECONDS = 1.5;
|
||||
export const MEDIA_PRESEEK_SECONDS = 1.5;
|
||||
|
||||
export function mediaSourceTimeAt(clip: RuntimeMediaClip, timeSeconds: number): number {
|
||||
const elapsed = Math.max(0, Math.min(timeSeconds, clip.end) - (clip.origin ?? clip.start));
|
||||
@@ -280,7 +287,7 @@ export function mediaSourceTimeAt(clip: RuntimeMediaClip, timeSeconds: number):
|
||||
return target;
|
||||
}
|
||||
|
||||
function prepareUpcomingVideo(
|
||||
function prepareUpcomingMedia(
|
||||
clip: RuntimeMediaClip,
|
||||
time: number,
|
||||
playing: boolean,
|
||||
@@ -290,9 +297,8 @@ function prepareUpcomingVideo(
|
||||
const dueIn = clip.start - time;
|
||||
if (
|
||||
!playing ||
|
||||
el.tagName !== "VIDEO" ||
|
||||
dueIn <= 0 ||
|
||||
dueIn > VIDEO_PRESEEK_SECONDS * rate ||
|
||||
dueIn > MEDIA_PRESEEK_SECONDS * rate ||
|
||||
!el.paused ||
|
||||
el.seeking ||
|
||||
el.readyState < el.HAVE_METADATA ||
|
||||
@@ -303,7 +309,7 @@ function prepareUpcomingVideo(
|
||||
const target = mediaSourceTimeAt(clip, clip.start);
|
||||
if (!Number.isFinite(target) || target < 0 || Math.abs(el.currentTime - target) <= 0.001) return;
|
||||
try {
|
||||
// A trimmed video exposed during its in-point seek has no current frame at the cut.
|
||||
// A trimmed clip seeked as it opens shows no frame and plays late, then is seeked again while it sounds.
|
||||
// Hidden preparation must not hold the visible-frame seek barrier.
|
||||
el.currentTime = mediaElementTime(target);
|
||||
} catch (err) {
|
||||
@@ -311,39 +317,9 @@ function prepareUpcomingVideo(
|
||||
}
|
||||
}
|
||||
|
||||
/** Drift a playing audio element may carry before sync pulls it back onto the playhead. */
|
||||
export const MEDIA_SYNC_TOLERANCE_SECONDS = 0.04;
|
||||
|
||||
// Chromium refuses a slower playbackRate (NotSupportedError).
|
||||
const MIN_NATIVE_PLAYBACK_RATE = 1 / 16;
|
||||
|
||||
// A playing video is steered back by rate, not seeked (a seek resets its decoder).
|
||||
// Its rate is written only when steering starts or stops: every write costs a frame.
|
||||
const VIDEO_STEER = 0.03;
|
||||
// A video that starts a cut late closes its lag over one clip at 3%; a bigger lag steers harder, in steps so the rate is written rarely.
|
||||
const VIDEO_STEER_STEPS: readonly (readonly [lag: number, steer: number])[] = [
|
||||
[0.3, 0.25],
|
||||
[0.1, 0.1],
|
||||
];
|
||||
const VIDEO_STEER_RELEASE_SECONDS = 0.01;
|
||||
|
||||
/** Direction (+1 fast, -1 slow) a video is being steered in; absent when it plays at its authored rate. */
|
||||
const videoSteering = new WeakMap<HTMLMediaElement, number>();
|
||||
|
||||
/** Rate for a playing video `offset` seconds behind (+) or ahead (-) of the playhead. */
|
||||
function steeredVideoRate(el: HTMLMediaElement, offset: number, baseRate: number): number {
|
||||
const direction = Math.sign(offset);
|
||||
const steering = videoSteering.get(el);
|
||||
if (Math.abs(offset) > MEDIA_SYNC_TOLERANCE_SECONDS) videoSteering.set(el, direction);
|
||||
else if (steering !== direction || Math.abs(offset) <= VIDEO_STEER_RELEASE_SECONDS) {
|
||||
videoSteering.delete(el);
|
||||
return baseRate;
|
||||
}
|
||||
const lag = Math.abs(offset);
|
||||
const steer = VIDEO_STEER_STEPS.find(([from]) => lag > from)?.[1] ?? VIDEO_STEER;
|
||||
return baseRate * (1 + direction * steer);
|
||||
}
|
||||
|
||||
// fallow-ignore-next-line complexity
|
||||
export function syncRuntimeMedia(params: {
|
||||
clips: RuntimeMediaClip[];
|
||||
@@ -389,6 +365,8 @@ export function syncRuntimeMedia(params: {
|
||||
forceSync?: boolean;
|
||||
/** How far the next tick will move the playhead: an audio clip due within it starts now. */
|
||||
cueAheadSeconds?: number;
|
||||
/** Seeks a sounding element under a gain dip; false when it has no dip stage and the sync seeks it plainly. */
|
||||
seekUnderDip?: (el: HTMLMediaElement, target: number) => boolean;
|
||||
/** Lets a video clip that runs to the composition end hold its last frame at the terminal time.
|
||||
* A thunk, because deriving the duration is only worth it for a clip past its own end. */
|
||||
getCompositionDuration: () => number;
|
||||
@@ -589,12 +567,11 @@ export function syncRuntimeMedia(params: {
|
||||
(el.ended && canSeekEndedMediaBackward && drift > 0.001) ||
|
||||
staleAudioOnFirstTick ||
|
||||
(drift > MEDIA_HARD_SYNC_SECONDS && (firstTickOfClip || offsetJumped || catastrophicDrift));
|
||||
// Playing videos use the browser's decoder for timing. Seeking one resets the decoder: a
|
||||
// ~150ms freeze while it re-buffers, as the monotonic clock advances, which loops into a
|
||||
// seek→freeze→drift→seek stutter. So a playing video skips strict and force sync; only hard
|
||||
// sync (>0.5s) warrants the decoder-reset cost. A paused transport pauses this video below,
|
||||
// so a seek that pauses mid-playback still lands it.
|
||||
const isPlayingVideo = el.tagName === "VIDEO" && !el.paused && playing;
|
||||
// Seeking playing media resets a video's decoder (a ~150ms freeze that loops into a
|
||||
// seek→freeze→drift→seek stutter) and jumps sounding audio (a click). So playing media skips
|
||||
// strict and force sync and is steered by rate; hard sync (>0.5s) and audio past its steer
|
||||
// limit seek. A paused transport pauses it below, so a seek that pauses mid-playback lands it.
|
||||
const isPlayingMedia = !el.paused && playing;
|
||||
// Only apply strict sync when offset has stabilized (not growing).
|
||||
// During initial buffering, offset grows ~16ms/tick as the timeline
|
||||
// advances while media stays at 0. Accumulated drift from pause/play
|
||||
@@ -604,7 +581,7 @@ export function syncRuntimeMedia(params: {
|
||||
heldBelowFloor || (prevOffset !== undefined && Math.abs(offset - prevOffset) < 0.004);
|
||||
let strictSync = false;
|
||||
if (
|
||||
!isPlayingVideo &&
|
||||
!isPlayingMedia &&
|
||||
!hardSync &&
|
||||
!firstTickOfClip &&
|
||||
offsetStabilized &&
|
||||
@@ -619,21 +596,22 @@ export function syncRuntimeMedia(params: {
|
||||
} else if (drift <= MEDIA_SYNC_TOLERANCE_SECONDS) {
|
||||
strictDriftSamples.set(el, 0);
|
||||
}
|
||||
const forceSync = !isPlayingVideo && params.forceSync && drift > 0.02;
|
||||
const forceSync = !isPlayingMedia && params.forceSync && drift > 0.02;
|
||||
const steered =
|
||||
isPlayingMedia && !hardSync ? steerMedia(el, offset, baseRate, offsetStabilized) : null;
|
||||
const steerSeeks = steered?.seek === true;
|
||||
const dipped = steerSeeks && params.seekUnderDip?.(el, relTime) === true;
|
||||
try {
|
||||
// A hard sync lands the video on the playhead, so its pre-seek offset says nothing.
|
||||
if (!isPlayingVideo || hardSync) videoSteering.delete(el);
|
||||
const rate = Math.max(
|
||||
MIN_NATIVE_PLAYBACK_RATE,
|
||||
isPlayingVideo && !hardSync ? steeredVideoRate(el, offset, baseRate) : baseRate,
|
||||
);
|
||||
// A hard sync lands the element on the playhead, so its pre-seek offset says nothing.
|
||||
if (!steered) forgetSteering(el);
|
||||
const rate = Math.max(MIN_NATIVE_PLAYBACK_RATE, steered?.rate ?? baseRate);
|
||||
// Some engines read a rate back at lower precision; an equal-enough rate is not rewritten.
|
||||
if (Math.abs(el.playbackRate - rate) > 1e-6) el.playbackRate = rate;
|
||||
} catch (err) {
|
||||
// ignore unsupported playbackRate
|
||||
swallow("runtime.media.site1", err);
|
||||
}
|
||||
if (hardSync || strictSync || forceSync) {
|
||||
if (hardSync || strictSync || forceSync || (steerSeeks && !dipped)) {
|
||||
// Skip the per-tick seek (and the `el.load()` drift-recovery retry
|
||||
// below) for `<video>` elements that have a sibling
|
||||
// `<img id="__render_frame_<id>__">`. The sibling is created only
|
||||
@@ -712,6 +690,6 @@ export function syncRuntimeMedia(params: {
|
||||
evictMediaSyncState(el);
|
||||
if (remainsInsideAuthoredWindow) lastRelativeTime.set(el, relTime);
|
||||
if (!el.paused) el.pause();
|
||||
prepareUpcomingVideo(clip, params.timeSeconds, params.playing, params.playbackRate);
|
||||
prepareUpcomingMedia(clip, params.timeSeconds, params.playing, params.playbackRate);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -0,0 +1,32 @@
|
||||
import { describe, expect, it } from "vitest";
|
||||
import { forgetSteering, isSteering, steerMedia } from "./mediaSteer";
|
||||
|
||||
const media = (tag: "audio" | "video") => document.createElement(tag);
|
||||
|
||||
describe("steerMedia", () => {
|
||||
it("leaves media inside the tolerance at its rate", () => {
|
||||
expect(steerMedia(media("audio"), 0.03, 1, true)).toEqual({ rate: 1, seek: false });
|
||||
});
|
||||
|
||||
it("steers video by its steps", () => {
|
||||
expect(steerMedia(media("video"), 0.35, 2, true)).toEqual({ rate: 2.5, seek: false });
|
||||
expect(steerMedia(media("video"), -0.05, 1, true).rate).toBeCloseTo(0.97, 9);
|
||||
});
|
||||
|
||||
it("caps audio at 10% while its lag still grows, and seeks it once the lag holds past 100 ms", () => {
|
||||
const el = media("audio");
|
||||
expect(steerMedia(el, 0.35, 1, false).rate).toBeCloseTo(1.1, 9);
|
||||
expect(steerMedia(el, 0.35, 1, true)).toEqual({ rate: 1, seek: true });
|
||||
expect(isSteering(el)).toBe(false);
|
||||
});
|
||||
|
||||
it("keeps steering until nearly back, and forgets on request", () => {
|
||||
const el = media("audio");
|
||||
steerMedia(el, 0.06, 1, true);
|
||||
expect(steerMedia(el, 0.02, 1, true).rate).toBeCloseTo(1.03, 9);
|
||||
expect(steerMedia(el, 0.005, 1, true)).toEqual({ rate: 1, seek: false });
|
||||
steerMedia(el, 0.06, 1, true);
|
||||
forgetSteering(el);
|
||||
expect(steerMedia(el, 0.02, 1, true).rate).toBe(1);
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,49 @@
|
||||
/** Drift a playing element may carry before sync pulls it back onto the playhead. */
|
||||
export const MEDIA_SYNC_TOLERANCE_SECONDS = 0.04;
|
||||
|
||||
// Steered by rate, not seeked (a seek resets a video's decoder and clicks in audio); steps keep rate writes rare.
|
||||
const MEDIA_STEER = 0.03;
|
||||
const MEDIA_STEER_STEPS: readonly (readonly [lag: number, steer: number])[] = [
|
||||
[0.3, 0.25],
|
||||
[0.1, 0.1],
|
||||
];
|
||||
const MEDIA_STEER_RELEASE_SECONDS = 0.01;
|
||||
// Audio never steers past 10%, where speech changes tempo; a lag held past 100 ms is seeked instead.
|
||||
const AUDIO_MAX_STEER = 0.1;
|
||||
const AUDIO_STEER_LIMIT_SECONDS = 0.1;
|
||||
|
||||
/** Direction (+1 fast, -1 slow) an element is being steered in; absent when it plays at its authored rate. */
|
||||
const steering = new WeakMap<HTMLMediaElement, number>();
|
||||
|
||||
/** Rate or seek for media `offset` s behind (+) or ahead (-); a lag still growing (`settled` false) never seeks. */
|
||||
export function steerMedia(
|
||||
el: HTMLMediaElement,
|
||||
offset: number,
|
||||
baseRate: number,
|
||||
settled: boolean,
|
||||
): { rate: number; seek: boolean } {
|
||||
const lag = Math.abs(offset);
|
||||
if (el.tagName === "AUDIO" && settled && lag > AUDIO_STEER_LIMIT_SECONDS) {
|
||||
steering.delete(el);
|
||||
return { rate: baseRate, seek: true };
|
||||
}
|
||||
const direction = Math.sign(offset);
|
||||
const current = steering.get(el);
|
||||
if (lag > MEDIA_SYNC_TOLERANCE_SECONDS) steering.set(el, direction);
|
||||
else if (current !== direction || lag <= MEDIA_STEER_RELEASE_SECONDS) {
|
||||
steering.delete(el);
|
||||
return { rate: baseRate, seek: false };
|
||||
}
|
||||
const step = MEDIA_STEER_STEPS.find(([from]) => lag > from)?.[1] ?? MEDIA_STEER;
|
||||
const steer = el.tagName === "AUDIO" ? Math.min(step, AUDIO_MAX_STEER) : step;
|
||||
return { rate: baseRate * (1 + direction * steer), seek: false };
|
||||
}
|
||||
|
||||
/** Drop `el`'s steering: it was seeked, paused or replaced, so its old offset says nothing. */
|
||||
export function forgetSteering(el: HTMLMediaElement): void {
|
||||
steering.delete(el);
|
||||
}
|
||||
|
||||
export function isSteering(el: HTMLMediaElement): boolean {
|
||||
return steering.has(el);
|
||||
}
|
||||
@@ -1,6 +1,7 @@
|
||||
// fallow-ignore-file code-duplication complexity
|
||||
import { beforeEach, describe, it, expect, vi } from "vitest";
|
||||
import { afterEach, beforeEach, describe, it, expect, vi } from "vitest";
|
||||
import { MAX_AUDIO_GAIN } from "../audioGain.js";
|
||||
import { mediaElementTime } from "./mediaClock";
|
||||
import { WebAudioTransport } from "./webAudioTransport";
|
||||
|
||||
function createMockAudioContext(currentTime = 100) {
|
||||
@@ -70,6 +71,7 @@ const mockEl = {
|
||||
volume: 0.4,
|
||||
paused: true,
|
||||
addEventListener: () => {},
|
||||
removeEventListener: () => {},
|
||||
getAttribute: (name: string) => (name === "data-playback-rate" ? "1" : null),
|
||||
} as unknown as HTMLMediaElement;
|
||||
|
||||
@@ -205,17 +207,12 @@ describe("WebAudioTransport", () => {
|
||||
expect(mock.ctx.createMediaElementSource).toHaveBeenCalledTimes(1);
|
||||
});
|
||||
|
||||
it("re-aims FX automation on a global-rate change without resampling the native source", async () => {
|
||||
it("does not resample the native source on a global-rate change", async () => {
|
||||
const { transport, mock, gen } = setupTransport(100);
|
||||
await transport.scheduleMediaElementPlayback(mockEl, 0, 0, 0, 1, gen, 1);
|
||||
const active = (transport as unknown as { _activeSources: { fx?: unknown }[] })
|
||||
._activeSources;
|
||||
const setRate = vi.fn();
|
||||
active[0]!.fx = { dispose: vi.fn(), setRate };
|
||||
|
||||
transport.setRate(2);
|
||||
|
||||
expect(setRate).toHaveBeenCalledWith(2);
|
||||
expect(mock.ctx.createBufferSource).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
@@ -992,6 +989,7 @@ describe("WebAudioTransport", () => {
|
||||
const settle = () => new Promise((r) => setTimeout(r, 0));
|
||||
|
||||
// An unbounded media-element clip: the runtime does not stopAll and reschedule it on a rate change.
|
||||
// Its gains in creation order: element gain, seek dip, lane, clip gain (then fades, when it has them).
|
||||
async function playBed(automation?: string, volume?: string) {
|
||||
const setup = setupGroupTransport();
|
||||
const el = groupedAudioEl("bed");
|
||||
@@ -999,7 +997,7 @@ describe("WebAudioTransport", () => {
|
||||
if (volume) el.setAttribute("data-volume", volume);
|
||||
await setup.transport.scheduleMediaElementPlayback(el, 0, 0, 0, 1, setup.gen);
|
||||
expect(setup.transport.hasBoundedActiveSources()).toBe(false);
|
||||
return { ...setup, el, laneGain: setup.mock.gainNodes[1]! };
|
||||
return { ...setup, el, laneGain: setup.mock.gainNodes[2]! };
|
||||
}
|
||||
|
||||
// Gain 2 times a 0 -> 3 ramp crosses the ceiling at 6.635 s; the product holds there after.
|
||||
@@ -1014,7 +1012,7 @@ describe("WebAudioTransport", () => {
|
||||
transport.setElementVolume(el, 1, 2);
|
||||
const booked = (laneGain.gain as unknown as { setValueAtTime: ReturnType<typeof vi.fn> })
|
||||
.setValueAtTime.mock.calls;
|
||||
expect(mock.gainNodes[2]!.gain.value * booked[0]![0]).toBeCloseTo(MAX_AUDIO_GAIN, 9);
|
||||
expect(mock.gainNodes[3]!.gain.value * booked[0]![0]).toBeCloseTo(MAX_AUDIO_GAIN, 9);
|
||||
});
|
||||
|
||||
it("holds a clip's gain times its lane under the ceiling, with data-volume played ahead", async () => {
|
||||
@@ -1049,7 +1047,7 @@ describe("WebAudioTransport", () => {
|
||||
const booked = (laneGain.gain as unknown as { setValueAtTime: ReturnType<typeof vi.fn> })
|
||||
.setValueAtTime.mock.calls;
|
||||
expect(booked.at(-1)).toEqual([1, 100]);
|
||||
expect(mock.gainNodes[2]!.gain.value * booked.at(-1)![0]).toBe(2);
|
||||
expect(mock.gainNodes[3]!.gain.value * booked.at(-1)![0]).toBe(2);
|
||||
});
|
||||
|
||||
it("restores the static group gain after its playing volume lane is removed", async () => {
|
||||
@@ -1080,12 +1078,18 @@ describe("WebAudioTransport", () => {
|
||||
]);
|
||||
});
|
||||
|
||||
it("re-aims a clip's lane at a new rate", async () => {
|
||||
const { transport, laneGain } = await playBed(FADE);
|
||||
it("re-aims a clip's lane at a new rate when its element takes it, not before", async () => {
|
||||
const { transport, el, laneGain } = await playBed(FADE);
|
||||
expect(rampsAfter(laneGain, 0)).toContainEqual([1, 103]);
|
||||
const before = param(laneGain).linearRampToValueAtTime.mock.calls.length;
|
||||
|
||||
transport.setRate(2);
|
||||
expect(rampsAfter(laneGain, before)).toEqual([]);
|
||||
Object.defineProperties(el, {
|
||||
paused: { value: false, configurable: true },
|
||||
playbackRate: { value: 2, configurable: true },
|
||||
});
|
||||
el.dispatchEvent(new Event("ratechange"));
|
||||
|
||||
expect(rampsAfter(laneGain, before)).toContainEqual([1, 101.5]);
|
||||
});
|
||||
@@ -1151,6 +1155,149 @@ describe("WebAudioTransport", () => {
|
||||
await settle();
|
||||
expect(rampsAfter(fader, before)).toContainEqual([0.25, 102]);
|
||||
});
|
||||
|
||||
describe("a routed clip's lane and fades follow its element", () => {
|
||||
// Routed at context 100, 0.1 s before its window opens at timeline 4 and source 8.
|
||||
async function routeTake(extra: Record<string, string> = {}) {
|
||||
const setup = setupGroupTransport();
|
||||
const el = groupedAudioEl("take");
|
||||
el.setAttribute("data-automation", FADE);
|
||||
for (const [name, value] of Object.entries(extra)) el.setAttribute(name, value);
|
||||
await setup.transport.scheduleMediaElementPlayback(el, 4, 8, 3.9, 1, setup.gen, 1, 2);
|
||||
const [laneGain, fadeGain] = [setup.mock.gainNodes[2]!, setup.mock.gainNodes[4]];
|
||||
const at = (ctxTime: number, currentTime: number, playbackRate = 1, paused = false) => {
|
||||
setup.mock.ctx.currentTime = ctxTime;
|
||||
Object.defineProperties(el, {
|
||||
currentTime: { value: currentTime, configurable: true },
|
||||
playbackRate: { value: playbackRate, configurable: true },
|
||||
paused: { value: paused, configurable: true },
|
||||
});
|
||||
};
|
||||
return { ...setup, el, laneGain, fadeGain, at };
|
||||
}
|
||||
const peakAfter = (node: { gain: unknown }, from: number) =>
|
||||
rampsAfter(node, from).find(([value]) => value === 1)?.[1];
|
||||
|
||||
it("books the lane where the element starts sounding, not where the clock put it", async () => {
|
||||
const { el, laneGain, at } = await routeTake();
|
||||
expect(peakAfter(laneGain, 0)).toBeCloseTo(103.1, 9);
|
||||
|
||||
const before = param(laneGain).linearRampToValueAtTime.mock.calls.length;
|
||||
at(100.155, 8);
|
||||
el.dispatchEvent(new Event("playing"));
|
||||
|
||||
expect(peakAfter(laneGain, before)).toBeCloseTo(103.155, 9);
|
||||
});
|
||||
|
||||
it("re-books the fades from the element too", async () => {
|
||||
const { el, fadeGain, at } = await routeTake({ "data-fade-out": "0.03" });
|
||||
expect(rampsAfter(fadeGain!, 0).at(-1)).toEqual([0, expect.closeTo(102.1, 9)]);
|
||||
|
||||
const before = param(fadeGain!).linearRampToValueAtTime.mock.calls.length;
|
||||
at(100.155, 8);
|
||||
el.dispatchEvent(new Event("playing"));
|
||||
|
||||
expect(rampsAfter(fadeGain!, before).at(-1)).toEqual([0, expect.closeTo(102.155, 9)]);
|
||||
});
|
||||
|
||||
it("re-books the lane after the element is seeked and when its rate is steered", async () => {
|
||||
const { el, laneGain, at } = await routeTake();
|
||||
let before = param(laneGain).linearRampToValueAtTime.mock.calls.length;
|
||||
at(100.2, 8.2);
|
||||
el.dispatchEvent(new Event("seeked"));
|
||||
expect(peakAfter(laneGain, before)).toBeCloseTo(103, 9);
|
||||
|
||||
before = param(laneGain).linearRampToValueAtTime.mock.calls.length;
|
||||
at(100.3, 8.3, 1.03);
|
||||
el.dispatchEvent(new Event("ratechange"));
|
||||
expect(peakAfter(laneGain, before)).toBeCloseTo(100.3 + 2.7 / 1.03, 9);
|
||||
});
|
||||
|
||||
it("books a looping element's lane on the pass the transport is in", async () => {
|
||||
const { el, laneGain, at } = await routeTake();
|
||||
Object.defineProperties(el, {
|
||||
loop: { value: true, configurable: true },
|
||||
duration: { value: 10, configurable: true },
|
||||
});
|
||||
const before = param(laneGain).linearRampToValueAtTime.mock.calls.length;
|
||||
at(102.2, 8.1);
|
||||
el.dispatchEvent(new Event("playing"));
|
||||
|
||||
expect(peakAfter(laneGain, before)).toBeCloseTo(103.1, 9);
|
||||
});
|
||||
|
||||
it("leaves the lane alone while the element is paused and once the clip is stopped", async () => {
|
||||
const { transport, el, laneGain, fadeGain, at } = await routeTake({
|
||||
"data-fade-out": "0.03",
|
||||
});
|
||||
const booked = () =>
|
||||
[laneGain, fadeGain!].map(
|
||||
(node) => param(node).linearRampToValueAtTime.mock.calls.length,
|
||||
);
|
||||
const before = booked();
|
||||
at(100.2, 8, 1, true);
|
||||
el.dispatchEvent(new Event("seeked"));
|
||||
transport.stopAll();
|
||||
at(100.3, 8.1);
|
||||
el.dispatchEvent(new Event("playing"));
|
||||
|
||||
expect(booked()).toEqual(before);
|
||||
});
|
||||
|
||||
describe("a seek while it sounds", () => {
|
||||
beforeEach(() => vi.useFakeTimers());
|
||||
afterEach(() => vi.useRealTimers());
|
||||
type Dip = Record<"linearRampToValueAtTime", ReturnType<typeof vi.fn>>;
|
||||
const dipRamps = (node: { gain: unknown }) =>
|
||||
(node.gain as Dip).linearRampToValueAtTime.mock.calls;
|
||||
|
||||
it("dips, seeks once the dip is down, and comes back up when the element lands", async () => {
|
||||
const { transport, el, mock, at } = await routeTake();
|
||||
const dip = mock.gainNodes[1]!;
|
||||
at(100.5, 8.35);
|
||||
const seek = vi.fn();
|
||||
Object.defineProperty(el, "currentTime", {
|
||||
get: () => 8.35,
|
||||
set: seek,
|
||||
configurable: true,
|
||||
});
|
||||
|
||||
expect(transport.seekUnderDip(el, 8.5)).toBe(true);
|
||||
expect(dipRamps(dip)).toEqual([[0, expect.closeTo(100.515, 9)]]);
|
||||
expect(transport.seekUnderDip(el, 8.51)).toBe(true);
|
||||
vi.advanceTimersByTime(14);
|
||||
expect(seek).not.toHaveBeenCalled();
|
||||
vi.advanceTimersByTime(1);
|
||||
expect(seek).toHaveBeenCalledExactlyOnceWith(mediaElementTime(8.5 + 0.015));
|
||||
|
||||
mock.ctx.currentTime = 100.54;
|
||||
el.dispatchEvent(new Event("seeked"));
|
||||
expect(dipRamps(dip).at(-1)).toEqual([1, expect.closeTo(100.555, 9)]);
|
||||
el.dispatchEvent(new Event("seeked"));
|
||||
expect(dipRamps(dip)).toHaveLength(2);
|
||||
});
|
||||
|
||||
it("leaves the element alone when the clip is stopped during the dip", async () => {
|
||||
const { transport, el, at } = await routeTake();
|
||||
at(100.5, 8.35);
|
||||
const seek = vi.fn();
|
||||
Object.defineProperty(el, "currentTime", {
|
||||
get: () => 8.35,
|
||||
set: seek,
|
||||
configurable: true,
|
||||
});
|
||||
transport.seekUnderDip(el, 8.5);
|
||||
transport.stopAll();
|
||||
vi.advanceTimersByTime(15);
|
||||
expect(seek).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it("has no dip for an element it does not route", () => {
|
||||
const { transport } = setupGroupTransport();
|
||||
expect(transport.seekUnderDip(groupedAudioEl("loose"), 1)).toBe(false);
|
||||
});
|
||||
});
|
||||
});
|
||||
});
|
||||
|
||||
it("routes an ungrouped clip straight to master through its own gain", async () => {
|
||||
@@ -1219,7 +1366,7 @@ describe("WebAudioTransport", () => {
|
||||
|
||||
transport.setElementVolume(el, 0.8, 0.5);
|
||||
|
||||
const [gain, , clipGain] = mock.gainNodes;
|
||||
const [gain, , , clipGain] = mock.gainNodes;
|
||||
expect(clipGain!.gain.value).toBe(0.5);
|
||||
expect(gain!.gain.value).toBe(0.8);
|
||||
});
|
||||
@@ -1230,10 +1377,14 @@ describe("WebAudioTransport", () => {
|
||||
|
||||
await transport.scheduleMediaElementPlayback(el, 0, 0, 0, 1, gen, 1);
|
||||
|
||||
expect(mock.gainNodes).toHaveLength(3);
|
||||
expect(mock.gainNodes[0]!.connect).toHaveBeenCalledWith(mock.masterGain);
|
||||
expect(mock.gainNodes[1]!.connect).toHaveBeenCalledWith(mock.gainNodes[0]);
|
||||
expect(mock.gainNodes[2]!.connect).toHaveBeenCalledWith(mock.gainNodes[1]);
|
||||
// Element gain to master, seek dip, lane into the element gain, clip gain into the lane.
|
||||
const [gain, dip, lane, clipGain] = mock.gainNodes;
|
||||
expect(mock.gainNodes).toHaveLength(4);
|
||||
expect(gain!.connect).toHaveBeenCalledWith(mock.masterGain);
|
||||
expect(lane!.connect).toHaveBeenCalledWith(gain);
|
||||
expect(clipGain!.connect).toHaveBeenCalledWith(lane);
|
||||
expect(mock.mediaElementSource.connect).toHaveBeenCalledWith(dip);
|
||||
expect(dip!.connect).toHaveBeenCalledWith(clipGain);
|
||||
});
|
||||
|
||||
it("two members of the same group land on ONE shared group gain, not master directly", async () => {
|
||||
@@ -1810,9 +1961,9 @@ describe("WebAudioTransport clip fades on the audio clock", () => {
|
||||
expect(gains[0]!.gain.value).toBeCloseTo(0.4, 9);
|
||||
});
|
||||
|
||||
it("a rate change rebooks the fades on the new rate from where the clip is", async () => {
|
||||
it("a rate change rebooks a decoded clip's fades on the new rate from where the clip is", async () => {
|
||||
const { transport, mock, gen, booked } = setup(100);
|
||||
await transport.scheduleMediaElementPlayback(faded("0.02", "1"), 5, 0, 5, 1, gen, 2, 10);
|
||||
await transport.schedulePlayback(faded("0.02", "1"), mockBuffer, 5, 0, () => 5, 1, gen, 2, 10);
|
||||
const gain = booked()!;
|
||||
mock.ctx.currentTime = 101;
|
||||
transport.setRate(1);
|
||||
|
||||
@@ -17,9 +17,11 @@ import { swallow } from "./diagnostics";
|
||||
import { createLevelTap, type LevelTap, type StereoLevel } from "./levelTap.js";
|
||||
import { clampAudioGain } from "../audioGain.js";
|
||||
import { getDebugSurface } from "./globals.js";
|
||||
import { readElementPlaybackRate } from "./media.js";
|
||||
import { readElementPlaybackRate, readElementRateSpec } from "./media.js";
|
||||
import { rateAt, sourceTimeAt, timeAtSourceTime } from "../speedRamp.js";
|
||||
import { classifyWebAudioMediaRoute, reportWebAudioMediaRoute } from "./webAudioRoute.js";
|
||||
import { assetUrl } from "./assetUrl.js";
|
||||
import { mediaElementTime } from "./mediaClock.js";
|
||||
import { clampFadesToDuration, fadeGain, readElementFades } from "../audioFade.js";
|
||||
|
||||
const MASTER_FX_ATTRS: FxAttributes = { automation: null };
|
||||
@@ -177,6 +179,11 @@ type ScheduledSourceBase = {
|
||||
fx?: ElementFxHandle | null;
|
||||
/** The clip's fades booked on the audio clock, between the source and its clip gain; absent without a fade. */
|
||||
fades?: BookedFades | null;
|
||||
/** Stops re-aiming the lane and fades at a routed element's own position. */
|
||||
unfollow?: () => void;
|
||||
/** A routed element's first stage: unity except while `seekUnderDip` hides a seek. */
|
||||
dipNode?: GainNode;
|
||||
dipping?: boolean;
|
||||
compositionStart: number;
|
||||
mediaStart: number;
|
||||
scheduledAt: number;
|
||||
@@ -211,6 +218,13 @@ const decodeKey = (el: HTMLMediaElement): string | null => {
|
||||
}
|
||||
};
|
||||
|
||||
const DIP_SECONDS = 0.015;
|
||||
/** A dipped seek comes back up when the element lands, or when it can no longer land. */
|
||||
const DIP_END_EVENTS = ["seeked", "error", "emptied", "abort"] as const;
|
||||
|
||||
/** Where a routed element starts sounding or jumps; a pause needs nothing, no audio flows until `playing`. */
|
||||
const FOLLOWED_EVENTS = ["playing", "seeked", "ratechange"] as const;
|
||||
|
||||
// A pause to scrub or a reschedule mid-play keeps the output awake; a real idle lets it sleep.
|
||||
const IDLE_SUSPEND_MS = 10000;
|
||||
|
||||
@@ -479,7 +493,7 @@ export class WebAudioTransport {
|
||||
async scheduleMediaElementPlayback(
|
||||
el: HTMLMediaElement,
|
||||
compositionStart: number,
|
||||
_mediaStart: number,
|
||||
mediaStart: number,
|
||||
compositionTime: number,
|
||||
volume: number,
|
||||
generation: number,
|
||||
@@ -511,10 +525,12 @@ export class WebAudioTransport {
|
||||
gainNode.connect(
|
||||
this.resolveDestination(el, scheduledAt, compositionTime, safeRate) ?? this._masterGain,
|
||||
);
|
||||
const dipNode = this._ctx.createGain();
|
||||
sourceNode.connect(dipNode);
|
||||
const { fx, laneNode, clipGainNode, fades } = attachClipStages(
|
||||
this._ctx,
|
||||
el,
|
||||
sourceNode,
|
||||
dipNode,
|
||||
gainNode,
|
||||
timing,
|
||||
{ from: Math.max(elapsed, 0), clipDuration },
|
||||
@@ -527,6 +543,7 @@ export class WebAudioTransport {
|
||||
const scheduled: ScheduledSource = {
|
||||
fx,
|
||||
fades,
|
||||
dipNode,
|
||||
el,
|
||||
sourceNode,
|
||||
sourceKind: "media-element",
|
||||
@@ -534,12 +551,13 @@ export class WebAudioTransport {
|
||||
laneNode,
|
||||
clipGainNode,
|
||||
compositionStart,
|
||||
mediaStart: _mediaStart,
|
||||
mediaStart,
|
||||
scheduledAt,
|
||||
priorMuted: el.muted,
|
||||
priorVolume: el.volume,
|
||||
mediaPlaybackRate: readElementPlaybackRate(el),
|
||||
bounded: false,
|
||||
unfollow: this.followElement(el, compositionStart, mediaStart, fx, fades),
|
||||
};
|
||||
// Chrome applies HTMLMediaElement.volume before MediaElementAudioSource.
|
||||
// Keep that upstream stage at unity so the existing downstream gain is
|
||||
@@ -554,6 +572,79 @@ export class WebAudioTransport {
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* A routed element keeps its own time: it starts late, is seeked and is steered by rate. Its lane and fades were
|
||||
* booked against the context clock, so each time it starts or moves they are re-aimed at where it actually is.
|
||||
*/
|
||||
private followElement(
|
||||
el: HTMLMediaElement,
|
||||
compositionStart: number,
|
||||
mediaStart: number,
|
||||
fx: ElementFxHandle | null,
|
||||
fades: BookedFades | null,
|
||||
): () => void {
|
||||
const retime = () => {
|
||||
if (el.paused || !this._ctx) return;
|
||||
const spec = readElementRateSpec(el);
|
||||
let source = el.currentTime - mediaStart;
|
||||
const loopLength = el.duration - mediaStart;
|
||||
// A looping element's position wraps: it is on the pass nearest where the transport expects it.
|
||||
if (el.loop && loopLength > 0) {
|
||||
const expected = sourceTimeAt(spec, this.getTime() - compositionStart);
|
||||
source += Math.round((expected - source) / loopLength) * loopLength;
|
||||
}
|
||||
const elapsed = timeAtSourceTime(spec, source);
|
||||
const rate = el.playbackRate / rateAt(spec, elapsed);
|
||||
// Stopped at rate 0, or with no position yet: no audio moves, so there is nothing to aim at.
|
||||
if (!(rate > 0) || !Number.isFinite(elapsed)) return;
|
||||
const timing: AutomationTiming = { scheduledAt: this._ctx.currentTime, elapsed, rate };
|
||||
fx?.retime(timing);
|
||||
fades?.rebook(timing);
|
||||
};
|
||||
for (const type of FOLLOWED_EVENTS) el.addEventListener(type, retime);
|
||||
return () => {
|
||||
for (const type of FOLLOWED_EVENTS) el.removeEventListener(type, retime);
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Seek a sounding routed element under a short gain dip: down, seek, and back up once it has landed, so the jump
|
||||
* is neither a click nor a tempo change. False when the element is not routed and has no dip to hide it under.
|
||||
*/
|
||||
seekUnderDip(el: HTMLMediaElement, target: number): boolean {
|
||||
const source = this._activeSources.find((s) => s.el === el && s.sourceKind === "media-element");
|
||||
const ctx = this._ctx;
|
||||
if (!ctx || !source?.dipNode) return false;
|
||||
if (source.dipping) return true;
|
||||
source.dipping = true;
|
||||
const gain = source.dipNode.gain;
|
||||
const ramp = (to: number) => {
|
||||
const now = ctx.currentTime;
|
||||
gain.cancelScheduledValues(now);
|
||||
gain.setValueAtTime(gain.value, now);
|
||||
gain.linearRampToValueAtTime(to, now + DIP_SECONDS);
|
||||
};
|
||||
const rise = () => {
|
||||
for (const type of DIP_END_EVENTS) el.removeEventListener(type, rise);
|
||||
source.dipping = false;
|
||||
ramp(1);
|
||||
};
|
||||
ramp(0);
|
||||
setTimeout(() => {
|
||||
// A stop or a new pass in the meantime owns the element now.
|
||||
if (!this._activeSources.includes(source)) return;
|
||||
for (const type of DIP_END_EVENTS) el.addEventListener(type, rise);
|
||||
try {
|
||||
// Where the element would be by now had it been on time.
|
||||
el.currentTime = mediaElementTime(target + DIP_SECONDS * el.playbackRate);
|
||||
} catch (err) {
|
||||
swallow("webAudioTransport.seekUnderDip", err);
|
||||
rise();
|
||||
}
|
||||
}, DIP_SECONDS * 1000);
|
||||
return true;
|
||||
}
|
||||
|
||||
/**
|
||||
* The gain a grouped member's signal should land on, building it on first
|
||||
* use. A group's clock is COMPOSITION time (design doc §1.3) — it has no
|
||||
@@ -923,7 +1014,7 @@ export class WebAudioTransport {
|
||||
* start in the future keep their original wallclock start time — callers
|
||||
* that need rate-correct future starts should `stopAll()` and reschedule.
|
||||
*
|
||||
* Each source's FX automation is re-aimed too. Lanes are committed to
|
||||
* Each decoded source's FX automation is re-aimed too. Lanes are committed to
|
||||
* absolute context times when the source is scheduled, so bumping only
|
||||
* `playbackRate` left every automated parameter running its original plan
|
||||
* over audio moving at a different speed. The `stopAll()`+reschedule recovery
|
||||
@@ -939,11 +1030,11 @@ export class WebAudioTransport {
|
||||
this._rateAnchorCtx = this._ctx.currentTime;
|
||||
}
|
||||
this._rate = safeRate;
|
||||
// A routed element is not touched: its rate change re-aims its own lane and fades (`followElement`).
|
||||
for (const source of this._activeSources) {
|
||||
if (!isBufferSource(source)) continue;
|
||||
try {
|
||||
if (isBufferSource(source)) {
|
||||
source.sourceNode.playbackRate.value = safeRate * source.mediaPlaybackRate;
|
||||
}
|
||||
source.sourceNode.playbackRate.value = safeRate * source.mediaPlaybackRate;
|
||||
source.fx?.setRate(safeRate);
|
||||
if (this._ctx && !this._paused)
|
||||
source.fades?.rebook({
|
||||
@@ -998,11 +1089,13 @@ export class WebAudioTransport {
|
||||
}
|
||||
|
||||
private dispose(source: ScheduledSource, keepsSounding = true): void {
|
||||
source.unfollow?.();
|
||||
try {
|
||||
if (isBufferSource(source)) source.sourceNode.stop();
|
||||
source.sourceNode.disconnect();
|
||||
source.fx?.dispose();
|
||||
source.fades?.node.disconnect();
|
||||
source.dipNode?.disconnect();
|
||||
source.clipGainNode.disconnect();
|
||||
source.laneNode.disconnect();
|
||||
source.gainNode.disconnect();
|
||||
|
||||
Reference in New Issue
Block a user