mirror of
https://github.com/heygen-com/hyperframes.git
synced 2026-10-31 02:45:40 +00:00
fix(core): audio clips join on time and their volume lane follows the audio (#6419)
* fix(core): audio clips join on time and their volume lane follows the audio An incoming <audio> clip is now pre-seeked to its in-point like a video, and a playing audio element that drifts is steered by rate instead of seeked while it sounds. A routed clip's volume lane and fades are re-aimed at the element's own position whenever it starts, is seeked or changes rate. * fix(core): sounding audio is nudged at most 10%, and a larger lag seeks under a short dip Speech audibly changes tempo past a 10% rate nudge, and a bare seek clicks. Audio that holds a lag rate cannot close in about a second is seeked under a 15 ms dip on its routed graph; an unrouted element is seeked plainly. Video keeps its steps. * fix(core): a routed clip's automation has one owner, its element, looping included A rate change re-aimed a routed clip's lane on the context clock and again when the element took the new rate. Only the element re-aims it now. A looping element's wrapped position is placed on the pass the transport is in, so a looping bed's lane follows it too. * style(core): wrap a steering comment to the line limit * refactor(core): one drift policy for playing media, exported for the parent proxy The steering steps, the audio cap and limit, and the steering state move to runtime/mediaSteer.ts as steerMedia(el, offset, baseRate, settled) and forgetSteering(el), exported as @hyperframes/core/runtime/media-steer so the player's parent-frame proxy uses the same policy instead of a second one. * style(core): trim the drift policy's comments to the package's share * fix(core): the dipped seek lands on the frame-start media time Routes the one currentTime write this branch adds through mediaElementTime, so Chrome's truncated currentTime cannot land it a frame early. * test(core): expect the plain audio seek at the frame-start media time
This commit is contained in:
@@ -392,6 +392,12 @@
|
|||||||
"types": "./dist/runtime/mediaClock.d.ts",
|
"types": "./dist/runtime/mediaClock.d.ts",
|
||||||
"environments": ["browser", "bun", "node"]
|
"environments": ["browser", "bun", "node"]
|
||||||
},
|
},
|
||||||
|
"./runtime/media-steer": {
|
||||||
|
"source": "./src/runtime/mediaSteer.ts",
|
||||||
|
"runtime": "./dist/runtime/mediaSteer.js",
|
||||||
|
"types": "./dist/runtime/mediaSteer.d.ts",
|
||||||
|
"environments": ["browser", "bun", "node"]
|
||||||
|
},
|
||||||
"./runtime/clip-reveal": {
|
"./runtime/clip-reveal": {
|
||||||
"source": "./src/runtime/clipReveal.ts",
|
"source": "./src/runtime/clipReveal.ts",
|
||||||
"runtime": "./dist/runtime/clipReveal.js",
|
"runtime": "./dist/runtime/clipReveal.js",
|
||||||
|
|||||||
@@ -399,6 +399,12 @@
|
|||||||
"import": "./src/runtime/mediaClock.ts",
|
"import": "./src/runtime/mediaClock.ts",
|
||||||
"types": "./src/runtime/mediaClock.ts"
|
"types": "./src/runtime/mediaClock.ts"
|
||||||
},
|
},
|
||||||
|
"./runtime/media-steer": {
|
||||||
|
"bun": "./src/runtime/mediaSteer.ts",
|
||||||
|
"node": "./dist/runtime/mediaSteer.js",
|
||||||
|
"import": "./src/runtime/mediaSteer.ts",
|
||||||
|
"types": "./src/runtime/mediaSteer.ts"
|
||||||
|
},
|
||||||
"./runtime/clip-reveal": {
|
"./runtime/clip-reveal": {
|
||||||
"bun": "./src/runtime/clipReveal.ts",
|
"bun": "./src/runtime/clipReveal.ts",
|
||||||
"node": "./dist/runtime/clipReveal.js",
|
"node": "./dist/runtime/clipReveal.js",
|
||||||
@@ -845,6 +851,10 @@
|
|||||||
"import": "./dist/runtime/mediaClock.js",
|
"import": "./dist/runtime/mediaClock.js",
|
||||||
"types": "./dist/runtime/mediaClock.d.ts"
|
"types": "./dist/runtime/mediaClock.d.ts"
|
||||||
},
|
},
|
||||||
|
"./runtime/media-steer": {
|
||||||
|
"import": "./dist/runtime/mediaSteer.js",
|
||||||
|
"types": "./dist/runtime/mediaSteer.d.ts"
|
||||||
|
},
|
||||||
"./runtime/clip-reveal": {
|
"./runtime/clip-reveal": {
|
||||||
"import": "./dist/runtime/clipReveal.js",
|
"import": "./dist/runtime/clipReveal.js",
|
||||||
"types": "./dist/runtime/clipReveal.d.ts"
|
"types": "./dist/runtime/clipReveal.d.ts"
|
||||||
|
|||||||
@@ -656,7 +656,7 @@ describe("attachElementFxChain", () => {
|
|||||||
}
|
}
|
||||||
});
|
});
|
||||||
|
|
||||||
it("clears a limiter's state when a persistent chain is re-anchored for a new pass", async () => {
|
it("clears a limiter's state for a new pass, not when the same pass is re-timed", async () => {
|
||||||
const messages: unknown[] = [];
|
const messages: unknown[] = [];
|
||||||
class RecordingWorkletStub extends WorkletNodeStub {
|
class RecordingWorkletStub extends WorkletNodeStub {
|
||||||
override port = { postMessage: (...sent: unknown[]) => void messages.push(...sent) };
|
override port = { postMessage: (...sent: unknown[]) => void messages.push(...sent) };
|
||||||
@@ -679,6 +679,7 @@ describe("attachElementFxChain", () => {
|
|||||||
timing,
|
timing,
|
||||||
);
|
);
|
||||||
await settle();
|
await settle();
|
||||||
|
fx?.retime({ scheduledAt: 1, elapsed: 0.9, rate: 1.03 });
|
||||||
expect(messages).not.toContainEqual({ __hfReset: true });
|
expect(messages).not.toContainEqual({ __hfReset: true });
|
||||||
fx?.reanchor(timing);
|
fx?.reanchor(timing);
|
||||||
expect(messages).toContainEqual({ __hfReset: true });
|
expect(messages).toContainEqual({ __hfReset: true });
|
||||||
|
|||||||
@@ -112,6 +112,8 @@ export interface ElementFxHandle {
|
|||||||
* stuck value, which for a fade-out is silence for the rest of the session.
|
* stuck value, which for a fade-out is silence for the rest of the session.
|
||||||
*/
|
*/
|
||||||
reanchor(timing: AutomationTiming): void;
|
reanchor(timing: AutomationTiming): void;
|
||||||
|
/** Re-aim every booked envelope at `timing` without clearing effect state: the same pass, re-measured. */
|
||||||
|
retime(timing: AutomationTiming): void;
|
||||||
/**
|
/**
|
||||||
* Re-aim every booked envelope at a new playback rate.
|
* Re-aim every booked envelope at a new playback rate.
|
||||||
*
|
*
|
||||||
@@ -343,15 +345,21 @@ export function attachElementFxChain(
|
|||||||
});
|
});
|
||||||
}
|
}
|
||||||
|
|
||||||
|
const retime = (next: AutomationTiming): void => {
|
||||||
|
if (disposed) return;
|
||||||
|
const at = timingNow();
|
||||||
|
if (at) cancelParamLane(automated, at.scheduledAt);
|
||||||
|
frame = { ...next };
|
||||||
|
scheduleFor(readChain(el).chain, frame);
|
||||||
|
};
|
||||||
|
|
||||||
return {
|
return {
|
||||||
reanchor: (next: AutomationTiming) => {
|
reanchor: (next: AutomationTiming) => {
|
||||||
if (disposed) return;
|
if (disposed) return;
|
||||||
const at = timingNow();
|
|
||||||
if (at) cancelParamLane(automated, at.scheduledAt);
|
|
||||||
frame = { ...next };
|
|
||||||
handle?.reset();
|
handle?.reset();
|
||||||
scheduleFor(readChain(el).chain, frame);
|
retime(next);
|
||||||
},
|
},
|
||||||
|
retime,
|
||||||
setRate: (rate: number) => {
|
setRate: (rate: number) => {
|
||||||
const at = timingNow();
|
const at = timingNow();
|
||||||
if (disposed || !at || !Number.isFinite(rate) || rate <= 0 || rate === at.rate) return;
|
if (disposed || !at || !Number.isFinite(rate) || rate <= 0 || rate === at.rate) return;
|
||||||
|
|||||||
@@ -90,7 +90,7 @@ import {
|
|||||||
resolveNaturalMediaTimelineDuration,
|
resolveNaturalMediaTimelineDuration,
|
||||||
resolveRuntimeMediaClipDuration,
|
resolveRuntimeMediaClipDuration,
|
||||||
syncRuntimeMedia,
|
syncRuntimeMedia,
|
||||||
VIDEO_PRESEEK_SECONDS,
|
MEDIA_PRESEEK_SECONDS,
|
||||||
type RuntimeMediaClip,
|
type RuntimeMediaClip,
|
||||||
} from "./media";
|
} from "./media";
|
||||||
import {
|
import {
|
||||||
@@ -3493,14 +3493,14 @@ export function initSandboxRuntimeModular(): void {
|
|||||||
visiting.add(clip.el);
|
visiting.add(clip.el);
|
||||||
}
|
}
|
||||||
if (state.isPlaying && !window.__HF_EXPORT_RENDER_SEEK_CONFIG) {
|
if (state.isPlaying && !window.__HF_EXPORT_RENDER_SEEK_CONFIG) {
|
||||||
// An endpoint-only pass reaches trimmed video too late: its first visible tick starts a seek.
|
// An endpoint-only pass reaches a trimmed clip too late: its first active tick starts a seek.
|
||||||
for (const clip of clipsWithEndpointBetween(
|
for (const clip of clipsWithEndpointBetween(
|
||||||
index.byStart,
|
index.byStart,
|
||||||
(c) => c.start,
|
(c) => c.start,
|
||||||
toSeconds,
|
toSeconds,
|
||||||
toSeconds + VIDEO_PRESEEK_SECONDS * state.playbackRate,
|
toSeconds + MEDIA_PRESEEK_SECONDS * state.playbackRate,
|
||||||
)) {
|
)) {
|
||||||
if (clip.el.tagName === "VIDEO") visiting.add(clip.el);
|
visiting.add(clip.el);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
return [...visiting];
|
return [...visiting];
|
||||||
@@ -3659,6 +3659,7 @@ export function initSandboxRuntimeModular(): void {
|
|||||||
webAudio.setElementVolume(el, authorVolume, clipGain, unfadedVolume),
|
webAudio.setElementVolume(el, authorVolume, clipGain, unfadedVolume),
|
||||||
isWebAudioOwned: (el) => webAudio.ownsElement(el),
|
isWebAudioOwned: (el) => webAudio.ownsElement(el),
|
||||||
isWebAudioRouted: (el) => webAudio.routesElement(el),
|
isWebAudioRouted: (el) => webAudio.routesElement(el),
|
||||||
|
seekUnderDip: (el, target) => webAudio.seekUnderDip(el, target),
|
||||||
onAutoplayBlocked: () => {
|
onAutoplayBlocked: () => {
|
||||||
if (state.mediaAutoplayBlockedPosted) return;
|
if (state.mediaAutoplayBlockedPosted) return;
|
||||||
state.mediaAutoplayBlockedPosted = true;
|
state.mediaAutoplayBlockedPosted = true;
|
||||||
|
|||||||
@@ -3,56 +3,58 @@ import { mediaElementTime } from "./mediaClock";
|
|||||||
import { initSandboxRuntimeModular } from "./init";
|
import { initSandboxRuntimeModular } from "./init";
|
||||||
import { createMockTimeline, resetRuntimeFixtureDom } from "./runtimeSeekFixture.test-helpers";
|
import { createMockTimeline, resetRuntimeFixtureDom } from "./runtimeSeekFixture.test-helpers";
|
||||||
|
|
||||||
it("prepares a hidden video when playback enters its lookahead, after metadata arrives", () => {
|
it.each(["video", "audio"])(
|
||||||
resetRuntimeFixtureDom();
|
"prepares an upcoming %s when playback enters its lookahead, after metadata arrives",
|
||||||
let now = 0;
|
(tag) => {
|
||||||
let id = 0;
|
resetRuntimeFixtureDom();
|
||||||
const callbacks = new Map<number, FrameRequestCallback>();
|
let now = 0;
|
||||||
vi.spyOn(performance, "now").mockImplementation(() => now);
|
let id = 0;
|
||||||
vi.spyOn(window, "requestAnimationFrame").mockImplementation((callback) => {
|
const callbacks = new Map<number, FrameRequestCallback>();
|
||||||
callbacks.set(++id, callback);
|
vi.spyOn(performance, "now").mockImplementation(() => now);
|
||||||
return id;
|
vi.spyOn(window, "requestAnimationFrame").mockImplementation((callback) => {
|
||||||
});
|
callbacks.set(++id, callback);
|
||||||
vi.spyOn(window, "cancelAnimationFrame").mockImplementation((key) => {
|
return id;
|
||||||
callbacks.delete(key);
|
});
|
||||||
});
|
vi.spyOn(window, "cancelAnimationFrame").mockImplementation((key) => {
|
||||||
const step = () => {
|
callbacks.delete(key);
|
||||||
now += 100;
|
});
|
||||||
const pending = [...callbacks.values()];
|
const step = () => {
|
||||||
callbacks.clear();
|
now += 100;
|
||||||
for (const callback of pending) callback(now);
|
const pending = [...callbacks.values()];
|
||||||
};
|
callbacks.clear();
|
||||||
document.body.innerHTML =
|
for (const callback of pending) callback(now);
|
||||||
'<div data-composition-id="main" data-root="true" data-duration="6"><video id="incoming" data-start="4" data-duration="2" data-media-start="8" src="/movie.mp4"></video></div>';
|
};
|
||||||
const video = document.querySelector("video")!;
|
document.body.innerHTML = `<div data-composition-id="main" data-root="true" data-duration="6"><${tag} id="incoming" data-start="4" data-duration="2" data-media-start="8" src="/movie.mp4"></${tag}></div>`;
|
||||||
Object.defineProperty(video, "duration", { value: 20, configurable: true });
|
const video = document.querySelector<HTMLMediaElement>(tag)!;
|
||||||
Object.defineProperty(video, "readyState", { value: 0, configurable: true });
|
Object.defineProperty(video, "duration", { value: 20, configurable: true });
|
||||||
Object.defineProperty(video, "paused", { value: true, writable: true, configurable: true });
|
Object.defineProperty(video, "readyState", { value: 0, configurable: true });
|
||||||
const play = vi.fn(() => {
|
Object.defineProperty(video, "paused", { value: true, writable: true, configurable: true });
|
||||||
Object.assign(video, { paused: false });
|
const play = vi.fn(() => {
|
||||||
return Promise.resolve();
|
Object.assign(video, { paused: false });
|
||||||
});
|
return Promise.resolve();
|
||||||
video.play = play;
|
});
|
||||||
video.pause = vi.fn(() => {
|
video.play = play;
|
||||||
Object.assign(video, { paused: true });
|
video.pause = vi.fn(() => {
|
||||||
});
|
Object.assign(video, { paused: true });
|
||||||
video.load = vi.fn();
|
});
|
||||||
const seek = vi.spyOn(video, "currentTime", "set");
|
video.load = vi.fn();
|
||||||
window.__timelines = { main: createMockTimeline(6) };
|
const seek = vi.spyOn(video, "currentTime", "set");
|
||||||
initSandboxRuntimeModular();
|
window.__timelines = { main: createMockTimeline(6) };
|
||||||
window.__player!.play();
|
initSandboxRuntimeModular();
|
||||||
for (let i = 0; i < 28; i++) step();
|
window.__player!.play();
|
||||||
expect(seek).not.toHaveBeenCalled();
|
for (let i = 0; i < 28; i++) step();
|
||||||
Object.defineProperty(video, "readyState", { value: 4, configurable: true });
|
expect(seek).not.toHaveBeenCalled();
|
||||||
step();
|
Object.defineProperty(video, "readyState", { value: 4, configurable: true });
|
||||||
expect(video.style.visibility).toBe("hidden");
|
step();
|
||||||
expect(video.currentTime).toBe(mediaElementTime(8));
|
if (tag === "video") expect(video.style.visibility).toBe("hidden");
|
||||||
expect(play).not.toHaveBeenCalled();
|
expect(video.currentTime).toBe(mediaElementTime(8));
|
||||||
for (let i = 0; i < 11; i++) step();
|
expect(play).not.toHaveBeenCalled();
|
||||||
expect(video.style.visibility).toBe("visible");
|
for (let i = 0; i < 11; i++) step();
|
||||||
expect(seek).toHaveBeenCalledTimes(1);
|
if (tag === "video") expect(video.style.visibility).toBe("visible");
|
||||||
expect(play).toHaveBeenCalledTimes(1);
|
expect(seek).toHaveBeenCalledTimes(1);
|
||||||
});
|
expect(play).toHaveBeenCalledTimes(1);
|
||||||
|
},
|
||||||
|
);
|
||||||
|
|
||||||
afterEach(() => {
|
afterEach(() => {
|
||||||
window.__hfRuntimeTeardown?.();
|
window.__hfRuntimeTeardown?.();
|
||||||
|
|||||||
@@ -38,19 +38,22 @@ afterEach(() => {
|
|||||||
vi.restoreAllMocks();
|
vi.restoreAllMocks();
|
||||||
});
|
});
|
||||||
|
|
||||||
describe("hidden incoming video preparation", () => {
|
describe("incoming media preparation", () => {
|
||||||
it("lands the in-point before the cut without playing or seeking it again at activation", () => {
|
it.each(["video", "audio"])(
|
||||||
const { el, seek, tick } = fixture();
|
"lands a %s clip's in-point before the cut without playing or seeking it again at activation",
|
||||||
tick(0);
|
(tag) => {
|
||||||
expect(seek).not.toHaveBeenCalled();
|
const { el, seek, tick } = fixture({}, tag);
|
||||||
tick(2.5);
|
tick(0);
|
||||||
expect(el.currentTime).toBe(mediaElementTime(8));
|
expect(seek).not.toHaveBeenCalled();
|
||||||
expect(el.play).not.toHaveBeenCalled();
|
tick(2.5);
|
||||||
tick(3);
|
expect(el.currentTime).toBe(mediaElementTime(8));
|
||||||
tick(4);
|
expect(el.play).not.toHaveBeenCalled();
|
||||||
expect(seek).toHaveBeenCalledTimes(1);
|
tick(3);
|
||||||
expect(el.play).toHaveBeenCalledTimes(1);
|
tick(4);
|
||||||
});
|
expect(seek).toHaveBeenCalledTimes(1);
|
||||||
|
expect(el.play).toHaveBeenCalledTimes(1);
|
||||||
|
},
|
||||||
|
);
|
||||||
|
|
||||||
it("retries after metadata arrives without interrupting an outstanding seek", () => {
|
it("retries after metadata arrives without interrupting an outstanding seek", () => {
|
||||||
const { el, seek, tick } = fixture();
|
const { el, seek, tick } = fixture();
|
||||||
@@ -84,10 +87,10 @@ describe("hidden incoming video preparation", () => {
|
|||||||
expect(el.currentTime).toBe(mediaElementTime(8));
|
expect(el.currentTime).toBe(mediaElementTime(8));
|
||||||
});
|
});
|
||||||
|
|
||||||
it.each(["audio", "paused", "past", "export", "injected", "playing", "disconnected"])(
|
it.each(["paused", "past", "export", "injected", "playing", "disconnected"])(
|
||||||
"leaves %s media untouched",
|
"leaves %s media untouched",
|
||||||
(kind) => {
|
(kind) => {
|
||||||
const { el, tick, seek } = fixture({}, kind === "audio" ? "audio" : "video");
|
const { el, tick, seek } = fixture();
|
||||||
if (kind === "export") window.__HF_EXPORT_RENDER_SEEK_CONFIG = {};
|
if (kind === "export") window.__HF_EXPORT_RENDER_SEEK_CONFIG = {};
|
||||||
if (kind === "injected") {
|
if (kind === "injected") {
|
||||||
el.id = "incoming";
|
el.id = "incoming";
|
||||||
|
|||||||
@@ -1984,9 +1984,13 @@ describe("syncRuntimeMedia", () => {
|
|||||||
expect(clip.el.currentTime).toBe(mediaElementTime(5));
|
expect(clip.el.currentTime).toBe(mediaElementTime(5));
|
||||||
});
|
});
|
||||||
|
|
||||||
describe("playing video drift", () => {
|
describe("playing media drift", () => {
|
||||||
function playingVideoAt(currentTime: number, playbackRate = 1) {
|
function playingVideoAt(
|
||||||
const clip = createMockClip({ start: 0, end: 20, duration: 20 });
|
currentTime: number,
|
||||||
|
playbackRate = 1,
|
||||||
|
tag: "audio" | "video" = "video",
|
||||||
|
) {
|
||||||
|
const clip = createMockClip({ start: 0, end: 20, duration: 20 }, tag);
|
||||||
Object.defineProperty(clip.el, "paused", { value: false, writable: true });
|
Object.defineProperty(clip.el, "paused", { value: false, writable: true });
|
||||||
Object.defineProperty(clip.el, "currentTime", { value: currentTime, writable: true });
|
Object.defineProperty(clip.el, "currentTime", { value: currentTime, writable: true });
|
||||||
clip.el.playbackRate = playbackRate;
|
clip.el.playbackRate = playbackRate;
|
||||||
@@ -2001,13 +2005,15 @@ describe("syncRuntimeMedia", () => {
|
|||||||
getCompositionDuration: () => 20,
|
getCompositionDuration: () => 20,
|
||||||
});
|
});
|
||||||
|
|
||||||
it("runs a video lagging past the sync tolerance 3% fast instead of seeking it", () => {
|
it.each(["video", "audio"] as const)(
|
||||||
const clip = playingVideoAt(4.908);
|
"runs a playing %s lagging past the sync tolerance 3% fast instead of seeking it",
|
||||||
tick(clip, 5);
|
(tag) => {
|
||||||
tick(clip, 5);
|
const clip = playingVideoAt(4.908, 1, tag);
|
||||||
expect(clip.el.playbackRate).toBeCloseTo(1.03, 9);
|
for (let i = 0; i < 4; i++) tick(clip, 5);
|
||||||
expect(clip.el.currentTime).toBe(4.908);
|
expect(clip.el.playbackRate).toBeCloseTo(1.03, 9);
|
||||||
});
|
expect(clip.el.currentTime).toBe(4.908);
|
||||||
|
},
|
||||||
|
);
|
||||||
|
|
||||||
it("runs a video that is ahead 3% slow", () => {
|
it("runs a video that is ahead 3% slow", () => {
|
||||||
const clip = playingVideoAt(5.06);
|
const clip = playingVideoAt(5.06);
|
||||||
@@ -2107,6 +2113,53 @@ describe("syncRuntimeMedia", () => {
|
|||||||
expect(clip.el.playbackRate).toBeCloseTo(1.03, 9);
|
expect(clip.el.playbackRate).toBeCloseTo(1.03, 9);
|
||||||
});
|
});
|
||||||
|
|
||||||
|
describe("sounding audio past a gentle nudge", () => {
|
||||||
|
const tickDipping = (clip: RuntimeMediaClip, timeSeconds: number, dips: boolean) => {
|
||||||
|
const seekUnderDip = vi.fn(() => dips);
|
||||||
|
syncRuntimeMedia({
|
||||||
|
clips: [clip],
|
||||||
|
timeSeconds,
|
||||||
|
playing: true,
|
||||||
|
playbackRate: 1,
|
||||||
|
getCompositionDuration: () => 20,
|
||||||
|
seekUnderDip,
|
||||||
|
});
|
||||||
|
return seekUnderDip;
|
||||||
|
};
|
||||||
|
|
||||||
|
it("never nudges audio past 10% while its lag still grows", () => {
|
||||||
|
const clip = playingVideoAt(4.6, 1, "audio");
|
||||||
|
tickDipping(clip, 5, true);
|
||||||
|
const seekUnderDip = tickDipping(clip, 5.02, true);
|
||||||
|
expect(clip.el.playbackRate).toBeCloseTo(1.1, 9);
|
||||||
|
expect(seekUnderDip).not.toHaveBeenCalled();
|
||||||
|
});
|
||||||
|
|
||||||
|
it("seeks audio that holds a lag past 100 ms under a dip, at its authored rate", () => {
|
||||||
|
const clip = playingVideoAt(4.85, 1, "audio");
|
||||||
|
tickDipping(clip, 5, true);
|
||||||
|
const seekUnderDip = tickDipping(clip, 5, true);
|
||||||
|
expect(seekUnderDip).toHaveBeenCalledWith(clip.el, 5);
|
||||||
|
expect(clip.el.currentTime).toBe(4.85);
|
||||||
|
expect(clip.el.playbackRate).toBe(1);
|
||||||
|
});
|
||||||
|
|
||||||
|
it("seeks it plainly when it has no dip to hide the seek under", () => {
|
||||||
|
const clip = playingVideoAt(4.85, 1, "audio");
|
||||||
|
tickDipping(clip, 5, false);
|
||||||
|
tickDipping(clip, 5, false);
|
||||||
|
expect(clip.el.currentTime).toBe(mediaElementTime(5));
|
||||||
|
});
|
||||||
|
|
||||||
|
it("leaves video on its steps", () => {
|
||||||
|
const clip = playingVideoAt(4.85);
|
||||||
|
tickDipping(clip, 5, true);
|
||||||
|
const seekUnderDip = tickDipping(clip, 5, true);
|
||||||
|
expect(seekUnderDip).not.toHaveBeenCalled();
|
||||||
|
expect(clip.el.playbackRate).toBeCloseTo(1.1, 9);
|
||||||
|
});
|
||||||
|
});
|
||||||
|
|
||||||
it("steers a video that is far ahead slower, by the same steps", () => {
|
it("steers a video that is far ahead slower, by the same steps", () => {
|
||||||
const clip = playingVideoAt(5.2);
|
const clip = playingVideoAt(5.2);
|
||||||
tick(clip, 5);
|
tick(clip, 5);
|
||||||
|
|||||||
@@ -13,6 +13,13 @@ import { findInjectedRenderFrame } from "./renderFrameSibling.js";
|
|||||||
import { isRenderFrameVideo } from "./preloadMedia.js";
|
import { isRenderFrameVideo } from "./preloadMedia.js";
|
||||||
import { registerSeekCompletion } from "./adapters/seek-dispatch.js";
|
import { registerSeekCompletion } from "./adapters/seek-dispatch.js";
|
||||||
import { cutToHostSlots, type HostSlot } from "../mediaTiming.js";
|
import { cutToHostSlots, type HostSlot } from "../mediaTiming.js";
|
||||||
|
import {
|
||||||
|
forgetSteering,
|
||||||
|
isSteering,
|
||||||
|
MEDIA_SYNC_TOLERANCE_SECONDS,
|
||||||
|
steerMedia,
|
||||||
|
} from "./mediaSteer.js";
|
||||||
|
export { MEDIA_SYNC_TOLERANCE_SECONDS } from "./mediaSteer.js";
|
||||||
export {
|
export {
|
||||||
readElementPlaybackRate,
|
readElementPlaybackRate,
|
||||||
readElementRateSpec,
|
readElementRateSpec,
|
||||||
@@ -250,7 +257,7 @@ export function evictMediaSyncState(el: HTMLMediaElement): void {
|
|||||||
strictDriftSamples.delete(el);
|
strictDriftSamples.delete(el);
|
||||||
seekLoadRetried.delete(el);
|
seekLoadRetried.delete(el);
|
||||||
lastRuntimeAppliedVolume.delete(el);
|
lastRuntimeAppliedVolume.delete(el);
|
||||||
videoSteering.delete(el);
|
forgetSteering(el);
|
||||||
}
|
}
|
||||||
|
|
||||||
/** Test-only seam: whether any per-source sync state is still tracked for `el`. */
|
/** Test-only seam: whether any per-source sync state is still tracked for `el`. */
|
||||||
@@ -261,12 +268,12 @@ export function hasMediaSyncStateForTest(el: HTMLMediaElement): boolean {
|
|||||||
strictDriftSamples.has(el) ||
|
strictDriftSamples.has(el) ||
|
||||||
seekLoadRetried.has(el) ||
|
seekLoadRetried.has(el) ||
|
||||||
lastRuntimeAppliedVolume.has(el) ||
|
lastRuntimeAppliedVolume.has(el) ||
|
||||||
videoSteering.has(el)
|
isSteering(el)
|
||||||
);
|
);
|
||||||
}
|
}
|
||||||
|
|
||||||
export const MEDIA_HARD_SYNC_SECONDS = 0.5;
|
export const MEDIA_HARD_SYNC_SECONDS = 0.5;
|
||||||
export const VIDEO_PRESEEK_SECONDS = 1.5;
|
export const MEDIA_PRESEEK_SECONDS = 1.5;
|
||||||
|
|
||||||
export function mediaSourceTimeAt(clip: RuntimeMediaClip, timeSeconds: number): number {
|
export function mediaSourceTimeAt(clip: RuntimeMediaClip, timeSeconds: number): number {
|
||||||
const elapsed = Math.max(0, Math.min(timeSeconds, clip.end) - (clip.origin ?? clip.start));
|
const elapsed = Math.max(0, Math.min(timeSeconds, clip.end) - (clip.origin ?? clip.start));
|
||||||
@@ -280,7 +287,7 @@ export function mediaSourceTimeAt(clip: RuntimeMediaClip, timeSeconds: number):
|
|||||||
return target;
|
return target;
|
||||||
}
|
}
|
||||||
|
|
||||||
function prepareUpcomingVideo(
|
function prepareUpcomingMedia(
|
||||||
clip: RuntimeMediaClip,
|
clip: RuntimeMediaClip,
|
||||||
time: number,
|
time: number,
|
||||||
playing: boolean,
|
playing: boolean,
|
||||||
@@ -290,9 +297,8 @@ function prepareUpcomingVideo(
|
|||||||
const dueIn = clip.start - time;
|
const dueIn = clip.start - time;
|
||||||
if (
|
if (
|
||||||
!playing ||
|
!playing ||
|
||||||
el.tagName !== "VIDEO" ||
|
|
||||||
dueIn <= 0 ||
|
dueIn <= 0 ||
|
||||||
dueIn > VIDEO_PRESEEK_SECONDS * rate ||
|
dueIn > MEDIA_PRESEEK_SECONDS * rate ||
|
||||||
!el.paused ||
|
!el.paused ||
|
||||||
el.seeking ||
|
el.seeking ||
|
||||||
el.readyState < el.HAVE_METADATA ||
|
el.readyState < el.HAVE_METADATA ||
|
||||||
@@ -303,7 +309,7 @@ function prepareUpcomingVideo(
|
|||||||
const target = mediaSourceTimeAt(clip, clip.start);
|
const target = mediaSourceTimeAt(clip, clip.start);
|
||||||
if (!Number.isFinite(target) || target < 0 || Math.abs(el.currentTime - target) <= 0.001) return;
|
if (!Number.isFinite(target) || target < 0 || Math.abs(el.currentTime - target) <= 0.001) return;
|
||||||
try {
|
try {
|
||||||
// A trimmed video exposed during its in-point seek has no current frame at the cut.
|
// A trimmed clip seeked as it opens shows no frame and plays late, then is seeked again while it sounds.
|
||||||
// Hidden preparation must not hold the visible-frame seek barrier.
|
// Hidden preparation must not hold the visible-frame seek barrier.
|
||||||
el.currentTime = mediaElementTime(target);
|
el.currentTime = mediaElementTime(target);
|
||||||
} catch (err) {
|
} catch (err) {
|
||||||
@@ -311,39 +317,9 @@ function prepareUpcomingVideo(
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
/** Drift a playing audio element may carry before sync pulls it back onto the playhead. */
|
|
||||||
export const MEDIA_SYNC_TOLERANCE_SECONDS = 0.04;
|
|
||||||
|
|
||||||
// Chromium refuses a slower playbackRate (NotSupportedError).
|
// Chromium refuses a slower playbackRate (NotSupportedError).
|
||||||
const MIN_NATIVE_PLAYBACK_RATE = 1 / 16;
|
const MIN_NATIVE_PLAYBACK_RATE = 1 / 16;
|
||||||
|
|
||||||
// A playing video is steered back by rate, not seeked (a seek resets its decoder).
|
|
||||||
// Its rate is written only when steering starts or stops: every write costs a frame.
|
|
||||||
const VIDEO_STEER = 0.03;
|
|
||||||
// A video that starts a cut late closes its lag over one clip at 3%; a bigger lag steers harder, in steps so the rate is written rarely.
|
|
||||||
const VIDEO_STEER_STEPS: readonly (readonly [lag: number, steer: number])[] = [
|
|
||||||
[0.3, 0.25],
|
|
||||||
[0.1, 0.1],
|
|
||||||
];
|
|
||||||
const VIDEO_STEER_RELEASE_SECONDS = 0.01;
|
|
||||||
|
|
||||||
/** Direction (+1 fast, -1 slow) a video is being steered in; absent when it plays at its authored rate. */
|
|
||||||
const videoSteering = new WeakMap<HTMLMediaElement, number>();
|
|
||||||
|
|
||||||
/** Rate for a playing video `offset` seconds behind (+) or ahead (-) of the playhead. */
|
|
||||||
function steeredVideoRate(el: HTMLMediaElement, offset: number, baseRate: number): number {
|
|
||||||
const direction = Math.sign(offset);
|
|
||||||
const steering = videoSteering.get(el);
|
|
||||||
if (Math.abs(offset) > MEDIA_SYNC_TOLERANCE_SECONDS) videoSteering.set(el, direction);
|
|
||||||
else if (steering !== direction || Math.abs(offset) <= VIDEO_STEER_RELEASE_SECONDS) {
|
|
||||||
videoSteering.delete(el);
|
|
||||||
return baseRate;
|
|
||||||
}
|
|
||||||
const lag = Math.abs(offset);
|
|
||||||
const steer = VIDEO_STEER_STEPS.find(([from]) => lag > from)?.[1] ?? VIDEO_STEER;
|
|
||||||
return baseRate * (1 + direction * steer);
|
|
||||||
}
|
|
||||||
|
|
||||||
// fallow-ignore-next-line complexity
|
// fallow-ignore-next-line complexity
|
||||||
export function syncRuntimeMedia(params: {
|
export function syncRuntimeMedia(params: {
|
||||||
clips: RuntimeMediaClip[];
|
clips: RuntimeMediaClip[];
|
||||||
@@ -389,6 +365,8 @@ export function syncRuntimeMedia(params: {
|
|||||||
forceSync?: boolean;
|
forceSync?: boolean;
|
||||||
/** How far the next tick will move the playhead: an audio clip due within it starts now. */
|
/** How far the next tick will move the playhead: an audio clip due within it starts now. */
|
||||||
cueAheadSeconds?: number;
|
cueAheadSeconds?: number;
|
||||||
|
/** Seeks a sounding element under a gain dip; false when it has no dip stage and the sync seeks it plainly. */
|
||||||
|
seekUnderDip?: (el: HTMLMediaElement, target: number) => boolean;
|
||||||
/** Lets a video clip that runs to the composition end hold its last frame at the terminal time.
|
/** Lets a video clip that runs to the composition end hold its last frame at the terminal time.
|
||||||
* A thunk, because deriving the duration is only worth it for a clip past its own end. */
|
* A thunk, because deriving the duration is only worth it for a clip past its own end. */
|
||||||
getCompositionDuration: () => number;
|
getCompositionDuration: () => number;
|
||||||
@@ -589,12 +567,11 @@ export function syncRuntimeMedia(params: {
|
|||||||
(el.ended && canSeekEndedMediaBackward && drift > 0.001) ||
|
(el.ended && canSeekEndedMediaBackward && drift > 0.001) ||
|
||||||
staleAudioOnFirstTick ||
|
staleAudioOnFirstTick ||
|
||||||
(drift > MEDIA_HARD_SYNC_SECONDS && (firstTickOfClip || offsetJumped || catastrophicDrift));
|
(drift > MEDIA_HARD_SYNC_SECONDS && (firstTickOfClip || offsetJumped || catastrophicDrift));
|
||||||
// Playing videos use the browser's decoder for timing. Seeking one resets the decoder: a
|
// Seeking playing media resets a video's decoder (a ~150ms freeze that loops into a
|
||||||
// ~150ms freeze while it re-buffers, as the monotonic clock advances, which loops into a
|
// seek→freeze→drift→seek stutter) and jumps sounding audio (a click). So playing media skips
|
||||||
// seek→freeze→drift→seek stutter. So a playing video skips strict and force sync; only hard
|
// strict and force sync and is steered by rate; hard sync (>0.5s) and audio past its steer
|
||||||
// sync (>0.5s) warrants the decoder-reset cost. A paused transport pauses this video below,
|
// limit seek. A paused transport pauses it below, so a seek that pauses mid-playback lands it.
|
||||||
// so a seek that pauses mid-playback still lands it.
|
const isPlayingMedia = !el.paused && playing;
|
||||||
const isPlayingVideo = el.tagName === "VIDEO" && !el.paused && playing;
|
|
||||||
// Only apply strict sync when offset has stabilized (not growing).
|
// Only apply strict sync when offset has stabilized (not growing).
|
||||||
// During initial buffering, offset grows ~16ms/tick as the timeline
|
// During initial buffering, offset grows ~16ms/tick as the timeline
|
||||||
// advances while media stays at 0. Accumulated drift from pause/play
|
// advances while media stays at 0. Accumulated drift from pause/play
|
||||||
@@ -604,7 +581,7 @@ export function syncRuntimeMedia(params: {
|
|||||||
heldBelowFloor || (prevOffset !== undefined && Math.abs(offset - prevOffset) < 0.004);
|
heldBelowFloor || (prevOffset !== undefined && Math.abs(offset - prevOffset) < 0.004);
|
||||||
let strictSync = false;
|
let strictSync = false;
|
||||||
if (
|
if (
|
||||||
!isPlayingVideo &&
|
!isPlayingMedia &&
|
||||||
!hardSync &&
|
!hardSync &&
|
||||||
!firstTickOfClip &&
|
!firstTickOfClip &&
|
||||||
offsetStabilized &&
|
offsetStabilized &&
|
||||||
@@ -619,21 +596,22 @@ export function syncRuntimeMedia(params: {
|
|||||||
} else if (drift <= MEDIA_SYNC_TOLERANCE_SECONDS) {
|
} else if (drift <= MEDIA_SYNC_TOLERANCE_SECONDS) {
|
||||||
strictDriftSamples.set(el, 0);
|
strictDriftSamples.set(el, 0);
|
||||||
}
|
}
|
||||||
const forceSync = !isPlayingVideo && params.forceSync && drift > 0.02;
|
const forceSync = !isPlayingMedia && params.forceSync && drift > 0.02;
|
||||||
|
const steered =
|
||||||
|
isPlayingMedia && !hardSync ? steerMedia(el, offset, baseRate, offsetStabilized) : null;
|
||||||
|
const steerSeeks = steered?.seek === true;
|
||||||
|
const dipped = steerSeeks && params.seekUnderDip?.(el, relTime) === true;
|
||||||
try {
|
try {
|
||||||
// A hard sync lands the video on the playhead, so its pre-seek offset says nothing.
|
// A hard sync lands the element on the playhead, so its pre-seek offset says nothing.
|
||||||
if (!isPlayingVideo || hardSync) videoSteering.delete(el);
|
if (!steered) forgetSteering(el);
|
||||||
const rate = Math.max(
|
const rate = Math.max(MIN_NATIVE_PLAYBACK_RATE, steered?.rate ?? baseRate);
|
||||||
MIN_NATIVE_PLAYBACK_RATE,
|
|
||||||
isPlayingVideo && !hardSync ? steeredVideoRate(el, offset, baseRate) : baseRate,
|
|
||||||
);
|
|
||||||
// Some engines read a rate back at lower precision; an equal-enough rate is not rewritten.
|
// Some engines read a rate back at lower precision; an equal-enough rate is not rewritten.
|
||||||
if (Math.abs(el.playbackRate - rate) > 1e-6) el.playbackRate = rate;
|
if (Math.abs(el.playbackRate - rate) > 1e-6) el.playbackRate = rate;
|
||||||
} catch (err) {
|
} catch (err) {
|
||||||
// ignore unsupported playbackRate
|
// ignore unsupported playbackRate
|
||||||
swallow("runtime.media.site1", err);
|
swallow("runtime.media.site1", err);
|
||||||
}
|
}
|
||||||
if (hardSync || strictSync || forceSync) {
|
if (hardSync || strictSync || forceSync || (steerSeeks && !dipped)) {
|
||||||
// Skip the per-tick seek (and the `el.load()` drift-recovery retry
|
// Skip the per-tick seek (and the `el.load()` drift-recovery retry
|
||||||
// below) for `<video>` elements that have a sibling
|
// below) for `<video>` elements that have a sibling
|
||||||
// `<img id="__render_frame_<id>__">`. The sibling is created only
|
// `<img id="__render_frame_<id>__">`. The sibling is created only
|
||||||
@@ -712,6 +690,6 @@ export function syncRuntimeMedia(params: {
|
|||||||
evictMediaSyncState(el);
|
evictMediaSyncState(el);
|
||||||
if (remainsInsideAuthoredWindow) lastRelativeTime.set(el, relTime);
|
if (remainsInsideAuthoredWindow) lastRelativeTime.set(el, relTime);
|
||||||
if (!el.paused) el.pause();
|
if (!el.paused) el.pause();
|
||||||
prepareUpcomingVideo(clip, params.timeSeconds, params.playing, params.playbackRate);
|
prepareUpcomingMedia(clip, params.timeSeconds, params.playing, params.playbackRate);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -0,0 +1,32 @@
|
|||||||
|
import { describe, expect, it } from "vitest";
|
||||||
|
import { forgetSteering, isSteering, steerMedia } from "./mediaSteer";
|
||||||
|
|
||||||
|
const media = (tag: "audio" | "video") => document.createElement(tag);
|
||||||
|
|
||||||
|
describe("steerMedia", () => {
|
||||||
|
it("leaves media inside the tolerance at its rate", () => {
|
||||||
|
expect(steerMedia(media("audio"), 0.03, 1, true)).toEqual({ rate: 1, seek: false });
|
||||||
|
});
|
||||||
|
|
||||||
|
it("steers video by its steps", () => {
|
||||||
|
expect(steerMedia(media("video"), 0.35, 2, true)).toEqual({ rate: 2.5, seek: false });
|
||||||
|
expect(steerMedia(media("video"), -0.05, 1, true).rate).toBeCloseTo(0.97, 9);
|
||||||
|
});
|
||||||
|
|
||||||
|
it("caps audio at 10% while its lag still grows, and seeks it once the lag holds past 100 ms", () => {
|
||||||
|
const el = media("audio");
|
||||||
|
expect(steerMedia(el, 0.35, 1, false).rate).toBeCloseTo(1.1, 9);
|
||||||
|
expect(steerMedia(el, 0.35, 1, true)).toEqual({ rate: 1, seek: true });
|
||||||
|
expect(isSteering(el)).toBe(false);
|
||||||
|
});
|
||||||
|
|
||||||
|
it("keeps steering until nearly back, and forgets on request", () => {
|
||||||
|
const el = media("audio");
|
||||||
|
steerMedia(el, 0.06, 1, true);
|
||||||
|
expect(steerMedia(el, 0.02, 1, true).rate).toBeCloseTo(1.03, 9);
|
||||||
|
expect(steerMedia(el, 0.005, 1, true)).toEqual({ rate: 1, seek: false });
|
||||||
|
steerMedia(el, 0.06, 1, true);
|
||||||
|
forgetSteering(el);
|
||||||
|
expect(steerMedia(el, 0.02, 1, true).rate).toBe(1);
|
||||||
|
});
|
||||||
|
});
|
||||||
@@ -0,0 +1,49 @@
|
|||||||
|
/** Drift a playing element may carry before sync pulls it back onto the playhead. */
|
||||||
|
export const MEDIA_SYNC_TOLERANCE_SECONDS = 0.04;
|
||||||
|
|
||||||
|
// Steered by rate, not seeked (a seek resets a video's decoder and clicks in audio); steps keep rate writes rare.
|
||||||
|
const MEDIA_STEER = 0.03;
|
||||||
|
const MEDIA_STEER_STEPS: readonly (readonly [lag: number, steer: number])[] = [
|
||||||
|
[0.3, 0.25],
|
||||||
|
[0.1, 0.1],
|
||||||
|
];
|
||||||
|
const MEDIA_STEER_RELEASE_SECONDS = 0.01;
|
||||||
|
// Audio never steers past 10%, where speech changes tempo; a lag held past 100 ms is seeked instead.
|
||||||
|
const AUDIO_MAX_STEER = 0.1;
|
||||||
|
const AUDIO_STEER_LIMIT_SECONDS = 0.1;
|
||||||
|
|
||||||
|
/** Direction (+1 fast, -1 slow) an element is being steered in; absent when it plays at its authored rate. */
|
||||||
|
const steering = new WeakMap<HTMLMediaElement, number>();
|
||||||
|
|
||||||
|
/** Rate or seek for media `offset` s behind (+) or ahead (-); a lag still growing (`settled` false) never seeks. */
|
||||||
|
export function steerMedia(
|
||||||
|
el: HTMLMediaElement,
|
||||||
|
offset: number,
|
||||||
|
baseRate: number,
|
||||||
|
settled: boolean,
|
||||||
|
): { rate: number; seek: boolean } {
|
||||||
|
const lag = Math.abs(offset);
|
||||||
|
if (el.tagName === "AUDIO" && settled && lag > AUDIO_STEER_LIMIT_SECONDS) {
|
||||||
|
steering.delete(el);
|
||||||
|
return { rate: baseRate, seek: true };
|
||||||
|
}
|
||||||
|
const direction = Math.sign(offset);
|
||||||
|
const current = steering.get(el);
|
||||||
|
if (lag > MEDIA_SYNC_TOLERANCE_SECONDS) steering.set(el, direction);
|
||||||
|
else if (current !== direction || lag <= MEDIA_STEER_RELEASE_SECONDS) {
|
||||||
|
steering.delete(el);
|
||||||
|
return { rate: baseRate, seek: false };
|
||||||
|
}
|
||||||
|
const step = MEDIA_STEER_STEPS.find(([from]) => lag > from)?.[1] ?? MEDIA_STEER;
|
||||||
|
const steer = el.tagName === "AUDIO" ? Math.min(step, AUDIO_MAX_STEER) : step;
|
||||||
|
return { rate: baseRate * (1 + direction * steer), seek: false };
|
||||||
|
}
|
||||||
|
|
||||||
|
/** Drop `el`'s steering: it was seeked, paused or replaced, so its old offset says nothing. */
|
||||||
|
export function forgetSteering(el: HTMLMediaElement): void {
|
||||||
|
steering.delete(el);
|
||||||
|
}
|
||||||
|
|
||||||
|
export function isSteering(el: HTMLMediaElement): boolean {
|
||||||
|
return steering.has(el);
|
||||||
|
}
|
||||||
@@ -1,6 +1,7 @@
|
|||||||
// fallow-ignore-file code-duplication complexity
|
// fallow-ignore-file code-duplication complexity
|
||||||
import { beforeEach, describe, it, expect, vi } from "vitest";
|
import { afterEach, beforeEach, describe, it, expect, vi } from "vitest";
|
||||||
import { MAX_AUDIO_GAIN } from "../audioGain.js";
|
import { MAX_AUDIO_GAIN } from "../audioGain.js";
|
||||||
|
import { mediaElementTime } from "./mediaClock";
|
||||||
import { WebAudioTransport } from "./webAudioTransport";
|
import { WebAudioTransport } from "./webAudioTransport";
|
||||||
|
|
||||||
function createMockAudioContext(currentTime = 100) {
|
function createMockAudioContext(currentTime = 100) {
|
||||||
@@ -70,6 +71,7 @@ const mockEl = {
|
|||||||
volume: 0.4,
|
volume: 0.4,
|
||||||
paused: true,
|
paused: true,
|
||||||
addEventListener: () => {},
|
addEventListener: () => {},
|
||||||
|
removeEventListener: () => {},
|
||||||
getAttribute: (name: string) => (name === "data-playback-rate" ? "1" : null),
|
getAttribute: (name: string) => (name === "data-playback-rate" ? "1" : null),
|
||||||
} as unknown as HTMLMediaElement;
|
} as unknown as HTMLMediaElement;
|
||||||
|
|
||||||
@@ -205,17 +207,12 @@ describe("WebAudioTransport", () => {
|
|||||||
expect(mock.ctx.createMediaElementSource).toHaveBeenCalledTimes(1);
|
expect(mock.ctx.createMediaElementSource).toHaveBeenCalledTimes(1);
|
||||||
});
|
});
|
||||||
|
|
||||||
it("re-aims FX automation on a global-rate change without resampling the native source", async () => {
|
it("does not resample the native source on a global-rate change", async () => {
|
||||||
const { transport, mock, gen } = setupTransport(100);
|
const { transport, mock, gen } = setupTransport(100);
|
||||||
await transport.scheduleMediaElementPlayback(mockEl, 0, 0, 0, 1, gen, 1);
|
await transport.scheduleMediaElementPlayback(mockEl, 0, 0, 0, 1, gen, 1);
|
||||||
const active = (transport as unknown as { _activeSources: { fx?: unknown }[] })
|
|
||||||
._activeSources;
|
|
||||||
const setRate = vi.fn();
|
|
||||||
active[0]!.fx = { dispose: vi.fn(), setRate };
|
|
||||||
|
|
||||||
transport.setRate(2);
|
transport.setRate(2);
|
||||||
|
|
||||||
expect(setRate).toHaveBeenCalledWith(2);
|
|
||||||
expect(mock.ctx.createBufferSource).not.toHaveBeenCalled();
|
expect(mock.ctx.createBufferSource).not.toHaveBeenCalled();
|
||||||
});
|
});
|
||||||
|
|
||||||
@@ -992,6 +989,7 @@ describe("WebAudioTransport", () => {
|
|||||||
const settle = () => new Promise((r) => setTimeout(r, 0));
|
const settle = () => new Promise((r) => setTimeout(r, 0));
|
||||||
|
|
||||||
// An unbounded media-element clip: the runtime does not stopAll and reschedule it on a rate change.
|
// An unbounded media-element clip: the runtime does not stopAll and reschedule it on a rate change.
|
||||||
|
// Its gains in creation order: element gain, seek dip, lane, clip gain (then fades, when it has them).
|
||||||
async function playBed(automation?: string, volume?: string) {
|
async function playBed(automation?: string, volume?: string) {
|
||||||
const setup = setupGroupTransport();
|
const setup = setupGroupTransport();
|
||||||
const el = groupedAudioEl("bed");
|
const el = groupedAudioEl("bed");
|
||||||
@@ -999,7 +997,7 @@ describe("WebAudioTransport", () => {
|
|||||||
if (volume) el.setAttribute("data-volume", volume);
|
if (volume) el.setAttribute("data-volume", volume);
|
||||||
await setup.transport.scheduleMediaElementPlayback(el, 0, 0, 0, 1, setup.gen);
|
await setup.transport.scheduleMediaElementPlayback(el, 0, 0, 0, 1, setup.gen);
|
||||||
expect(setup.transport.hasBoundedActiveSources()).toBe(false);
|
expect(setup.transport.hasBoundedActiveSources()).toBe(false);
|
||||||
return { ...setup, el, laneGain: setup.mock.gainNodes[1]! };
|
return { ...setup, el, laneGain: setup.mock.gainNodes[2]! };
|
||||||
}
|
}
|
||||||
|
|
||||||
// Gain 2 times a 0 -> 3 ramp crosses the ceiling at 6.635 s; the product holds there after.
|
// Gain 2 times a 0 -> 3 ramp crosses the ceiling at 6.635 s; the product holds there after.
|
||||||
@@ -1014,7 +1012,7 @@ describe("WebAudioTransport", () => {
|
|||||||
transport.setElementVolume(el, 1, 2);
|
transport.setElementVolume(el, 1, 2);
|
||||||
const booked = (laneGain.gain as unknown as { setValueAtTime: ReturnType<typeof vi.fn> })
|
const booked = (laneGain.gain as unknown as { setValueAtTime: ReturnType<typeof vi.fn> })
|
||||||
.setValueAtTime.mock.calls;
|
.setValueAtTime.mock.calls;
|
||||||
expect(mock.gainNodes[2]!.gain.value * booked[0]![0]).toBeCloseTo(MAX_AUDIO_GAIN, 9);
|
expect(mock.gainNodes[3]!.gain.value * booked[0]![0]).toBeCloseTo(MAX_AUDIO_GAIN, 9);
|
||||||
});
|
});
|
||||||
|
|
||||||
it("holds a clip's gain times its lane under the ceiling, with data-volume played ahead", async () => {
|
it("holds a clip's gain times its lane under the ceiling, with data-volume played ahead", async () => {
|
||||||
@@ -1049,7 +1047,7 @@ describe("WebAudioTransport", () => {
|
|||||||
const booked = (laneGain.gain as unknown as { setValueAtTime: ReturnType<typeof vi.fn> })
|
const booked = (laneGain.gain as unknown as { setValueAtTime: ReturnType<typeof vi.fn> })
|
||||||
.setValueAtTime.mock.calls;
|
.setValueAtTime.mock.calls;
|
||||||
expect(booked.at(-1)).toEqual([1, 100]);
|
expect(booked.at(-1)).toEqual([1, 100]);
|
||||||
expect(mock.gainNodes[2]!.gain.value * booked.at(-1)![0]).toBe(2);
|
expect(mock.gainNodes[3]!.gain.value * booked.at(-1)![0]).toBe(2);
|
||||||
});
|
});
|
||||||
|
|
||||||
it("restores the static group gain after its playing volume lane is removed", async () => {
|
it("restores the static group gain after its playing volume lane is removed", async () => {
|
||||||
@@ -1080,12 +1078,18 @@ describe("WebAudioTransport", () => {
|
|||||||
]);
|
]);
|
||||||
});
|
});
|
||||||
|
|
||||||
it("re-aims a clip's lane at a new rate", async () => {
|
it("re-aims a clip's lane at a new rate when its element takes it, not before", async () => {
|
||||||
const { transport, laneGain } = await playBed(FADE);
|
const { transport, el, laneGain } = await playBed(FADE);
|
||||||
expect(rampsAfter(laneGain, 0)).toContainEqual([1, 103]);
|
expect(rampsAfter(laneGain, 0)).toContainEqual([1, 103]);
|
||||||
const before = param(laneGain).linearRampToValueAtTime.mock.calls.length;
|
const before = param(laneGain).linearRampToValueAtTime.mock.calls.length;
|
||||||
|
|
||||||
transport.setRate(2);
|
transport.setRate(2);
|
||||||
|
expect(rampsAfter(laneGain, before)).toEqual([]);
|
||||||
|
Object.defineProperties(el, {
|
||||||
|
paused: { value: false, configurable: true },
|
||||||
|
playbackRate: { value: 2, configurable: true },
|
||||||
|
});
|
||||||
|
el.dispatchEvent(new Event("ratechange"));
|
||||||
|
|
||||||
expect(rampsAfter(laneGain, before)).toContainEqual([1, 101.5]);
|
expect(rampsAfter(laneGain, before)).toContainEqual([1, 101.5]);
|
||||||
});
|
});
|
||||||
@@ -1151,6 +1155,149 @@ describe("WebAudioTransport", () => {
|
|||||||
await settle();
|
await settle();
|
||||||
expect(rampsAfter(fader, before)).toContainEqual([0.25, 102]);
|
expect(rampsAfter(fader, before)).toContainEqual([0.25, 102]);
|
||||||
});
|
});
|
||||||
|
|
||||||
|
describe("a routed clip's lane and fades follow its element", () => {
|
||||||
|
// Routed at context 100, 0.1 s before its window opens at timeline 4 and source 8.
|
||||||
|
async function routeTake(extra: Record<string, string> = {}) {
|
||||||
|
const setup = setupGroupTransport();
|
||||||
|
const el = groupedAudioEl("take");
|
||||||
|
el.setAttribute("data-automation", FADE);
|
||||||
|
for (const [name, value] of Object.entries(extra)) el.setAttribute(name, value);
|
||||||
|
await setup.transport.scheduleMediaElementPlayback(el, 4, 8, 3.9, 1, setup.gen, 1, 2);
|
||||||
|
const [laneGain, fadeGain] = [setup.mock.gainNodes[2]!, setup.mock.gainNodes[4]];
|
||||||
|
const at = (ctxTime: number, currentTime: number, playbackRate = 1, paused = false) => {
|
||||||
|
setup.mock.ctx.currentTime = ctxTime;
|
||||||
|
Object.defineProperties(el, {
|
||||||
|
currentTime: { value: currentTime, configurable: true },
|
||||||
|
playbackRate: { value: playbackRate, configurable: true },
|
||||||
|
paused: { value: paused, configurable: true },
|
||||||
|
});
|
||||||
|
};
|
||||||
|
return { ...setup, el, laneGain, fadeGain, at };
|
||||||
|
}
|
||||||
|
const peakAfter = (node: { gain: unknown }, from: number) =>
|
||||||
|
rampsAfter(node, from).find(([value]) => value === 1)?.[1];
|
||||||
|
|
||||||
|
it("books the lane where the element starts sounding, not where the clock put it", async () => {
|
||||||
|
const { el, laneGain, at } = await routeTake();
|
||||||
|
expect(peakAfter(laneGain, 0)).toBeCloseTo(103.1, 9);
|
||||||
|
|
||||||
|
const before = param(laneGain).linearRampToValueAtTime.mock.calls.length;
|
||||||
|
at(100.155, 8);
|
||||||
|
el.dispatchEvent(new Event("playing"));
|
||||||
|
|
||||||
|
expect(peakAfter(laneGain, before)).toBeCloseTo(103.155, 9);
|
||||||
|
});
|
||||||
|
|
||||||
|
it("re-books the fades from the element too", async () => {
|
||||||
|
const { el, fadeGain, at } = await routeTake({ "data-fade-out": "0.03" });
|
||||||
|
expect(rampsAfter(fadeGain!, 0).at(-1)).toEqual([0, expect.closeTo(102.1, 9)]);
|
||||||
|
|
||||||
|
const before = param(fadeGain!).linearRampToValueAtTime.mock.calls.length;
|
||||||
|
at(100.155, 8);
|
||||||
|
el.dispatchEvent(new Event("playing"));
|
||||||
|
|
||||||
|
expect(rampsAfter(fadeGain!, before).at(-1)).toEqual([0, expect.closeTo(102.155, 9)]);
|
||||||
|
});
|
||||||
|
|
||||||
|
it("re-books the lane after the element is seeked and when its rate is steered", async () => {
|
||||||
|
const { el, laneGain, at } = await routeTake();
|
||||||
|
let before = param(laneGain).linearRampToValueAtTime.mock.calls.length;
|
||||||
|
at(100.2, 8.2);
|
||||||
|
el.dispatchEvent(new Event("seeked"));
|
||||||
|
expect(peakAfter(laneGain, before)).toBeCloseTo(103, 9);
|
||||||
|
|
||||||
|
before = param(laneGain).linearRampToValueAtTime.mock.calls.length;
|
||||||
|
at(100.3, 8.3, 1.03);
|
||||||
|
el.dispatchEvent(new Event("ratechange"));
|
||||||
|
expect(peakAfter(laneGain, before)).toBeCloseTo(100.3 + 2.7 / 1.03, 9);
|
||||||
|
});
|
||||||
|
|
||||||
|
it("books a looping element's lane on the pass the transport is in", async () => {
|
||||||
|
const { el, laneGain, at } = await routeTake();
|
||||||
|
Object.defineProperties(el, {
|
||||||
|
loop: { value: true, configurable: true },
|
||||||
|
duration: { value: 10, configurable: true },
|
||||||
|
});
|
||||||
|
const before = param(laneGain).linearRampToValueAtTime.mock.calls.length;
|
||||||
|
at(102.2, 8.1);
|
||||||
|
el.dispatchEvent(new Event("playing"));
|
||||||
|
|
||||||
|
expect(peakAfter(laneGain, before)).toBeCloseTo(103.1, 9);
|
||||||
|
});
|
||||||
|
|
||||||
|
it("leaves the lane alone while the element is paused and once the clip is stopped", async () => {
|
||||||
|
const { transport, el, laneGain, fadeGain, at } = await routeTake({
|
||||||
|
"data-fade-out": "0.03",
|
||||||
|
});
|
||||||
|
const booked = () =>
|
||||||
|
[laneGain, fadeGain!].map(
|
||||||
|
(node) => param(node).linearRampToValueAtTime.mock.calls.length,
|
||||||
|
);
|
||||||
|
const before = booked();
|
||||||
|
at(100.2, 8, 1, true);
|
||||||
|
el.dispatchEvent(new Event("seeked"));
|
||||||
|
transport.stopAll();
|
||||||
|
at(100.3, 8.1);
|
||||||
|
el.dispatchEvent(new Event("playing"));
|
||||||
|
|
||||||
|
expect(booked()).toEqual(before);
|
||||||
|
});
|
||||||
|
|
||||||
|
describe("a seek while it sounds", () => {
|
||||||
|
beforeEach(() => vi.useFakeTimers());
|
||||||
|
afterEach(() => vi.useRealTimers());
|
||||||
|
type Dip = Record<"linearRampToValueAtTime", ReturnType<typeof vi.fn>>;
|
||||||
|
const dipRamps = (node: { gain: unknown }) =>
|
||||||
|
(node.gain as Dip).linearRampToValueAtTime.mock.calls;
|
||||||
|
|
||||||
|
it("dips, seeks once the dip is down, and comes back up when the element lands", async () => {
|
||||||
|
const { transport, el, mock, at } = await routeTake();
|
||||||
|
const dip = mock.gainNodes[1]!;
|
||||||
|
at(100.5, 8.35);
|
||||||
|
const seek = vi.fn();
|
||||||
|
Object.defineProperty(el, "currentTime", {
|
||||||
|
get: () => 8.35,
|
||||||
|
set: seek,
|
||||||
|
configurable: true,
|
||||||
|
});
|
||||||
|
|
||||||
|
expect(transport.seekUnderDip(el, 8.5)).toBe(true);
|
||||||
|
expect(dipRamps(dip)).toEqual([[0, expect.closeTo(100.515, 9)]]);
|
||||||
|
expect(transport.seekUnderDip(el, 8.51)).toBe(true);
|
||||||
|
vi.advanceTimersByTime(14);
|
||||||
|
expect(seek).not.toHaveBeenCalled();
|
||||||
|
vi.advanceTimersByTime(1);
|
||||||
|
expect(seek).toHaveBeenCalledExactlyOnceWith(mediaElementTime(8.5 + 0.015));
|
||||||
|
|
||||||
|
mock.ctx.currentTime = 100.54;
|
||||||
|
el.dispatchEvent(new Event("seeked"));
|
||||||
|
expect(dipRamps(dip).at(-1)).toEqual([1, expect.closeTo(100.555, 9)]);
|
||||||
|
el.dispatchEvent(new Event("seeked"));
|
||||||
|
expect(dipRamps(dip)).toHaveLength(2);
|
||||||
|
});
|
||||||
|
|
||||||
|
it("leaves the element alone when the clip is stopped during the dip", async () => {
|
||||||
|
const { transport, el, at } = await routeTake();
|
||||||
|
at(100.5, 8.35);
|
||||||
|
const seek = vi.fn();
|
||||||
|
Object.defineProperty(el, "currentTime", {
|
||||||
|
get: () => 8.35,
|
||||||
|
set: seek,
|
||||||
|
configurable: true,
|
||||||
|
});
|
||||||
|
transport.seekUnderDip(el, 8.5);
|
||||||
|
transport.stopAll();
|
||||||
|
vi.advanceTimersByTime(15);
|
||||||
|
expect(seek).not.toHaveBeenCalled();
|
||||||
|
});
|
||||||
|
|
||||||
|
it("has no dip for an element it does not route", () => {
|
||||||
|
const { transport } = setupGroupTransport();
|
||||||
|
expect(transport.seekUnderDip(groupedAudioEl("loose"), 1)).toBe(false);
|
||||||
|
});
|
||||||
|
});
|
||||||
|
});
|
||||||
});
|
});
|
||||||
|
|
||||||
it("routes an ungrouped clip straight to master through its own gain", async () => {
|
it("routes an ungrouped clip straight to master through its own gain", async () => {
|
||||||
@@ -1219,7 +1366,7 @@ describe("WebAudioTransport", () => {
|
|||||||
|
|
||||||
transport.setElementVolume(el, 0.8, 0.5);
|
transport.setElementVolume(el, 0.8, 0.5);
|
||||||
|
|
||||||
const [gain, , clipGain] = mock.gainNodes;
|
const [gain, , , clipGain] = mock.gainNodes;
|
||||||
expect(clipGain!.gain.value).toBe(0.5);
|
expect(clipGain!.gain.value).toBe(0.5);
|
||||||
expect(gain!.gain.value).toBe(0.8);
|
expect(gain!.gain.value).toBe(0.8);
|
||||||
});
|
});
|
||||||
@@ -1230,10 +1377,14 @@ describe("WebAudioTransport", () => {
|
|||||||
|
|
||||||
await transport.scheduleMediaElementPlayback(el, 0, 0, 0, 1, gen, 1);
|
await transport.scheduleMediaElementPlayback(el, 0, 0, 0, 1, gen, 1);
|
||||||
|
|
||||||
expect(mock.gainNodes).toHaveLength(3);
|
// Element gain to master, seek dip, lane into the element gain, clip gain into the lane.
|
||||||
expect(mock.gainNodes[0]!.connect).toHaveBeenCalledWith(mock.masterGain);
|
const [gain, dip, lane, clipGain] = mock.gainNodes;
|
||||||
expect(mock.gainNodes[1]!.connect).toHaveBeenCalledWith(mock.gainNodes[0]);
|
expect(mock.gainNodes).toHaveLength(4);
|
||||||
expect(mock.gainNodes[2]!.connect).toHaveBeenCalledWith(mock.gainNodes[1]);
|
expect(gain!.connect).toHaveBeenCalledWith(mock.masterGain);
|
||||||
|
expect(lane!.connect).toHaveBeenCalledWith(gain);
|
||||||
|
expect(clipGain!.connect).toHaveBeenCalledWith(lane);
|
||||||
|
expect(mock.mediaElementSource.connect).toHaveBeenCalledWith(dip);
|
||||||
|
expect(dip!.connect).toHaveBeenCalledWith(clipGain);
|
||||||
});
|
});
|
||||||
|
|
||||||
it("two members of the same group land on ONE shared group gain, not master directly", async () => {
|
it("two members of the same group land on ONE shared group gain, not master directly", async () => {
|
||||||
@@ -1810,9 +1961,9 @@ describe("WebAudioTransport clip fades on the audio clock", () => {
|
|||||||
expect(gains[0]!.gain.value).toBeCloseTo(0.4, 9);
|
expect(gains[0]!.gain.value).toBeCloseTo(0.4, 9);
|
||||||
});
|
});
|
||||||
|
|
||||||
it("a rate change rebooks the fades on the new rate from where the clip is", async () => {
|
it("a rate change rebooks a decoded clip's fades on the new rate from where the clip is", async () => {
|
||||||
const { transport, mock, gen, booked } = setup(100);
|
const { transport, mock, gen, booked } = setup(100);
|
||||||
await transport.scheduleMediaElementPlayback(faded("0.02", "1"), 5, 0, 5, 1, gen, 2, 10);
|
await transport.schedulePlayback(faded("0.02", "1"), mockBuffer, 5, 0, () => 5, 1, gen, 2, 10);
|
||||||
const gain = booked()!;
|
const gain = booked()!;
|
||||||
mock.ctx.currentTime = 101;
|
mock.ctx.currentTime = 101;
|
||||||
transport.setRate(1);
|
transport.setRate(1);
|
||||||
|
|||||||
@@ -17,9 +17,11 @@ import { swallow } from "./diagnostics";
|
|||||||
import { createLevelTap, type LevelTap, type StereoLevel } from "./levelTap.js";
|
import { createLevelTap, type LevelTap, type StereoLevel } from "./levelTap.js";
|
||||||
import { clampAudioGain } from "../audioGain.js";
|
import { clampAudioGain } from "../audioGain.js";
|
||||||
import { getDebugSurface } from "./globals.js";
|
import { getDebugSurface } from "./globals.js";
|
||||||
import { readElementPlaybackRate } from "./media.js";
|
import { readElementPlaybackRate, readElementRateSpec } from "./media.js";
|
||||||
|
import { rateAt, sourceTimeAt, timeAtSourceTime } from "../speedRamp.js";
|
||||||
import { classifyWebAudioMediaRoute, reportWebAudioMediaRoute } from "./webAudioRoute.js";
|
import { classifyWebAudioMediaRoute, reportWebAudioMediaRoute } from "./webAudioRoute.js";
|
||||||
import { assetUrl } from "./assetUrl.js";
|
import { assetUrl } from "./assetUrl.js";
|
||||||
|
import { mediaElementTime } from "./mediaClock.js";
|
||||||
import { clampFadesToDuration, fadeGain, readElementFades } from "../audioFade.js";
|
import { clampFadesToDuration, fadeGain, readElementFades } from "../audioFade.js";
|
||||||
|
|
||||||
const MASTER_FX_ATTRS: FxAttributes = { automation: null };
|
const MASTER_FX_ATTRS: FxAttributes = { automation: null };
|
||||||
@@ -177,6 +179,11 @@ type ScheduledSourceBase = {
|
|||||||
fx?: ElementFxHandle | null;
|
fx?: ElementFxHandle | null;
|
||||||
/** The clip's fades booked on the audio clock, between the source and its clip gain; absent without a fade. */
|
/** The clip's fades booked on the audio clock, between the source and its clip gain; absent without a fade. */
|
||||||
fades?: BookedFades | null;
|
fades?: BookedFades | null;
|
||||||
|
/** Stops re-aiming the lane and fades at a routed element's own position. */
|
||||||
|
unfollow?: () => void;
|
||||||
|
/** A routed element's first stage: unity except while `seekUnderDip` hides a seek. */
|
||||||
|
dipNode?: GainNode;
|
||||||
|
dipping?: boolean;
|
||||||
compositionStart: number;
|
compositionStart: number;
|
||||||
mediaStart: number;
|
mediaStart: number;
|
||||||
scheduledAt: number;
|
scheduledAt: number;
|
||||||
@@ -211,6 +218,13 @@ const decodeKey = (el: HTMLMediaElement): string | null => {
|
|||||||
}
|
}
|
||||||
};
|
};
|
||||||
|
|
||||||
|
const DIP_SECONDS = 0.015;
|
||||||
|
/** A dipped seek comes back up when the element lands, or when it can no longer land. */
|
||||||
|
const DIP_END_EVENTS = ["seeked", "error", "emptied", "abort"] as const;
|
||||||
|
|
||||||
|
/** Where a routed element starts sounding or jumps; a pause needs nothing, no audio flows until `playing`. */
|
||||||
|
const FOLLOWED_EVENTS = ["playing", "seeked", "ratechange"] as const;
|
||||||
|
|
||||||
// A pause to scrub or a reschedule mid-play keeps the output awake; a real idle lets it sleep.
|
// A pause to scrub or a reschedule mid-play keeps the output awake; a real idle lets it sleep.
|
||||||
const IDLE_SUSPEND_MS = 10000;
|
const IDLE_SUSPEND_MS = 10000;
|
||||||
|
|
||||||
@@ -479,7 +493,7 @@ export class WebAudioTransport {
|
|||||||
async scheduleMediaElementPlayback(
|
async scheduleMediaElementPlayback(
|
||||||
el: HTMLMediaElement,
|
el: HTMLMediaElement,
|
||||||
compositionStart: number,
|
compositionStart: number,
|
||||||
_mediaStart: number,
|
mediaStart: number,
|
||||||
compositionTime: number,
|
compositionTime: number,
|
||||||
volume: number,
|
volume: number,
|
||||||
generation: number,
|
generation: number,
|
||||||
@@ -511,10 +525,12 @@ export class WebAudioTransport {
|
|||||||
gainNode.connect(
|
gainNode.connect(
|
||||||
this.resolveDestination(el, scheduledAt, compositionTime, safeRate) ?? this._masterGain,
|
this.resolveDestination(el, scheduledAt, compositionTime, safeRate) ?? this._masterGain,
|
||||||
);
|
);
|
||||||
|
const dipNode = this._ctx.createGain();
|
||||||
|
sourceNode.connect(dipNode);
|
||||||
const { fx, laneNode, clipGainNode, fades } = attachClipStages(
|
const { fx, laneNode, clipGainNode, fades } = attachClipStages(
|
||||||
this._ctx,
|
this._ctx,
|
||||||
el,
|
el,
|
||||||
sourceNode,
|
dipNode,
|
||||||
gainNode,
|
gainNode,
|
||||||
timing,
|
timing,
|
||||||
{ from: Math.max(elapsed, 0), clipDuration },
|
{ from: Math.max(elapsed, 0), clipDuration },
|
||||||
@@ -527,6 +543,7 @@ export class WebAudioTransport {
|
|||||||
const scheduled: ScheduledSource = {
|
const scheduled: ScheduledSource = {
|
||||||
fx,
|
fx,
|
||||||
fades,
|
fades,
|
||||||
|
dipNode,
|
||||||
el,
|
el,
|
||||||
sourceNode,
|
sourceNode,
|
||||||
sourceKind: "media-element",
|
sourceKind: "media-element",
|
||||||
@@ -534,12 +551,13 @@ export class WebAudioTransport {
|
|||||||
laneNode,
|
laneNode,
|
||||||
clipGainNode,
|
clipGainNode,
|
||||||
compositionStart,
|
compositionStart,
|
||||||
mediaStart: _mediaStart,
|
mediaStart,
|
||||||
scheduledAt,
|
scheduledAt,
|
||||||
priorMuted: el.muted,
|
priorMuted: el.muted,
|
||||||
priorVolume: el.volume,
|
priorVolume: el.volume,
|
||||||
mediaPlaybackRate: readElementPlaybackRate(el),
|
mediaPlaybackRate: readElementPlaybackRate(el),
|
||||||
bounded: false,
|
bounded: false,
|
||||||
|
unfollow: this.followElement(el, compositionStart, mediaStart, fx, fades),
|
||||||
};
|
};
|
||||||
// Chrome applies HTMLMediaElement.volume before MediaElementAudioSource.
|
// Chrome applies HTMLMediaElement.volume before MediaElementAudioSource.
|
||||||
// Keep that upstream stage at unity so the existing downstream gain is
|
// Keep that upstream stage at unity so the existing downstream gain is
|
||||||
@@ -554,6 +572,79 @@ export class WebAudioTransport {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* A routed element keeps its own time: it starts late, is seeked and is steered by rate. Its lane and fades were
|
||||||
|
* booked against the context clock, so each time it starts or moves they are re-aimed at where it actually is.
|
||||||
|
*/
|
||||||
|
private followElement(
|
||||||
|
el: HTMLMediaElement,
|
||||||
|
compositionStart: number,
|
||||||
|
mediaStart: number,
|
||||||
|
fx: ElementFxHandle | null,
|
||||||
|
fades: BookedFades | null,
|
||||||
|
): () => void {
|
||||||
|
const retime = () => {
|
||||||
|
if (el.paused || !this._ctx) return;
|
||||||
|
const spec = readElementRateSpec(el);
|
||||||
|
let source = el.currentTime - mediaStart;
|
||||||
|
const loopLength = el.duration - mediaStart;
|
||||||
|
// A looping element's position wraps: it is on the pass nearest where the transport expects it.
|
||||||
|
if (el.loop && loopLength > 0) {
|
||||||
|
const expected = sourceTimeAt(spec, this.getTime() - compositionStart);
|
||||||
|
source += Math.round((expected - source) / loopLength) * loopLength;
|
||||||
|
}
|
||||||
|
const elapsed = timeAtSourceTime(spec, source);
|
||||||
|
const rate = el.playbackRate / rateAt(spec, elapsed);
|
||||||
|
// Stopped at rate 0, or with no position yet: no audio moves, so there is nothing to aim at.
|
||||||
|
if (!(rate > 0) || !Number.isFinite(elapsed)) return;
|
||||||
|
const timing: AutomationTiming = { scheduledAt: this._ctx.currentTime, elapsed, rate };
|
||||||
|
fx?.retime(timing);
|
||||||
|
fades?.rebook(timing);
|
||||||
|
};
|
||||||
|
for (const type of FOLLOWED_EVENTS) el.addEventListener(type, retime);
|
||||||
|
return () => {
|
||||||
|
for (const type of FOLLOWED_EVENTS) el.removeEventListener(type, retime);
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Seek a sounding routed element under a short gain dip: down, seek, and back up once it has landed, so the jump
|
||||||
|
* is neither a click nor a tempo change. False when the element is not routed and has no dip to hide it under.
|
||||||
|
*/
|
||||||
|
seekUnderDip(el: HTMLMediaElement, target: number): boolean {
|
||||||
|
const source = this._activeSources.find((s) => s.el === el && s.sourceKind === "media-element");
|
||||||
|
const ctx = this._ctx;
|
||||||
|
if (!ctx || !source?.dipNode) return false;
|
||||||
|
if (source.dipping) return true;
|
||||||
|
source.dipping = true;
|
||||||
|
const gain = source.dipNode.gain;
|
||||||
|
const ramp = (to: number) => {
|
||||||
|
const now = ctx.currentTime;
|
||||||
|
gain.cancelScheduledValues(now);
|
||||||
|
gain.setValueAtTime(gain.value, now);
|
||||||
|
gain.linearRampToValueAtTime(to, now + DIP_SECONDS);
|
||||||
|
};
|
||||||
|
const rise = () => {
|
||||||
|
for (const type of DIP_END_EVENTS) el.removeEventListener(type, rise);
|
||||||
|
source.dipping = false;
|
||||||
|
ramp(1);
|
||||||
|
};
|
||||||
|
ramp(0);
|
||||||
|
setTimeout(() => {
|
||||||
|
// A stop or a new pass in the meantime owns the element now.
|
||||||
|
if (!this._activeSources.includes(source)) return;
|
||||||
|
for (const type of DIP_END_EVENTS) el.addEventListener(type, rise);
|
||||||
|
try {
|
||||||
|
// Where the element would be by now had it been on time.
|
||||||
|
el.currentTime = mediaElementTime(target + DIP_SECONDS * el.playbackRate);
|
||||||
|
} catch (err) {
|
||||||
|
swallow("webAudioTransport.seekUnderDip", err);
|
||||||
|
rise();
|
||||||
|
}
|
||||||
|
}, DIP_SECONDS * 1000);
|
||||||
|
return true;
|
||||||
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* The gain a grouped member's signal should land on, building it on first
|
* The gain a grouped member's signal should land on, building it on first
|
||||||
* use. A group's clock is COMPOSITION time (design doc §1.3) — it has no
|
* use. A group's clock is COMPOSITION time (design doc §1.3) — it has no
|
||||||
@@ -923,7 +1014,7 @@ export class WebAudioTransport {
|
|||||||
* start in the future keep their original wallclock start time — callers
|
* start in the future keep their original wallclock start time — callers
|
||||||
* that need rate-correct future starts should `stopAll()` and reschedule.
|
* that need rate-correct future starts should `stopAll()` and reschedule.
|
||||||
*
|
*
|
||||||
* Each source's FX automation is re-aimed too. Lanes are committed to
|
* Each decoded source's FX automation is re-aimed too. Lanes are committed to
|
||||||
* absolute context times when the source is scheduled, so bumping only
|
* absolute context times when the source is scheduled, so bumping only
|
||||||
* `playbackRate` left every automated parameter running its original plan
|
* `playbackRate` left every automated parameter running its original plan
|
||||||
* over audio moving at a different speed. The `stopAll()`+reschedule recovery
|
* over audio moving at a different speed. The `stopAll()`+reschedule recovery
|
||||||
@@ -939,11 +1030,11 @@ export class WebAudioTransport {
|
|||||||
this._rateAnchorCtx = this._ctx.currentTime;
|
this._rateAnchorCtx = this._ctx.currentTime;
|
||||||
}
|
}
|
||||||
this._rate = safeRate;
|
this._rate = safeRate;
|
||||||
|
// A routed element is not touched: its rate change re-aims its own lane and fades (`followElement`).
|
||||||
for (const source of this._activeSources) {
|
for (const source of this._activeSources) {
|
||||||
|
if (!isBufferSource(source)) continue;
|
||||||
try {
|
try {
|
||||||
if (isBufferSource(source)) {
|
source.sourceNode.playbackRate.value = safeRate * source.mediaPlaybackRate;
|
||||||
source.sourceNode.playbackRate.value = safeRate * source.mediaPlaybackRate;
|
|
||||||
}
|
|
||||||
source.fx?.setRate(safeRate);
|
source.fx?.setRate(safeRate);
|
||||||
if (this._ctx && !this._paused)
|
if (this._ctx && !this._paused)
|
||||||
source.fades?.rebook({
|
source.fades?.rebook({
|
||||||
@@ -998,11 +1089,13 @@ export class WebAudioTransport {
|
|||||||
}
|
}
|
||||||
|
|
||||||
private dispose(source: ScheduledSource, keepsSounding = true): void {
|
private dispose(source: ScheduledSource, keepsSounding = true): void {
|
||||||
|
source.unfollow?.();
|
||||||
try {
|
try {
|
||||||
if (isBufferSource(source)) source.sourceNode.stop();
|
if (isBufferSource(source)) source.sourceNode.stop();
|
||||||
source.sourceNode.disconnect();
|
source.sourceNode.disconnect();
|
||||||
source.fx?.dispose();
|
source.fx?.dispose();
|
||||||
source.fades?.node.disconnect();
|
source.fades?.node.disconnect();
|
||||||
|
source.dipNode?.disconnect();
|
||||||
source.clipGainNode.disconnect();
|
source.clipGainNode.disconnect();
|
||||||
source.laneNode.disconnect();
|
source.laneNode.disconnect();
|
||||||
source.gainNode.disconnect();
|
source.gainNode.disconnect();
|
||||||
|
|||||||
Reference in New Issue
Block a user