ext/hyperframes
2
0
Fork 0
mirror of https://github.com/heygen-com/hyperframes.git synced 2026-09-03 04:38:33 +00:00
Code Issues Packages Projects Releases Wiki Activity

fix(slideshow): harden media controls in present decks (#1619)

Browse Source
This commit is contained in:
Vance Ingalls
2026-06-20 23:35:58 -07:00
committed by GitHub
parent 0d2198e654
commit 341e65aea2
16 changed files with 1713 additions and 81 deletions
Download Patch File Download Diff File Expand all files Collapse all files
packages/core/src/runtime/bridge.test.ts
+34
View File
@@ -11,6 +11,8 @@ function createMockDeps() {
onSetMuted: vi.fn(),
onSetVolume: vi.fn(),
onSetMediaOutputMuted: vi.fn(),
onSetNativeMediaSyncDisabled: vi.fn(),
onSetWebAudioMediaDisabled: vi.fn(),
onSetPlaybackRate: vi.fn(),
onSetColorGrading: vi.fn(),
onSetColorGradingCompare: vi.fn(),
@@ -107,6 +109,38 @@ describe("installRuntimeControlBridge", () => {
expect(deps.onSetMediaOutputMuted).toHaveBeenCalledWith(false);
});
it("dispatches set-native-media-sync-disabled command", () => {
const deps = createMockDeps();
const handler = installRuntimeControlBridge(deps);
handler(makeControlMessage("set-native-media-sync-disabled", { disabled: true }));
expect(deps.onSetNativeMediaSyncDisabled).toHaveBeenCalledWith(true);
handler(makeControlMessage("set-native-media-sync-disabled", { disabled: false }));
expect(deps.onSetNativeMediaSyncDisabled).toHaveBeenCalledWith(false);
});
it("set-native-media-sync-disabled coerces absent flag to false", () => {
const deps = createMockDeps();
const handler = installRuntimeControlBridge(deps);
handler(makeControlMessage("set-native-media-sync-disabled"));
expect(deps.onSetNativeMediaSyncDisabled).toHaveBeenCalledWith(false);
});
it("dispatches set-web-audio-media-disabled command", () => {
const deps = createMockDeps();
const handler = installRuntimeControlBridge(deps);
handler(makeControlMessage("set-web-audio-media-disabled", { disabled: true }));
expect(deps.onSetWebAudioMediaDisabled).toHaveBeenCalledWith(true);
handler(makeControlMessage("set-web-audio-media-disabled", { disabled: false }));
expect(deps.onSetWebAudioMediaDisabled).toHaveBeenCalledWith(false);
});
it("set-web-audio-media-disabled coerces absent flag to false", () => {
const deps = createMockDeps();
const handler = installRuntimeControlBridge(deps);
handler(makeControlMessage("set-web-audio-media-disabled"));
expect(deps.onSetWebAudioMediaDisabled).toHaveBeenCalledWith(false);
});
it("dispatches set-playback-rate command", () => {
const deps = createMockDeps();
const handler = installRuntimeControlBridge(deps);
packages/core/src/runtime/bridge.ts
+6
View File
@@ -11,6 +11,8 @@ type BridgeDeps = {
onSetMuted: (muted: boolean) => void;
onSetVolume: (volume: number) => void;
onSetMediaOutputMuted: (muted: boolean) => void;
onSetNativeMediaSyncDisabled: (disabled: boolean) => void;
onSetWebAudioMediaDisabled: (disabled: boolean) => void;
onSetPlaybackRate: (rate: number) => void;
onSetColorGrading: (target: HfColorGradingTarget | string | null, grading: unknown) => void;
onSetColorGradingCompare: (
@@ -46,6 +48,10 @@ const CONTROL_HANDLERS: Record<string, ControlHandler> = {
"set-volume": (data, deps) =>
deps.onSetVolume(Math.max(0, Math.min(1, Number(data.volume ?? 1)))),
"set-media-output-muted": (data, deps) => deps.onSetMediaOutputMuted(Boolean(data.muted)),
"set-native-media-sync-disabled": (data, deps) =>
deps.onSetNativeMediaSyncDisabled(Boolean(data.disabled)),
"set-web-audio-media-disabled": (data, deps) =>
deps.onSetWebAudioMediaDisabled(Boolean(data.disabled)),
"set-playback-rate": (data, deps) => deps.onSetPlaybackRate(Number(data.playbackRate ?? 1)),
"set-color-grading": (data, deps) =>
deps.onSetColorGrading(data.target ?? null, data.grading ?? null),
packages/core/src/runtime/init.test.ts
+51
View File
@@ -1147,6 +1147,57 @@ describe("initSandboxRuntimeModular", () => {
expect(audio.muted).toBe(false);
});
it("native media sync opt-out leaves user-started media playing while timeline is paused", () => {
const root = document.createElement("div");
root.setAttribute("data-composition-id", "root");
root.setAttribute("data-root", "true");
root.setAttribute("data-start", "0");
root.setAttribute("data-duration", "10");
root.setAttribute("data-width", "1920");
root.setAttribute("data-height", "1080");
document.body.appendChild(root);
const audio = document.createElement("audio");
audio.setAttribute("data-start", "0");
audio.setAttribute("data-duration", "10");
audio.setAttribute("src", "voiceover.mp3");
Object.defineProperty(audio, "duration", { value: 10, configurable: true });
Object.defineProperty(audio, "readyState", {
value: HTMLMediaElement.HAVE_FUTURE_DATA,
configurable: true,
});
Object.defineProperty(audio, "currentTime", { value: 0, writable: true, configurable: true });
Object.defineProperty(audio, "paused", { value: true, writable: true, configurable: true });
audio.pause = vi.fn(() => {
Object.defineProperty(audio, "paused", {
value: true,
writable: true,
configurable: true,
});
});
root.appendChild(audio);
window.__timelines = { root: createMockTimeline(10) };
initSandboxRuntimeModular();
window.dispatchEvent(
new MessageEvent("message", {
data: {
source: "hf-parent",
type: "control",
action: "set-native-media-sync-disabled",
disabled: true,
},
}),
);
Object.defineProperty(audio, "paused", { value: false, writable: true, configurable: true });
vi.mocked(audio.pause).mockClear();
window.__player?.renderSeek(5);
expect(audio.pause).not.toHaveBeenCalled();
});
it("skips the per-frame transport re-seek while a Studio manual-edit gesture is active", () => {
const raf = createManualRaf();
vi.spyOn(performance, "now").mockImplementation(() => raf.now());
packages/core/src/runtime/init.ts
+62 -20
View File
@@ -1501,23 +1501,27 @@ export function initSandboxRuntimeModular(): void {
const forceSync = state.mediaForceSyncNextTick;
if (forceSync) state.mediaForceSyncNextTick = false;
syncRuntimeMedia({
clips: cache.mediaClips,
timeSeconds: state.currentTime,
playing: state.isPlaying,
playbackRate: state.playbackRate,
outputMuted: state.mediaOutputMuted,
userMuted: state.bridgeMuted,
userVolume: state.bridgeVolume,
forceSync,
onElementVolume: (el, volume) => webAudio.setElementVolume(el, volume),
isWebAudioOwned: (el) => webAudio.ownsElement(el),
onAutoplayBlocked: () => {
if (state.mediaAutoplayBlockedPosted) return;
state.mediaAutoplayBlockedPosted = true;
postRuntimeMessage({ source: "hf-preview", type: "media-autoplay-blocked" });
},
});
if (!state.nativeMediaSyncDisabled) {
syncRuntimeMedia({
clips: cache.mediaClips,
timeSeconds: state.currentTime,
playing: state.isPlaying,
playbackRate: state.playbackRate,
outputMuted:
state.mediaOutputMuted ||
(!state.webAudioMediaDisabled && !state.nativeMediaSyncDisabled && webAudio.isActive()),
userMuted: state.bridgeMuted,
userVolume: state.bridgeVolume,
forceSync,
onElementVolume: (el, volume) => webAudio.setElementVolume(el, volume),
isWebAudioOwned: (el) => webAudio.ownsElement(el),
onAutoplayBlocked: () => {
if (state.mediaAutoplayBlockedPosted) return;
state.mediaAutoplayBlockedPosted = true;
postRuntimeMessage({ source: "hf-preview", type: "media-autoplay-blocked" });
},
});
}
const visibilityNodes = Array.from(document.querySelectorAll("[data-start]"));
const rootComp = resolveRootCompositionElement();
for (const rawNode of visibilityNodes) {
@@ -1882,6 +1886,29 @@ export function initSandboxRuntimeModular(): void {
el.muted = effective || el.defaultMuted;
}
},
onSetNativeMediaSyncDisabled: (disabled) => {
if (state.nativeMediaSyncDisabled === disabled) return;
state.nativeMediaSyncDisabled = disabled;
state.mediaForceSyncNextTick = true;
if (disabled) {
webAudio.stopAll();
clock.detachAudioSource();
} else {
syncMediaForCurrentState();
}
},
onSetWebAudioMediaDisabled: (disabled) => {
if (state.webAudioMediaDisabled === disabled) return;
state.webAudioMediaDisabled = disabled;
state.mediaForceSyncNextTick = true;
if (disabled) {
webAudio.stopAll();
clock.detachAudioSource();
syncMediaForCurrentState();
} else {
syncMediaForCurrentState();
}
},
onSetPlaybackRate: (rate) => {
applyPlaybackRate(rate);
if (state.transportClock) state.transportClock.setRate(state.playbackRate);
@@ -2201,7 +2228,12 @@ export function initSandboxRuntimeModular(): void {
// 2. HTMLMediaElement (audio.currentTime): ~33ms, frame-accurate
// 3. Monotonic (performance.now()): ~1ms, no audio coupling
if (clock.isPlaying() && !state.mediaOutputMuted) {
if (webAudio.isActive() && webAudio.context) {
if (
!state.nativeMediaSyncDisabled &&
!state.webAudioMediaDisabled &&
webAudio.isActive() &&
webAudio.context
) {
const webAudioTime = webAudio.getTime();
if (webAudioTime >= 0) {
clock.attachAudioSource({ currentTimeSeconds: webAudioTime });
@@ -2313,6 +2345,7 @@ export function initSandboxRuntimeModular(): void {
// same edge as the HTMLMedia path. Reused by play() and by the rate-change
// handler (a rate change can't rescale a bounded source in place).
const scheduleWebAudioForActiveClips = () => {
if (state.nativeMediaSyncDisabled || state.webAudioMediaDisabled) return;
const gen = webAudio.startGeneration();
const audioEls = document.querySelectorAll("audio[data-start]");
for (const rawEl of audioEls) {
@@ -2362,7 +2395,14 @@ export function initSandboxRuntimeModular(): void {
// stopAll()+reschedule at the new rate to keep trimmed clips ending on time.
const applyWebAudioRate = () => {
const changed = webAudio.setRate(state.playbackRate);
if (changed && webAudioReady && clock.isPlaying() && webAudio.hasBoundedActiveSources()) {
if (
changed &&
!state.nativeMediaSyncDisabled &&
!state.webAudioMediaDisabled &&
webAudioReady &&
clock.isPlaying() &&
webAudio.hasBoundedActiveSources()
) {
webAudio.stopAll();
scheduleWebAudioForActiveClips();
}
@@ -2392,7 +2432,9 @@ export function initSandboxRuntimeModular(): void {
// Schedule audio through WebAudio for sample-accurate timing.
// Falls back to HTMLMediaElement playback if WebAudio isn't ready
// or decoding fails (the syncRuntimeMedia path handles that).
if (webAudioReady) scheduleWebAudioForActiveClips();
if (webAudioReady && !state.nativeMediaSyncDisabled && !state.webAudioMediaDisabled) {
scheduleWebAudioForActiveClips();
}
runAdapters("play");
syncMediaForCurrentState();
colorGrading.redraw();
packages/core/src/runtime/state.test.ts
+2
View File
@@ -9,6 +9,8 @@ describe("createRuntimeState", () => {
expect(state.canonicalFps).toBe(30);
expect(state.playbackRate).toBe(1);
expect(state.bridgeMuted).toBe(false);
expect(state.nativeMediaSyncDisabled).toBe(false);
expect(state.webAudioMediaDisabled).toBe(false);
expect(state.capturedTimeline).toBeNull();
expect(state.tornDown).toBe(false);
});
packages/core/src/runtime/state.ts
+16
View File
@@ -19,6 +19,20 @@ export type RuntimeState = {
* accuracy but produces no audio of its own.
*/
mediaOutputMuted: boolean;
/**
* Disable runtime ownership of native media elements. Slideshow presenter
* mode keeps the slide timeline paused while users interact with embedded
* media, so the runtime must not auto-play, auto-pause, seek, or volume-sync
* those native elements on every transport tick.
*/
nativeMediaSyncDisabled: boolean;
/**
* Disable the runtime's WebAudio replacement for native <audio> elements.
* Slideshow presenter/audience windows mirror native media element events
* across browsers, so muting those elements for WebAudio ownership breaks
* audible presenter playback and remote sync.
*/
webAudioMediaDisabled: boolean;
/**
* Latch so the `media-autoplay-blocked` outbound message is posted at most
* once per runtime session. The parent only needs the first signal — it
@@ -88,6 +102,8 @@ export function createRuntimeState(): RuntimeState {
bridgeMuted: false,
bridgeVolume: 1,
mediaOutputMuted: false,
nativeMediaSyncDisabled: false,
webAudioMediaDisabled: false,
mediaAutoplayBlockedPosted: false,
mediaForceSyncNextTick: false,
playbackRate: 1,
packages/core/src/runtime/types.ts
+3
View File
@@ -16,6 +16,8 @@ export type RuntimeBridgeControlAction =
| "tick"
| "set-volume"
| "set-media-output-muted"
| "set-native-media-sync-disabled"
| "set-web-audio-media-disabled"
| "stop-media"
| "flash-elements";
@@ -26,6 +28,7 @@ export type RuntimeBridgeControlMessage = {
frame?: number;
muted?: boolean;
volume?: number;
disabled?: boolean;
playbackRate?: number;
target?: HfColorGradingTarget | string | null;
grading?: RuntimeJson;