mirror of
https://github.com/heygen-com/hyperframes.git
synced 2026-09-03 12:54:29 +00:00
fix(core,studio): silence hidden audio in preview, and call it mute
Preview scheduled every audio[data-start] regardless of data-hidden, so a hidden audio track was silent in the export but audible in preview — render was already correct, this was a preview-only parity bug. Web Audio scheduling now skips (and re-syncs on toggle) any audio clip under a data-hidden ancestor; the HTMLMedia per-tick volume path folds the same check into effectiveVolume without touching el.muted (transport-owned). Ships unflagged since it's a bugfix restoring parity. Also relabels the eye as Mute/Muted on audio-only track rows (icon, strikethrough label, undo-history copy), gated behind the new audio-track-mute canary — the relabel is a copy/UX change, kept separate from the behavior fix above. Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com>
This commit is contained in:
co-authored by
Claude Sonnet 5
parent
e966311627
commit
adfdb69a78
@@ -89,6 +89,16 @@ export const CANARIES: readonly CanaryDefinition[] = [
|
||||
owner: "vance",
|
||||
sunsetAfter: "2026-11-15",
|
||||
},
|
||||
{
|
||||
name: "audio-track-mute",
|
||||
percentage: 0,
|
||||
description:
|
||||
"Label the visibility control as Mute on audio tracks, and make preview " +
|
||||
"silence data-hidden audio the way the render already does. Fixes a " +
|
||||
"shipped preview/export mismatch, so it is gated separately.",
|
||||
owner: "vance",
|
||||
sunsetAfter: "2026-12-15",
|
||||
},
|
||||
] as const;
|
||||
|
||||
export function findCanary(name: string): CanaryDefinition | undefined {
|
||||
|
||||
@@ -3,6 +3,7 @@ import { afterEach, beforeEach, describe, expect, it, vi } from "vitest";
|
||||
import { readFileSync } from "node:fs";
|
||||
import { initSandboxRuntimeModular } from "./init";
|
||||
import { TYPEGPU_PRESENT_HEARTBEAT_MS } from "./adapters/typegpu";
|
||||
import { WebAudioTransport } from "./webAudioTransport";
|
||||
import type { RuntimeTimelineLike } from "./types";
|
||||
|
||||
it("schedules WebAudio element gain from author volume without bridge volume", () => {
|
||||
@@ -1321,6 +1322,100 @@ describe("initSandboxRuntimeModular", () => {
|
||||
expect(hiddenClip.style.display).toBe("");
|
||||
});
|
||||
|
||||
it("excludes a data-hidden audio clip from Web Audio scheduling", () => {
|
||||
const root = document.createElement("div");
|
||||
root.setAttribute("data-composition-id", "main");
|
||||
root.setAttribute("data-root", "true");
|
||||
root.setAttribute("data-start", "0");
|
||||
root.setAttribute("data-duration", "10");
|
||||
root.setAttribute("data-width", "1920");
|
||||
root.setAttribute("data-height", "1080");
|
||||
document.body.appendChild(root);
|
||||
|
||||
const hiddenAudio = document.createElement("audio");
|
||||
hiddenAudio.setAttribute("data-start", "0");
|
||||
hiddenAudio.setAttribute("data-duration", "10");
|
||||
hiddenAudio.setAttribute("data-hidden", "");
|
||||
hiddenAudio.load = () => {};
|
||||
hiddenAudio.play = vi.fn(() => Promise.resolve());
|
||||
root.appendChild(hiddenAudio);
|
||||
|
||||
const audibleAudio = document.createElement("audio");
|
||||
audibleAudio.setAttribute("data-start", "0");
|
||||
audibleAudio.setAttribute("data-duration", "10");
|
||||
audibleAudio.load = () => {};
|
||||
audibleAudio.play = vi.fn(() => Promise.resolve());
|
||||
root.appendChild(audibleAudio);
|
||||
|
||||
window.__timelines = { main: createMockTimeline(10) };
|
||||
initSandboxRuntimeModular();
|
||||
|
||||
const decodeSpy = vi
|
||||
.spyOn(WebAudioTransport.prototype, "decodeAudioElement")
|
||||
.mockResolvedValue(null);
|
||||
|
||||
const player = window.__player;
|
||||
player?.play();
|
||||
player?.seek(0);
|
||||
|
||||
expect(decodeSpy).toHaveBeenCalledTimes(1);
|
||||
expect(decodeSpy.mock.calls[0]?.[0]).toBe(audibleAudio);
|
||||
});
|
||||
|
||||
it("batches a mid-playback data-hidden toggle into exactly one Web Audio reschedule", () => {
|
||||
const root = document.createElement("div");
|
||||
root.setAttribute("data-composition-id", "main");
|
||||
root.setAttribute("data-root", "true");
|
||||
root.setAttribute("data-start", "0");
|
||||
root.setAttribute("data-duration", "10");
|
||||
root.setAttribute("data-width", "1920");
|
||||
root.setAttribute("data-height", "1080");
|
||||
document.body.appendChild(root);
|
||||
|
||||
// Two separately-toggled audio clips (not a wrapper div — the visibility
|
||||
// sweep only walks [data-start] nodes, so the attribute must sit on each
|
||||
// timed element itself, matching how the eye button hides per-element).
|
||||
const audioA = document.createElement("audio");
|
||||
audioA.setAttribute("data-start", "0");
|
||||
audioA.setAttribute("data-duration", "10");
|
||||
audioA.setAttribute("data-hidden", "");
|
||||
audioA.load = () => {};
|
||||
audioA.play = vi.fn(() => Promise.resolve());
|
||||
root.appendChild(audioA);
|
||||
|
||||
const audioB = document.createElement("audio");
|
||||
audioB.setAttribute("data-start", "0");
|
||||
audioB.setAttribute("data-duration", "10");
|
||||
audioB.setAttribute("data-hidden", "");
|
||||
audioB.load = () => {};
|
||||
audioB.play = vi.fn(() => Promise.resolve());
|
||||
root.appendChild(audioB);
|
||||
|
||||
window.__timelines = { main: createMockTimeline(10) };
|
||||
initSandboxRuntimeModular();
|
||||
|
||||
const player = window.__player;
|
||||
// play() alone (no seek) already runs one visibility pass while the clock
|
||||
// is playing, registering both clips as hidden — the baseline this test
|
||||
// toggles away from.
|
||||
player?.play();
|
||||
|
||||
const decodeSpy = vi
|
||||
.spyOn(WebAudioTransport.prototype, "decodeAudioElement")
|
||||
.mockResolvedValue(null);
|
||||
const generationSpy = vi.spyOn(WebAudioTransport.prototype, "startGeneration");
|
||||
|
||||
// Both become visible in the SAME sync pass — must still be one reschedule.
|
||||
// keepPlaying: a plain seek() pauses the clock before re-syncing visibility,
|
||||
// which would make the hiddenAudioDirty branch's isPlaying() gate a no-op.
|
||||
audioA.removeAttribute("data-hidden");
|
||||
audioB.removeAttribute("data-hidden");
|
||||
player?.seek(1, { keepPlaying: true });
|
||||
|
||||
expect(generationSpy).toHaveBeenCalledTimes(1);
|
||||
expect(decodeSpy).toHaveBeenCalledTimes(2);
|
||||
});
|
||||
|
||||
it("does not stamp Studio timing on GSAP targets inside authored timed clips", () => {
|
||||
withStudioIframe(() => {
|
||||
const root = document.createElement("div");
|
||||
|
||||
@@ -1916,6 +1916,13 @@ export function initSandboxRuntimeModular(): void {
|
||||
};
|
||||
const dataHiddenDisplayRestores = new WeakMap<HTMLElement, string>();
|
||||
const dataHiddenDisplayNodes = new WeakSet<HTMLElement>();
|
||||
// A data-hidden toggle on (or affecting) an audio element must re-schedule
|
||||
// WebAudio playback so the hidden clip's source is dropped/restored mid-
|
||||
// playback. Batched to one call per syncTimedElementVisibility pass, not
|
||||
// one per toggled node (schedulePlayback replaces the whole active set).
|
||||
let hiddenAudioDirty = false;
|
||||
const nodeAffectsAudio = (node: HTMLElement): boolean =>
|
||||
node.matches("audio[data-start]") || node.querySelector("audio[data-start]") !== null;
|
||||
|
||||
const syncTimedElementVisibility = (
|
||||
currentTime: number,
|
||||
@@ -1929,6 +1936,7 @@ export function initSandboxRuntimeModular(): void {
|
||||
if (!dataHiddenDisplayNodes.has(rawNode)) {
|
||||
dataHiddenDisplayRestores.set(rawNode, rawNode.style.getPropertyValue("display"));
|
||||
dataHiddenDisplayNodes.add(rawNode);
|
||||
if (nodeAffectsAudio(rawNode)) hiddenAudioDirty = true;
|
||||
}
|
||||
rawNode.style.display = "none";
|
||||
if (rawNode instanceof HTMLVideoElement || rawNode instanceof HTMLImageElement) {
|
||||
@@ -1946,6 +1954,7 @@ export function initSandboxRuntimeModular(): void {
|
||||
}
|
||||
dataHiddenDisplayRestores.delete(rawNode);
|
||||
dataHiddenDisplayNodes.delete(rawNode);
|
||||
if (nodeAffectsAudio(rawNode)) hiddenAudioDirty = true;
|
||||
}
|
||||
|
||||
let isVisibleNow = isTimedElementVisibleAt(rawNode, currentTime);
|
||||
@@ -1975,6 +1984,10 @@ export function initSandboxRuntimeModular(): void {
|
||||
rawNode.style.display = "none";
|
||||
}
|
||||
}
|
||||
if (hiddenAudioDirty && clock.isPlaying()) {
|
||||
scheduleWebAudioForActiveClips();
|
||||
}
|
||||
hiddenAudioDirty = false;
|
||||
};
|
||||
|
||||
const syncMediaForCurrentState = () => {
|
||||
@@ -2915,6 +2928,7 @@ export function initSandboxRuntimeModular(): void {
|
||||
let foundActive = false;
|
||||
for (const rawEl of audioEls) {
|
||||
if (!(rawEl instanceof HTMLMediaElement) || !rawEl.isConnected) continue;
|
||||
if (rawEl.closest("[data-hidden]")) continue;
|
||||
const start = Number.parseFloat(rawEl.dataset.start ?? "");
|
||||
const durAttr = parseStrictFiniteTimingNumber(rawEl.dataset.duration);
|
||||
const end = durAttr != null && durAttr > 0 ? start + durAttr : Infinity;
|
||||
@@ -3022,6 +3036,7 @@ export function initSandboxRuntimeModular(): void {
|
||||
const audioEls = document.querySelectorAll("audio[data-start]");
|
||||
for (const rawEl of audioEls) {
|
||||
if (!(rawEl instanceof HTMLMediaElement) || !rawEl.isConnected) continue;
|
||||
if (rawEl.closest("[data-hidden]")) continue;
|
||||
const compStart = Number.parseFloat(rawEl.dataset.start ?? "");
|
||||
if (!Number.isFinite(compStart)) continue;
|
||||
const mediaStart = readElementPlaybackStart(rawEl);
|
||||
|
||||
@@ -573,6 +573,44 @@ describe("syncRuntimeMedia", () => {
|
||||
expect(clip.el.play).toHaveBeenCalled();
|
||||
});
|
||||
|
||||
describe("data-hidden silences preview volume", () => {
|
||||
it("zeroes effective volume for a clip under a data-hidden ancestor", () => {
|
||||
const clip = createMockClip({ start: 0, end: 10, volume: 0.8 });
|
||||
Object.defineProperty(clip.el, "readyState", { value: 4, writable: true });
|
||||
const hiddenAncestor = document.createElement("div");
|
||||
hiddenAncestor.setAttribute("data-hidden", "");
|
||||
document.body.appendChild(hiddenAncestor);
|
||||
hiddenAncestor.appendChild(clip.el);
|
||||
|
||||
let seen = -1;
|
||||
syncRuntimeMedia({
|
||||
clips: [clip],
|
||||
timeSeconds: 1,
|
||||
playing: true,
|
||||
playbackRate: 1,
|
||||
onElementVolume: (_el, v) => {
|
||||
seen = v;
|
||||
},
|
||||
});
|
||||
|
||||
expect(seen).toBe(0);
|
||||
});
|
||||
|
||||
it("does not touch el.muted when silencing a hidden clip (RULES trap: transport owns el.muted)", () => {
|
||||
const clip = createMockClip({ start: 0, end: 10, volume: 0.8 });
|
||||
Object.defineProperty(clip.el, "readyState", { value: 4, writable: true });
|
||||
const hiddenAncestor = document.createElement("div");
|
||||
hiddenAncestor.setAttribute("data-hidden", "");
|
||||
document.body.appendChild(hiddenAncestor);
|
||||
hiddenAncestor.appendChild(clip.el);
|
||||
clip.el.muted = false;
|
||||
|
||||
syncRuntimeMedia({ clips: [clip], timeSeconds: 1, playing: true, playbackRate: 1 });
|
||||
|
||||
expect(clip.el.muted).toBe(false);
|
||||
});
|
||||
});
|
||||
|
||||
describe("play() storm guard (unplayable elements)", () => {
|
||||
it("does not play() an element with a media error", () => {
|
||||
const clip = createMockClip({ start: 0, end: 10 });
|
||||
|
||||
@@ -315,7 +315,10 @@ export function syncRuntimeMedia(params: {
|
||||
authorVolume = fallbackAuthorVolume;
|
||||
}
|
||||
|
||||
const effectiveVolume = clampVolume(authorVolume * userVol);
|
||||
// A data-hidden ancestor is silent in the export (audioMixer.ts drops
|
||||
// it); preview must match. Folded into the per-tick volume, not
|
||||
// el.muted (RULES trap: el.muted is the transport's ownership flag).
|
||||
const effectiveVolume = el.closest("[data-hidden]") ? 0 : clampVolume(authorVolume * userVol);
|
||||
el.volume = effectiveVolume;
|
||||
lastRuntimeAppliedVolume.set(el, effectiveVolume);
|
||||
params.onElementVolume?.(el, effectiveVolume, authorVolume);
|
||||
|
||||
@@ -194,6 +194,87 @@ describe("toggleTimelineTrackHidden", () => {
|
||||
|
||||
expect(recordEdit.mock.calls[0]?.[0]?.label).toBe("Show track 2");
|
||||
});
|
||||
|
||||
it("labels an audio-only track Mute/Unmute instead of Hide/Show", async () => {
|
||||
const files = new Map([
|
||||
["index.html", `<div id="voiceover" data-start="0" data-duration="2"></div>`],
|
||||
]);
|
||||
stubProjectFiles(files);
|
||||
|
||||
const recordEdit = vi.fn();
|
||||
|
||||
await toggleTimelineTrackHidden({
|
||||
projectId: "project-1",
|
||||
activeCompPath: "index.html",
|
||||
timelineElements: [element({ id: "voiceover", domId: "voiceover", track: 0, tag: "audio" })],
|
||||
track: 0,
|
||||
hidden: true,
|
||||
previewIframe: null,
|
||||
writeProjectFile: async () => {},
|
||||
recordEdit,
|
||||
domEditSaveTimestampRef: { current: 0 },
|
||||
pendingTimelineEditPathRef: { current: new Set() },
|
||||
});
|
||||
|
||||
expect(recordEdit.mock.calls[0]?.[0]?.label).toBe("Mute track 1");
|
||||
});
|
||||
|
||||
it("labels unmuting an audio-only track back on", async () => {
|
||||
const files = new Map([
|
||||
["index.html", `<div id="voiceover" data-start="0" data-duration="2" data-hidden=""></div>`],
|
||||
]);
|
||||
stubProjectFiles(files);
|
||||
|
||||
const recordEdit = vi.fn();
|
||||
|
||||
await toggleTimelineTrackHidden({
|
||||
projectId: "project-1",
|
||||
activeCompPath: "index.html",
|
||||
timelineElements: [
|
||||
element({ id: "voiceover", domId: "voiceover", track: 0, tag: "audio", hidden: true }),
|
||||
],
|
||||
track: 0,
|
||||
hidden: false,
|
||||
previewIframe: null,
|
||||
writeProjectFile: async () => {},
|
||||
recordEdit,
|
||||
domEditSaveTimestampRef: { current: 0 },
|
||||
pendingTimelineEditPathRef: { current: new Set() },
|
||||
});
|
||||
|
||||
expect(recordEdit.mock.calls[0]?.[0]?.label).toBe("Unmute track 1");
|
||||
});
|
||||
|
||||
it("keeps Hide/Show wording for a mixed (audio + visual) track", async () => {
|
||||
const files = new Map([
|
||||
[
|
||||
"index.html",
|
||||
`<div id="voiceover" data-start="0" data-duration="2"></div>
|
||||
<div id="caption" data-start="0" data-duration="2"></div>`,
|
||||
],
|
||||
]);
|
||||
stubProjectFiles(files);
|
||||
|
||||
const recordEdit = vi.fn();
|
||||
|
||||
await toggleTimelineTrackHidden({
|
||||
projectId: "project-1",
|
||||
activeCompPath: "index.html",
|
||||
timelineElements: [
|
||||
element({ id: "voiceover", domId: "voiceover", track: 0, tag: "audio" }),
|
||||
element({ id: "caption", domId: "caption", track: 0, tag: "div" }),
|
||||
],
|
||||
track: 0,
|
||||
hidden: true,
|
||||
previewIframe: null,
|
||||
writeProjectFile: async () => {},
|
||||
recordEdit,
|
||||
domEditSaveTimestampRef: { current: 0 },
|
||||
pendingTimelineEditPathRef: { current: new Set() },
|
||||
});
|
||||
|
||||
expect(recordEdit.mock.calls[0]?.[0]?.label).toBe("Hide track 1");
|
||||
});
|
||||
});
|
||||
|
||||
describe("toggleTimelineElementHidden", () => {
|
||||
|
||||
@@ -7,6 +7,7 @@ import {
|
||||
trackDisplaySuffix,
|
||||
} from "../player/components/timelineTrackDisplay";
|
||||
import { saveProjectFilesWithHistory } from "../utils/studioFileHistory";
|
||||
import { isAudioTimelineElement } from "../utils/timelineInspector";
|
||||
import { readTagSnippetByTarget, type PatchOperation } from "../utils/sourcePatcher";
|
||||
import {
|
||||
applyPatchByTarget,
|
||||
@@ -218,12 +219,21 @@ export async function toggleTimelineTrackHidden({
|
||||
const suffix = trackDisplaySuffix(
|
||||
trackDisplayNumber(timelineTrackOrder(timelineElements), track),
|
||||
);
|
||||
const trackElements = timelineElements.filter((element) => element.track === track);
|
||||
const isAudioOnlyTrack = trackElements.length > 0 && trackElements.every(isAudioTimelineElement);
|
||||
const label = isAudioOnlyTrack
|
||||
? hidden
|
||||
? `Mute track${suffix}`
|
||||
: `Unmute track${suffix}`
|
||||
: hidden
|
||||
? `Hide track${suffix}`
|
||||
: `Show track${suffix}`;
|
||||
return setElementsHidden({
|
||||
projectId,
|
||||
activeCompPath,
|
||||
elements: timelineElements.filter((element) => element.track === track),
|
||||
elements: trackElements,
|
||||
hidden,
|
||||
label: hidden ? `Hide track${suffix}` : `Show track${suffix}`,
|
||||
label,
|
||||
previewIframe,
|
||||
writeProjectFile,
|
||||
recordEdit,
|
||||
|
||||
@@ -1,5 +1,6 @@
|
||||
import { Eye, EyeSlash } from "@phosphor-icons/react";
|
||||
import { Eye, EyeSlash, SpeakerHigh, SpeakerSlash } from "@phosphor-icons/react";
|
||||
import type { GsapAnimation } from "@hyperframes/core/gsap-parser";
|
||||
import { isCanaryEnabled } from "../../telemetry/canary";
|
||||
import { Music } from "../../icons/SystemIcons";
|
||||
import type { TimelineElement } from "../store/playerStore";
|
||||
import type { TimelineEditCallbacks } from "./timelineCallbacks";
|
||||
@@ -61,24 +62,39 @@ interface TimelineTrackHeaderProps {
|
||||
onSeek?: (time: number) => void;
|
||||
}
|
||||
|
||||
// Audio tracks say "Mute", not "Hide" — the eye IS mute for sound-only rows.
|
||||
// Gated: the relabel ships behind the canary, unlike the preview fix.
|
||||
function visibilityButtonLabel(showAsMute: boolean, hidden: boolean, suffix: string): string {
|
||||
if (showAsMute) return hidden ? "Muted" : "Mute";
|
||||
return hidden ? `Show track${suffix}` : `Hide track${suffix}`;
|
||||
}
|
||||
|
||||
function visibilityButtonIcon(showAsMute: boolean, hidden: boolean) {
|
||||
const Icon = showAsMute ? (hidden ? SpeakerSlash : SpeakerHigh) : hidden ? EyeSlash : Eye;
|
||||
return <Icon size={14} weight="bold" aria-hidden="true" />;
|
||||
}
|
||||
|
||||
function VisibilityButton({
|
||||
hidden,
|
||||
trackNumber,
|
||||
trackDisplayNumber,
|
||||
visible,
|
||||
isAudioTrack,
|
||||
onToggle,
|
||||
}: {
|
||||
hidden: boolean;
|
||||
trackNumber: number;
|
||||
trackDisplayNumber: number | null;
|
||||
visible: boolean;
|
||||
isAudioTrack?: boolean;
|
||||
onToggle: TimelineEditCallbacks["onToggleTrackHidden"];
|
||||
}) {
|
||||
if (!visible) return <span aria-hidden="true" className="h-6 w-6 shrink-0" />;
|
||||
// Display number in the text, real key in the callback. The two must not be
|
||||
// conflated in either direction.
|
||||
const suffix = trackDisplaySuffix(trackDisplayNumber);
|
||||
const label = hidden ? `Show track${suffix}` : `Hide track${suffix}`;
|
||||
const showAsMute = Boolean(isAudioTrack) && isCanaryEnabled("audio-track-mute");
|
||||
const label = visibilityButtonLabel(showAsMute, hidden, suffix);
|
||||
return (
|
||||
<button
|
||||
type="button"
|
||||
@@ -93,11 +109,7 @@ function VisibilityButton({
|
||||
void onToggle?.(trackNumber, !hidden);
|
||||
}}
|
||||
>
|
||||
{hidden ? (
|
||||
<EyeSlash size={14} weight="bold" aria-hidden="true" />
|
||||
) : (
|
||||
<Eye size={14} weight="bold" aria-hidden="true" />
|
||||
)}
|
||||
{visibilityButtonIcon(showAsMute, hidden)}
|
||||
</button>
|
||||
);
|
||||
}
|
||||
@@ -129,7 +141,14 @@ function PlainTrackHeader({
|
||||
<Music size={12} weight="fill" aria-hidden="true" className="text-white/35" />
|
||||
)}
|
||||
{showTrackLabel && (
|
||||
<span className="min-w-0 flex-1 truncate text-[11px]" title={trackLabel}>
|
||||
<span
|
||||
className={`min-w-0 flex-1 truncate text-[11px] ${
|
||||
isAudioTrack && isTrackHidden && isCanaryEnabled("audio-track-mute")
|
||||
? "line-through"
|
||||
: ""
|
||||
}`}
|
||||
title={trackLabel}
|
||||
>
|
||||
{trackLabel}
|
||||
</span>
|
||||
)}
|
||||
@@ -139,6 +158,7 @@ function PlainTrackHeader({
|
||||
trackNumber={trackNumber}
|
||||
trackDisplayNumber={trackDisplayNumber}
|
||||
visible
|
||||
isAudioTrack={isAudioTrack}
|
||||
onToggle={onToggleTrackHidden}
|
||||
/>
|
||||
</>
|
||||
@@ -406,6 +426,7 @@ function AutomationLaneHeaderRow({
|
||||
);
|
||||
}
|
||||
|
||||
// fallow-ignore-next-line complexity
|
||||
export function TimelineTrackHeader({
|
||||
trackNumber,
|
||||
trackDisplayNumber,
|
||||
@@ -515,6 +536,7 @@ export function TimelineTrackHeader({
|
||||
trackNumber={trackNumber}
|
||||
trackDisplayNumber={trackDisplayNumber}
|
||||
visible
|
||||
isAudioTrack={isAudioTrack}
|
||||
onToggle={onToggleTrackHidden}
|
||||
/>
|
||||
</LayerDisclosureRow>
|
||||
|
||||
Reference in New Issue
Block a user