fix(studio): the live clip list carries mute, volume and sound, and refreshes on change (#4626)

* fix(studio): the live clip list says whether each clip is heard and refreshes when that changes

* fix(studio): a muted clip reads as muted and a timed wrapper does not take its video's sound

* test(studio): a timed wrapper ignores the data-has-audio of the video inside it

* refactor(studio): keep the clip change check and audio reader under the complexity gate

* fix(studio): a fade-only change refreshes the live clip list

packages/studio/src/player/hooks/timelinePlayerSync.test.ts

@@ -0,0 +1,20 @@
+import { describe, expect, it } from "vitest";
+import { timelineElementsChanged } from "./timelinePlayerSync";
+
+describe("timelineElementsChanged", () => {
+  const clip = { id: "v", tag: "video", start: 0, duration: 4, track: 1 };
+  it.each([
+    ["muted", { muted: true }],
+    ["sound", { hasAudio: true }],
+    ["volume", { volume: 0 }],
+    ["speed", { playbackRate: 2 }],
+    ["hidden", { hidden: true }],
+    ["audio group volume", { audioGroupVolume: 0 }],
+    ["audio group hidden", { audioGroupHidden: true }],
+    ["fade in", { fadeIn: 1 }],
+    ["fade out", { fadeOut: 1 }],
+  ])("sees a clip whose %s changed with no timing change", (_name, change) => {
+    expect(timelineElementsChanged([clip], [{ ...clip, ...change }])).toBe(true);
+    expect(timelineElementsChanged([{ ...clip, ...change }], [{ ...clip, ...change }])).toBe(false);
+  });
+});

packages/studio/src/player/hooks/timelinePlayerSync.ts

@@ -1,5 +1,22 @@
 import type { TimelineElement } from "../store/playerStore";
 
+const RENDERED_FIELDS: readonly (keyof TimelineElement)[] = [
+  "id",
+  "start",
+  "duration",
+  "track",
+  "sourceDuration",
+  "muted",
+  "hasAudio",
+  "volume",
+  "playbackRate",
+  "hidden",
+  "audioGroupVolume",
+  "audioGroupHidden",
+  "fadeIn",
+  "fadeOut",
+];
+
 /** Whether a derived timeline changes any field that affects rendering. */
 export function timelineElementsChanged(
   previous: TimelineElement[],
@@ -8,13 +25,6 @@ export function timelineElementsChanged(
   if (next.length !== previous.length) return true;
   return next.some((element, index) => {
     const prior = previous[index];
-    return (
-      !prior ||
-      element.id !== prior.id ||
-      element.start !== prior.start ||
-      element.duration !== prior.duration ||
-      element.track !== prior.track ||
-      element.sourceDuration !== prior.sourceDuration
-    );
+    return !prior || RENDERED_FIELDS.some((key) => element[key] !== prior[key]);
   });
 }

packages/studio/src/player/lib/timelineDOM.test.ts

@@ -547,3 +547,74 @@ describe("a composition clip's source length", () => {
     expect(card?.sourceDuration).toBeUndefined();
   });
 });
+
+describe("what the live clip list says a clip plays", () => {
+  function manifestVideo(attrs: string): TimelineElement {
+    const doc = makeDoc(
+      `<div data-composition-id="root"><video id="v" src="a.mp4" data-start="0" data-duration="4" ${attrs}></video></div>`,
+    );
+    return createTimelineElementFromManifestClip({
+      clip: {
+        id: "v",
+        label: "v",
+        kind: "video",
+        tagName: "video",
+        start: 0,
+        duration: 4,
+        track: 0,
+        assetUrl: null,
+      },
+      fallbackIndex: 0,
+      doc,
+      hostEl: doc.getElementById("v"),
+    });
+  }
+
+  it("carries data-volume and muted from the element", () => {
+    expect(manifestVideo('data-volume="0" muted')).toMatchObject({ volume: 0, muted: true });
+    const plain = manifestVideo("");
+    expect(plain.volume).toBeUndefined();
+    expect(plain.muted).toBeUndefined();
+  });
+
+  it("carries the clip-edge fades, and drops a zero one", () => {
+    const faded = manifestVideo('data-fade-in="1.5" data-fade-out="0"');
+    expect(faded.fadeIn).toBe(1.5);
+    expect(faded.fadeOut).toBeUndefined();
+  });
+
+  it("gives a video with neither muted nor data-has-audio sound, as the compiler does", () => {
+    expect(manifestVideo("").hasAudio).toBe(true);
+    expect(manifestVideo("muted").hasAudio).toBeUndefined();
+    expect(manifestVideo('data-has-audio="false"').hasAudio).toBeUndefined();
+    expect(manifestVideo('muted data-has-audio="true"')).toMatchObject({
+      hasAudio: true,
+      muted: true,
+    });
+  });
+
+  it.each(["", ' data-has-audio="true"'])(
+    "does not give a timed wrapper the sound of an untimed video%s inside it",
+    (videoAttrs) => {
+      const doc = makeDoc(
+        `<div data-composition-id="root"><div id="w" data-start="0" data-duration="4"><video src="a.mp4"${videoAttrs}></video></div></div>`,
+      );
+      const wrapper = createTimelineElementFromManifestClip({
+        clip: {
+          id: "w",
+          label: "w",
+          kind: "element",
+          tagName: "div",
+          start: 0,
+          duration: 4,
+          track: 0,
+          assetUrl: null,
+        },
+        fallbackIndex: 0,
+        doc,
+        hostEl: doc.getElementById("w"),
+      });
+      expect(wrapper.hasAudio).toBeUndefined();
+    },
+  );
+});

packages/studio/src/player/lib/timelineDOM.ts

@@ -304,8 +304,6 @@ export function parseTimelineFromDOM(
       if (mediaEl.tagName === "IMG") {
         entry.tag = "img";
       }
-      const vol = el.getAttribute("data-volume") ?? mediaEl.getAttribute("data-volume");
-      if (vol) entry.volume = parseFloat(vol);
       // Override AFTER the helper (which sets the raw relative attribute) so the
       // resolved absolute URL wins — the Studio can then fetch the asset
       // regardless of whether the attribute value was relative or absolute.

packages/studio/src/player/lib/timelineElementHelpers.ts

@@ -213,9 +213,25 @@ function setOptional<K extends keyof TimelineElement>(
   else entry[key] = value;
 }
 
-/** `data-has-audio` and the clip-edge fades: what the mixer hears and how it enters and leaves. */
+function readVolume(el: Element, media: Element): number | undefined {
+  const volume = Number.parseFloat(
+    el.getAttribute("data-volume") ?? media.getAttribute("data-volume") ?? "",
+  );
+  return Number.isFinite(volume) ? volume : undefined;
+}
+
+/** What the mixer gets: the compiler's `data-has-audio` rule, muted and volume. */
 function applyAudioMetadataFromElement(entry: TimelineElement, el: Element): void {
-  setOptional(entry, "hasAudio", el.getAttribute("data-has-audio") === "true" ? true : undefined);
+  const media = resolveMediaElement(el) ?? el;
+  const muted = el.hasAttribute("muted") || media.hasAttribute("muted");
+  const hasAudio = el.getAttribute("data-has-audio");
+  const sound = hasAudio === null ? el.tagName === "VIDEO" && !muted : hasAudio === "true";
+  setOptional(entry, "hasAudio", sound ? true : undefined);
+  setOptional(entry, "muted", muted ? true : undefined);
+  setOptional(entry, "volume", readVolume(el, media));
+}
+
+function applyFadeMetadataFromElement(entry: TimelineElement, el: Element): void {
   const fades = readElementFades(el);
   setOptional(entry, "fadeIn", fades.fadeIn > 0 ? fades.fadeIn : undefined);
   setOptional(entry, "fadeOut", fades.fadeOut > 0 ? fades.fadeOut : undefined);
@@ -224,6 +240,7 @@ function applyAudioMetadataFromElement(entry: TimelineElement, el: Element): voi
 export function applyMediaMetadataFromElement(entry: TimelineElement, el: Element): void {
   applyPlaybackMetadataFromElement(entry, el);
   applyAudioMetadataFromElement(entry, el);
+  applyFadeMetadataFromElement(entry, el);
 
   const mediaEl = resolveMediaElement(el);
   if (!mediaEl) return;

packages/studio/src/player/store/timelineElement.ts

@@ -55,8 +55,9 @@ export interface TimelineElement {
   playbackRate?: number;
   sourceDuration?: number;
   volume?: number;
-  /** `data-has-audio="true"` on a video: its sound is mixed, so it takes audio affordances (fades). */
+  /** A video with sound to mix (`data-has-audio`, or unmuted without it); `muted` silences it. */
   hasAudio?: boolean;
+  muted?: boolean;
   /** Clip-edge fades from `data-fade-in` / `data-fade-out`, seconds; absent means none. */
   fadeIn?: number;
   fadeOut?: number;