feat(studio): the agent sees every timeline clip, audio included (#4123)

packages/studio/src/components/editor/domEditingAgentPrompt.ts

@@ -96,6 +96,7 @@ export function buildElementAgentPrompt({
   selectionContext,
   userInstruction,
   sourceFilePath,
+  timeline,
 }: {
   selection: DomEditSelection;
   currentTime: number;
@@ -103,6 +104,8 @@ export function buildElementAgentPrompt({
   selectionContext?: string;
   userInstruction?: string;
   sourceFilePath?: string;
+  /** The `formatTimelineBlock` text; omitted or empty when the timeline has no clips. */
+  timeline?: string;
 }): string {
   const displayedSourceFile = sourceFilePath?.trim() || selection.sourceFile;
   const info: AgentPromptElementInfo = {
@@ -136,6 +139,8 @@ export function buildElementAgentPrompt({
 
   lines.push(...buildElementDetailLines(info));
 
+  if (timeline) lines.push("", timeline);
+
   if (tagSnippet) {
     lines.push("", "Target HTML:", tagSnippet);
   }

packages/studio/src/hooks/useAskAgentModal.ts

@@ -1,3 +1,4 @@
+import { formatTimelineBlock } from "../player/lib/describeClips";
 import { buildProjectApiPath } from "../utils/projectRouting";
 import { useState, useCallback, useRef, useEffect } from "react";
 import { copyTextToClipboard } from "../utils/clipboard";
@@ -92,6 +93,7 @@ export function useAskAgentModal({
       const prompt = buildElementAgentPrompt({
         selection: domEditSelection,
         currentTime: usePlayerStore.getState().currentTime,
+        timeline: formatTimelineBlock(usePlayerStore.getState().elements),
         tagSnippet,
         selectionContext: agentPromptSelectionContext,
         userInstruction,

packages/studio/src/player/lib/describeClips.test.ts

@@ -0,0 +1,95 @@
+import { describe, expect, it } from "vitest";
+import type { TimelineElement } from "../store/timelineElement";
+import { describeClips, formatTimelineBlock } from "./describeClips";
+
+function audio(id: string, start: number, duration: number, extra: Partial<TimelineElement> = {}) {
+  return {
+    id,
+    domId: id,
+    tag: "audio",
+    kind: "audio",
+    start,
+    duration,
+    track: 0,
+    ...extra,
+  } satisfies TimelineElement;
+}
+
+// Shaped like claude-paper-launch: two voiceovers, clicks, and dozens of keystroke sounds, each on its own track.
+function paperLaunchClips(): TimelineElement[] {
+  const clips = [
+    audio("vo", 29.6, 2.17, { src: "voiceover.mp3", authoredTrack: 8, volume: 0.009772 }),
+    audio("sfx-click-0", 1.6, 0.07, { src: "click.mp3", authoredTrack: 100, volume: 0.85 }),
+    audio("sfx-type-0", 7.8, 0.57, { src: "typenew.mp3", authoredTrack: 187, volume: 0.2 }),
+  ];
+  for (let i = 1; i <= 84; i++) {
+    clips.push(audio(`sfx-extra-${i}`, 8 + i * 0.1, 0.5, { authoredTrack: 200 + i, volume: 0.2 }));
+  }
+  return clips;
+}
+
+describe("formatTimelineBlock", () => {
+  it("lists every audio clip with its start and volume", () => {
+    const block = formatTimelineBlock(paperLaunchClips());
+    const audioLines = block.split("\n").filter((line) => line.startsWith("- audio "));
+    expect(audioLines).toHaveLength(87);
+    expect(
+      audioLines.every((line) => /start=[\d.]+/.test(line) && /volume=[\d.]+/.test(line)),
+    ).toBe(true);
+    expect(block).toContain(
+      '- audio "sfx-click-0" src=click.mp3 start=1.6 duration=0.07 end=1.67 track=100 volume=0.85',
+    );
+    expect(block).toContain(
+      '- audio "vo" src=voiceover.mp3 start=29.6 duration=2.17 end=31.77 track=8 volume=0.01',
+    );
+  });
+
+  it("orders clips by start time", () => {
+    const lines = formatTimelineBlock(paperLaunchClips()).split("\n");
+    expect(lines[1]).toContain('"sfx-click-0"');
+  });
+
+  it("prints volume automation points, rate and group", () => {
+    const block = formatTimelineBlock([
+      audio("bed", 8, 20, {
+        automation:
+          '{"version":1,"lanes":[{"target":"volume","points":[{"t":0,"v":0.1},{"t":10,"v":0.5}]}]}',
+        playbackRate: 0.5,
+        audioGroup: "music",
+        timelineRole: "music",
+      }),
+    ]);
+    expect(block).toContain("rate=0.5 group=music role=music volume-lane=[0:0.1, 10:0.5]");
+  });
+
+  it("prints volume 0 and a project-relative src, and omits rate at normal speed", () => {
+    const block = formatTimelineBlock([
+      audio("mute", 0, 1, {
+        src: "http://localhost:5190/api/projects/p1/preview/assets/a.mp3",
+        volume: 0,
+        playbackRate: 1,
+      }),
+    ]);
+    expect(block).toContain(
+      '- audio "mute" src=assets/a.mp3 start=0 duration=1 end=1 track=0 volume=0',
+    );
+    expect(block).not.toContain("rate=");
+  });
+
+  it("is empty for an empty timeline and caps long ones", () => {
+    expect(formatTimelineBlock([])).toBe("");
+    const many = Array.from({ length: 250 }, (_, i) => audio(`a${i}`, i, 1));
+    expect(formatTimelineBlock(many)).toContain("(50 more clips not listed)");
+  });
+});
+
+describe("describeClips", () => {
+  it("keeps 'not authored' distinct from zero for volume and rate", () => {
+    const [plain, silent] = describeClips([
+      audio("a", 0, 1),
+      audio("b", 1, 1, { volume: 0, playbackRate: 0 }),
+    ]);
+    expect([plain?.volume, plain?.playbackRate]).toEqual([null, null]);
+    expect([silent?.volume, silent?.playbackRate]).toEqual([0, 0]);
+  });
+});

packages/studio/src/player/lib/describeClips.ts

@@ -0,0 +1,103 @@
+/** Plain-field clip facts from the player store; feeds the Ask-agent prompt and `studio_look`. */
+import type { TimelineElement } from "../store/playerStore";
+import { elementAutomationLanes } from "../components/automationLaneData";
+import { roundTo3 } from "../../utils/rounding";
+
+export interface ClipLane {
+  /** `volume`, or `fx.<nodeId>.<param>`. */
+  target: string;
+  /** `t` is seconds from the start of the clip, not the composition. */
+  points: { t: number; v: number }[];
+}
+
+export interface ClipFact {
+  id: string;
+  label: string | null;
+  kind: string;
+  start: number;
+  duration: number;
+  end: number;
+  /** The `data-track-index` as written in the source file. */
+  trackIndex: number;
+  src: string | null;
+  sourceFile: string | null;
+  /** `null` when `data-volume` is not authored (the clip plays at 1). */
+  volume: number | null;
+  lanes: ClipLane[];
+  /** `null` at normal speed: not authored, or 1 (the manifest defaults it to 1). */
+  playbackRate: number | null;
+  audioGroup: string | null;
+  role: string | null;
+}
+
+/** The store holds preview URLs; the agent edits project files, so drop the origin and preview prefix. */
+function projectRelativeSrc(src: string): string {
+  return src.replace(/^https?:\/\/[^/]+/, "").replace(/^\/api\/projects\/[^/]+\/preview\//, "");
+}
+
+export function describeClip(element: TimelineElement): ClipFact {
+  return {
+    id: element.domId ?? element.id,
+    label: element.label ?? null,
+    kind: element.kind ?? element.tag.toLowerCase(),
+    start: element.start,
+    duration: element.duration,
+    end: element.start + element.duration,
+    trackIndex: element.authoredTrack ?? element.track,
+    src: element.src ? projectRelativeSrc(element.src) : null,
+    sourceFile: element.sourceFile ?? null,
+    volume: element.volume ?? null,
+    lanes: elementAutomationLanes(element).map((lane) => ({
+      target: lane.target,
+      points: lane.points.map(({ t, v }) => ({ t, v })),
+    })),
+    playbackRate: element.playbackRate === 1 ? null : (element.playbackRate ?? null),
+    audioGroup: element.audioGroup ?? null,
+    role: element.timelineRole ?? null,
+  };
+}
+
+export const byStart = (a: ClipFact, b: ClipFact) =>
+  a.start - b.start || a.trackIndex - b.trackIndex;
+
+export function describeClips(elements: readonly TimelineElement[]): ClipFact[] {
+  return elements.map(describeClip).sort(byStart);
+}
+
+const num = (n: number) => String(roundTo3(n));
+
+function formatClipLine(clip: ClipFact): string {
+  const parts = [
+    `${clip.kind} "${clip.id}"`,
+    clip.src && `src=${clip.src}`,
+    `start=${num(clip.start)}`,
+    `duration=${num(clip.duration)}`,
+    `end=${num(clip.end)}`,
+    `track=${clip.trackIndex}`,
+    clip.volume !== null && `volume=${num(clip.volume)}`,
+    clip.playbackRate !== null && `rate=${num(clip.playbackRate)}`,
+    clip.audioGroup && `group=${clip.audioGroup}`,
+    clip.role && `role=${clip.role}`,
+    clip.sourceFile && `file=${clip.sourceFile}`,
+    ...clip.lanes.map(
+      (lane) =>
+        `${lane.target}-lane=[${lane.points.map((p) => `${num(p.t)}:${num(p.v)}`).join(", ")}]`,
+    ),
+  ];
+  return `- ${parts.filter(Boolean).join(" ")}`;
+}
+
+const PROMPT_CLIP_CAP = 200;
+
+/** The Ask-agent prompt's Timeline block; empty when the timeline has no clips. */
+export function formatTimelineBlock(elements: readonly TimelineElement[]): string {
+  const clips = describeClips(elements);
+  if (clips.length === 0) return "";
+  const shown = clips.slice(0, PROMPT_CLIP_CAP).map(formatClipLine);
+  const more = clips.length - shown.length;
+  return [
+    "Timeline (composition seconds; lane points are seconds from the clip start):",
+    ...shown,
+    ...(more > 0 ? [`(${more} more clips not listed)`] : []),
+  ].join("\n");
+}

packages/studio/src/webmcp/tools/lookTools.test.ts

@@ -370,3 +370,43 @@ function sceneDoc(): Document {
   </main>`;
   return doc;
 }
+
+describe("buildStudioLook clips", () => {
+  const clip = (overrides: Partial<TimelineElement>) =>
+    element({ tag: "audio", kind: "audio", domId: "sfx", start: 2, duration: 0.5, ...overrides });
+
+  it("returns timeline clips that are not scene layers, audio included", () => {
+    const result = buildStudioLook(
+      snapshot({
+        elements: [clip({ domId: "late", start: 9 }), clip({ domId: "early", volume: 0.85 })],
+      }),
+    );
+    if (!result.ok) throw new Error("expected ok");
+    expect(result.clipCount).toBe(2);
+    expect(result.clips.map((c) => c.id)).toEqual(["early", "late"]);
+    expect(result.clips[0]).toMatchObject({ kind: "audio", start: 2, volume: 0.85 });
+    expect(result.elements).toEqual([]);
+  });
+
+  it("filters clips by id, kind or src", () => {
+    const result = buildStudioLook(
+      snapshot({
+        elements: [
+          clip({ domId: "click", src: "click.mp3" }),
+          clip({ domId: "vo", src: "voice.mp3" }),
+        ],
+      }),
+      { filter: "voice" },
+    );
+    if (!result.ok) throw new Error("expected ok");
+    expect(result.clips.map((c) => c.id)).toEqual(["vo"]);
+  });
+
+  it("caps clips at the limit and reports the full count and truncation", () => {
+    const elements = Array.from({ length: 5 }, (_, i) => clip({ domId: `c${i}`, start: i }));
+    const result = buildStudioLook(snapshot({ elements }), { limit: 2 });
+    if (!result.ok) throw new Error("expected ok");
+    expect(result.clips.map((c) => c.id)).toEqual(["c0", "c1"]);
+    expect([result.clipCount, result.clipsTruncated]).toEqual([5, true]);
+  });
+});

packages/studio/src/webmcp/tools/lookTools.ts

@@ -11,6 +11,7 @@
 import { collectDomEditLayerItems } from "../../components/editor/domEditingLayers";
 import type { DomEditLayerItem, DomEditSelection } from "../../components/editor/domEditingTypes";
 import type { TimelineElement } from "../../player/store/timelineElement";
+import { byStart, describeClip, type ClipFact } from "../../player/lib/describeClips";
 import { mintElementHandle, patchTargetAddress, timelineElementAddress } from "../handles";
 import { toolOk, type ToolResult } from "../toolResult";
 
@@ -107,8 +108,15 @@ export interface StudioLook {
   elementCount: number;
   truncated: boolean;
   elements: LookElement[];
+  /** Every timeline clip, audio included, whether or not it is a visual layer. */
+  clipCount: number;
+  clipsTruncated: boolean;
+  clips: LookClip[];
 }
 
+/** A timeline clip; `handle` is null when the clip has no addressable source element. */
+type LookClip = ClipFact & { handle: string | null };
+
 export interface StudioLookInput {
   /** Case-insensitive substring match against label, tag, and handle. */
   filter?: string;
@@ -154,6 +162,22 @@ function describeSelection(
   };
 }
 
+function matchesClipFilter(clip: LookClip, needle: string): boolean {
+  return [clip.id, clip.label, clip.kind, clip.src].some((field) =>
+    field?.toLowerCase().includes(needle),
+  );
+}
+
+function describeClipsWithHandles(snapshot: StudioLookSnapshot): LookClip[] {
+  const activeCompositionPath = snapshot.compositionPath ?? "index.html";
+  return snapshot.elements.map((element) => ({
+    ...describeClip(element),
+    handle: mintElementHandle(
+      timelineElementAddress(element, activeCompositionPath, snapshot.projectId),
+    ),
+  }));
+}
+
 function matchesFilter(element: LookElement, needle: string): boolean {
   return (
     element.label.toLowerCase().includes(needle) ||
@@ -258,6 +282,8 @@ export function buildStudioLook(
   const requested =
     Number.isInteger(input.limit) && input.limit! > 0 ? input.limit! : DEFAULT_LIMIT;
   const limit = Math.min(requested, DEFAULT_LIMIT);
+  const clips = describeClipsWithHandles(snapshot).sort(byStart);
+  const matchedClips = needle ? clips.filter((clip) => matchesClipFilter(clip, needle)) : clips;
 
   return toolOk<StudioLook>({
     projectId: snapshot.projectId,
@@ -281,6 +307,9 @@ export function buildStudioLook(
     elementCount: matched.length,
     truncated: matched.length > limit,
     elements: matched.slice(0, limit),
+    clipCount: matchedClips.length,
+    clipsTruncated: matchedClips.length > limit,
+    clips: matchedClips.slice(0, limit),
   });
 }
 
@@ -296,7 +325,7 @@ export const STUDIO_LOOK_INPUT_SCHEMA = {
       type: "integer",
       minimum: 1,
       maximum: DEFAULT_LIMIT,
-      description: `Cap the returned elements (default and max ${DEFAULT_LIMIT}). elementCount always reports the full match count.`,
+      description: `Cap the returned elements and clips (default and max ${DEFAULT_LIMIT} each). elementCount and clipCount always report the full match count.`,
     },
   },
   additionalProperties: false,
@@ -307,6 +336,8 @@ export const STUDIO_LOOK_DESCRIPTION = [
   "the playhead and duration, what the human currently has selected (including what that",
   "element will and will not accept), and the live nested scene in DOM preorder.",
   "Each scene element includes source ownership, hierarchy, and optional timeline timing.",
+  "`clips` lists every timeline clip, audio included: kind, start, duration, track, volume,",
+  "volume/effect automation points, playback rate, audio group and role.",
   "Pass a handle back to any tool that edits an element.",
   "Returns an object with `ok: true`, or `ok: false` with `kind`, `reason` and often a `hint`.",
   "`history.undoLabel` is worth checkpointing before a batch: if it later names something",

skills-manifest.json

@@ -34,7 +34,7 @@
       "files": 11
     },
     "hyperframes-core": {
-      "hash": "dffc4485ca8cff24",
+      "hash": "3e29cc2454953b05",
       "files": 11
     },
     "hyperframes-creative": {

skills/hyperframes-core/references/variables-and-media.md

@@ -100,7 +100,7 @@ Video elements must be muted and inline. Audio must be a separate `<audio>` elem
 - **Never** add `crossorigin` to `<video>`/`<audio>`. `lint` rejects it unconditionally (`media_crossorigin_breaks_preview`, error) because a media host without `Access-Control-Allow-Origin` then fails silently in preview while renders still work, hiding the bug. There is no suppression, so this holds even for the canvas/WebGL/WebAudio readback case.
 - **Every `<audio>` needs an `id`.** The mixer selects `audio[id][src]`, so an id-less `<audio>` is never mixed and the render is **silent**. `lint` catches it as `media_missing_id`.
 - Audio always lives on a separate `<audio>` element — even if its source file is the same as a `<video>`. The `<video>` is muted; the `<audio>` carries sound.
-- For volume fades/ducking, animate `volume` on the timeline (`tl.to("#bgm", { volume: 0, duration: 1 }, "outro")`) rather than swapping `data-volume`. The runtime probes the timeline's volume keyframes and applies them identically in preview and render; `data-volume` is the static baseline for elements no tween touches. A tween's values REPLACE that baseline rather than scaling it, so on a clip whose gain is not `1` you scale the tween's targets instead (`{ volume: 1.95 }`, not `{ volume: 1 }`) — `lint` warns with `audio_volume_tween_overrides_gain` when the two disagree.
+- For volume fades, ducking and ramps, write a `data-automation` volume lane on the `<audio>` (`data-automation='{"version":1,"lanes":[{"target":"volume","points":[{"t":0,"v":0.1},{"t":10,"v":0.5}]}]}'`, `t` in seconds from the clip start; details in `hyperframes-audio`). Studio draws and edits that lane, and preview and render apply it identically. A GSAP `volume` tween on the same clip is ignored once a lane exists (`lint`: `audio_volume_double_automation`), so never author both; `data-volume` is the static baseline for a clip with no lane.
 
 For media duration: `<video>` and `<audio>` can omit `data-duration` if the media's intrinsic length is known and you want the full clip. Otherwise provide `data-duration` explicitly.