From 3337cc899071c40e0bf3c6bc030d91c499c639d0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Miguel=20=C3=81ngel?= Date: Sun, 30 Aug 2026 13:00:23 -0400 Subject: [PATCH] feat(studio): give an agent eyes with studio_frame (#3516) * feat(studio): let an agent drive Studio's selection and playhead Adds `studio_select` and `studio_seek`, so an agent and the human are looking at the same element and the same instant. Selecting reveals the inspector, exactly as a click does, which is what makes the agent's move visible. Selection is shared state, not a per-call argument, and that is forced rather than chosen. Most of Studio's edit handlers read the ambient React selection, and `applyDomSelection` only schedules a state update, so selecting and committing inside ONE call would write to whatever was selected before. Two tool calls are separated by a render, so the contract is select first, then act. That is also how a human works: click, then type. `studio_seek` uses `requestSeek`, not `setCurrentTime`. The latter only moves the timeline's displayed number and leaves the composition where it was. Two things the tools refuse to fake: Seek does not clamp. `seek()` already clamps against the adapter's duration, which can differ from the store's, and clamping again would give that invariant two owners that can disagree. The tool reports where the playhead actually landed instead, read back afterwards. `requestSeek` is fire-and-forget, so it cannot report that no adapter was mounted to receive it. The tool compares the playhead before and after and fails rather than claiming a seek that never happened. Select separates three failures that a single message would have merged: the preview is not mounted yet (wait), no element matches the handle (re-read), and the element cannot be selected (try a neighbour). The agent's next move differs for each, so collapsing them would cost it a round trip or a retry loop. * feat(studio): give an agent eyes with studio_frame Renders the composition to a PNG at a given time and returns the URL. This is what turns the tool set from a remote control into a loop: author a change, capture the instant it affects, look, adjust. No agent can judge motion from source, because "what does this look like at 2.4 seconds" is not a question a file answers. Reuses Studio's existing capture endpoint via `buildFrameCaptureUrl` rather than inventing a second one. Two things this does not fake: It reports the time the playhead LANDED on, not the time requested. The player clamps, so those differ at the ends, and attaching the wrong time to a frame is how an agent draws a confident wrong conclusion about motion. It waits before capturing, by default 150ms. The frame is rendered from the file on disk, and the render cache is cleared by a file watcher with a 40ms write-stability threshold, so a capture that beats the watcher renders the PRE-edit composition. That exact staleness was a real bug here once. An agent reading a stale frame as "my edit failed" would thrash, so the wait is on by default, `settleMs` makes it tunable, and the tool description names the failure rather than leaving it to be rediscovered. It probes with HEAD before returning, so a URL that 404s comes back as a failure with a hint instead of as a link the agent cannot render. * feat(studio): add studio_inspect, so an agent reads before it writes (#3517) Everything about one element in one call: resolved styles, text fields, box, data attributes, GSAP animations, and what the element will and will not accept. The point is to prevent a failed write rather than to satisfy curiosity. `can.reasonIfDisabled` is passed through verbatim from Studio's own capabilities, so an agent that reads first should never attempt an edit the element would refuse. Three things it refuses to get wrong: Animations are reported ONLY for the current selection, because that is the only element Studio parses them for. Attributing them to any other element would be reporting the wrong element's motion, which is worse than reporting none. When a handle names something else the field is empty and `animationEditingBlocked` says why. `animationEditingBlocked` also carries the two states where animation editing is off entirely, multiple timelines and an unsupported timeline pattern. Both live on the selection context. Learning them from a read costs one call; learning them from a failed write costs a retry loop. Inspecting a handle does NOT change what is selected. It is a read, and stealing the human's selection would be a side effect they did not ask for. There is a test asserting `applySelection` is never called. Nothing selected and no handle given is a failure, not an empty result. An empty result would assert "this element has nothing", which is a different and false claim. --------- Co-authored-by: miga-heygen Co-authored-by: Claude Opus 4.6 (1M context) --- .../studio/src/webmcp/StudioAgentTools.tsx | 40 +++- .../src/webmcp/tools/frameTools.test.ts | 136 ++++++++++++ .../studio/src/webmcp/tools/frameTools.ts | 133 +++++++++++ .../src/webmcp/tools/inspectTools.test.ts | 197 +++++++++++++++++ .../studio/src/webmcp/tools/inspectTools.ts | 208 ++++++++++++++++++ .../src/webmcp/tools/selectionTools.test.ts | 52 +---- .../src/webmcp/useStudioAgentTools.test.tsx | 26 ++- .../studio/src/webmcp/useStudioAgentTools.ts | 38 +++- packages/studio/src/webmcp/webmcpTestUtils.ts | 91 ++++++++ 9 files changed, 860 insertions(+), 61 deletions(-) create mode 100644 packages/studio/src/webmcp/tools/frameTools.test.ts create mode 100644 packages/studio/src/webmcp/tools/frameTools.ts create mode 100644 packages/studio/src/webmcp/tools/inspectTools.test.ts create mode 100644 packages/studio/src/webmcp/tools/inspectTools.ts create mode 100644 packages/studio/src/webmcp/webmcpTestUtils.ts diff --git a/packages/studio/src/webmcp/StudioAgentTools.tsx b/packages/studio/src/webmcp/StudioAgentTools.tsx index 0130ce1a9..4fbf55e68 100644 --- a/packages/studio/src/webmcp/StudioAgentTools.tsx +++ b/packages/studio/src/webmcp/StudioAgentTools.tsx @@ -18,7 +18,12 @@ import type { StudioLookSnapshot } from "./tools/lookTools"; */ export function StudioAgentTools() { const { projectId, activeCompPath, editHistory } = useStudioShellContext(); - const { domEditSelection, selectedGsapAnimations } = useDomEditSelectionContext(); + const { + domEditSelection, + selectedGsapAnimations, + gsapMultipleTimelines, + gsapUnsupportedTimelinePattern, + } = useDomEditSelectionContext(); const { previewIframeRef, buildDomSelectionFromTarget, applyDomSelection } = useDomEditActionsContext(); @@ -57,8 +62,39 @@ export function StudioAgentTools() { isPlaying: player.isPlaying, }; }, + getProjectId: () => projectId, + getCompositionPath: () => activeCompPath, + // HEAD, not GET: the tool only needs to know the frame renders. Pulling + // the PNG here would download it once for nothing, since the agent + // fetches the URL itself. + probeFrame: async (url) => { + try { + const response = await fetch(url, { method: "HEAD" }); + return { ok: response.ok, status: response.status }; + } catch { + return { ok: false, status: 0 }; + } + }, + wait: (ms) => new Promise((resolve) => setTimeout(resolve, ms)), + getCurrentSelection: () => domEditSelection, + getGsapDiagnostics: () => ({ + animations: selectedGsapAnimations, + multipleTimelines: gsapMultipleTimelines, + unsupportedTimelinePattern: gsapUnsupportedTimelinePattern, + }), }), - [getSnapshot, previewIframeRef, buildDomSelectionFromTarget, applyDomSelection], + [ + getSnapshot, + previewIframeRef, + buildDomSelectionFromTarget, + applyDomSelection, + projectId, + activeCompPath, + domEditSelection, + selectedGsapAnimations, + gsapMultipleTimelines, + gsapUnsupportedTimelinePattern, + ], ); useStudioAgentTools(deps); diff --git a/packages/studio/src/webmcp/tools/frameTools.test.ts b/packages/studio/src/webmcp/tools/frameTools.test.ts new file mode 100644 index 000000000..8ac0b5a43 --- /dev/null +++ b/packages/studio/src/webmcp/tools/frameTools.test.ts @@ -0,0 +1,136 @@ +// @vitest-environment jsdom +import { describe, expect, it, vi } from "vitest"; +import { studioFrame, type FrameToolDeps, type StudioFrameResult } from "./frameTools"; +import { expectFailure, expectOk } from "../webmcpTestUtils"; + +function frameDeps(overrides: Partial = {}): FrameToolDeps { + return { + getProjectId: () => "demo", + getCompositionPath: () => "index.html", + readPlayhead: () => ({ currentTime: 2.4, duration: 10, isPlaying: false }), + requestSeek: () => undefined, + probeFrame: async () => ({ ok: true, status: 200 }), + wait: async () => undefined, + ...overrides, + }; +} + +describe("studioFrame", () => { + it("returns a URL for the composition at the playhead", async () => { + const result = await studioFrame(frameDeps()); + + const ok = expectOk(result); + expect(ok.time).toBe(2.4); + expect(ok.compositionPath).toBe("index.html"); + expect(ok.url).toContain("/thumbnail/"); + expect(ok.url).toContain("t=2.400"); + expect(ok.url).toContain("format=png"); + }); + + it("seeks first when given a time", async () => { + const requestSeek = vi.fn(); + + await studioFrame(frameDeps({ requestSeek }), { time: 5 }); + + expect(requestSeek).toHaveBeenCalledWith(5); + }); + + it("captures where the playhead LANDED, not what was asked for", async () => { + // The player clamps. Reporting the request would attach the wrong time to + // the frame, and an agent judging motion would draw the wrong conclusion. + const result = await studioFrame( + frameDeps({ readPlayhead: () => ({ currentTime: 10, duration: 10, isPlaying: false }) }), + { time: 999 }, + ); + + const ok = expectOk(result); + expect(ok.time).toBe(10); + expect(ok.url).toContain("t=10.000"); + }); + + it("waits before capturing, so a just-made edit is in the frame", async () => { + // The render cache is cleared by a file watcher with a write-stability + // threshold. Capturing faster than that renders the PRE-edit composition. + const wait = vi.fn(async () => undefined); + const order: string[] = []; + + await studioFrame( + frameDeps({ + wait: async (ms) => { + order.push(`wait:${ms}`); + await wait(); + }, + probeFrame: async () => { + order.push("probe"); + return { ok: true, status: 200 }; + }, + }), + ); + + expect(order).toEqual(["wait:150", "probe"]); + }); + + it("honours a caller-supplied settle time and reports it", async () => { + const result = await studioFrame(frameDeps(), { settleMs: 800 }); + + expect(expectOk(result).settledMs).toBe(800); + }); + + it("clamps an absurd settle time rather than hanging", async () => { + const result = await studioFrame(frameDeps(), { settleMs: 10 * 60 * 1000 }); + + expect(expectOk(result).settledMs).toBe(5000); + }); + + it("falls back to the default for a nonsense settle time", async () => { + for (const settleMs of [-1, Number.NaN]) { + const result = await studioFrame(frameDeps(), { settleMs }); + expect(expectOk(result).settledMs).toBe(150); + } + }); + + it("skips the wait entirely when asked for zero", async () => { + const wait = vi.fn(async () => undefined); + + await studioFrame(frameDeps({ wait }), { settleMs: 0 }); + + expect(wait).not.toHaveBeenCalled(); + }); + + it("reports a renderer failure instead of handing back a dead URL", async () => { + const result = expectFailure( + await studioFrame(frameDeps({ probeFrame: async () => ({ ok: false, status: 500 }) })), + ); + + expect(result.kind).toBe("failed"); + expect(result.reason).toContain("500"); + expect(result.hint).toBeDefined(); + }); + + it("fails when no project is open, before touching the renderer", async () => { + const probeFrame = vi.fn(); + + const result = expectFailure( + await studioFrame(frameDeps({ getProjectId: () => null, probeFrame })), + ); + + expect(result.kind).toBe("blocked"); + expect(probeFrame).not.toHaveBeenCalled(); + }); + + it("rejects a negative or non-finite time without seeking", async () => { + const requestSeek = vi.fn(); + + for (const time of [-1, Number.NaN, Number.POSITIVE_INFINITY]) { + const result = expectFailure(await studioFrame(frameDeps({ requestSeek }), { time })); + expect(result.kind).toBe("invalid"); + } + expect(requestSeek).not.toHaveBeenCalled(); + }); + + it("captures the master composition when no path is active", async () => { + const result = await studioFrame(frameDeps({ getCompositionPath: () => null })); + + expect(expectOk(result).compositionPath).toBe("index.html"); + }); +}); diff --git a/packages/studio/src/webmcp/tools/frameTools.ts b/packages/studio/src/webmcp/tools/frameTools.ts new file mode 100644 index 000000000..839e4c231 --- /dev/null +++ b/packages/studio/src/webmcp/tools/frameTools.ts @@ -0,0 +1,133 @@ +/** + * `studio_frame`: the eyes. + * + * Without this the tool set is a remote control. With it an agent can author a + * change, look at the instant it affects, judge it, and adjust. That loop is the + * one thing source alone cannot support, because "what does this look like at + * 2.4 seconds" is not a question a file can answer. + * + * Reuses Studio's existing capture endpoint (`utils/frameCapture`) rather than + * inventing a second one. The server renders the composition with Puppeteer, so + * the frame reflects the file on disk, not the live preview DOM. + */ + +import { buildFrameCaptureUrl } from "../../utils/frameCapture"; +import { toolFailure, toolOk, type ToolResult } from "../toolResult"; + +export interface FrameToolDeps { + getProjectId: () => string | null; + getCompositionPath: () => string | null; + readPlayhead: () => { currentTime: number; duration: number; isPlaying: boolean }; + requestSeek: (time: number) => void; + /** Confirms the URL renders. Injected so tests need no network. */ + probeFrame: (url: string) => Promise<{ ok: boolean; status: number }>; + wait: (ms: number) => Promise; +} + +export interface StudioFrameResult { + /** Fetch this to see the frame. A PNG of the composition at `time`. */ + url: string; + time: number; + compositionPath: string; + /** How long the tool waited for a pending write to settle before capturing. */ + settledMs: number; +} + +export interface StudioFrameInput { + /** Seconds. Omit to capture wherever the playhead already is. */ + time?: number; + /** + * Milliseconds to wait before capturing, so a just-written edit is visible. + * See the staleness note in the description. + */ + settleMs?: number; +} + +/** + * Long enough to cover the project watcher's 40ms write-stability threshold + * plus filesystem latency, short enough not to be felt. This is the mitigation + * for a real, previously-fixed bug: the preview signature is invalidated by a + * file watcher, and a capture that beats the watcher renders the PRE-edit + * composition. An agent reading that as "my edit failed" would thrash. + */ +const DEFAULT_SETTLE_MS = 150; +const MAX_SETTLE_MS = 5_000; + +export async function studioFrame( + deps: FrameToolDeps, + input: StudioFrameInput = {}, +): Promise> { + const projectId = deps.getProjectId(); + if (!projectId) { + return toolFailure("blocked", "no project is open"); + } + + if (input.time !== undefined) { + if (typeof input.time !== "number" || !Number.isFinite(input.time) || input.time < 0) { + return toolFailure("invalid", "time must be a non-negative, finite number of seconds"); + } + deps.requestSeek(input.time); + } + + const settledMs = clampSettle(input.settleMs); + if (settledMs > 0) await deps.wait(settledMs); + + // Capture whatever the playhead now reads, rather than what was requested: + // the player clamps, so those can differ and the frame belongs to the former. + const { currentTime } = deps.readPlayhead(); + const compositionPath = deps.getCompositionPath(); + const url = buildFrameCaptureUrl({ projectId, compositionPath, currentTime }); + + const probe = await deps.probeFrame(url); + if (!probe.ok) { + return toolFailure( + "failed", + `the renderer returned ${probe.status} for this frame`, + "The composition may not build. Try `hyperframes check`.", + ); + } + + return toolOk({ + url, + time: currentTime, + compositionPath: compositionPath ?? "index.html", + settledMs, + }); +} + +function clampSettle(requested: number | undefined): number { + if (requested === undefined) return DEFAULT_SETTLE_MS; + if (typeof requested !== "number" || !Number.isFinite(requested) || requested < 0) { + return DEFAULT_SETTLE_MS; + } + return Math.min(requested, MAX_SETTLE_MS); +} + +export const STUDIO_FRAME_INPUT_SCHEMA = { + type: "object", + properties: { + time: { + type: "number", + minimum: 0, + description: "Seconds. Omit to capture wherever the playhead already is.", + }, + settleMs: { + type: "integer", + minimum: 0, + maximum: MAX_SETTLE_MS, + description: `Wait this long before capturing so a just-made edit is included. Default ${DEFAULT_SETTLE_MS}.`, + }, + }, + additionalProperties: false, +} as const; + +export const STUDIO_FRAME_DESCRIPTION = [ + "Render the composition to a PNG at a given time and return its URL, so you can", + "SEE the result instead of inferring it from source. Use this to judge a change:", + "edit, capture the instant it affects, look, adjust.", + "The frame is rendered from the file on disk, not the live preview.", + "A capture taken immediately after an edit can therefore predate that edit, because", + "the render cache is cleared by a file watcher. The tool waits briefly to cover that;", + "raise `settleMs` if a frame still looks stale, rather than concluding the edit failed.", + "Returns `ok: true` with `url` and the `time` actually captured, or `ok: false`.", +].join(" "); diff --git a/packages/studio/src/webmcp/tools/inspectTools.test.ts b/packages/studio/src/webmcp/tools/inspectTools.test.ts new file mode 100644 index 000000000..d59034a45 --- /dev/null +++ b/packages/studio/src/webmcp/tools/inspectTools.test.ts @@ -0,0 +1,197 @@ +// @vitest-environment jsdom +import { describe, expect, it, vi } from "vitest"; +import type { GsapAnimation } from "@hyperframes/parsers/gsap-parser"; +import { studioInspect, type InspectToolDeps, type StudioInspectResult } from "./inspectTools"; +import { + expectFailure, + expectOk, + previewDoc, + previewElement, + selectionFor, +} from "../webmcpTestUtils"; + +function animation(overrides: Partial = {}): GsapAnimation { + return { + id: "anim-1", + targetSelector: "#headline", + method: "from", + position: 0, + properties: { y: -50, opacity: 0 }, + duration: 1, + ease: "power2.out", + ...overrides, + } as GsapAnimation; +} + +function inspectDeps(overrides: Partial = {}): InspectToolDeps { + return { + getPreviewDocument: () => null, + buildSelection: async (element) => selectionFor(element), + applySelection: () => undefined, + requestSeek: () => undefined, + readPlayhead: () => ({ currentTime: 0, duration: 10, isPlaying: false }), + getCurrentSelection: () => null, + getGsapDiagnostics: () => ({ + animations: [], + multipleTimelines: false, + unsupportedTimelinePattern: false, + }), + ...overrides, + }; +} + +describe("studioInspect", () => { + it("returns the resolved styles, not the authored ones", async () => { + const element = previewElement('

Ship it

', "headline"); + const selection = selectionFor(element); + + const result = await studioInspect(inspectDeps({ getCurrentSelection: () => selection })); + + const ok = expectOk(result); + // The authored value is a clamp(); the resolved one is what actually renders. + expect(ok.styles["font-size"]).toBe("42.7px"); + expect(ok.inlineStyles.color).toBe("red"); + expect(ok.box.width).toBe(880); + }); + + it("reports capabilities and the disabled reason verbatim", async () => { + const element = previewElement('

Ship it

', "headline"); + const locked = selectionFor(element, { + capabilities: { + canSelect: true, + canEditStyles: false, + canCrop: false, + canMove: false, + canResize: false, + canApplyManualOffset: false, + canApplyManualSize: false, + canApplyManualRotation: false, + reasonIfDisabled: "Element is inside a locked composition", + }, + }); + + const result = await studioInspect(inspectDeps({ getCurrentSelection: () => locked })); + + const ok = expectOk(result); + expect(ok.can.editStyles).toBe(false); + expect(ok.can.move).toBe(false); + expect(ok.can.reasonIfDisabled).toBe("Element is inside a locked composition"); + }); + + it("lists the animations on the current selection", async () => { + const element = previewElement('

Ship it

', "headline"); + + const result = await studioInspect( + inspectDeps({ + getCurrentSelection: () => selectionFor(element), + getGsapDiagnostics: () => ({ + animations: [animation()], + multipleTimelines: false, + unsupportedTimelinePattern: false, + }), + }), + ); + + const ok = expectOk(result); + expect(ok.animations).toHaveLength(1); + expect(ok.animations[0]?.animationId).toBe("anim-1"); + expect(ok.animations[0]?.ease).toBe("power2.out"); + expect(ok.animationEditingBlocked).toBeNull(); + }); + + it("says WHY animation editing is unavailable, so a write is not attempted", async () => { + const element = previewElement('

Ship it

', "headline"); + const base = { + getCurrentSelection: () => selectionFor(element), + }; + + const multiple = await studioInspect( + inspectDeps({ + ...base, + getGsapDiagnostics: () => ({ + animations: [], + multipleTimelines: true, + unsupportedTimelinePattern: false, + }), + }), + ); + const unsupported = await studioInspect( + inspectDeps({ + ...base, + getGsapDiagnostics: () => ({ + animations: [], + multipleTimelines: false, + unsupportedTimelinePattern: true, + }), + }), + ); + + expect(expectOk(multiple).animationEditingBlocked).toMatch( + /multiple GSAP timelines/, + ); + expect(expectOk(unsupported).animationEditingBlocked).toMatch( + /not editable/, + ); + }); + + it("does not attribute the selection's animations to a different element", async () => { + // Studio only parses animations for the CURRENT selection. Reporting them + // against another element would report the wrong element's motion. + const headline = previewElement('

A

B

', "headline"); + const doc = headline.ownerDocument; + + const result = await studioInspect( + inspectDeps({ + getPreviewDocument: () => doc, + getCurrentSelection: () => selectionFor(headline), + getGsapDiagnostics: () => ({ + animations: [animation()], + multipleTimelines: false, + unsupportedTimelinePattern: false, + }), + }), + { handle: "dom:body" }, + ); + + const ok = expectOk(result); + expect(ok.isCurrentSelection).toBe(false); + expect(ok.animations).toEqual([]); + expect(ok.animationEditingBlocked).toMatch(/only readable for the current selection/); + }); + + it("inspects a handle without changing what is selected", async () => { + const doc = previewDoc('

A

'); + const applySelection = vi.fn(); + + const result = await studioInspect( + inspectDeps({ getPreviewDocument: () => doc, applySelection }), + { handle: "dom:headline" }, + ); + + expect(result.ok).toBe(true); + // Inspecting is a read. It must not steal the human's selection. + expect(applySelection).not.toHaveBeenCalled(); + }); + + it("fails rather than returning an empty result when nothing is selected", async () => { + const result = expectFailure(await studioInspect(inspectDeps())); + + // An empty result would assert "this element has nothing", a different and + // false claim from "you did not say which element". + expect(result.kind).toBe("invalid"); + expect(result.reason).toMatch(/nothing is selected/); + expect(result.hint).toMatch(/studio_select/); + }); + + it("reports an unknown handle distinctly from an unmounted preview", async () => { + const notMounted = expectFailure(await studioInspect(inspectDeps(), { handle: "dom:x" })); + expect(notMounted.kind).toBe("blocked"); + + const doc = previewDoc('

A

'); + const unknown = expectFailure( + await studioInspect(inspectDeps({ getPreviewDocument: () => doc }), { handle: "dom:x" }), + ); + expect(unknown.kind).toBe("invalid"); + expect(unknown.reason).not.toBe(notMounted.reason); + }); +}); diff --git a/packages/studio/src/webmcp/tools/inspectTools.ts b/packages/studio/src/webmcp/tools/inspectTools.ts new file mode 100644 index 000000000..f31b9fc83 --- /dev/null +++ b/packages/studio/src/webmcp/tools/inspectTools.ts @@ -0,0 +1,208 @@ +/** + * `studio_inspect`: everything about one element, in one call. + * + * The point is to prevent a failed write. Every field here either tells the + * agent what it can change (`can`, with `reasonIfDisabled` verbatim) or what it + * would be changing (the resolved styles, the text fields, the animations). + * An agent that reads this first should never attempt an edit the element will + * refuse. + * + * The GSAP diagnostics are here for the same reason: `multipleTimelines` and + * `unsupportedTimelinePattern` are states where animation editing is off, and + * learning that from a read is cheaper than learning it from a failed write. + */ + +import type { GsapAnimation } from "@hyperframes/parsers/gsap-parser"; +import type { DomEditSelection } from "../../components/editor/domEditingTypes"; +import { mintElementHandle, patchTargetAddress, resolveElementHandle } from "../handles"; +import { toolFailure, toolOk, type ToolResult } from "../toolResult"; +import type { SelectionToolDeps } from "./selectionTools"; + +export interface InspectToolDeps extends SelectionToolDeps { + /** What the human currently has selected, used when no handle is given. */ + getCurrentSelection: () => DomEditSelection | null; + getGsapDiagnostics: () => { + animations: readonly GsapAnimation[]; + multipleTimelines: boolean; + unsupportedTimelinePattern: boolean; + }; +} + +interface InspectAnimation { + animationId: string; + method: string; + target: string; + position: number | string; + duration: number | null; + ease: string | null; + properties: Record; + hasKeyframes: boolean; + hasArcPath: boolean; +} + +interface InspectTextField { + key: string; + label: string; + value: string; + tagName: string; +} + +export interface StudioInspectResult { + handle: string | null; + label: string; + tagName: string; + sourceFile: string; + box: { x: number; y: number; width: number; height: number }; + text: string | null; + textFields: InspectTextField[]; + /** The styles Studio itself surfaces, resolved, not as authored. */ + styles: Record; + inlineStyles: Record; + dataAttributes: Record; + can: { + editStyles: boolean; + move: boolean; + resize: boolean; + rotate: boolean; + crop: boolean; + editText: boolean; + reasonIfDisabled: string | null; + }; + animations: InspectAnimation[]; + /** Present only when animation editing is unavailable, with the reason. */ + animationEditingBlocked: string | null; + /** True when this element is the one the human currently has selected. */ + isCurrentSelection: boolean; +} + +export interface StudioInspectInput { + /** Omit to inspect the current selection. */ + handle?: string; +} + +function describeAnimation(animation: GsapAnimation): InspectAnimation { + return { + animationId: animation.id, + method: animation.method, + target: animation.targetSelector, + position: animation.position, + duration: animation.duration ?? null, + ease: animation.ease ?? null, + properties: animation.properties, + hasKeyframes: animation.keyframes !== undefined, + hasArcPath: animation.arcPath !== undefined, + }; +} + +function describe( + selection: DomEditSelection, + deps: InspectToolDeps, + isCurrentSelection: boolean, +): ToolResult { + const { capabilities } = selection; + const gsap = deps.getGsapDiagnostics(); + + // Only the CURRENT selection's animations are parsed by Studio. Reporting + // them for some other element would be reporting the wrong element's motion, + // which is worse than reporting none. + const animations = isCurrentSelection ? gsap.animations.map(describeAnimation) : []; + + let animationEditingBlocked: string | null = null; + if (!isCurrentSelection) { + animationEditingBlocked = "animations are only readable for the current selection"; + } else if (gsap.multipleTimelines) { + animationEditingBlocked = "this composition has multiple GSAP timelines"; + } else if (gsap.unsupportedTimelinePattern) { + animationEditingBlocked = "this composition's timeline pattern is not editable by Studio"; + } + + return toolOk({ + handle: mintElementHandle(patchTargetAddress(selection)), + label: selection.label, + tagName: selection.tagName, + sourceFile: selection.sourceFile, + box: selection.boundingBox, + text: selection.textContent, + textFields: selection.textFields.map((field) => ({ + key: field.key, + label: field.label, + value: field.value, + tagName: field.tagName, + })), + styles: selection.computedStyles, + inlineStyles: selection.inlineStyles, + dataAttributes: selection.dataAttributes, + can: { + editStyles: capabilities.canEditStyles, + move: capabilities.canMove || capabilities.canApplyManualOffset, + resize: capabilities.canResize || capabilities.canApplyManualSize, + rotate: capabilities.canApplyManualRotation, + crop: capabilities.canCrop, + editText: selection.textFields.length > 0, + reasonIfDisabled: capabilities.reasonIfDisabled ?? null, + }, + animations, + animationEditingBlocked, + isCurrentSelection, + }); +} + +export async function studioInspect( + deps: InspectToolDeps, + input: StudioInspectInput = {}, +): Promise> { + const current = deps.getCurrentSelection(); + + if (!input.handle) { + // An empty result here would assert "this element has nothing", which is a + // different and false claim from "you did not tell me which element". + if (!current) { + return toolFailure( + "invalid", + "nothing is selected and no handle was given", + "Pass a handle from studio_look, or call studio_select first.", + ); + } + return describe(current, deps, true); + } + + const doc = deps.getPreviewDocument(); + if (!doc) return toolFailure("blocked", "the preview is not mounted yet"); + + const element = resolveElementHandle(doc, input.handle); + if (!element) { + return toolFailure( + "invalid", + `no element matches handle ${input.handle}`, + "Call studio_look for current handles.", + ); + } + + const selection = await deps.buildSelection(element); + if (!selection) { + return toolFailure("blocked", `${input.handle} resolved to an element Studio cannot inspect`); + } + + return describe(selection, deps, current?.element === element); +} + +export const STUDIO_INSPECT_INPUT_SCHEMA = { + type: "object", + properties: { + handle: { + type: "string", + description: "An element handle from studio_look. Omit to inspect the current selection.", + }, + }, + additionalProperties: false, +} as const; + +export const STUDIO_INSPECT_DESCRIPTION = [ + "Everything about one element: its resolved styles, its text fields, its box,", + "its GSAP animations, and crucially what it will and will not accept.", + "Read this BEFORE editing. `can` tells you which edits are possible and", + "`can.reasonIfDisabled` says why one is not, so you can avoid a write that would be refused.", + "Animations are only readable for the CURRENT selection; `animationEditingBlocked` says when", + "and why animation editing is unavailable.", + "Returns `ok: true`, or `ok: false` with `kind`, `reason` and a `hint`.", +].join(" "); diff --git a/packages/studio/src/webmcp/tools/selectionTools.test.ts b/packages/studio/src/webmcp/tools/selectionTools.test.ts index c6ec5fdd2..07168529a 100644 --- a/packages/studio/src/webmcp/tools/selectionTools.test.ts +++ b/packages/studio/src/webmcp/tools/selectionTools.test.ts @@ -1,6 +1,5 @@ // @vitest-environment jsdom import { describe, expect, it, vi } from "vitest"; -import type { DomEditSelection } from "../../components/editor/domEditingTypes"; import { studioSeek, studioSelect, @@ -8,46 +7,7 @@ import { type StudioSeekResult, type StudioSelectResult, } from "./selectionTools"; -import type { ToolFailure, ToolResult } from "../toolResult"; - -function previewDoc(html: string): Document { - const iframe = document.createElement("iframe"); - document.body.append(iframe); - const doc = iframe.contentDocument; - if (!doc) throw new Error("expected iframe document"); - doc.body.innerHTML = html; - return doc; -} - -function selectionFor(element: HTMLElement): DomEditSelection { - return { - id: element.id || undefined, - hfId: element.getAttribute("data-hf-id") ?? undefined, - element, - label: "Headline", - tagName: element.tagName.toLowerCase(), - sourceFile: "index.html", - compositionPath: "index.html", - isCompositionHost: false, - isInsideLockedComposition: false, - boundingBox: { x: 40, y: 12, width: 880, height: 96 }, - textContent: element.textContent, - dataAttributes: {}, - inlineStyles: {}, - computedStyles: {}, - textFields: [], - capabilities: { - canSelect: true, - canEditStyles: true, - canCrop: true, - canMove: true, - canResize: true, - canApplyManualOffset: true, - canApplyManualSize: true, - canApplyManualRotation: true, - }, - }; -} +import { expectFailure, expectOk, previewDoc, selectionFor } from "../webmcpTestUtils"; function selectionDeps(overrides: Partial = {}): SelectionToolDeps { return { @@ -60,16 +20,6 @@ function selectionDeps(overrides: Partial = {}): SelectionToo }; } -function expectFailure(result: ToolResult): ToolFailure { - if (result.ok) throw new Error(`expected failure, got ${JSON.stringify(result)}`); - return result; -} - -function expectOk(result: ToolResult): { ok: true } & T { - if (!result.ok) throw new Error(`expected ok, got ${JSON.stringify(result)}`); - return result; -} - describe("studioSelect", () => { it("applies the selection a click would produce and reports it back", async () => { const doc = previewDoc('

Ship it

'); diff --git a/packages/studio/src/webmcp/useStudioAgentTools.test.tsx b/packages/studio/src/webmcp/useStudioAgentTools.test.tsx index 3386e5f3a..3a9a3ee98 100644 --- a/packages/studio/src/webmcp/useStudioAgentTools.test.tsx +++ b/packages/studio/src/webmcp/useStudioAgentTools.test.tsx @@ -38,6 +38,16 @@ function deps(overrides: Partial = {}): StudioAgentToolsDe applySelection: () => undefined, requestSeek: () => undefined, readPlayhead: () => ({ currentTime: 0, duration: 10, isPlaying: false }), + getProjectId: () => "demo", + getCompositionPath: () => "index.html", + probeFrame: async () => ({ ok: true, status: 200 }), + wait: async () => undefined, + getCurrentSelection: () => null, + getGsapDiagnostics: () => ({ + animations: [], + multipleTimelines: false, + unsupportedTimelinePattern: false, + }), ...overrides, }; } @@ -102,6 +112,8 @@ describe("useStudioAgentTools", () => { "studio_look", "studio_select", "studio_seek", + "studio_frame", + "studio_inspect", ]); expect(trackEvent).toHaveBeenCalledWith("webmcp.native_present"); }); @@ -116,14 +128,14 @@ describe("useStudioAgentTools", () => { await act(async () => { harness = mountTools(deps({ getSnapshot: () => snapshot() })); }); - expect(registerTool).toHaveBeenCalledTimes(3); + expect(registerTool).toHaveBeenCalledTimes(5); await act(async () => { harness?.rerenderWith(deps({ getSnapshot: () => snapshot({ currentTime: 5 }) })); harness?.rerenderWith(deps({ getSnapshot: () => snapshot({ currentTime: 6 }) })); }); - expect(registerTool).toHaveBeenCalledTimes(3); + expect(registerTool).toHaveBeenCalledTimes(5); }); it("executes against the LATEST deps, not the ones present at registration", async () => { @@ -166,16 +178,16 @@ describe("useStudioAgentTools", () => { expect(signal?.aborted).toBe(true); }); - it("boots cleanly when the browser has no native WebMCP", async () => { + it("registers nothing when the browser has no WebMCP", async () => { removeModelContext(); await act(async () => { mountTools(deps({ getSnapshot: () => snapshot() })); }); - // The assertion is that mounting did not throw; a browser without the - // native API must still boot Studio. The polyfill may install - // document.modelContext as a fallback — that is expected. + // The assertion is that mounting did not throw; a browser without the API + // must still boot Studio. + expect(document).not.toHaveProperty("modelContext"); }); it("registers nothing when the preference is turned off", async () => { @@ -196,7 +208,7 @@ describe("useStudioAgentTools", () => { mountTools(deps({ getSnapshot: () => snapshot() })); }); - expect(registerTool).toHaveBeenCalledTimes(3); + expect(registerTool).toHaveBeenCalledTimes(5); }); it("reports a non-abort registration failure through production telemetry", async () => { diff --git a/packages/studio/src/webmcp/useStudioAgentTools.ts b/packages/studio/src/webmcp/useStudioAgentTools.ts index 2c67d2332..c5e1a444e 100644 --- a/packages/studio/src/webmcp/useStudioAgentTools.ts +++ b/packages/studio/src/webmcp/useStudioAgentTools.ts @@ -25,6 +25,22 @@ import { type StudioSeekResult, type StudioSelectResult, } from "./tools/selectionTools"; +import { + studioFrame, + STUDIO_FRAME_DESCRIPTION, + STUDIO_FRAME_INPUT_SCHEMA, + type FrameToolDeps, + type StudioFrameInput, + type StudioFrameResult, +} from "./tools/frameTools"; +import { + studioInspect, + STUDIO_INSPECT_DESCRIPTION, + STUDIO_INSPECT_INPUT_SCHEMA, + type InspectToolDeps, + type StudioInspectInput, + type StudioInspectResult, +} from "./tools/inspectTools"; const log = makeStudioDebugLogger("webmcp"); @@ -38,7 +54,7 @@ function reportRegistration(report: ToolRegistrationReport, native: boolean): vo } } -export interface StudioAgentToolsDeps extends SelectionToolDeps { +export interface StudioAgentToolsDeps extends SelectionToolDeps, FrameToolDeps, InspectToolDeps { /** Read Studio's current state. Called per tool invocation, never cached. */ getSnapshot: () => StudioLookSnapshot; } @@ -90,6 +106,26 @@ function buildStudioTools(depsRef: { readonly current: StudioAgentToolsDeps }): studioSeek(depsRef.current, readNumberInput(input, "time")), ), }, + { + name: "studio_frame", + title: "See the composition", + description: STUDIO_FRAME_DESCRIPTION, + inputSchema: STUDIO_FRAME_INPUT_SCHEMA, + annotations: { readOnlyHint: true, untrustedContentHint: true }, + execute: (input): Promise> => + runToolBody("studio_frame", () => studioFrame(depsRef.current, input as StudioFrameInput)), + }, + { + name: "studio_inspect", + title: "Inspect one element", + description: STUDIO_INSPECT_DESCRIPTION, + inputSchema: STUDIO_INSPECT_INPUT_SCHEMA, + annotations: { readOnlyHint: true, untrustedContentHint: true }, + execute: (input): Promise> => + runToolBody("studio_inspect", () => + studioInspect(depsRef.current, input as StudioInspectInput), + ), + }, ]; } diff --git a/packages/studio/src/webmcp/webmcpTestUtils.ts b/packages/studio/src/webmcp/webmcpTestUtils.ts new file mode 100644 index 000000000..8f4f5e96b --- /dev/null +++ b/packages/studio/src/webmcp/webmcpTestUtils.ts @@ -0,0 +1,91 @@ +/** + * Shared fixtures for the WebMCP tool tests. + * + * Not a `.test` file so vitest does not collect it as a suite. Mirrors the + * existing `hooks/domSelectionTestHarness.ts` convention. + */ + +import { expect } from "vitest"; +import type { DomEditSelection } from "../components/editor/domEditingTypes"; +import type { ToolFailure, ToolResult } from "./toolResult"; + +/** + * An element inside a real iframe, which is where Studio's chrome expects to + * find preview elements. The separate realm matters: a preview element is not + * an instance of Studio's own `HTMLElement`. + */ +export function previewDoc(html: string): Document { + const iframe = document.createElement("iframe"); + document.body.append(iframe); + const doc = iframe.contentDocument; + if (!doc) throw new Error("expected iframe document"); + doc.body.innerHTML = html; + return doc; +} + +export function previewElement(html: string, id: string): HTMLElement { + const doc = previewDoc(html); + const element = doc.getElementById(id); + const HTMLElementCtor = doc.defaultView?.HTMLElement; + if (!HTMLElementCtor || !(element instanceof HTMLElementCtor)) { + throw new Error(`expected preview element #${id}`); + } + return element; +} + +export function selectionFor( + element: HTMLElement, + overrides: Partial = {}, +): DomEditSelection { + return { + id: element.id || undefined, + hfId: element.getAttribute("data-hf-id") ?? undefined, + element, + label: "Headline", + tagName: element.tagName.toLowerCase(), + sourceFile: "index.html", + compositionPath: "index.html", + isCompositionHost: false, + isInsideLockedComposition: false, + boundingBox: { x: 40, y: 12, width: 880, height: 96 }, + textContent: element.textContent, + dataAttributes: { "data-role": "title" }, + inlineStyles: { color: "red" }, + computedStyles: { "font-size": "42.7px", color: "rgb(255, 0, 0)" }, + textFields: [ + { + key: "self", + label: "Text", + value: element.textContent ?? "", + tagName: element.tagName.toLowerCase(), + attributes: [], + inlineStyles: {}, + computedStyles: {}, + source: "self", + }, + ], + capabilities: { + canSelect: true, + canEditStyles: true, + canCrop: true, + canMove: true, + canResize: true, + canApplyManualOffset: true, + canApplyManualSize: true, + canApplyManualRotation: true, + }, + ...overrides, + }; +} + +export function expectOk(result: ToolResult): { ok: true } & T { + expect(result.ok, `expected ok, got ${JSON.stringify(result)}`).toBe(true); + if (!result.ok) throw new Error("unreachable"); + return result; +} + +export function expectFailure(result: ToolResult): ToolFailure { + expect(result.ok, `expected failure, got ${JSON.stringify(result)}`).toBe(false); + if (result.ok) throw new Error("unreachable"); + return result; +}