mirror of
https://github.com/heygen-com/hyperframes.git
synced 2026-09-03 23:03:09 +00:00
feat(studio): let an agent drive Studio's selection and playhead (#3515)
Adds `studio_select` and `studio_seek`, so an agent and the human are looking at the same element and the same instant. Selecting reveals the inspector, exactly as a click does, which is what makes the agent's move visible. Selection is shared state, not a per-call argument, and that is forced rather than chosen. Most of Studio's edit handlers read the ambient React selection, and `applyDomSelection` only schedules a state update, so selecting and committing inside ONE call would write to whatever was selected before. Two tool calls are separated by a render, so the contract is select first, then act. That is also how a human works: click, then type. `studio_seek` uses `requestSeek`, not `setCurrentTime`. The latter only moves the timeline's displayed number and leaves the composition where it was. Two things the tools refuse to fake: Seek does not clamp. `seek()` already clamps against the adapter's duration, which can differ from the store's, and clamping again would give that invariant two owners that can disagree. The tool reports where the playhead actually landed instead, read back afterwards. `requestSeek` is fire-and-forget, so it cannot report that no adapter was mounted to receive it. The tool compares the playhead before and after and fails rather than claiming a seek that never happened. Select separates three failures that a single message would have merged: the preview is not mounted yet (wait), no element matches the handle (re-read), and the element cannot be selected (try a neighbour). The agent's next move differs for each, so collapsing them would cost it a round trip or a retry loop.
This commit is contained in:
@@ -1,8 +1,8 @@
|
||||
import { useCallback } from "react";
|
||||
import { useDomEditSelectionContext } from "../contexts/DomEditContext";
|
||||
import { useCallback, useMemo } from "react";
|
||||
import { useDomEditActionsContext, useDomEditSelectionContext } from "../contexts/DomEditContext";
|
||||
import { useStudioShellContext } from "../contexts/StudioContext";
|
||||
import { usePlayerStore } from "../player";
|
||||
import { useStudioAgentTools } from "./useStudioAgentTools";
|
||||
import { useStudioAgentTools, type StudioAgentToolsDeps } from "./useStudioAgentTools";
|
||||
import type { StudioLookSnapshot } from "./tools/lookTools";
|
||||
|
||||
/**
|
||||
@@ -12,14 +12,15 @@ import type { StudioLookSnapshot } from "./tools/lookTools";
|
||||
* contexts are only readable below `DomEditProvider`, which `App` renders, and
|
||||
* `App.tsx` sits three lines under the 600-line cap.
|
||||
*
|
||||
* The player store is read IMPERATIVELY through `getState()` inside the
|
||||
* snapshot callback rather than subscribed to. Subscribing to `currentTime`
|
||||
* would re-render this component on every animation frame during playback for
|
||||
* a value nothing here displays.
|
||||
* The player store is read IMPERATIVELY through `getState()` rather than
|
||||
* subscribed to. Subscribing to `currentTime` would re-render this component on
|
||||
* every animation frame during playback for a value nothing here displays.
|
||||
*/
|
||||
export function StudioAgentTools() {
|
||||
const { projectId, activeCompPath, editHistory } = useStudioShellContext();
|
||||
const { domEditSelection, selectedGsapAnimations } = useDomEditSelectionContext();
|
||||
const { previewIframeRef, buildDomSelectionFromTarget, applyDomSelection } =
|
||||
useDomEditActionsContext();
|
||||
|
||||
const getSnapshot = useCallback((): StudioLookSnapshot => {
|
||||
const player = usePlayerStore.getState();
|
||||
@@ -41,6 +42,25 @@ export function StudioAgentTools() {
|
||||
};
|
||||
}, [projectId, activeCompPath, domEditSelection, selectedGsapAnimations, editHistory]);
|
||||
|
||||
useStudioAgentTools({ getSnapshot });
|
||||
const deps = useMemo<StudioAgentToolsDeps>(
|
||||
() => ({
|
||||
getSnapshot,
|
||||
getPreviewDocument: () => previewIframeRef.current?.contentDocument ?? null,
|
||||
buildSelection: (element) => buildDomSelectionFromTarget(element),
|
||||
applySelection: (selection) => applyDomSelection(selection, { revealPanel: true }),
|
||||
requestSeek: (time) => usePlayerStore.getState().requestSeek(time),
|
||||
readPlayhead: () => {
|
||||
const player = usePlayerStore.getState();
|
||||
return {
|
||||
currentTime: player.currentTime,
|
||||
duration: player.duration,
|
||||
isPlaying: player.isPlaying,
|
||||
};
|
||||
},
|
||||
}),
|
||||
[getSnapshot, previewIframeRef, buildDomSelectionFromTarget, applyDomSelection],
|
||||
);
|
||||
|
||||
useStudioAgentTools(deps);
|
||||
return null;
|
||||
}
|
||||
|
||||
@@ -36,7 +36,7 @@ export function toolOk<T extends object>(value: T): { ok: true } & T {
|
||||
return { ok: true, ...value };
|
||||
}
|
||||
|
||||
function toolFailure(kind: ToolFailureKind, reason: string, hint?: string): ToolFailure {
|
||||
export function toolFailure(kind: ToolFailureKind, reason: string, hint?: string): ToolFailure {
|
||||
return hint ? { ok: false, kind, reason, hint } : { ok: false, kind, reason };
|
||||
}
|
||||
|
||||
|
||||
@@ -0,0 +1,214 @@
|
||||
// @vitest-environment jsdom
|
||||
import { describe, expect, it, vi } from "vitest";
|
||||
import type { DomEditSelection } from "../../components/editor/domEditingTypes";
|
||||
import {
|
||||
studioSeek,
|
||||
studioSelect,
|
||||
type SelectionToolDeps,
|
||||
type StudioSeekResult,
|
||||
type StudioSelectResult,
|
||||
} from "./selectionTools";
|
||||
import type { ToolFailure, ToolResult } from "../toolResult";
|
||||
|
||||
function previewDoc(html: string): Document {
|
||||
const iframe = document.createElement("iframe");
|
||||
document.body.append(iframe);
|
||||
const doc = iframe.contentDocument;
|
||||
if (!doc) throw new Error("expected iframe document");
|
||||
doc.body.innerHTML = html;
|
||||
return doc;
|
||||
}
|
||||
|
||||
function selectionFor(element: HTMLElement): DomEditSelection {
|
||||
return {
|
||||
id: element.id || undefined,
|
||||
hfId: element.getAttribute("data-hf-id") ?? undefined,
|
||||
element,
|
||||
label: "Headline",
|
||||
tagName: element.tagName.toLowerCase(),
|
||||
sourceFile: "index.html",
|
||||
compositionPath: "index.html",
|
||||
isCompositionHost: false,
|
||||
isInsideLockedComposition: false,
|
||||
boundingBox: { x: 40, y: 12, width: 880, height: 96 },
|
||||
textContent: element.textContent,
|
||||
dataAttributes: {},
|
||||
inlineStyles: {},
|
||||
computedStyles: {},
|
||||
textFields: [],
|
||||
capabilities: {
|
||||
canSelect: true,
|
||||
canEditStyles: true,
|
||||
canCrop: true,
|
||||
canMove: true,
|
||||
canResize: true,
|
||||
canApplyManualOffset: true,
|
||||
canApplyManualSize: true,
|
||||
canApplyManualRotation: true,
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
function selectionDeps(overrides: Partial<SelectionToolDeps> = {}): SelectionToolDeps {
|
||||
return {
|
||||
getPreviewDocument: () => null,
|
||||
buildSelection: async (element) => selectionFor(element),
|
||||
applySelection: () => undefined,
|
||||
requestSeek: () => undefined,
|
||||
readPlayhead: () => ({ currentTime: 0, duration: 10, isPlaying: false }),
|
||||
...overrides,
|
||||
};
|
||||
}
|
||||
|
||||
function expectFailure(result: ToolResult<unknown>): ToolFailure {
|
||||
if (result.ok) throw new Error(`expected failure, got ${JSON.stringify(result)}`);
|
||||
return result;
|
||||
}
|
||||
|
||||
function expectOk<T>(result: ToolResult<T>): { ok: true } & T {
|
||||
if (!result.ok) throw new Error(`expected ok, got ${JSON.stringify(result)}`);
|
||||
return result;
|
||||
}
|
||||
|
||||
describe("studioSelect", () => {
|
||||
it("applies the selection a click would produce and reports it back", async () => {
|
||||
const doc = previewDoc('<h1 id="headline" data-hf-id="abc">Ship it</h1>');
|
||||
const applySelection = vi.fn();
|
||||
|
||||
const result = await studioSelect(
|
||||
selectionDeps({ getPreviewDocument: () => doc, applySelection }),
|
||||
"hf:abc",
|
||||
);
|
||||
|
||||
const ok = expectOk<StudioSelectResult>(result);
|
||||
expect(ok.handle).toBe("hf:abc");
|
||||
expect(ok.label).toBe("Headline");
|
||||
expect(ok.box.width).toBe(880);
|
||||
// Reveals the inspector, which is what makes the human see what the agent did.
|
||||
expect(applySelection).toHaveBeenCalledTimes(1);
|
||||
});
|
||||
|
||||
it("distinguishes a preview that is not mounted from a handle that does not match", async () => {
|
||||
const notMounted = expectFailure(await studioSelect(selectionDeps(), "dom:headline"));
|
||||
expect(notMounted.kind).toBe("blocked");
|
||||
expect(notMounted.reason).toMatch(/not mounted/);
|
||||
|
||||
const doc = previewDoc('<h1 id="headline">Ship it</h1>');
|
||||
const noMatch = expectFailure(
|
||||
await studioSelect(selectionDeps({ getPreviewDocument: () => doc }), "dom:missing"),
|
||||
);
|
||||
expect(noMatch.kind).toBe("invalid");
|
||||
expect(noMatch.reason).toMatch(/no element matches/);
|
||||
// The two must not be the same message: waiting and re-reading are different fixes.
|
||||
expect(noMatch.reason).not.toBe(notMounted.reason);
|
||||
});
|
||||
|
||||
it("reports an element Studio cannot build a selection for, as a third case", async () => {
|
||||
const doc = previewDoc('<h1 id="headline">Ship it</h1>');
|
||||
|
||||
const result = expectFailure(
|
||||
await studioSelect(
|
||||
selectionDeps({ getPreviewDocument: () => doc, buildSelection: async () => null }),
|
||||
"dom:headline",
|
||||
),
|
||||
);
|
||||
|
||||
expect(result.kind).toBe("blocked");
|
||||
expect(result.reason).toMatch(/cannot select/);
|
||||
});
|
||||
|
||||
it("rejects a missing handle without touching the preview", async () => {
|
||||
const getPreviewDocument = vi.fn(() => null);
|
||||
|
||||
const result = expectFailure(await studioSelect(selectionDeps({ getPreviewDocument }), " "));
|
||||
|
||||
expect(result.kind).toBe("invalid");
|
||||
expect(getPreviewDocument).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it("leaves the existing selection alone when it fails", async () => {
|
||||
const doc = previewDoc('<h1 id="headline">Ship it</h1>');
|
||||
const applySelection = vi.fn();
|
||||
|
||||
await studioSelect(
|
||||
selectionDeps({ getPreviewDocument: () => doc, applySelection }),
|
||||
"dom:missing",
|
||||
);
|
||||
|
||||
expect(applySelection).not.toHaveBeenCalled();
|
||||
});
|
||||
});
|
||||
|
||||
describe("studioSeek", () => {
|
||||
it("reports where the playhead landed, not what was requested", () => {
|
||||
// The player clamps against the ADAPTER's duration, which the wrapper
|
||||
// deliberately does not second-guess.
|
||||
let currentTime = 0;
|
||||
const result = studioSeek(
|
||||
selectionDeps({
|
||||
requestSeek: () => {
|
||||
currentTime = 10;
|
||||
},
|
||||
readPlayhead: () => ({ currentTime, duration: 10, isPlaying: false }),
|
||||
}),
|
||||
999,
|
||||
);
|
||||
|
||||
const ok = expectOk<StudioSeekResult>(result);
|
||||
expect(ok.playhead).toBe(10);
|
||||
expect(ok.moved).toBe(true);
|
||||
});
|
||||
|
||||
it("reports that playback stopped", () => {
|
||||
let isPlaying = true;
|
||||
let currentTime = 0;
|
||||
const result = studioSeek(
|
||||
selectionDeps({
|
||||
requestSeek: () => {
|
||||
currentTime = 2;
|
||||
isPlaying = false;
|
||||
},
|
||||
readPlayhead: () => ({ currentTime, duration: 10, isPlaying }),
|
||||
}),
|
||||
2,
|
||||
);
|
||||
|
||||
expect(expectOk<StudioSeekResult>(result).isPlaying).toBe(false);
|
||||
});
|
||||
|
||||
it("fails rather than claiming a seek the player never received", () => {
|
||||
// `requestSeek` is fire-and-forget: with no adapter mounted it silently does
|
||||
// nothing, and reporting ok would be a lie the agent builds on.
|
||||
const result = expectFailure(
|
||||
studioSeek(
|
||||
selectionDeps({ readPlayhead: () => ({ currentTime: 0, duration: 10, isPlaying: false }) }),
|
||||
5,
|
||||
),
|
||||
);
|
||||
|
||||
expect(result.kind).toBe("blocked");
|
||||
expect(result.reason).toMatch(/did not move/);
|
||||
});
|
||||
|
||||
it("succeeds when asked to seek to where the playhead already is", () => {
|
||||
const result = studioSeek(
|
||||
selectionDeps({ readPlayhead: () => ({ currentTime: 3, duration: 10, isPlaying: false }) }),
|
||||
3,
|
||||
);
|
||||
|
||||
// Nothing moved, but nothing failed either, and `moved` says which.
|
||||
const ok = expectOk<StudioSeekResult>(result);
|
||||
expect(ok.moved).toBe(false);
|
||||
expect(ok.playhead).toBe(3);
|
||||
});
|
||||
|
||||
it("rejects a non-finite time without calling the player", () => {
|
||||
const requestSeek = vi.fn();
|
||||
|
||||
for (const time of [Number.NaN, Number.POSITIVE_INFINITY]) {
|
||||
const result = expectFailure(studioSeek(selectionDeps({ requestSeek }), time));
|
||||
expect(result.kind).toBe("invalid");
|
||||
}
|
||||
expect(requestSeek).not.toHaveBeenCalled();
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,154 @@
|
||||
/**
|
||||
* `studio_select` and `studio_seek`: pointing the human and the agent at the
|
||||
* same thing.
|
||||
*
|
||||
* Selection is shared state, not a per-call argument. That is deliberate and it
|
||||
* is also forced: most of Studio's edit handlers read the ambient React
|
||||
* selection, and `applyDomSelection` only schedules a state update, so
|
||||
* selecting and committing inside ONE call would write to whatever was selected
|
||||
* before. Two tool calls are separated by a render, so the contract is select
|
||||
* first, then act, which is also how a human works: click, then type.
|
||||
*/
|
||||
|
||||
import type { DomEditSelection } from "../../components/editor/domEditingTypes";
|
||||
import { mintElementHandle, patchTargetAddress, resolveElementHandle } from "../handles";
|
||||
import { toolFailure, toolOk, type ToolResult } from "../toolResult";
|
||||
|
||||
export interface SelectionToolDeps {
|
||||
/** The preview iframe's document, or null before it mounts. */
|
||||
getPreviewDocument: () => Document | null;
|
||||
buildSelection: (element: HTMLElement) => Promise<DomEditSelection | null>;
|
||||
applySelection: (selection: DomEditSelection) => void;
|
||||
/** Out-of-loop seek. `requestSeek`, not `setCurrentTime`. */
|
||||
requestSeek: (time: number) => void;
|
||||
readPlayhead: () => { currentTime: number; duration: number; isPlaying: boolean };
|
||||
}
|
||||
|
||||
export interface StudioSelectResult {
|
||||
handle: string | null;
|
||||
label: string;
|
||||
tagName: string;
|
||||
box: { x: number; y: number; width: number; height: number };
|
||||
}
|
||||
|
||||
export async function studioSelect(
|
||||
deps: SelectionToolDeps,
|
||||
handle: string,
|
||||
): Promise<ToolResult<StudioSelectResult>> {
|
||||
if (typeof handle !== "string" || !handle.trim()) {
|
||||
return toolFailure("invalid", "handle must be a non-empty string", "Call studio_look first.");
|
||||
}
|
||||
|
||||
// Three distinct failures, deliberately not collapsed: "the preview is not up
|
||||
// yet" is a wait, "no such element" is a stale handle, and "could not build a
|
||||
// selection" is an element Studio cannot drive. The agent's next move differs
|
||||
// for each.
|
||||
const doc = deps.getPreviewDocument();
|
||||
if (!doc) {
|
||||
return toolFailure(
|
||||
"blocked",
|
||||
"the preview is not mounted yet",
|
||||
"Wait for the composition to load, then retry.",
|
||||
);
|
||||
}
|
||||
|
||||
const element = resolveElementHandle(doc, handle);
|
||||
if (!element) {
|
||||
return toolFailure(
|
||||
"invalid",
|
||||
`no element matches handle ${handle}`,
|
||||
"The composition may have changed. Call studio_look for current handles.",
|
||||
);
|
||||
}
|
||||
|
||||
const selection = await deps.buildSelection(element);
|
||||
if (!selection) {
|
||||
return toolFailure(
|
||||
"blocked",
|
||||
`${handle} resolved to an element Studio cannot select`,
|
||||
"Try a parent or child element from studio_look.",
|
||||
);
|
||||
}
|
||||
|
||||
deps.applySelection(selection);
|
||||
return toolOk<StudioSelectResult>({
|
||||
handle: mintElementHandle(patchTargetAddress(selection)),
|
||||
label: selection.label,
|
||||
tagName: selection.tagName,
|
||||
box: selection.boundingBox,
|
||||
});
|
||||
}
|
||||
|
||||
export interface StudioSeekResult {
|
||||
/** Where the playhead ACTUALLY landed, which may differ from the request. */
|
||||
playhead: number;
|
||||
duration: number;
|
||||
isPlaying: boolean;
|
||||
moved: boolean;
|
||||
}
|
||||
|
||||
export function studioSeek(deps: SelectionToolDeps, time: number): ToolResult<StudioSeekResult> {
|
||||
if (typeof time !== "number" || !Number.isFinite(time)) {
|
||||
return toolFailure("invalid", "time must be a finite number of seconds");
|
||||
}
|
||||
|
||||
const before = deps.readPlayhead();
|
||||
// Deliberately NOT clamped here. `seek()` already clamps against the
|
||||
// adapter's duration, which can differ from the store's, and a second clamp
|
||||
// would give that invariant two owners that can disagree. Report where it
|
||||
// landed instead.
|
||||
deps.requestSeek(time);
|
||||
const after = deps.readPlayhead();
|
||||
|
||||
// `requestSeek` is fire-and-forget: it cannot report that no adapter was
|
||||
// mounted to receive it. Reading back is the only way to avoid claiming a
|
||||
// seek that never happened.
|
||||
const moved = after.currentTime !== before.currentTime;
|
||||
if (!moved && before.currentTime !== time) {
|
||||
return toolFailure(
|
||||
"blocked",
|
||||
`the playhead did not move; it is still at ${after.currentTime}`,
|
||||
"The preview may not be ready. Check studio_look, then retry.",
|
||||
);
|
||||
}
|
||||
|
||||
return toolOk<StudioSeekResult>({
|
||||
playhead: after.currentTime,
|
||||
duration: after.duration,
|
||||
isPlaying: after.isPlaying,
|
||||
moved,
|
||||
});
|
||||
}
|
||||
|
||||
export const STUDIO_SELECT_INPUT_SCHEMA = {
|
||||
type: "object",
|
||||
properties: {
|
||||
handle: { type: "string", description: "An element handle from studio_look." },
|
||||
},
|
||||
required: ["handle"],
|
||||
additionalProperties: false,
|
||||
} as const;
|
||||
|
||||
export const STUDIO_SELECT_DESCRIPTION = [
|
||||
"Select an element in HyperFrames Studio, exactly as clicking it would:",
|
||||
"the human sees the same selection box and inspector.",
|
||||
"Takes a handle from studio_look. Most editing tools act on the CURRENT selection,",
|
||||
"so call this first, then the edit.",
|
||||
"Returns `ok: true` with the resulting selection, or `ok: false` with `kind`, `reason` and a `hint`.",
|
||||
].join(" ");
|
||||
|
||||
export const STUDIO_SEEK_INPUT_SCHEMA = {
|
||||
type: "object",
|
||||
properties: {
|
||||
time: { type: "number", minimum: 0, description: "Playhead position in seconds." },
|
||||
},
|
||||
required: ["time"],
|
||||
additionalProperties: false,
|
||||
} as const;
|
||||
|
||||
export const STUDIO_SEEK_DESCRIPTION = [
|
||||
"Move the playhead to a time in seconds. Pauses playback.",
|
||||
"Out-of-range times are clamped by the player, so check the returned `playhead`",
|
||||
"for where it actually landed rather than assuming it matched your request.",
|
||||
"Returns `ok: true`, or `ok: false` with `kind`, `reason` and a `hint`.",
|
||||
].join(" ");
|
||||
@@ -29,6 +29,19 @@ function snapshot(overrides: Partial<StudioLookSnapshot> = {}): StudioLookSnapsh
|
||||
};
|
||||
}
|
||||
|
||||
/** Full deps with inert defaults; override only what the test is about. */
|
||||
function deps(overrides: Partial<StudioAgentToolsDeps> = {}): StudioAgentToolsDeps {
|
||||
return {
|
||||
getSnapshot: () => snapshot(),
|
||||
getPreviewDocument: () => null,
|
||||
buildSelection: async () => null,
|
||||
applySelection: () => undefined,
|
||||
requestSeek: () => undefined,
|
||||
readPlayhead: () => ({ currentTime: 0, duration: 10, isPlaying: false }),
|
||||
...overrides,
|
||||
};
|
||||
}
|
||||
|
||||
/** Install a fake `document.modelContext` and report what got registered. */
|
||||
function installModelContext() {
|
||||
const registered: ModelContextTool[] = [];
|
||||
@@ -50,12 +63,12 @@ function removeModelContext() {
|
||||
Reflect.deleteProperty(document, "modelContext");
|
||||
}
|
||||
|
||||
function mountTools(deps: StudioAgentToolsDeps) {
|
||||
function mountTools(initial: StudioAgentToolsDeps) {
|
||||
function Probe({ current }: { current: StudioAgentToolsDeps }) {
|
||||
useStudioAgentTools(current);
|
||||
return null;
|
||||
}
|
||||
const root = mountReactHarness(<Probe current={deps} />);
|
||||
const root = mountReactHarness(<Probe current={initial} />);
|
||||
cleanup = () => act(() => root.unmount());
|
||||
return {
|
||||
rerenderWith(next: StudioAgentToolsDeps) {
|
||||
@@ -82,10 +95,14 @@ describe("useStudioAgentTools", () => {
|
||||
const { registered } = installModelContext();
|
||||
|
||||
await act(async () => {
|
||||
mountTools({ getSnapshot: () => snapshot() });
|
||||
mountTools(deps({ getSnapshot: () => snapshot() }));
|
||||
});
|
||||
|
||||
expect(registered.map((tool) => tool.name)).toEqual(["studio_look"]);
|
||||
expect(registered.map((tool) => tool.name)).toEqual([
|
||||
"studio_look",
|
||||
"studio_select",
|
||||
"studio_seek",
|
||||
]);
|
||||
expect(trackEvent).toHaveBeenCalledWith("webmcp.native_present");
|
||||
});
|
||||
|
||||
@@ -97,16 +114,16 @@ describe("useStudioAgentTools", () => {
|
||||
|
||||
let harness: ReturnType<typeof mountTools> | null = null;
|
||||
await act(async () => {
|
||||
harness = mountTools({ getSnapshot: () => snapshot() });
|
||||
harness = mountTools(deps({ getSnapshot: () => snapshot() }));
|
||||
});
|
||||
expect(registerTool).toHaveBeenCalledTimes(1);
|
||||
expect(registerTool).toHaveBeenCalledTimes(3);
|
||||
|
||||
await act(async () => {
|
||||
harness?.rerenderWith({ getSnapshot: () => snapshot({ currentTime: 5 }) });
|
||||
harness?.rerenderWith({ getSnapshot: () => snapshot({ currentTime: 6 }) });
|
||||
harness?.rerenderWith(deps({ getSnapshot: () => snapshot({ currentTime: 5 }) }));
|
||||
harness?.rerenderWith(deps({ getSnapshot: () => snapshot({ currentTime: 6 }) }));
|
||||
});
|
||||
|
||||
expect(registerTool).toHaveBeenCalledTimes(1);
|
||||
expect(registerTool).toHaveBeenCalledTimes(3);
|
||||
});
|
||||
|
||||
it("executes against the LATEST deps, not the ones present at registration", async () => {
|
||||
@@ -116,11 +133,11 @@ describe("useStudioAgentTools", () => {
|
||||
|
||||
let harness: ReturnType<typeof mountTools> | null = null;
|
||||
await act(async () => {
|
||||
harness = mountTools({ getSnapshot: () => snapshot({ currentTime: 1 }) });
|
||||
harness = mountTools(deps({ getSnapshot: () => snapshot({ currentTime: 1 }) }));
|
||||
});
|
||||
|
||||
await act(async () => {
|
||||
harness?.rerenderWith({ getSnapshot: () => snapshot({ currentTime: 42 }) });
|
||||
harness?.rerenderWith(deps({ getSnapshot: () => snapshot({ currentTime: 42 }) }));
|
||||
});
|
||||
|
||||
const look = registered[0];
|
||||
@@ -138,7 +155,7 @@ describe("useStudioAgentTools", () => {
|
||||
const { registerTool } = installModelContext();
|
||||
|
||||
await act(async () => {
|
||||
mountTools({ getSnapshot: () => snapshot() });
|
||||
mountTools(deps({ getSnapshot: () => snapshot() }));
|
||||
});
|
||||
const signal = registerTool.mock.calls[0]?.[1]?.signal;
|
||||
expect(signal?.aborted).toBe(false);
|
||||
@@ -153,7 +170,7 @@ describe("useStudioAgentTools", () => {
|
||||
removeModelContext();
|
||||
|
||||
await act(async () => {
|
||||
mountTools({ getSnapshot: () => snapshot() });
|
||||
mountTools(deps({ getSnapshot: () => snapshot() }));
|
||||
});
|
||||
|
||||
// The assertion is that mounting did not throw; a browser without the
|
||||
@@ -166,7 +183,7 @@ describe("useStudioAgentTools", () => {
|
||||
const { registerTool } = installModelContext();
|
||||
|
||||
await act(async () => {
|
||||
mountTools({ getSnapshot: () => snapshot() });
|
||||
mountTools(deps({ getSnapshot: () => snapshot() }));
|
||||
});
|
||||
|
||||
expect(registerTool).not.toHaveBeenCalled();
|
||||
@@ -176,10 +193,10 @@ describe("useStudioAgentTools", () => {
|
||||
const { registerTool } = installModelContext();
|
||||
|
||||
await act(async () => {
|
||||
mountTools({ getSnapshot: () => snapshot() });
|
||||
mountTools(deps({ getSnapshot: () => snapshot() }));
|
||||
});
|
||||
|
||||
expect(registerTool).toHaveBeenCalledTimes(1);
|
||||
expect(registerTool).toHaveBeenCalledTimes(3);
|
||||
});
|
||||
|
||||
it("reports a non-abort registration failure through production telemetry", async () => {
|
||||
@@ -187,7 +204,7 @@ describe("useStudioAgentTools", () => {
|
||||
registerTool.mockRejectedValue(new DOMException("blocked", "NotAllowedError"));
|
||||
|
||||
await act(async () => {
|
||||
mountTools({ getSnapshot: () => snapshot() });
|
||||
mountTools(deps({ getSnapshot: () => snapshot() }));
|
||||
});
|
||||
|
||||
expect(trackEvent).toHaveBeenCalledWith("webmcp_registration_failed", {
|
||||
@@ -200,11 +217,13 @@ describe("useStudioAgentTools", () => {
|
||||
const { registered } = installModelContext();
|
||||
|
||||
await act(async () => {
|
||||
mountTools({
|
||||
getSnapshot: () => {
|
||||
throw new TypeError("handler signature moved");
|
||||
},
|
||||
});
|
||||
mountTools(
|
||||
deps({
|
||||
getSnapshot: () => {
|
||||
throw new TypeError("handler signature moved");
|
||||
},
|
||||
}),
|
||||
);
|
||||
});
|
||||
vi.spyOn(console, "error").mockImplementation(() => {});
|
||||
|
||||
|
||||
@@ -14,6 +14,17 @@ import {
|
||||
type StudioLookInput,
|
||||
type StudioLookSnapshot,
|
||||
} from "./tools/lookTools";
|
||||
import {
|
||||
studioSeek,
|
||||
studioSelect,
|
||||
STUDIO_SEEK_DESCRIPTION,
|
||||
STUDIO_SEEK_INPUT_SCHEMA,
|
||||
STUDIO_SELECT_DESCRIPTION,
|
||||
STUDIO_SELECT_INPUT_SCHEMA,
|
||||
type SelectionToolDeps,
|
||||
type StudioSeekResult,
|
||||
type StudioSelectResult,
|
||||
} from "./tools/selectionTools";
|
||||
|
||||
const log = makeStudioDebugLogger("webmcp");
|
||||
|
||||
@@ -27,7 +38,7 @@ function reportRegistration(report: ToolRegistrationReport, native: boolean): vo
|
||||
}
|
||||
}
|
||||
|
||||
export interface StudioAgentToolsDeps {
|
||||
export interface StudioAgentToolsDeps extends SelectionToolDeps {
|
||||
/** Read Studio's current state. Called per tool invocation, never cached. */
|
||||
getSnapshot: () => StudioLookSnapshot;
|
||||
}
|
||||
@@ -57,9 +68,46 @@ function buildStudioTools(depsRef: { readonly current: StudioAgentToolsDeps }):
|
||||
buildStudioLook(depsRef.current.getSnapshot(), input as StudioLookInput),
|
||||
),
|
||||
},
|
||||
{
|
||||
name: "studio_select",
|
||||
title: "Select an element",
|
||||
description: STUDIO_SELECT_DESCRIPTION,
|
||||
inputSchema: STUDIO_SELECT_INPUT_SCHEMA,
|
||||
annotations: { readOnlyHint: false, untrustedContentHint: true },
|
||||
execute: (input): Promise<ToolResult<StudioSelectResult>> =>
|
||||
runToolBody("studio_select", () =>
|
||||
studioSelect(depsRef.current, readStringInput(input, "handle")),
|
||||
),
|
||||
},
|
||||
{
|
||||
name: "studio_seek",
|
||||
title: "Move the playhead",
|
||||
description: STUDIO_SEEK_DESCRIPTION,
|
||||
inputSchema: STUDIO_SEEK_INPUT_SCHEMA,
|
||||
annotations: { readOnlyHint: false },
|
||||
execute: (input): Promise<ToolResult<StudioSeekResult>> =>
|
||||
runToolBody("studio_seek", async () =>
|
||||
studioSeek(depsRef.current, readNumberInput(input, "time")),
|
||||
),
|
||||
},
|
||||
];
|
||||
}
|
||||
|
||||
/**
|
||||
* Nothing in the platform validates the input object against `inputSchema`, so
|
||||
* a tool receives whatever the agent sent. These read a field without asserting
|
||||
* its type; the tools themselves reject what they cannot use.
|
||||
*/
|
||||
function readStringInput(input: object, key: string): string {
|
||||
const value = Reflect.get(input, key);
|
||||
return typeof value === "string" ? value : "";
|
||||
}
|
||||
|
||||
function readNumberInput(input: object, key: string): number {
|
||||
const value = Reflect.get(input, key);
|
||||
return typeof value === "number" ? value : Number.NaN;
|
||||
}
|
||||
|
||||
/**
|
||||
* Register Studio's tools with the browser, exactly once per mount.
|
||||
*
|
||||
|
||||
Reference in New Issue
Block a user