mirror of
https://github.com/heygen-com/hyperframes.git
synced 2026-08-31 02:41:44 +00:00
Merge 72cb25939e into 0fffa66202
This commit is contained in:
@@ -433,6 +433,8 @@ export function StudioApp() {
|
||||
handleRedo: appHotkeys.handleRedo,
|
||||
renderQueue,
|
||||
compositionDimensions,
|
||||
domEditSaveQueuePaused: previewPersistence.domEditSaveQueuePaused,
|
||||
externalFileConflict: externalFileChanges.blocked !== null,
|
||||
waitForPendingDomEditSaves: previewPersistence.waitForPendingDomEditSaves,
|
||||
handlePreviewIframeRef,
|
||||
refreshPreviewDocumentVersion,
|
||||
|
||||
@@ -16,6 +16,13 @@ export interface StudioShellValue {
|
||||
undoLabel: string | undefined;
|
||||
redoLabel: string | undefined;
|
||||
};
|
||||
/**
|
||||
* Why a composition write would be refused right now, or null when writes
|
||||
* are possible. Derived from the paused save queue and the external-file
|
||||
* conflict state, both of which are otherwise banners with no lock behind
|
||||
* them. One field rather than two, so there is one owner of the question.
|
||||
*/
|
||||
writeBlockedReason: string | null;
|
||||
handleUndo: () => Promise<void>;
|
||||
handleRedo: () => Promise<void>;
|
||||
renderQueue: {
|
||||
@@ -106,6 +113,7 @@ export function StudioShellProvider({
|
||||
showToast,
|
||||
previewIframeRef,
|
||||
editHistory,
|
||||
writeBlockedReason,
|
||||
handleUndo,
|
||||
handleRedo,
|
||||
renderQueue,
|
||||
@@ -122,6 +130,7 @@ export function StudioShellProvider({
|
||||
showToast,
|
||||
previewIframeRef,
|
||||
editHistory,
|
||||
writeBlockedReason,
|
||||
handleUndo,
|
||||
handleRedo,
|
||||
renderQueue,
|
||||
@@ -138,6 +147,7 @@ export function StudioShellProvider({
|
||||
setActiveCompPath,
|
||||
showToast,
|
||||
previewIframeRef,
|
||||
writeBlockedReason,
|
||||
handleUndo,
|
||||
handleRedo,
|
||||
waitForPendingDomEditSaves,
|
||||
|
||||
@@ -25,6 +25,10 @@ interface StudioContextInput {
|
||||
// fields around it: the context type owns it.
|
||||
renderQueue: StudioContextValue["renderQueue"];
|
||||
compositionDimensions: { width: number; height: number } | null;
|
||||
/** Message from `usePreviewPersistence` when auto-save is paused. */
|
||||
domEditSaveQueuePaused: string | null;
|
||||
/** True when an external edit to the open file is awaiting the user's decision. */
|
||||
externalFileConflict: boolean;
|
||||
waitForPendingDomEditSaves: () => Promise<void>;
|
||||
handlePreviewIframeRef: (iframe: HTMLIFrameElement | null) => void;
|
||||
refreshPreviewDocumentVersion: () => void;
|
||||
@@ -46,6 +50,11 @@ export function buildStudioContextValue(input: StudioContextInput): StudioContex
|
||||
timelineElements: input.timelineElements,
|
||||
isPlaying: input.isPlaying,
|
||||
editHistory: input.editHistory,
|
||||
// Conflict first: when both are true the conflict is the one the user has
|
||||
// been asked to decide, and resolving it is what unblocks the queue.
|
||||
writeBlockedReason: input.externalFileConflict
|
||||
? "an external change to this file is waiting to be resolved"
|
||||
: input.domEditSaveQueuePaused,
|
||||
handleUndo: input.handleUndo,
|
||||
handleRedo: input.handleRedo,
|
||||
renderQueue: input.renderQueue,
|
||||
|
||||
@@ -17,15 +17,23 @@ import type { StudioLookSnapshot } from "./tools/lookTools";
|
||||
* every animation frame during playback for a value nothing here displays.
|
||||
*/
|
||||
export function StudioAgentTools() {
|
||||
const { projectId, activeCompPath, editHistory } = useStudioShellContext();
|
||||
const { projectId, activeCompPath, editHistory, writeBlockedReason } = useStudioShellContext();
|
||||
const {
|
||||
domEditSelection,
|
||||
selectedGsapAnimations,
|
||||
gsapMultipleTimelines,
|
||||
gsapUnsupportedTimelinePattern,
|
||||
} = useDomEditSelectionContext();
|
||||
const { previewIframeRef, buildDomSelectionFromTarget, applyDomSelection } =
|
||||
useDomEditActionsContext();
|
||||
const {
|
||||
previewIframeRef,
|
||||
buildDomSelectionFromTarget,
|
||||
applyDomSelection,
|
||||
handleDomTextCommit,
|
||||
handleDomStyleCommit,
|
||||
handleDomPathOffsetCommit,
|
||||
handleDomBoxSizeCommit,
|
||||
handleDomRotationCommit,
|
||||
} = useDomEditActionsContext();
|
||||
|
||||
const getSnapshot = useCallback((): StudioLookSnapshot => {
|
||||
const player = usePlayerStore.getState();
|
||||
@@ -77,6 +85,18 @@ export function StudioAgentTools() {
|
||||
},
|
||||
wait: (ms) => new Promise((resolve) => setTimeout(resolve, ms)),
|
||||
getCurrentSelection: () => domEditSelection,
|
||||
getWriteBlockedReason: () => writeBlockedReason,
|
||||
setText: (value, fieldKey) => handleDomTextCommit(value, fieldKey),
|
||||
setStyle: (property, value) => handleDomStyleCommit(property, value),
|
||||
// Measured, not authored: the tool compares this before and after to
|
||||
// tell a real change from a handler that did nothing and resolved.
|
||||
readBox: (selection) => {
|
||||
const rect = selection.element.getBoundingClientRect();
|
||||
return { x: rect.x, y: rect.y, width: rect.width, height: rect.height };
|
||||
},
|
||||
moveTo: (selection, next) => handleDomPathOffsetCommit(selection, next),
|
||||
resizeTo: (selection, next) => handleDomBoxSizeCommit(selection, next),
|
||||
rotateTo: (selection, next) => handleDomRotationCommit(selection, next),
|
||||
getGsapDiagnostics: () => ({
|
||||
animations: selectedGsapAnimations,
|
||||
multipleTimelines: gsapMultipleTimelines,
|
||||
@@ -90,6 +110,12 @@ export function StudioAgentTools() {
|
||||
applyDomSelection,
|
||||
projectId,
|
||||
activeCompPath,
|
||||
writeBlockedReason,
|
||||
handleDomTextCommit,
|
||||
handleDomStyleCommit,
|
||||
handleDomPathOffsetCommit,
|
||||
handleDomBoxSizeCommit,
|
||||
handleDomRotationCommit,
|
||||
domEditSelection,
|
||||
selectedGsapAnimations,
|
||||
gsapMultipleTimelines,
|
||||
|
||||
@@ -0,0 +1,201 @@
|
||||
// @vitest-environment jsdom
|
||||
import { describe, expect, it, vi } from "vitest";
|
||||
import {
|
||||
studioSetStyle,
|
||||
studioSetText,
|
||||
type ContentToolDeps,
|
||||
type StudioSetStyleResult,
|
||||
type StudioSetTextResult,
|
||||
} from "./contentTools";
|
||||
import { expectFailure, expectOk, previewElement, selectionFor } from "../webmcpTestUtils";
|
||||
|
||||
function contentDeps(overrides: Partial<ContentToolDeps> = {}): ContentToolDeps {
|
||||
const element = previewElement('<h1 id="headline">Ship it</h1>', "headline");
|
||||
return {
|
||||
getCurrentSelection: () => selectionFor(element),
|
||||
getWriteBlockedReason: () => null,
|
||||
setText: async () => ({ ok: true }),
|
||||
setStyle: async () => ({ ok: true }),
|
||||
...overrides,
|
||||
};
|
||||
}
|
||||
|
||||
describe("studioSetText", () => {
|
||||
it("writes the text and reports what it now is", async () => {
|
||||
const setText = vi.fn(async () => ({ ok: true }) as const);
|
||||
|
||||
const result = await studioSetText(contentDeps({ setText }), { text: "Ship it faster" });
|
||||
|
||||
const ok = expectOk<StudioSetTextResult>(result);
|
||||
expect(ok.text).toBe("Ship it faster");
|
||||
expect(ok.changed).toBe(true);
|
||||
expect(setText).toHaveBeenCalledWith("Ship it faster", undefined);
|
||||
});
|
||||
|
||||
it("reports changed:false when the text already said that", async () => {
|
||||
const result = await studioSetText(contentDeps(), { text: "Ship it" });
|
||||
|
||||
expect(expectOk<StudioSetTextResult>(result).changed).toBe(false);
|
||||
});
|
||||
|
||||
it("refuses to write while a conflict is waiting for the user", async () => {
|
||||
// The paused-save and conflict states are banners with no lock behind them.
|
||||
// Nothing else stops a programmatic write landing on top of a decision the
|
||||
// user has been asked to make.
|
||||
const setText = vi.fn();
|
||||
|
||||
const result = expectFailure(
|
||||
await studioSetText(
|
||||
contentDeps({
|
||||
getWriteBlockedReason: () => "an external change to this file is waiting to be resolved",
|
||||
setText,
|
||||
}),
|
||||
{ text: "Ship it faster" },
|
||||
),
|
||||
);
|
||||
|
||||
expect(result.kind).toBe("blocked");
|
||||
expect(result.reason).toMatch(/external change/);
|
||||
expect(setText).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it("does not report success when the commit declined", async () => {
|
||||
// The whole reason the handlers now return an outcome: they resolve on
|
||||
// failure, so awaiting them proves nothing.
|
||||
const result = expectFailure(
|
||||
await studioSetText(
|
||||
contentDeps({ setText: async () => ({ ok: false, reason: "persist-failed" }) }),
|
||||
{ text: "Ship it faster" },
|
||||
),
|
||||
);
|
||||
|
||||
expect(result.kind).toBe("failed");
|
||||
expect(result.reason).toMatch(/persist-failed/);
|
||||
});
|
||||
|
||||
it("turns a decline reason into a hint naming what to do instead", async () => {
|
||||
const result = expectFailure(
|
||||
await studioSetText(
|
||||
contentDeps({ setText: async () => ({ ok: false, reason: "not-text-editable" }) }),
|
||||
{ text: "x" },
|
||||
),
|
||||
);
|
||||
|
||||
expect(result.kind).toBe("blocked");
|
||||
expect(result.hint).toMatch(/studio_inspect/);
|
||||
});
|
||||
|
||||
it("rejects a non-string text without dispatching", async () => {
|
||||
const setText = vi.fn();
|
||||
|
||||
const result = expectFailure(await studioSetText(contentDeps({ setText }), { text: 42 }));
|
||||
|
||||
expect(result.kind).toBe("invalid");
|
||||
expect(setText).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it("fails when nothing is selected", async () => {
|
||||
const setText = vi.fn();
|
||||
|
||||
const result = expectFailure(
|
||||
await studioSetText(contentDeps({ getCurrentSelection: () => null, setText }), { text: "x" }),
|
||||
);
|
||||
|
||||
expect(result.kind).toBe("invalid");
|
||||
expect(result.hint).toMatch(/studio_select/);
|
||||
expect(setText).not.toHaveBeenCalled();
|
||||
});
|
||||
});
|
||||
|
||||
describe("studioSetStyle", () => {
|
||||
it("applies every property and reports them", async () => {
|
||||
const setStyle = vi.fn(async () => ({ ok: true }) as const);
|
||||
|
||||
const result = await studioSetStyle(contentDeps({ setStyle }), {
|
||||
styles: { color: "red", "font-size": "48px" },
|
||||
});
|
||||
|
||||
const ok = expectOk<StudioSetStyleResult>(result);
|
||||
expect(ok.applied).toEqual({ color: "red", "font-size": "48px" });
|
||||
expect(ok.rejected).toEqual({});
|
||||
expect(setStyle).toHaveBeenCalledTimes(2);
|
||||
});
|
||||
|
||||
it("commits sequentially, never concurrently", async () => {
|
||||
// Two commits racing through Studio's client-side read-modify-write can
|
||||
// record undo entries that both claim the same starting content.
|
||||
let inFlight = 0;
|
||||
let maxInFlight = 0;
|
||||
const setStyle = vi.fn(async () => {
|
||||
inFlight += 1;
|
||||
maxInFlight = Math.max(maxInFlight, inFlight);
|
||||
await Promise.resolve();
|
||||
inFlight -= 1;
|
||||
return { ok: true } as const;
|
||||
});
|
||||
|
||||
await studioSetStyle(contentDeps({ setStyle }), {
|
||||
styles: { color: "red", "font-size": "48px", opacity: "0.5" },
|
||||
});
|
||||
|
||||
expect(maxInFlight).toBe(1);
|
||||
});
|
||||
|
||||
it("reports a partial success as partial, not whole", async () => {
|
||||
const setStyle = vi.fn(async (property: string) =>
|
||||
property === "left"
|
||||
? ({ ok: false, reason: "geometry-property" } as const)
|
||||
: ({ ok: true } as const),
|
||||
);
|
||||
|
||||
const result = await studioSetStyle(contentDeps({ setStyle }), {
|
||||
styles: { color: "red", left: "10px" },
|
||||
});
|
||||
|
||||
const ok = expectOk<StudioSetStyleResult>(result);
|
||||
expect(ok.applied).toEqual({ color: "red" });
|
||||
expect(ok.rejected).toEqual({ left: "geometry-property" });
|
||||
});
|
||||
|
||||
it("fails when every property was refused", async () => {
|
||||
const result = expectFailure(
|
||||
await studioSetStyle(
|
||||
contentDeps({ setStyle: async () => ({ ok: false, reason: "styles-not-editable" }) }),
|
||||
{ styles: { color: "red" } },
|
||||
),
|
||||
);
|
||||
|
||||
expect(result.kind).toBe("blocked");
|
||||
expect(result.reason).toMatch(/styles-not-editable/);
|
||||
});
|
||||
|
||||
it("rejects an empty styles object rather than committing nothing", async () => {
|
||||
const setStyle = vi.fn();
|
||||
|
||||
const result = expectFailure(await studioSetStyle(contentDeps({ setStyle }), { styles: {} }));
|
||||
|
||||
expect(result.kind).toBe("invalid");
|
||||
expect(setStyle).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it("rejects a non-object styles value", async () => {
|
||||
for (const styles of ["color: red", 42, null, ["color"]]) {
|
||||
const result = expectFailure(await studioSetStyle(contentDeps(), { styles }));
|
||||
expect(result.kind).toBe("invalid");
|
||||
}
|
||||
});
|
||||
|
||||
it("refuses to write while a conflict is waiting for the user", async () => {
|
||||
const setStyle = vi.fn();
|
||||
|
||||
const result = expectFailure(
|
||||
await studioSetStyle(
|
||||
contentDeps({ getWriteBlockedReason: () => "Auto-save is paused", setStyle }),
|
||||
{ styles: { color: "red" } },
|
||||
),
|
||||
);
|
||||
|
||||
expect(result.kind).toBe("blocked");
|
||||
expect(setStyle).not.toHaveBeenCalled();
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,184 @@
|
||||
/**
|
||||
* `studio_set_text` and `studio_set_style`: the first tools that change the file.
|
||||
*
|
||||
* Both operate on the CURRENT selection and take no handle. That is not an
|
||||
* omission. `handleDomTextCommit(value, fieldKey?)` and
|
||||
* `handleDomStyleCommit(property, value)` read the ambient React selection, and
|
||||
* `applyDomSelection` only schedules a state update, so selecting and
|
||||
* committing inside one call would write to whatever was selected before.
|
||||
* Two tool calls are separated by a render. Select first, then edit.
|
||||
*
|
||||
* Every write here is guarded before dispatch and verified after. Studio has
|
||||
* several paths where a failed commit resolves anyway, so "the function did not
|
||||
* throw" proves nothing; the outcome the handler now returns is what proves it.
|
||||
*/
|
||||
|
||||
import type { DomEditCommitOutcome } from "../../hooks/domEditCommitRunner";
|
||||
import type { DomEditSelection } from "../../components/editor/domEditingTypes";
|
||||
import { toolFailure, toolOk, type ToolFailure, type ToolResult } from "../toolResult";
|
||||
|
||||
export interface ContentToolDeps {
|
||||
getCurrentSelection: () => DomEditSelection | null;
|
||||
/** Why a write would be refused right now, or null. Checked BEFORE dispatch. */
|
||||
getWriteBlockedReason: () => string | null;
|
||||
setText: (value: string, fieldKey?: string) => Promise<DomEditCommitOutcome>;
|
||||
setStyle: (property: string, value: string) => Promise<DomEditCommitOutcome>;
|
||||
}
|
||||
|
||||
/**
|
||||
* The reasons a commit declines, translated into something an agent can act on.
|
||||
* `persist-failed` is exogenous; the rest are states it should route around.
|
||||
*/
|
||||
const DECLINE_HINTS: Record<string, { kind: "blocked" | "invalid" | "failed"; hint?: string }> = {
|
||||
"no-selection": { kind: "invalid", hint: "Call studio_select first." },
|
||||
"no-project": { kind: "blocked" },
|
||||
"geometry-property": {
|
||||
kind: "blocked",
|
||||
hint: "Position and size are not editable as styles. Use the transform tools.",
|
||||
},
|
||||
"styles-not-editable": {
|
||||
kind: "blocked",
|
||||
hint: "studio_inspect reports why, in can.reasonIfDisabled.",
|
||||
},
|
||||
"not-text-editable": {
|
||||
kind: "blocked",
|
||||
hint: "This element has no editable text. studio_inspect lists its textFields.",
|
||||
},
|
||||
"persist-failed": { kind: "failed", hint: "The write did not reach the file. Check Studio." },
|
||||
};
|
||||
|
||||
function fromOutcome(outcome: DomEditCommitOutcome, what: string): ToolFailure | null {
|
||||
if (outcome.ok) return null;
|
||||
const mapped = DECLINE_HINTS[outcome.reason] ?? { kind: "failed" as const };
|
||||
return toolFailure(mapped.kind, `${what} was not applied: ${outcome.reason}`, mapped.hint);
|
||||
}
|
||||
|
||||
function guardWrite(deps: ContentToolDeps): ToolFailure | null {
|
||||
// Both blocked states are banners in Studio's UI with no lock behind them, so
|
||||
// nothing else stops a programmatic write from landing on top of a conflict
|
||||
// the user has been asked to adjudicate.
|
||||
const blocked = deps.getWriteBlockedReason();
|
||||
if (blocked) {
|
||||
return toolFailure("blocked", blocked, "Resolve it in Studio, then retry.");
|
||||
}
|
||||
if (!deps.getCurrentSelection()) {
|
||||
return toolFailure("invalid", "nothing is selected", "Call studio_select first.");
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
export interface StudioSetTextResult {
|
||||
text: string;
|
||||
changed: boolean;
|
||||
}
|
||||
|
||||
export async function studioSetText(
|
||||
deps: ContentToolDeps,
|
||||
input: { text?: unknown; field?: unknown },
|
||||
): Promise<ToolResult<StudioSetTextResult>> {
|
||||
if (typeof input.text !== "string") {
|
||||
return toolFailure("invalid", "text must be a string");
|
||||
}
|
||||
const field = typeof input.field === "string" && input.field ? input.field : undefined;
|
||||
|
||||
const blocked = guardWrite(deps);
|
||||
if (blocked) return blocked;
|
||||
|
||||
const before = deps.getCurrentSelection()?.textContent ?? null;
|
||||
const outcome = await deps.setText(input.text, field);
|
||||
const failure = fromOutcome(outcome, "the text");
|
||||
if (failure) return failure;
|
||||
|
||||
return toolOk<StudioSetTextResult>({ text: input.text, changed: before !== input.text });
|
||||
}
|
||||
|
||||
export interface StudioSetStyleResult {
|
||||
applied: Record<string, string>;
|
||||
/** Properties the element refused, with the reason. Empty when all landed. */
|
||||
rejected: Record<string, string>;
|
||||
}
|
||||
|
||||
export async function studioSetStyle(
|
||||
deps: ContentToolDeps,
|
||||
input: { styles?: unknown },
|
||||
): Promise<ToolResult<StudioSetStyleResult>> {
|
||||
const styles = input.styles;
|
||||
if (typeof styles !== "object" || styles === null || Array.isArray(styles)) {
|
||||
return toolFailure("invalid", "styles must be an object of CSS property to value");
|
||||
}
|
||||
const entries = Object.entries(styles).filter(
|
||||
(entry): entry is [string, string] => typeof entry[1] === "string",
|
||||
);
|
||||
if (entries.length === 0) {
|
||||
// An empty commit would report success having done nothing.
|
||||
return toolFailure("invalid", "styles must contain at least one string value");
|
||||
}
|
||||
|
||||
const blocked = guardWrite(deps);
|
||||
if (blocked) return blocked;
|
||||
|
||||
// `handleDomStyleCommit` is one property per call, so N properties are N
|
||||
// commits and N undo entries. Sequential, not concurrent: two commits racing
|
||||
// through Studio's client-side read-modify-write can record undo entries that
|
||||
// both claim the same starting content.
|
||||
const applied: Record<string, string> = {};
|
||||
const rejected: Record<string, string> = {};
|
||||
for (const [property, value] of entries) {
|
||||
const outcome = await deps.setStyle(property, value);
|
||||
if (outcome.ok) applied[property] = value;
|
||||
else rejected[property] = outcome.reason;
|
||||
}
|
||||
|
||||
if (Object.keys(applied).length === 0) {
|
||||
const reasons = Object.entries(rejected)
|
||||
.map(([property, reason]) => `${property}: ${reason}`)
|
||||
.join(", ");
|
||||
return toolFailure("blocked", `no style was applied (${reasons})`);
|
||||
}
|
||||
|
||||
return toolOk<StudioSetStyleResult>({ applied, rejected });
|
||||
}
|
||||
|
||||
export const STUDIO_SET_TEXT_INPUT_SCHEMA = {
|
||||
type: "object",
|
||||
properties: {
|
||||
text: { type: "string", description: "The new text content." },
|
||||
field: {
|
||||
type: "string",
|
||||
description:
|
||||
"Which text field to write, from studio_inspect. Omit for the element's own text.",
|
||||
},
|
||||
},
|
||||
required: ["text"],
|
||||
additionalProperties: false,
|
||||
} as const;
|
||||
|
||||
export const STUDIO_SET_TEXT_DESCRIPTION = [
|
||||
"Set the text of the CURRENTLY SELECTED element. Call studio_select first.",
|
||||
"This is the edit a synthetic double-click cannot reach, because Studio's canvas",
|
||||
"takes pointer capture and recognises the double press itself.",
|
||||
"Returns `ok: true` with the resulting text and whether it changed, or `ok: false`",
|
||||
"with `kind`, `reason` and usually a `hint` naming what to do instead.",
|
||||
].join(" ");
|
||||
|
||||
export const STUDIO_SET_STYLE_INPUT_SCHEMA = {
|
||||
type: "object",
|
||||
properties: {
|
||||
styles: {
|
||||
type: "object",
|
||||
description: 'CSS property to value, for example {"color": "red", "font-size": "48px"}.',
|
||||
additionalProperties: { type: "string" },
|
||||
},
|
||||
},
|
||||
required: ["styles"],
|
||||
additionalProperties: false,
|
||||
} as const;
|
||||
|
||||
export const STUDIO_SET_STYLE_DESCRIPTION = [
|
||||
"Set inline styles on the CURRENTLY SELECTED element. Call studio_select first.",
|
||||
"Each property is a separate commit, so N properties produce N undo entries.",
|
||||
"Position and size properties (left, top, width, height) are refused here on purpose;",
|
||||
"they belong to the transform tools.",
|
||||
"Returns `ok: true` with `applied` and `rejected` maps, so a partial success is visible",
|
||||
"as a partial success rather than reported as a whole one.",
|
||||
].join(" ");
|
||||
@@ -0,0 +1,179 @@
|
||||
// @vitest-environment jsdom
|
||||
import { describe, expect, it, vi } from "vitest";
|
||||
import {
|
||||
studioTransform,
|
||||
type ElementBox,
|
||||
type StudioTransformResult,
|
||||
type TransformToolDeps,
|
||||
} from "./transformTools";
|
||||
import { expectFailure, expectOk, previewElement, selectionFor } from "../webmcpTestUtils";
|
||||
|
||||
/**
|
||||
* A stand-in for the rendered box. happy-dom and jsdom report all-zero rects,
|
||||
* so the box is injected rather than measured; these tests are about what the
|
||||
* tool concludes from a box, not about layout.
|
||||
*/
|
||||
function boxStore(initial: ElementBox) {
|
||||
const box = { ...initial };
|
||||
return {
|
||||
read: () => ({ ...box }),
|
||||
set: (next: Partial<ElementBox>) => Object.assign(box, next),
|
||||
};
|
||||
}
|
||||
|
||||
function transformDeps(overrides: Partial<TransformToolDeps> = {}): TransformToolDeps {
|
||||
const element = previewElement('<h1 id="headline">Ship it</h1>', "headline");
|
||||
return {
|
||||
getCurrentSelection: () => selectionFor(element),
|
||||
getWriteBlockedReason: () => null,
|
||||
readBox: () => ({ x: 0, y: 0, width: 100, height: 50 }),
|
||||
moveTo: async () => undefined,
|
||||
resizeTo: async () => undefined,
|
||||
rotateTo: async () => undefined,
|
||||
...overrides,
|
||||
};
|
||||
}
|
||||
|
||||
describe("studioTransform", () => {
|
||||
it("reports the box read back, not the box requested", async () => {
|
||||
const store = boxStore({ x: 0, y: 0, width: 100, height: 50 });
|
||||
// The handler lands somewhere other than asked, which is what a clamp or a
|
||||
// layout constraint does.
|
||||
const resizeTo = vi.fn(async () => store.set({ width: 300, height: 120 }));
|
||||
|
||||
const result = await studioTransform(transformDeps({ readBox: store.read, resizeTo }), {
|
||||
width: 999,
|
||||
height: 999,
|
||||
});
|
||||
|
||||
const ok = expectOk<StudioTransformResult>(result);
|
||||
expect(ok.box.width).toBe(300);
|
||||
expect(ok.box.height).toBe(120);
|
||||
expect(ok.applied).toContain("resize");
|
||||
});
|
||||
|
||||
it("reports a silent no-op as unchanged instead of success", async () => {
|
||||
// handleGsapAwarePathOffsetCommit is `if (gsapCommitMutation) {...}` with no
|
||||
// else branch. Without GSAP it resolves having written nothing, and echoing
|
||||
// the request back would be a lie the agent builds on.
|
||||
const store = boxStore({ x: 10, y: 10, width: 100, height: 50 });
|
||||
const moveTo = vi.fn(async () => undefined);
|
||||
|
||||
const result = expectFailure(
|
||||
await studioTransform(transformDeps({ readBox: store.read, moveTo }), { x: 500, y: 400 }),
|
||||
);
|
||||
|
||||
expect(moveTo).toHaveBeenCalled();
|
||||
expect(result.kind).toBe("blocked");
|
||||
expect(result.reason).toMatch(/did not move/);
|
||||
expect(result.hint).toMatch(/GSAP/);
|
||||
});
|
||||
|
||||
it("separates what landed from what did not, in one call", async () => {
|
||||
const store = boxStore({ x: 0, y: 0, width: 100, height: 50 });
|
||||
const resizeTo = vi.fn(async () => store.set({ width: 200, height: 80 }));
|
||||
const moveTo = vi.fn(async () => undefined);
|
||||
|
||||
const result = await studioTransform(transformDeps({ readBox: store.read, resizeTo, moveTo }), {
|
||||
x: 40,
|
||||
y: 40,
|
||||
width: 200,
|
||||
height: 80,
|
||||
});
|
||||
|
||||
const ok = expectOk<StudioTransformResult>(result);
|
||||
expect(ok.applied).toEqual(["resize"]);
|
||||
expect(ok.unchanged.move).toMatch(/did not move/);
|
||||
});
|
||||
|
||||
it("re-reads between operations so a later one sees the earlier result", async () => {
|
||||
const store = boxStore({ x: 0, y: 0, width: 100, height: 50 });
|
||||
const resizeTo = vi.fn(async () => store.set({ width: 200, height: 80 }));
|
||||
const moveTo = vi.fn(async () => store.set({ x: 40, y: 40 }));
|
||||
|
||||
const result = await studioTransform(transformDeps({ readBox: store.read, resizeTo, moveTo }), {
|
||||
x: 40,
|
||||
y: 40,
|
||||
width: 200,
|
||||
height: 80,
|
||||
});
|
||||
|
||||
// Move is judged against the box AFTER the resize. Comparing against the
|
||||
// original would credit the resize's change to the move.
|
||||
const ok = expectOk<StudioTransformResult>(result);
|
||||
expect(ok.applied).toEqual(["resize", "move"]);
|
||||
expect(ok.unchanged).toEqual({});
|
||||
});
|
||||
|
||||
it("reports rotation as dispatched rather than verified", async () => {
|
||||
// `rotate` is an individual transform property and does not appear in the
|
||||
// computed transform, so there is no honest box-derived signal for it.
|
||||
const rotateTo = vi.fn(async () => undefined);
|
||||
|
||||
const result = await studioTransform(transformDeps({ rotateTo }), { rotate: 15 });
|
||||
|
||||
const ok = expectOk<StudioTransformResult>(result);
|
||||
expect(rotateTo).toHaveBeenCalledWith(expect.anything(), { angle: 15 });
|
||||
expect(ok.applied).toEqual(["rotate"]);
|
||||
});
|
||||
|
||||
it("refuses to write while a conflict is waiting for the user", async () => {
|
||||
const moveTo = vi.fn();
|
||||
|
||||
const result = expectFailure(
|
||||
await studioTransform(
|
||||
transformDeps({ getWriteBlockedReason: () => "Auto-save is paused", moveTo }),
|
||||
{ x: 10, y: 10 },
|
||||
),
|
||||
);
|
||||
|
||||
expect(result.kind).toBe("blocked");
|
||||
expect(moveTo).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it("requires x and y together, and width and height together", async () => {
|
||||
const moveTo = vi.fn();
|
||||
const resizeTo = vi.fn();
|
||||
const deps = transformDeps({ moveTo, resizeTo });
|
||||
|
||||
expect(expectFailure(await studioTransform(deps, { x: 10 })).reason).toMatch(/together/);
|
||||
expect(expectFailure(await studioTransform(deps, { width: 10 })).reason).toMatch(/together/);
|
||||
expect(moveTo).not.toHaveBeenCalled();
|
||||
expect(resizeTo).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it("rejects a negative size and an empty request", async () => {
|
||||
const deps = transformDeps();
|
||||
|
||||
expect(expectFailure(await studioTransform(deps, { width: -1, height: 10 })).kind).toBe(
|
||||
"invalid",
|
||||
);
|
||||
expect(expectFailure(await studioTransform(deps, {})).reason).toMatch(/at least one/);
|
||||
});
|
||||
|
||||
it("rejects non-finite numbers rather than passing them to a handler", async () => {
|
||||
const moveTo = vi.fn();
|
||||
|
||||
const result = expectFailure(
|
||||
await studioTransform(transformDeps({ moveTo }), { x: Number.NaN, y: 10 }),
|
||||
);
|
||||
|
||||
expect(result.kind).toBe("invalid");
|
||||
expect(moveTo).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it("fails when nothing is selected", async () => {
|
||||
const moveTo = vi.fn();
|
||||
|
||||
const result = expectFailure(
|
||||
await studioTransform(transformDeps({ getCurrentSelection: () => null, moveTo }), {
|
||||
x: 1,
|
||||
y: 1,
|
||||
}),
|
||||
);
|
||||
|
||||
expect(result.kind).toBe("invalid");
|
||||
expect(result.hint).toMatch(/studio_select/);
|
||||
expect(moveTo).not.toHaveBeenCalled();
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,205 @@
|
||||
/**
|
||||
* `studio_transform`: move, resize and rotate, as a drag would.
|
||||
*
|
||||
* This tool reads the element's box back after every write and reports what
|
||||
* ACTUALLY changed. That is not belt-and-braces, it is the only thing standing
|
||||
* between an agent and a silent lie, because two of the three handlers can do
|
||||
* nothing and resolve:
|
||||
*
|
||||
* - The handlers exposed on `DomEditActionsValue` are the GSAP-AWARE wrappers
|
||||
* (`useDomEditSession.ts` aliases them), not the CSS ones in
|
||||
* `useDomGeometryCommits.ts`.
|
||||
* - `handleGsapAwarePathOffsetCommit` and `handleGsapAwareRotationCommit` are
|
||||
* `if (gsapCommitMutation) { ...intercept... }` with NO else branch. In a
|
||||
* composition with no GSAP they return having done nothing. The adjacent
|
||||
* comments confirm that is deliberate: there is no CSS fallback to write to.
|
||||
* - `handleGsapAwareBoxSizeCommit` is different. It runs through
|
||||
* `runGestureTransaction` with a scale route and a width/height route, so
|
||||
* resize works more generally than the other two.
|
||||
*
|
||||
* Read back, do not assume.
|
||||
*/
|
||||
|
||||
import type { DomEditSelection } from "../../components/editor/domEditingTypes";
|
||||
import { toolFailure, toolOk, type ToolFailure, type ToolResult } from "../toolResult";
|
||||
|
||||
export interface ElementBox {
|
||||
x: number;
|
||||
y: number;
|
||||
width: number;
|
||||
height: number;
|
||||
}
|
||||
|
||||
export interface TransformToolDeps {
|
||||
getCurrentSelection: () => DomEditSelection | null;
|
||||
getWriteBlockedReason: () => string | null;
|
||||
/** The element's box as it renders right now. */
|
||||
readBox: (selection: DomEditSelection) => ElementBox;
|
||||
moveTo: (selection: DomEditSelection, next: { x: number; y: number }) => Promise<void>;
|
||||
resizeTo: (selection: DomEditSelection, next: { width: number; height: number }) => Promise<void>;
|
||||
rotateTo: (selection: DomEditSelection, next: { angle: number }) => Promise<void>;
|
||||
}
|
||||
|
||||
export interface StudioTransformInput {
|
||||
x?: unknown;
|
||||
y?: unknown;
|
||||
width?: unknown;
|
||||
height?: unknown;
|
||||
rotate?: unknown;
|
||||
}
|
||||
|
||||
export interface StudioTransformResult {
|
||||
/** The box as it renders after the write, read back, not echoed. */
|
||||
box: ElementBox;
|
||||
applied: string[];
|
||||
/** Requested operations whose effect could not be observed, with why. */
|
||||
unchanged: Record<string, string>;
|
||||
}
|
||||
|
||||
const NO_OP_HINT =
|
||||
"Move and rotate are written as GSAP code; a composition with no GSAP timeline has nothing to write to. studio_inspect reports the element's animations.";
|
||||
|
||||
function readNumber(value: unknown): number | null {
|
||||
return typeof value === "number" && Number.isFinite(value) ? value : null;
|
||||
}
|
||||
|
||||
function guard(deps: TransformToolDeps): ToolFailure | null {
|
||||
const blocked = deps.getWriteBlockedReason();
|
||||
if (blocked) return toolFailure("blocked", blocked, "Resolve it in Studio, then retry.");
|
||||
if (!deps.getCurrentSelection()) {
|
||||
return toolFailure("invalid", "nothing is selected", "Call studio_select first.");
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
interface TransformRequest {
|
||||
move: { x: number; y: number } | null;
|
||||
size: { width: number; height: number } | null;
|
||||
rotate: number | null;
|
||||
}
|
||||
|
||||
/**
|
||||
* Both or neither. Accepting one axis alone would mean inventing the other from
|
||||
* the current value, which moves the element somewhere the caller did not ask
|
||||
* for.
|
||||
*/
|
||||
function parsePair(
|
||||
a: unknown,
|
||||
b: unknown,
|
||||
names: [string, string],
|
||||
min = Number.NEGATIVE_INFINITY,
|
||||
): { pair: [number, number] | null } | ToolFailure {
|
||||
const first = readNumber(a);
|
||||
const second = readNumber(b);
|
||||
if (first === null && second === null) return { pair: null };
|
||||
if (first === null || second === null) {
|
||||
return toolFailure("invalid", `${names[0]} and ${names[1]} must be given together`);
|
||||
}
|
||||
if (first < min || second < min) {
|
||||
return toolFailure("invalid", `${names[0]} and ${names[1]} must be at least ${min}`);
|
||||
}
|
||||
return { pair: [first, second] };
|
||||
}
|
||||
|
||||
function isFailure(value: object): value is ToolFailure {
|
||||
return "ok" in value;
|
||||
}
|
||||
|
||||
function parseRequest(input: StudioTransformInput): TransformRequest | ToolFailure {
|
||||
const move = parsePair(input.x, input.y, ["x", "y"]);
|
||||
if (isFailure(move)) return move;
|
||||
const size = parsePair(input.width, input.height, ["width", "height"], 0);
|
||||
if (isFailure(size)) return size;
|
||||
const rotate = readNumber(input.rotate);
|
||||
|
||||
if (!move.pair && !size.pair && rotate === null) {
|
||||
return toolFailure(
|
||||
"invalid",
|
||||
"give at least one of x, y, width, height, rotate as a finite number",
|
||||
);
|
||||
}
|
||||
|
||||
return {
|
||||
move: move.pair ? { x: move.pair[0], y: move.pair[1] } : null,
|
||||
size: size.pair ? { width: size.pair[0], height: size.pair[1] } : null,
|
||||
rotate,
|
||||
};
|
||||
}
|
||||
|
||||
export async function studioTransform(
|
||||
deps: TransformToolDeps,
|
||||
input: StudioTransformInput,
|
||||
): Promise<ToolResult<StudioTransformResult>> {
|
||||
const request = parseRequest(input);
|
||||
if (isFailure(request)) return request;
|
||||
|
||||
const blocked = guard(deps);
|
||||
if (blocked) return blocked;
|
||||
|
||||
const selection = deps.getCurrentSelection();
|
||||
if (!selection) return toolFailure("invalid", "nothing is selected");
|
||||
|
||||
const applied: string[] = [];
|
||||
const unchanged: Record<string, string> = {};
|
||||
|
||||
// Sequential, and each one re-reads first, so a move is judged against the box
|
||||
// AFTER a resize in the same call rather than against the original.
|
||||
if (request.size) {
|
||||
const before = deps.readBox(selection);
|
||||
await deps.resizeTo(selection, request.size);
|
||||
const after = deps.readBox(selection);
|
||||
if (after.width !== before.width || after.height !== before.height) applied.push("resize");
|
||||
else unchanged.resize = "the element's size did not change";
|
||||
}
|
||||
|
||||
if (request.move) {
|
||||
const before = deps.readBox(selection);
|
||||
await deps.moveTo(selection, request.move);
|
||||
const after = deps.readBox(selection);
|
||||
if (after.x !== before.x || after.y !== before.y) applied.push("move");
|
||||
else unchanged.move = `the element did not move. ${NO_OP_HINT}`;
|
||||
}
|
||||
|
||||
if (request.rotate !== null) {
|
||||
// Rotation is written as the CSS `rotate` property, an individual transform
|
||||
// property that does NOT appear in getComputedStyle().transform. There is no
|
||||
// reliable box-derived signal, so this is reported as dispatched rather than
|
||||
// verified, and the description says so.
|
||||
await deps.rotateTo(selection, { angle: request.rotate });
|
||||
applied.push("rotate");
|
||||
}
|
||||
|
||||
if (applied.length === 0) {
|
||||
return toolFailure(
|
||||
"blocked",
|
||||
`nothing changed: ${Object.values(unchanged).join("; ")}`,
|
||||
NO_OP_HINT,
|
||||
);
|
||||
}
|
||||
|
||||
return toolOk<StudioTransformResult>({ box: deps.readBox(selection), applied, unchanged });
|
||||
}
|
||||
|
||||
export const STUDIO_TRANSFORM_INPUT_SCHEMA = {
|
||||
type: "object",
|
||||
properties: {
|
||||
x: { type: "number", description: "New x offset in pixels. Must be paired with y." },
|
||||
y: { type: "number", description: "New y offset in pixels. Must be paired with x." },
|
||||
width: { type: "number", minimum: 0, description: "New width. Must be paired with height." },
|
||||
height: { type: "number", minimum: 0, description: "New height. Must be paired with width." },
|
||||
rotate: { type: "number", description: "Rotation in degrees." },
|
||||
},
|
||||
additionalProperties: false,
|
||||
} as const;
|
||||
|
||||
export const STUDIO_TRANSFORM_DESCRIPTION = [
|
||||
"Move, resize or rotate the CURRENTLY SELECTED element, the way a drag would.",
|
||||
"Call studio_select first. Give x with y, and width with height.",
|
||||
"The result's `box` is READ BACK after the write, not echoed from your request, and",
|
||||
"`applied` lists what actually took effect. Check it.",
|
||||
"Move and rotate are written as GSAP code, so in a composition with no GSAP timeline they",
|
||||
"do nothing; that shows up in `unchanged` rather than as a false success.",
|
||||
"Rotation is reported as dispatched rather than verified, because the CSS `rotate` property",
|
||||
"does not appear in the element's computed transform.",
|
||||
"Returns `ok: true`, or `ok: false` with `kind`, `reason` and a `hint`.",
|
||||
].join(" ");
|
||||
@@ -43,6 +43,13 @@ function deps(overrides: Partial<StudioAgentToolsDeps> = {}): StudioAgentToolsDe
|
||||
probeFrame: async () => ({ ok: true, status: 200 }),
|
||||
wait: async () => undefined,
|
||||
getCurrentSelection: () => null,
|
||||
getWriteBlockedReason: () => null,
|
||||
setText: async () => ({ ok: true }),
|
||||
setStyle: async () => ({ ok: true }),
|
||||
readBox: () => ({ x: 0, y: 0, width: 100, height: 50 }),
|
||||
moveTo: async () => undefined,
|
||||
resizeTo: async () => undefined,
|
||||
rotateTo: async () => undefined,
|
||||
getGsapDiagnostics: () => ({
|
||||
animations: [],
|
||||
multipleTimelines: false,
|
||||
@@ -114,6 +121,9 @@ describe("useStudioAgentTools", () => {
|
||||
"studio_seek",
|
||||
"studio_frame",
|
||||
"studio_inspect",
|
||||
"studio_set_text",
|
||||
"studio_set_style",
|
||||
"studio_transform",
|
||||
]);
|
||||
expect(trackEvent).toHaveBeenCalledWith("webmcp.native_present");
|
||||
});
|
||||
@@ -128,14 +138,14 @@ describe("useStudioAgentTools", () => {
|
||||
await act(async () => {
|
||||
harness = mountTools(deps({ getSnapshot: () => snapshot() }));
|
||||
});
|
||||
expect(registerTool).toHaveBeenCalledTimes(5);
|
||||
expect(registerTool).toHaveBeenCalledTimes(8);
|
||||
|
||||
await act(async () => {
|
||||
harness?.rerenderWith(deps({ getSnapshot: () => snapshot({ currentTime: 5 }) }));
|
||||
harness?.rerenderWith(deps({ getSnapshot: () => snapshot({ currentTime: 6 }) }));
|
||||
});
|
||||
|
||||
expect(registerTool).toHaveBeenCalledTimes(5);
|
||||
expect(registerTool).toHaveBeenCalledTimes(8);
|
||||
});
|
||||
|
||||
it("executes against the LATEST deps, not the ones present at registration", async () => {
|
||||
@@ -208,7 +218,7 @@ describe("useStudioAgentTools", () => {
|
||||
mountTools(deps({ getSnapshot: () => snapshot() }));
|
||||
});
|
||||
|
||||
expect(registerTool).toHaveBeenCalledTimes(5);
|
||||
expect(registerTool).toHaveBeenCalledTimes(8);
|
||||
});
|
||||
|
||||
it("reports a non-abort registration failure through production telemetry", async () => {
|
||||
|
||||
@@ -41,6 +41,25 @@ import {
|
||||
type StudioInspectInput,
|
||||
type StudioInspectResult,
|
||||
} from "./tools/inspectTools";
|
||||
import {
|
||||
studioSetStyle,
|
||||
studioSetText,
|
||||
STUDIO_SET_STYLE_DESCRIPTION,
|
||||
STUDIO_SET_STYLE_INPUT_SCHEMA,
|
||||
STUDIO_SET_TEXT_DESCRIPTION,
|
||||
STUDIO_SET_TEXT_INPUT_SCHEMA,
|
||||
type ContentToolDeps,
|
||||
type StudioSetStyleResult,
|
||||
type StudioSetTextResult,
|
||||
} from "./tools/contentTools";
|
||||
import {
|
||||
studioTransform,
|
||||
STUDIO_TRANSFORM_DESCRIPTION,
|
||||
STUDIO_TRANSFORM_INPUT_SCHEMA,
|
||||
type StudioTransformInput,
|
||||
type StudioTransformResult,
|
||||
type TransformToolDeps,
|
||||
} from "./tools/transformTools";
|
||||
|
||||
const log = makeStudioDebugLogger("webmcp");
|
||||
|
||||
@@ -54,7 +73,8 @@ function reportRegistration(report: ToolRegistrationReport, native: boolean): vo
|
||||
}
|
||||
}
|
||||
|
||||
export interface StudioAgentToolsDeps extends SelectionToolDeps, FrameToolDeps, InspectToolDeps {
|
||||
export interface StudioAgentToolsDeps
|
||||
extends SelectionToolDeps, FrameToolDeps, InspectToolDeps, ContentToolDeps, TransformToolDeps {
|
||||
/** Read Studio's current state. Called per tool invocation, never cached. */
|
||||
getSnapshot: () => StudioLookSnapshot;
|
||||
}
|
||||
@@ -126,6 +146,35 @@ function buildStudioTools(depsRef: { readonly current: StudioAgentToolsDeps }):
|
||||
studioInspect(depsRef.current, input as StudioInspectInput),
|
||||
),
|
||||
},
|
||||
{
|
||||
name: "studio_set_text",
|
||||
title: "Set an element's text",
|
||||
description: STUDIO_SET_TEXT_DESCRIPTION,
|
||||
inputSchema: STUDIO_SET_TEXT_INPUT_SCHEMA,
|
||||
annotations: { readOnlyHint: false, untrustedContentHint: true },
|
||||
execute: (input): Promise<ToolResult<StudioSetTextResult>> =>
|
||||
runToolBody("studio_set_text", () => studioSetText(depsRef.current, input)),
|
||||
},
|
||||
{
|
||||
name: "studio_set_style",
|
||||
title: "Set an element's styles",
|
||||
description: STUDIO_SET_STYLE_DESCRIPTION,
|
||||
inputSchema: STUDIO_SET_STYLE_INPUT_SCHEMA,
|
||||
annotations: { readOnlyHint: false },
|
||||
execute: (input): Promise<ToolResult<StudioSetStyleResult>> =>
|
||||
runToolBody("studio_set_style", () => studioSetStyle(depsRef.current, input)),
|
||||
},
|
||||
{
|
||||
name: "studio_transform",
|
||||
title: "Move, resize or rotate",
|
||||
description: STUDIO_TRANSFORM_DESCRIPTION,
|
||||
inputSchema: STUDIO_TRANSFORM_INPUT_SCHEMA,
|
||||
annotations: { readOnlyHint: false },
|
||||
execute: (input): Promise<ToolResult<StudioTransformResult>> =>
|
||||
runToolBody("studio_transform", () =>
|
||||
studioTransform(depsRef.current, input as StudioTransformInput),
|
||||
),
|
||||
},
|
||||
];
|
||||
}
|
||||
|
||||
|
||||
Reference in New Issue
Block a user