feat(studio): composer follows the selection, and speaks for a whole one

Selecting an element now opens the composer on it — the selection was already
the ask, and waiting for a second gesture on the thing just clicked was a step
for nothing. A multi-selection carries through: the prompt names every
co-selected element and widens its guardrail to "change every one of them", and
the field says how many it is talking about.

The harness and model lists are capped and scroll on their own instead of
growing the composer, and they declare they own their wheel so the canvas
underneath stays put.

Adds usage telemetry for the whole surface — composer opened, run submitted,
harness and model chosen, custom harness added — carrying the harness, model
and element count, never the instruction text.
This commit is contained in:
Miguel Angel Simon Sierra
2026-08-04 15:19:15 -07:00
parent c1697c9b50
commit 87ad9804f8
5 changed files with 138 additions and 8 deletions
@@ -262,7 +262,10 @@ export function InlineAgentComposer({
</div>
{pickerOpen && onSelectAgent && (
<div className="mb-1.5 max-h-64 overflow-y-auto overscroll-contain rounded-[10px] bg-neutral-900/70 p-1 ring-1 ring-white/10">
<div
data-preview-overlay-scroll="true"
className="mb-1.5 max-h-44 scroll-py-1 overflow-y-auto overscroll-contain scroll-smooth rounded-[10px] bg-neutral-900/70 p-1 ring-1 ring-white/10"
>
{addingAgent && onAddCustomAgent ? (
<CustomAgentForm
onSubmit={onAddCustomAgent}
@@ -308,7 +311,10 @@ export function InlineAgentComposer({
)}
{modelPickerOpen && onSelectModel && agentModels.length > 0 && (
<ul className="mb-1.5 max-h-64 space-y-0.5 overflow-y-auto overscroll-contain rounded-[10px] bg-neutral-900/70 p-1 ring-1 ring-white/10">
<ul
data-preview-overlay-scroll="true"
className="mb-1.5 max-h-44 space-y-0.5 scroll-py-1 overflow-y-auto overscroll-contain scroll-smooth rounded-[10px] bg-neutral-900/70 p-1 ring-1 ring-white/10"
>
<li>
<button
className="flex w-full items-center gap-1.5 rounded-md px-1.5 py-1 text-left text-[11px] leading-none text-neutral-300 transition-colors duration-150 ease-out hover:bg-neutral-800/70"
@@ -418,6 +424,7 @@ export function InlineAgentComposerHost({
const {
domEditSelection,
domEditGroupSelections,
agentModalOpen,
agentRunLabel,
agentRunKind,
@@ -433,7 +440,11 @@ export function InlineAgentComposerHost({
<>
{agentModalOpen && domEditSelection && (
<InlineAgentComposer
selectionLabel={domEditSelection.label}
selectionLabel={
domEditGroupSelections.length > 1
? `${domEditGroupSelections.length} elements`
: domEditSelection.label
}
rect={rect}
canvas={canvas}
runLabel={agentRunLabel}
@@ -1319,3 +1319,46 @@ describe("hfId — find, key, capabilities (R7 fixes)", () => {
expect(result.canMove).toBe(true);
});
});
describe("buildElementAgentPrompt — multi-selection", () => {
function selectionFor(id: string): DomEditSelection {
// The builder only reads fields off the selection, never the live node.
return {
element: { outerHTML: `<div id="${id}"></div>` },
id,
selector: `#${id}`,
selectorIndex: 0,
tagName: "div",
label: id,
compositionPath: "index.html",
sourceFile: "index.html",
boundingBox: { x: 0, y: 0, width: 10, height: 10 },
textContent: null,
inlineStyles: {},
computedStyles: {},
dataAttributes: {},
textFields: [],
capabilities: {},
} as unknown as DomEditSelection;
}
it("names every co-selected element and widens the guardrail", () => {
const prompt = buildElementAgentPrompt({
selection: selectionFor("title"),
alsoSelected: [selectionFor("subtitle")],
currentTime: 0,
userInstruction: "make them blue",
});
expect(prompt).toContain("Also selected (1 more element)");
expect(prompt).toContain("selector=#subtitle");
expect(prompt).toContain("- Make the targeted change to every selected element");
expect(prompt).not.toContain("- Make a targeted change to this element only.");
});
it("keeps the single-element guardrail when nothing else is selected", () => {
const prompt = buildElementAgentPrompt({ selection: selectionFor("title"), currentTime: 0 });
expect(prompt).toContain("- Make a targeted change to this element only.");
expect(prompt).not.toContain("Also selected");
});
});
@@ -24,8 +24,19 @@ function formatTextFields(fields: DomEditTextField[]): string {
.join("\n");
}
/** One line per co-selected element, so a multi-select edit names every target. */
function formatAlsoSelected(selections: DomEditSelection[]): string {
return selections
.map((selection) => {
const text = selection.textContent ? `; text=${JSON.stringify(selection.textContent)}` : "";
return `- <${selection.tagName}> id=${selection.id ?? "(none)"}; selector=${selection.selector ?? "(none)"}; index=${selection.selectorIndex ?? 0}; bounds=${formatBoundingBox(selection.boundingBox)}${text}`;
})
.join("\n");
}
export function buildElementAgentPrompt({
selection,
alsoSelected = [],
currentTime,
tagSnippet,
selectionContext,
@@ -33,6 +44,8 @@ export function buildElementAgentPrompt({
sourceFilePath,
}: {
selection: DomEditSelection;
/** The rest of a multi-selection; the instruction applies to all of them. */
alsoSelected?: DomEditSelection[];
currentTime: number;
tagSnippet?: string;
selectionContext?: string;
@@ -84,10 +97,20 @@ export function buildElementAgentPrompt({
lines.push("", "Target HTML:", tagSnippet);
}
if (alsoSelected.length > 0) {
lines.push(
"",
`Also selected (${alsoSelected.length} more element${alsoSelected.length > 1 ? "s" : ""}) — apply the same change to each:`,
formatAlsoSelected(alsoSelected),
);
}
lines.push(
"",
"Guardrails:",
"- Make a targeted change to this element only.",
alsoSelected.length > 0
? "- Make the targeted change to every selected element, and nothing else."
: "- Make a targeted change to this element only.",
"- Preserve the rest of the composition and its timing.",
"- Do not modify other elements' data-* attributes or positioning.",
"- Prefer existing inline styles or existing CSS rules for this element over adding unrelated selectors.",
+56 -4
View File
@@ -13,6 +13,7 @@ import type {
import { findElementForSelection } from "../components/editor/domEditing";
import { readStudioUiPreferences, writeStudioUiPreferences } from "../utils/studioUiPreferences";
import { usePlayerStore } from "../player";
import { trackStudioEvent } from "../utils/studioTelemetry";
// ── Types ──
@@ -33,6 +34,8 @@ export interface UseAskAgentModalParams {
showToast: (message: string, tone?: "error" | "info") => void;
domEditSelectionRef: React.MutableRefObject<DomEditSelection | null>;
domEditSelection: DomEditSelection | null;
/** The whole marquee/shift selection; the instruction applies to all of it. */
domEditGroupSelections: DomEditSelection[];
}
// ── Hook ──
@@ -48,6 +51,7 @@ export function useAskAgentModal({
showToast,
domEditSelectionRef,
domEditSelection,
domEditGroupSelections,
}: UseAskAgentModalParams) {
// ── State ──
@@ -69,6 +73,7 @@ export function useAskAgentModal({
() => readStudioUiPreferences().agentId ?? null,
);
const setSelectedAgentId = useCallback((id: string | null) => {
trackStudioEvent("agent_harness_selected", { harness: id ?? "auto" });
setSelectedAgentIdState(id);
writeStudioUiPreferences({ agentId: id ?? undefined });
}, []);
@@ -168,6 +173,7 @@ export function useAskAgentModal({
);
const setSelectedModel = useCallback((kind: AgentKind, model: string | null) => {
trackStudioEvent("agent_model_selected", { harness: kind, model: model ?? "cheapest" });
setModelByKind((current) => {
const next = { ...current };
if (model) next[kind] = model;
@@ -184,15 +190,38 @@ export function useAskAgentModal({
setAgentPromptSelectionContext(undefined);
void preloadAgentPromptSnippet(selection);
setAgentModalOpen(true);
trackStudioEvent("agent_composer_opened", { elements: domEditGroupSelections.length || 1 });
void refreshAgentState();
void refreshAgentModels(activeKindRef.current);
}, [preloadAgentPromptSnippet, refreshAgentModels, refreshAgentState, resolveSelection]);
}, [
domEditGroupSelections.length,
preloadAgentPromptSnippet,
refreshAgentModels,
refreshAgentState,
resolveSelection,
]);
// Selecting an element is the ask: the composer follows the selection instead
// of waiting for a second gesture on the same thing the user just clicked.
// eslint-disable-next-line no-restricted-syntax
useEffect(() => {
if (!domEditSelection) return;
handleAskAgent();
}, [domEditSelection, handleAskAgent]);
/** Everything the instruction should touch: the anchor plus its co-selection. */
const resolveGroup = useCallback(
(anchor: DomEditSelection) =>
domEditGroupSelections.filter((entry) => entry.element !== anchor.element),
[domEditGroupSelections],
);
const buildPrompt = useCallback(
(selection: DomEditSelection, userInstruction: string) => {
const targetPath = selection.sourceFile || activeCompPath || "index.html";
return buildElementAgentPrompt({
selection,
alsoSelected: resolveGroup(selection),
currentTime: usePlayerStore.getState().currentTime,
tagSnippet: agentPromptTagSnippet ?? selection.element.outerHTML,
selectionContext: agentPromptSelectionContext,
@@ -200,7 +229,7 @@ export function useAskAgentModal({
sourceFilePath: toProjectAbsolutePath(projectDir, targetPath),
});
},
[activeCompPath, agentPromptSelectionContext, agentPromptTagSnippet, projectDir],
[activeCompPath, agentPromptSelectionContext, agentPromptTagSnippet, projectDir, resolveGroup],
);
/**
@@ -214,13 +243,25 @@ export function useAskAgentModal({
const pid = projectIdRef.current;
if (!selection || !pid) return;
const coSelected = resolveGroup(selection).length;
// Traction, not content: how often runs are started, with which harness
// and model, and whether multi-select is used. No instruction text.
trackStudioEvent("agent_run_submitted", {
harness: activeKindRef.current ?? "unknown",
model: activeKindRef.current
? (modelByKind[activeKindRef.current] ?? "default")
: "default",
elements: coSelected + 1,
instruction_length: userInstruction.length,
});
void fetch(`/api/projects/${pid}/agent`, {
method: "POST",
headers: { "content-type": "application/json" },
body: JSON.stringify({
prompt: buildPrompt(selection, userInstruction),
instruction: userInstruction,
target: selection.label,
target: coSelected > 0 ? `${coSelected + 1} elements` : selection.label,
agent: selectedAgentId ?? undefined,
model: activeKindRef.current ? modelByKind[activeKindRef.current] : undefined,
// Selection coordinates travel with the run so the tray can seek back
@@ -246,7 +287,15 @@ export function useAskAgentModal({
showToast(err instanceof Error ? err.message : "Could not start the agent.", "error");
});
},
[buildPrompt, modelByKind, projectIdRef, resolveSelection, selectedAgentId, showToast],
[
buildPrompt,
modelByKind,
projectIdRef,
resolveGroup,
resolveSelection,
selectedAgentId,
showToast,
],
);
/** Seek to when a run was asked for and re-select the element it edited. */
@@ -336,6 +385,9 @@ export function useAskAgentModal({
showToast(data.error ?? "Could not add that harness.", "error");
return false;
}
trackStudioEvent("agent_custom_harness_added", {
has_model_flag: Boolean(draft.modelFlag),
});
await refreshAgentState();
return true;
} catch {
@@ -181,6 +181,7 @@ export function useDomEditSession({
showToast,
domEditSelectionRef,
domEditSelection,
domEditGroupSelections,
});
useStudioSelectionPublisher({