mirror of
https://github.com/heygen-com/hyperframes.git
synced 2026-11-15 13:58:20 +00:00
feat(studio): composer follows the selection, and speaks for a whole one
Selecting an element now opens the composer on it — the selection was already the ask, and waiting for a second gesture on the thing just clicked was a step for nothing. A multi-selection carries through: the prompt names every co-selected element and widens its guardrail to "change every one of them", and the field says how many it is talking about. The harness and model lists are capped and scroll on their own instead of growing the composer, and they declare they own their wheel so the canvas underneath stays put. Adds usage telemetry for the whole surface — composer opened, run submitted, harness and model chosen, custom harness added — carrying the harness, model and element count, never the instruction text.
This commit is contained in:
@@ -262,7 +262,10 @@ export function InlineAgentComposer({
|
||||
</div>
|
||||
|
||||
{pickerOpen && onSelectAgent && (
|
||||
<div className="mb-1.5 max-h-64 overflow-y-auto overscroll-contain rounded-[10px] bg-neutral-900/70 p-1 ring-1 ring-white/10">
|
||||
<div
|
||||
data-preview-overlay-scroll="true"
|
||||
className="mb-1.5 max-h-44 scroll-py-1 overflow-y-auto overscroll-contain scroll-smooth rounded-[10px] bg-neutral-900/70 p-1 ring-1 ring-white/10"
|
||||
>
|
||||
{addingAgent && onAddCustomAgent ? (
|
||||
<CustomAgentForm
|
||||
onSubmit={onAddCustomAgent}
|
||||
@@ -308,7 +311,10 @@ export function InlineAgentComposer({
|
||||
)}
|
||||
|
||||
{modelPickerOpen && onSelectModel && agentModels.length > 0 && (
|
||||
<ul className="mb-1.5 max-h-64 space-y-0.5 overflow-y-auto overscroll-contain rounded-[10px] bg-neutral-900/70 p-1 ring-1 ring-white/10">
|
||||
<ul
|
||||
data-preview-overlay-scroll="true"
|
||||
className="mb-1.5 max-h-44 space-y-0.5 scroll-py-1 overflow-y-auto overscroll-contain scroll-smooth rounded-[10px] bg-neutral-900/70 p-1 ring-1 ring-white/10"
|
||||
>
|
||||
<li>
|
||||
<button
|
||||
className="flex w-full items-center gap-1.5 rounded-md px-1.5 py-1 text-left text-[11px] leading-none text-neutral-300 transition-colors duration-150 ease-out hover:bg-neutral-800/70"
|
||||
@@ -418,6 +424,7 @@ export function InlineAgentComposerHost({
|
||||
|
||||
const {
|
||||
domEditSelection,
|
||||
domEditGroupSelections,
|
||||
agentModalOpen,
|
||||
agentRunLabel,
|
||||
agentRunKind,
|
||||
@@ -433,7 +440,11 @@ export function InlineAgentComposerHost({
|
||||
<>
|
||||
{agentModalOpen && domEditSelection && (
|
||||
<InlineAgentComposer
|
||||
selectionLabel={domEditSelection.label}
|
||||
selectionLabel={
|
||||
domEditGroupSelections.length > 1
|
||||
? `${domEditGroupSelections.length} elements`
|
||||
: domEditSelection.label
|
||||
}
|
||||
rect={rect}
|
||||
canvas={canvas}
|
||||
runLabel={agentRunLabel}
|
||||
|
||||
@@ -1319,3 +1319,46 @@ describe("hfId — find, key, capabilities (R7 fixes)", () => {
|
||||
expect(result.canMove).toBe(true);
|
||||
});
|
||||
});
|
||||
|
||||
describe("buildElementAgentPrompt — multi-selection", () => {
|
||||
function selectionFor(id: string): DomEditSelection {
|
||||
// The builder only reads fields off the selection, never the live node.
|
||||
return {
|
||||
element: { outerHTML: `<div id="${id}"></div>` },
|
||||
id,
|
||||
selector: `#${id}`,
|
||||
selectorIndex: 0,
|
||||
tagName: "div",
|
||||
label: id,
|
||||
compositionPath: "index.html",
|
||||
sourceFile: "index.html",
|
||||
boundingBox: { x: 0, y: 0, width: 10, height: 10 },
|
||||
textContent: null,
|
||||
inlineStyles: {},
|
||||
computedStyles: {},
|
||||
dataAttributes: {},
|
||||
textFields: [],
|
||||
capabilities: {},
|
||||
} as unknown as DomEditSelection;
|
||||
}
|
||||
|
||||
it("names every co-selected element and widens the guardrail", () => {
|
||||
const prompt = buildElementAgentPrompt({
|
||||
selection: selectionFor("title"),
|
||||
alsoSelected: [selectionFor("subtitle")],
|
||||
currentTime: 0,
|
||||
userInstruction: "make them blue",
|
||||
});
|
||||
|
||||
expect(prompt).toContain("Also selected (1 more element)");
|
||||
expect(prompt).toContain("selector=#subtitle");
|
||||
expect(prompt).toContain("- Make the targeted change to every selected element");
|
||||
expect(prompt).not.toContain("- Make a targeted change to this element only.");
|
||||
});
|
||||
|
||||
it("keeps the single-element guardrail when nothing else is selected", () => {
|
||||
const prompt = buildElementAgentPrompt({ selection: selectionFor("title"), currentTime: 0 });
|
||||
expect(prompt).toContain("- Make a targeted change to this element only.");
|
||||
expect(prompt).not.toContain("Also selected");
|
||||
});
|
||||
});
|
||||
|
||||
@@ -24,8 +24,19 @@ function formatTextFields(fields: DomEditTextField[]): string {
|
||||
.join("\n");
|
||||
}
|
||||
|
||||
/** One line per co-selected element, so a multi-select edit names every target. */
|
||||
function formatAlsoSelected(selections: DomEditSelection[]): string {
|
||||
return selections
|
||||
.map((selection) => {
|
||||
const text = selection.textContent ? `; text=${JSON.stringify(selection.textContent)}` : "";
|
||||
return `- <${selection.tagName}> id=${selection.id ?? "(none)"}; selector=${selection.selector ?? "(none)"}; index=${selection.selectorIndex ?? 0}; bounds=${formatBoundingBox(selection.boundingBox)}${text}`;
|
||||
})
|
||||
.join("\n");
|
||||
}
|
||||
|
||||
export function buildElementAgentPrompt({
|
||||
selection,
|
||||
alsoSelected = [],
|
||||
currentTime,
|
||||
tagSnippet,
|
||||
selectionContext,
|
||||
@@ -33,6 +44,8 @@ export function buildElementAgentPrompt({
|
||||
sourceFilePath,
|
||||
}: {
|
||||
selection: DomEditSelection;
|
||||
/** The rest of a multi-selection; the instruction applies to all of them. */
|
||||
alsoSelected?: DomEditSelection[];
|
||||
currentTime: number;
|
||||
tagSnippet?: string;
|
||||
selectionContext?: string;
|
||||
@@ -84,10 +97,20 @@ export function buildElementAgentPrompt({
|
||||
lines.push("", "Target HTML:", tagSnippet);
|
||||
}
|
||||
|
||||
if (alsoSelected.length > 0) {
|
||||
lines.push(
|
||||
"",
|
||||
`Also selected (${alsoSelected.length} more element${alsoSelected.length > 1 ? "s" : ""}) — apply the same change to each:`,
|
||||
formatAlsoSelected(alsoSelected),
|
||||
);
|
||||
}
|
||||
|
||||
lines.push(
|
||||
"",
|
||||
"Guardrails:",
|
||||
"- Make a targeted change to this element only.",
|
||||
alsoSelected.length > 0
|
||||
? "- Make the targeted change to every selected element, and nothing else."
|
||||
: "- Make a targeted change to this element only.",
|
||||
"- Preserve the rest of the composition and its timing.",
|
||||
"- Do not modify other elements' data-* attributes or positioning.",
|
||||
"- Prefer existing inline styles or existing CSS rules for this element over adding unrelated selectors.",
|
||||
|
||||
@@ -13,6 +13,7 @@ import type {
|
||||
import { findElementForSelection } from "../components/editor/domEditing";
|
||||
import { readStudioUiPreferences, writeStudioUiPreferences } from "../utils/studioUiPreferences";
|
||||
import { usePlayerStore } from "../player";
|
||||
import { trackStudioEvent } from "../utils/studioTelemetry";
|
||||
|
||||
// ── Types ──
|
||||
|
||||
@@ -33,6 +34,8 @@ export interface UseAskAgentModalParams {
|
||||
showToast: (message: string, tone?: "error" | "info") => void;
|
||||
domEditSelectionRef: React.MutableRefObject<DomEditSelection | null>;
|
||||
domEditSelection: DomEditSelection | null;
|
||||
/** The whole marquee/shift selection; the instruction applies to all of it. */
|
||||
domEditGroupSelections: DomEditSelection[];
|
||||
}
|
||||
|
||||
// ── Hook ──
|
||||
@@ -48,6 +51,7 @@ export function useAskAgentModal({
|
||||
showToast,
|
||||
domEditSelectionRef,
|
||||
domEditSelection,
|
||||
domEditGroupSelections,
|
||||
}: UseAskAgentModalParams) {
|
||||
// ── State ──
|
||||
|
||||
@@ -69,6 +73,7 @@ export function useAskAgentModal({
|
||||
() => readStudioUiPreferences().agentId ?? null,
|
||||
);
|
||||
const setSelectedAgentId = useCallback((id: string | null) => {
|
||||
trackStudioEvent("agent_harness_selected", { harness: id ?? "auto" });
|
||||
setSelectedAgentIdState(id);
|
||||
writeStudioUiPreferences({ agentId: id ?? undefined });
|
||||
}, []);
|
||||
@@ -168,6 +173,7 @@ export function useAskAgentModal({
|
||||
);
|
||||
|
||||
const setSelectedModel = useCallback((kind: AgentKind, model: string | null) => {
|
||||
trackStudioEvent("agent_model_selected", { harness: kind, model: model ?? "cheapest" });
|
||||
setModelByKind((current) => {
|
||||
const next = { ...current };
|
||||
if (model) next[kind] = model;
|
||||
@@ -184,15 +190,38 @@ export function useAskAgentModal({
|
||||
setAgentPromptSelectionContext(undefined);
|
||||
void preloadAgentPromptSnippet(selection);
|
||||
setAgentModalOpen(true);
|
||||
trackStudioEvent("agent_composer_opened", { elements: domEditGroupSelections.length || 1 });
|
||||
void refreshAgentState();
|
||||
void refreshAgentModels(activeKindRef.current);
|
||||
}, [preloadAgentPromptSnippet, refreshAgentModels, refreshAgentState, resolveSelection]);
|
||||
}, [
|
||||
domEditGroupSelections.length,
|
||||
preloadAgentPromptSnippet,
|
||||
refreshAgentModels,
|
||||
refreshAgentState,
|
||||
resolveSelection,
|
||||
]);
|
||||
|
||||
// Selecting an element is the ask: the composer follows the selection instead
|
||||
// of waiting for a second gesture on the same thing the user just clicked.
|
||||
// eslint-disable-next-line no-restricted-syntax
|
||||
useEffect(() => {
|
||||
if (!domEditSelection) return;
|
||||
handleAskAgent();
|
||||
}, [domEditSelection, handleAskAgent]);
|
||||
|
||||
/** Everything the instruction should touch: the anchor plus its co-selection. */
|
||||
const resolveGroup = useCallback(
|
||||
(anchor: DomEditSelection) =>
|
||||
domEditGroupSelections.filter((entry) => entry.element !== anchor.element),
|
||||
[domEditGroupSelections],
|
||||
);
|
||||
|
||||
const buildPrompt = useCallback(
|
||||
(selection: DomEditSelection, userInstruction: string) => {
|
||||
const targetPath = selection.sourceFile || activeCompPath || "index.html";
|
||||
return buildElementAgentPrompt({
|
||||
selection,
|
||||
alsoSelected: resolveGroup(selection),
|
||||
currentTime: usePlayerStore.getState().currentTime,
|
||||
tagSnippet: agentPromptTagSnippet ?? selection.element.outerHTML,
|
||||
selectionContext: agentPromptSelectionContext,
|
||||
@@ -200,7 +229,7 @@ export function useAskAgentModal({
|
||||
sourceFilePath: toProjectAbsolutePath(projectDir, targetPath),
|
||||
});
|
||||
},
|
||||
[activeCompPath, agentPromptSelectionContext, agentPromptTagSnippet, projectDir],
|
||||
[activeCompPath, agentPromptSelectionContext, agentPromptTagSnippet, projectDir, resolveGroup],
|
||||
);
|
||||
|
||||
/**
|
||||
@@ -214,13 +243,25 @@ export function useAskAgentModal({
|
||||
const pid = projectIdRef.current;
|
||||
if (!selection || !pid) return;
|
||||
|
||||
const coSelected = resolveGroup(selection).length;
|
||||
// Traction, not content: how often runs are started, with which harness
|
||||
// and model, and whether multi-select is used. No instruction text.
|
||||
trackStudioEvent("agent_run_submitted", {
|
||||
harness: activeKindRef.current ?? "unknown",
|
||||
model: activeKindRef.current
|
||||
? (modelByKind[activeKindRef.current] ?? "default")
|
||||
: "default",
|
||||
elements: coSelected + 1,
|
||||
instruction_length: userInstruction.length,
|
||||
});
|
||||
|
||||
void fetch(`/api/projects/${pid}/agent`, {
|
||||
method: "POST",
|
||||
headers: { "content-type": "application/json" },
|
||||
body: JSON.stringify({
|
||||
prompt: buildPrompt(selection, userInstruction),
|
||||
instruction: userInstruction,
|
||||
target: selection.label,
|
||||
target: coSelected > 0 ? `${coSelected + 1} elements` : selection.label,
|
||||
agent: selectedAgentId ?? undefined,
|
||||
model: activeKindRef.current ? modelByKind[activeKindRef.current] : undefined,
|
||||
// Selection coordinates travel with the run so the tray can seek back
|
||||
@@ -246,7 +287,15 @@ export function useAskAgentModal({
|
||||
showToast(err instanceof Error ? err.message : "Could not start the agent.", "error");
|
||||
});
|
||||
},
|
||||
[buildPrompt, modelByKind, projectIdRef, resolveSelection, selectedAgentId, showToast],
|
||||
[
|
||||
buildPrompt,
|
||||
modelByKind,
|
||||
projectIdRef,
|
||||
resolveGroup,
|
||||
resolveSelection,
|
||||
selectedAgentId,
|
||||
showToast,
|
||||
],
|
||||
);
|
||||
|
||||
/** Seek to when a run was asked for and re-select the element it edited. */
|
||||
@@ -336,6 +385,9 @@ export function useAskAgentModal({
|
||||
showToast(data.error ?? "Could not add that harness.", "error");
|
||||
return false;
|
||||
}
|
||||
trackStudioEvent("agent_custom_harness_added", {
|
||||
has_model_flag: Boolean(draft.modelFlag),
|
||||
});
|
||||
await refreshAgentState();
|
||||
return true;
|
||||
} catch {
|
||||
|
||||
@@ -181,6 +181,7 @@ export function useDomEditSession({
|
||||
showToast,
|
||||
domEditSelectionRef,
|
||||
domEditSelection,
|
||||
domEditGroupSelections,
|
||||
});
|
||||
|
||||
useStudioSelectionPublisher({
|
||||
|
||||
Reference in New Issue
Block a user