diff --git a/packages/studio/src/App.tsx b/packages/studio/src/App.tsx index 5075ea1a69..6a825bbe94 100644 --- a/packages/studio/src/App.tsx +++ b/packages/studio/src/App.tsx @@ -4,6 +4,7 @@ import { useRef, useEffect, useMemo, + type CSSProperties, type MouseEvent, type ReactNode, } from "react"; @@ -91,10 +92,12 @@ import { findElementForTimelineElement, getDomEditLayerKey, getDomEditTargetKey, + isLargeRasterDomEditSelection, isTextEditableSelection, resolveVisualDomEditSelectionTarget, serializeDomEditTextFields, resolveDomEditSelection, + type DomEditViewport, type DomEditLayerItem, type DomEditTextField, type DomEditSelection, @@ -359,6 +362,59 @@ function isManualGeometryStyleProperty(property: string): boolean { return property === "left" || property === "top" || property === "width" || property === "height"; } +interface PreviewLocalPointer { + x: number; + y: number; + viewport: DomEditViewport; +} + +interface AgentModalAnchorPoint { + x: number; + y: number; +} + +function resolvePreviewLocalPointer( + iframe: HTMLIFrameElement, + doc: Document, + win: Window, + clientX: number, + clientY: number, +): PreviewLocalPointer | null { + const iframeRect = iframe.getBoundingClientRect(); + const root = + doc.querySelector("[data-composition-id]") ?? doc.documentElement ?? null; + const rootRect = root?.getBoundingClientRect(); + const rootWidth = rootRect?.width || win.innerWidth; + const rootHeight = rootRect?.height || win.innerHeight; + if (!rootWidth || !rootHeight) return null; + + const scaleX = iframeRect.width / rootWidth; + const scaleY = iframeRect.height / rootHeight; + return { + x: (clientX - iframeRect.left) / scaleX, + y: (clientY - iframeRect.top) / scaleY, + viewport: { width: rootWidth, height: rootHeight }, + }; +} + +function getPreviewLocalPointer( + iframe: HTMLIFrameElement, + clientX: number, + clientY: number, +): PreviewLocalPointer | null { + let doc: Document | null = null; + let win: Window | null = null; + try { + doc = iframe.contentDocument; + win = iframe.contentWindow; + } catch { + return null; + } + if (!doc || !win) return null; + + return resolvePreviewLocalPointer(iframe, doc, win, clientX, clientY); +} + function getPreviewTargetFromPointer( iframe: HTMLIFrameElement, clientX: number, @@ -375,22 +431,12 @@ function getPreviewTargetFromPointer( } if (!doc || !win) return null; - const iframeRect = iframe.getBoundingClientRect(); - const root = - doc.querySelector("[data-composition-id]") ?? doc.documentElement ?? null; - const rootRect = root?.getBoundingClientRect(); - const rootWidth = rootRect?.width || win.innerWidth; - const rootHeight = rootRect?.height || win.innerHeight; - if (!rootWidth || !rootHeight) return null; - - const scaleX = iframeRect.width / rootWidth; - const scaleY = iframeRect.height / rootHeight; - const localX = (clientX - iframeRect.left) / scaleX; - const localY = (clientY - iframeRect.top) / scaleY; + const localPointer = resolvePreviewLocalPointer(iframe, doc, win, clientX, clientY); + if (!localPointer) return null; if (typeof doc.elementsFromPoint === "function") { const visualTarget = resolveVisualDomEditSelectionTarget( - doc.elementsFromPoint(localX, localY), + doc.elementsFromPoint(localPointer.x, localPointer.y), { activeCompositionPath, }, @@ -398,7 +444,22 @@ function getPreviewTargetFromPointer( if (visualTarget) return visualTarget; } - return getEventTargetElement(doc.elementFromPoint(localX, localY)); + return getEventTargetElement(doc.elementFromPoint(localPointer.x, localPointer.y)); +} + +function buildRasterClickSelectionContext( + selection: DomEditSelection, + localPointer: PreviewLocalPointer, +): string { + return [ + "The user clicked a large raster/background element in the Studio preview.", + `Preview click: x=${Math.round(localPointer.x)}px, y=${Math.round(localPointer.y)}px in a ${Math.round( + localPointer.viewport.width, + )}x${Math.round(localPointer.viewport.height)} composition.`, + `Selected target: <${selection.tagName}> ${selection.selector ?? selection.id ?? selection.label}.`, + "Visible copy or artwork at that point may be baked into the selected image/background rather than a selectable DOM text layer.", + "If the request mentions text seen at the click location, inspect or replace the image asset, or recreate that visible copy as editable DOM.", + ].join("\n"); } function domEditSelectionsTargetSame( @@ -604,17 +665,47 @@ function pauseStudioPreviewPlayback(iframe: HTMLIFrameElement | null): number | // ── Ask Agent Modal ── +function clampNumber(value: number, min: number, max: number): number { + if (max < min) return min; + return Math.min(Math.max(value, min), max); +} + +function getAgentModalPositionStyle( + anchorPoint: AgentModalAnchorPoint | null, +): CSSProperties | undefined { + if (!anchorPoint || typeof window === "undefined") return undefined; + + const modalWidth = 480; + const estimatedModalHeight = 270; + const margin = 16; + const left = clampNumber( + anchorPoint.x, + margin + modalWidth / 2, + window.innerWidth - margin - modalWidth / 2, + ); + const top = clampNumber( + anchorPoint.y + 12, + margin, + window.innerHeight - margin - estimatedModalHeight, + ); + + return { left, top, transform: "translateX(-50%)" }; +} + function AskAgentModal({ selectionLabel, + anchorPoint = null, onSubmit, onClose, }: { selectionLabel: string; + anchorPoint?: AgentModalAnchorPoint | null; onSubmit: (instruction: string) => void; onClose: () => void; }) { const [value, setValue] = useState(""); const inputRef = useRef(null); + const modalPositionStyle = getAgentModalPositionStyle(anchorPoint); useMountEffect(() => { requestAnimationFrame(() => inputRef.current?.focus()); @@ -627,11 +718,18 @@ function AskAgentModal({ return (
e.stopPropagation()} >
@@ -786,6 +884,12 @@ export function StudioApp() { const [domEditGroupSelections, setDomEditGroupSelections] = useState([]); const [domEditHoverSelection, setDomEditHoverSelection] = useState(null); const [agentPromptTagSnippet, setAgentPromptTagSnippet] = useState(); + const [agentPromptSelectionContext, setAgentPromptSelectionContext] = useState< + string | undefined + >(); + const [agentModalAnchorPoint, setAgentModalAnchorPoint] = useState( + null, + ); const [copiedAgentPrompt, setCopiedAgentPrompt] = useState(false); const [agentModalOpen, setAgentModalOpen] = useState(false); const [previewIframe, setPreviewIframe] = useState(null); @@ -1780,6 +1884,8 @@ export function StudioApp() { options?: { revealPanel?: boolean; additive?: boolean; preserveGroup?: boolean }, ) => { setAgentPromptTagSnippet(undefined); + setAgentPromptSelectionContext(undefined); + setAgentModalAnchorPoint(null); setCopiedAgentPrompt(false); if (!selection) { domEditSelectionRef.current = null; @@ -2992,6 +3098,8 @@ export function StudioApp() { const handleAskAgent = useCallback(() => { if (!domEditSelection) return; setAgentPromptTagSnippet(undefined); + setAgentPromptSelectionContext(undefined); + setAgentModalAnchorPoint(null); void preloadAgentPromptSnippet(domEditSelection); setAgentModalOpen(true); }, [domEditSelection, preloadAgentPromptSnippet]); @@ -3006,6 +3114,7 @@ export function StudioApp() { selection: domEditSelection, currentTime, tagSnippet, + selectionContext: agentPromptSelectionContext, userInstruction, sourceFilePath: toProjectAbsolutePath(projectDir, targetPath), }); @@ -3017,11 +3126,21 @@ export function StudioApp() { } setAgentModalOpen(false); + setAgentPromptSelectionContext(undefined); + setAgentModalAnchorPoint(null); if (copiedAgentTimerRef.current) clearTimeout(copiedAgentTimerRef.current); setCopiedAgentPrompt(true); copiedAgentTimerRef.current = setTimeout(() => setCopiedAgentPrompt(false), 1600); }, - [activeCompPath, agentPromptTagSnippet, currentTime, domEditSelection, projectDir, showToast], + [ + activeCompPath, + agentPromptSelectionContext, + agentPromptTagSnippet, + currentTime, + domEditSelection, + projectDir, + showToast, + ], ); const handlePreviewIframeRef = useCallback( @@ -3049,9 +3168,29 @@ export function StudioApp() { } e.preventDefault(); e.stopPropagation(); + const localPointer = previewIframeRef.current + ? getPreviewLocalPointer(previewIframeRef.current, e.clientX, e.clientY) + : null; applyDomSelection(nextSelection, { additive: e.shiftKey }); + if ( + !e.shiftKey && + localPointer && + isLargeRasterDomEditSelection(nextSelection, localPointer.viewport) + ) { + setAgentPromptSelectionContext( + buildRasterClickSelectionContext(nextSelection, localPointer), + ); + setAgentModalAnchorPoint({ x: e.clientX, y: e.clientY }); + void preloadAgentPromptSnippet(nextSelection); + setAgentModalOpen(true); + } }, - [applyDomSelection, captionEditMode, resolveDomSelectionFromPreviewPoint], + [ + applyDomSelection, + captionEditMode, + preloadAgentPromptSnippet, + resolveDomSelectionFromPreviewPoint, + ], ); const handlePreviewCanvasPointerMove = useCallback( @@ -4117,8 +4256,13 @@ export function StudioApp() { {agentModalOpen && domEditSelection && ( setAgentModalOpen(false)} + onClose={() => { + setAgentModalOpen(false); + setAgentPromptSelectionContext(undefined); + setAgentModalAnchorPoint(null); + }} /> )} diff --git a/packages/studio/src/components/editor/domEditing.test.ts b/packages/studio/src/components/editor/domEditing.test.ts index 330d109724..a15d31083a 100644 --- a/packages/studio/src/components/editor/domEditing.test.ts +++ b/packages/studio/src/components/editor/domEditing.test.ts @@ -9,6 +9,7 @@ import { findElementForTimelineElement, getDomEditNonEditableReason, getDomEditTargetKey, + isLargeRasterDomEditSelection, isTextEditableSelection, resolveVisualDomEditSelectionTarget, serializeDomEditTextFields, @@ -322,6 +323,56 @@ describe("resolveVisualDomEditSelectionTarget", () => { }); }); +describe("isLargeRasterDomEditSelection", () => { + it("flags large image and background targets for raster click fallback", () => { + expect( + isLargeRasterDomEditSelection( + { + tagName: "img", + boundingBox: { x: 0, y: 0, width: 1920, height: 1080 }, + computedStyles: {}, + }, + { width: 1920, height: 1080 }, + ), + ).toBe(true); + + expect( + isLargeRasterDomEditSelection( + { + tagName: "div", + boundingBox: { x: 0, y: 0, width: 1280, height: 720 }, + computedStyles: { "background-image": 'url("hero.png")' }, + }, + { width: 1920, height: 1080 }, + ), + ).toBe(true); + }); + + it("does not flag small media or text selections", () => { + expect( + isLargeRasterDomEditSelection( + { + tagName: "img", + boundingBox: { x: 80, y: 80, width: 96, height: 96 }, + computedStyles: {}, + }, + { width: 1920, height: 1080 }, + ), + ).toBe(false); + + expect( + isLargeRasterDomEditSelection( + { + tagName: "h1", + boundingBox: { x: 0, y: 0, width: 1600, height: 300 }, + computedStyles: {}, + }, + { width: 1920, height: 1080 }, + ), + ).toBe(false); + }); +}); + describe("resolveDomEditSelection", () => { it("keeps composition host transforms disabled in master view", () => { expect( @@ -936,6 +987,50 @@ describe("patch builders and prompt builder", () => { expect(prompt).not.toContain("Source file: index.html"); }); + it("includes raster click context in copied agent prompts", () => { + const selection = { + element: {} as HTMLElement, + id: undefined, + selector: ".hero-bg", + selectorIndex: undefined, + sourceFile: "index.html", + compositionPath: "index.html", + compositionSrc: undefined, + isCompositionHost: false, + label: "Hero Bg", + tagName: "img", + boundingBox: { x: 0, y: 0, width: 1920, height: 1080 }, + textContent: null, + dataAttributes: {}, + inlineStyles: {}, + computedStyles: {}, + textFields: [], + capabilities: { + canSelect: true, + canEditStyles: true, + canMove: true, + canResize: true, + canApplyManualOffset: true, + canApplyManualSize: true, + canApplyManualRotation: true, + }, + } satisfies DomEditSelection; + + const prompt = buildElementAgentPrompt({ + selection, + currentTime: 3, + selectionContext: + "The user clicked visible text that is baked into the selected image/background.", + userInstruction: "Change the title copy.", + }); + + expect(prompt).toContain("Selection context:"); + expect(prompt).toContain( + "The user clicked visible text that is baked into the selected image/background.", + ); + expect(prompt).toContain("Change the title copy."); + }); + it("serializes child text fields back into HTML", () => { expect( serializeDomEditTextFields([ diff --git a/packages/studio/src/components/editor/domEditing.ts b/packages/studio/src/components/editor/domEditing.ts index f4ff9dabe9..f05adef722 100644 --- a/packages/studio/src/components/editor/domEditing.ts +++ b/packages/studio/src/components/editor/domEditing.ts @@ -105,6 +105,11 @@ export interface DomEditContextOptions { preferClipAncestor?: boolean; } +export interface DomEditViewport { + width: number; + height: number; +} + export interface TimelineElementDomTarget { id?: string; domId?: string; @@ -497,6 +502,31 @@ export function resolveVisualDomEditSelectionTarget( return best?.element ?? null; } +function hasRasterBackground(selection: Pick): boolean { + const backgroundImage = selection.computedStyles["background-image"]?.trim(); + return Boolean(backgroundImage && backgroundImage !== "none"); +} + +export function isLargeRasterDomEditSelection( + selection: Pick, + viewport?: DomEditViewport | null, +): boolean { + const tagName = selection.tagName.toLowerCase(); + const isRasterLike = tagName === "img" || hasRasterBackground(selection); + if (!isRasterLike) return false; + + const { width, height } = selection.boundingBox; + if (width <= 1 || height <= 1) return false; + if (!viewport || viewport.width <= 1 || viewport.height <= 1) { + return width >= 960 && height >= 540; + } + + const areaRatio = (width * height) / (viewport.width * viewport.height); + const widthRatio = width / viewport.width; + const heightRatio = height / viewport.height; + return areaRatio >= 0.4 || (widthRatio >= 0.7 && heightRatio >= 0.5); +} + function getDirectLayerChildren(el: HTMLElement, options: DomEditContextOptions): HTMLElement[] { return Array.from(el.children).filter( (child): child is HTMLElement => @@ -1013,12 +1043,14 @@ export function buildElementAgentPrompt({ selection, currentTime, tagSnippet, + selectionContext, userInstruction, sourceFilePath, }: { selection: DomEditSelection; currentTime: number; tagSnippet?: string; + selectionContext?: string; userInstruction?: string; sourceFilePath?: string; }): string { @@ -1043,6 +1075,11 @@ export function buildElementAgentPrompt({ lines.push(`Text: ${selection.textContent}`); } + const trimmedSelectionContext = selectionContext?.trim(); + if (trimmedSelectionContext) { + lines.push("", "Selection context:", trimmedSelectionContext); + } + const textFieldsBlock = formatTextFields(selection.textFields); if (textFieldsBlock) { lines.push("", "Text fields:", textFieldsBlock); diff --git a/qa-artifacts/studio-inspector-selection-ux/notion-showcase/raster-text-ask-agent-fallback.png b/qa-artifacts/studio-inspector-selection-ux/notion-showcase/raster-text-ask-agent-fallback.png new file mode 100644 index 0000000000..c82c10c95d Binary files /dev/null and b/qa-artifacts/studio-inspector-selection-ux/notion-showcase/raster-text-ask-agent-fallback.png differ diff --git a/qa-artifacts/studio-inspector-selection-ux/notion-showcase/raster-text-ask-agent-fallback.webm b/qa-artifacts/studio-inspector-selection-ux/notion-showcase/raster-text-ask-agent-fallback.webm new file mode 100644 index 0000000000..87a60a9bdf Binary files /dev/null and b/qa-artifacts/studio-inspector-selection-ux/notion-showcase/raster-text-ask-agent-fallback.webm differ