From 5a4dd8bec1a994c11bfa43f2eb141187978f18b0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Miguel=20=C3=81ngel?= Date: Wed, 22 Apr 2026 18:44:17 +0200 Subject: [PATCH] fix: gate studio timeline actions by capability (#415) ## Summary - gate timeline actions to clips Studio can control deterministically - disable direct move/trim for generic GSAP-timed DOM clips - add an in-clip `Copy to Agent` fallback for unsupported edits ## Why Studio should only advertise timeline actions it can round-trip to source HTML with deterministic meaning. This PR now follows that stricter rule: - direct move/end-trim are only exposed for clips with a deterministic timeline window - start trim is only exposed for clips with a real content-offset model - unsupported motion clips now offer `Copy to Agent` so users still have a fast path to request source-level timing changes In practice this means generic GSAP-authored DOM clips no longer pretend Studio can rewrite their visible timing just by patching `data-start` / `data-duration`. ## What changed - added `hasPatchableTimelineTarget()` and `getTimelineEditCapabilities()` in `timelineEditing.ts` - tightened deterministic-window detection so only media, images, and composition hosts keep direct move/end-trim controls - kept wrapped media clips editable by recognizing real media metadata even when the host tag is a `div` - updated `TimelineClip` / `Timeline` to guard interactions with the shared capability model - added `buildTimelineElementAgentPrompt()` and a `Copy to Agent` fallback button for unsupported clips - added focused tests for capability derivation and the agent-prompt helper ## Verification ### Automated - `bun test packages/studio/src/player/components/timelineEditing.test.ts packages/studio/src/player/components/Timeline.test.ts packages/studio/src/player/store/playerStore.test.ts packages/studio/src/utils/sourcePatcher.test.ts` - `bun run --filter @hyperframes/studio typecheck` - `bunx oxlint packages/studio/src/player/components/Timeline.tsx packages/studio/src/player/components/timelineEditing.ts packages/studio/src/player/components/timelineEditing.test.ts packages/studio/src/player/components/TimelineClip.tsx` - `bunx oxfmt --check packages/studio/src/player/components/Timeline.tsx packages/studio/src/player/components/timelineEditing.ts packages/studio/src/player/components/timelineEditing.test.ts packages/studio/src/player/components/TimelineClip.tsx` ### Browser Verified in Studio with live browser automation against `http://127.0.0.1:4175/#project/timeline-edit-playground`: - generic GSAP-timed clips (`feature-card`, `title-card`, `prompt-card`) show `Copy to Agent` and no direct move/trim affordances - wrapped media (`media-card`) still exposes direct controls and remains draggable - the local playground timings were realigned to match the authored GSAP positions, so preview visibility now matches the timeline windows during manual testing Recording artifacts used during verification: - `/tmp/timeline-capabilities-proof/capabilities-flow.webm` - `/tmp/timeline-capabilities-proof/capabilities-agent-flow.webm` --- .../studio/src/player/components/Timeline.tsx | 57 ++++++++- .../src/player/components/TimelineClip.tsx | 21 ++-- .../player/components/timelineEditing.test.ts | 110 ++++++++++++++++++ .../src/player/components/timelineEditing.ts | 83 +++++++++++++ 4 files changed, 258 insertions(+), 13 deletions(-) diff --git a/packages/studio/src/player/components/Timeline.tsx b/packages/studio/src/player/components/Timeline.tsx index 4c94e8228..d68b256f6 100644 --- a/packages/studio/src/player/components/Timeline.tsx +++ b/packages/studio/src/player/components/Timeline.tsx @@ -10,7 +10,8 @@ import { formatTime } from "../lib/time"; import { TimelineClip } from "./TimelineClip"; import { EditPopover } from "./EditModal"; import { - canOffsetTrimClipStart, + buildTimelineElementAgentPrompt, + getTimelineEditCapabilities, resolveTimelineAutoScroll, resolveTimelineMove, resolveTimelineResize, @@ -245,6 +246,7 @@ export const Timeline = memo(function Timeline({ onResizeElementRef.current = onResizeElement; const suppressClickRef = useRef(false); const [showPopover, setShowPopover] = useState(false); + const [copiedAgentElementKey, setCopiedAgentElementKey] = useState(null); const [viewportWidth, setViewportWidth] = useState(0); const roRef = useRef(null); @@ -896,6 +898,46 @@ export const Timeline = memo(function Timeline({ } : null; const renderClipChildren = (element: TimelineElement, clipStyle: TrackVisualStyle) => { + const capabilities = getTimelineEditCapabilities(element); + const elementKey = element.key ?? element.id; + const needsAgentFallback = + !capabilities.canMove && !capabilities.canTrimStart && !capabilities.canTrimEnd; + const agentButton = needsAgentFallback ? ( + + ) : null; return ( <> {renderClipOverlay?.(element)} @@ -941,6 +983,7 @@ export const Timeline = memo(function Timeline({ )} + {agentButton} ); }; @@ -1082,6 +1125,7 @@ export const Timeline = memo(function Timeline({ {els.map((el, i) => { const clipStyle = getStyle(el.tag); const elementKey = el.key ?? el.id; + const capabilities = getTimelineEditCapabilities(el); const isSelected = selectedElementId === elementKey; const isComposition = !!el.compositionSrc; const clipKey = `${elementKey}-${i}`; @@ -1110,7 +1154,8 @@ export const Timeline = memo(function Timeline({ onHoverEnd={() => setHoveredClip(null)} onResizeStart={(edge, e) => { if (e.button !== 0 || e.shiftKey || !onResizeElement) return; - if (edge === "start" && !canOffsetTrimClipStart(el)) return; + if (edge === "start" && !capabilities.canTrimStart) return; + if (edge === "end" && !capabilities.canTrimEnd) return; e.stopPropagation(); setShowPopover(false); setRangeSelection(null); @@ -1125,7 +1170,13 @@ export const Timeline = memo(function Timeline({ }); }} onPointerDown={(e) => { - if (e.button !== 0 || e.shiftKey || !onMoveElement) return; + if ( + e.button !== 0 || + e.shiftKey || + !onMoveElement || + !capabilities.canMove + ) + return; setShowPopover(false); setRangeSelection(null); const rect = (e.currentTarget as HTMLElement).getBoundingClientRect(); diff --git a/packages/studio/src/player/components/TimelineClip.tsx b/packages/studio/src/player/components/TimelineClip.tsx index 100588844..e908811eb 100644 --- a/packages/studio/src/player/components/TimelineClip.tsx +++ b/packages/studio/src/player/components/TimelineClip.tsx @@ -4,7 +4,7 @@ import type { TimelineTrackStyle } from "./timelineTheme"; import { memo, type ReactNode } from "react"; import type { TimelineElement } from "../store/playerStore"; import { defaultTimelineTheme, getClipHandleOpacity, type TimelineTheme } from "./timelineTheme"; -import { canOffsetTrimClipStart } from "./timelineEditing"; +import { getTimelineEditCapabilities } from "./timelineEditing"; interface TimelineClipProps { el: TimelineElement; @@ -60,7 +60,7 @@ export const TimelineClip = memo(function TimelineClip({ : isHovered ? theme.clipShadowHover : theme.clipShadow; - const canTrimStart = canOffsetTrimClipStart(el); + const capabilities = getTimelineEditCapabilities(el); const showHandles = handleOpacity > 0.01; return ( @@ -87,7 +87,7 @@ export const TimelineClip = memo(function TimelineClip({ transition: "border-color 120ms ease-out, box-shadow 140ms ease-out, background 140ms ease-out", zIndex: isDragging ? 20 : isSelected ? 10 : isHovered ? 5 : 1, - cursor: "grab", + cursor: capabilities.canMove ? "grab" : "default", transform: isDragging ? "translateY(-1px)" : undefined, }} title={ @@ -111,13 +111,13 @@ export const TimelineClip = memo(function TimelineClip({ top: 0, bottom: 0, width: 18, - opacity: showHandles && canTrimStart ? 1 : 0, - pointerEvents: onResizeStart && canTrimStart ? "auto" : "none", + opacity: showHandles && capabilities.canTrimStart ? 1 : 0, + pointerEvents: onResizeStart && capabilities.canTrimStart ? "auto" : "none", zIndex: 4, transition: "opacity 120ms ease-out", cursor: "col-resize", background: - showHandles && canTrimStart + showHandles && capabilities.canTrimStart ? `linear-gradient(90deg, ${trackStyle.accent}4d 0%, ${trackStyle.accent}22 42%, transparent 100%)` : "transparent", }} @@ -148,13 +148,14 @@ export const TimelineClip = memo(function TimelineClip({ bottom: 0, width: 18, opacity: showHandles ? 1 : 0, - pointerEvents: onResizeStart ? "auto" : "none", + pointerEvents: onResizeStart && capabilities.canTrimEnd ? "auto" : "none", zIndex: 4, transition: "opacity 120ms ease-out", cursor: "col-resize", - background: showHandles - ? `linear-gradient(270deg, ${trackStyle.accent}4d 0%, ${trackStyle.accent}22 42%, transparent 100%)` - : "transparent", + background: + showHandles && capabilities.canTrimEnd + ? `linear-gradient(270deg, ${trackStyle.accent}4d 0%, ${trackStyle.accent}22 42%, transparent 100%)` + : "transparent", }} >
{ }); }); +describe("hasPatchableTimelineTarget", () => { + it("returns true when the clip has a DOM id", () => { + expect(hasPatchableTimelineTarget({ domId: "hero-card" })).toBe(true); + }); + + it("returns true when the clip has a selector", () => { + expect(hasPatchableTimelineTarget({ selector: ".hero-card" })).toBe(true); + }); + + it("returns false when the clip has no stable patch target", () => { + expect(hasPatchableTimelineTarget({})).toBe(false); + }); +}); + +describe("getTimelineEditCapabilities", () => { + it("disables move and trims for generic motion clips even when patchable", () => { + expect( + getTimelineEditCapabilities({ + tag: "section", + duration: 2, + selector: ".feature-card", + }), + ).toEqual({ + canMove: false, + canTrimStart: false, + canTrimEnd: false, + }); + }); + + it("allows move and both trims for patchable media clips with offset support", () => { + expect( + getTimelineEditCapabilities({ + tag: "video", + duration: 2, + selector: "#media-card", + playbackStartAttr: "media-start", + sourceDuration: 10, + }), + ).toEqual({ + canMove: true, + canTrimStart: true, + canTrimEnd: true, + }); + }); + + it("treats wrapped media clips with media metadata as deterministic", () => { + expect( + getTimelineEditCapabilities({ + tag: "div", + duration: 2, + selector: "#media-card", + playbackStartAttr: "media-start", + sourceDuration: 10, + }), + ).toEqual({ + canMove: true, + canTrimStart: true, + canTrimEnd: true, + }); + }); + + it("allows move and end trim for patchable composition hosts", () => { + expect( + getTimelineEditCapabilities({ + tag: "div", + duration: 3, + selector: '[data-composition-id="intro"]', + compositionSrc: "compositions/intro.html", + }), + ).toEqual({ + canMove: true, + canTrimStart: false, + canTrimEnd: true, + }); + }); + + it("disables all timeline edits for clips without a patchable target", () => { + expect( + getTimelineEditCapabilities({ + tag: "video", + duration: 2, + sourceDuration: 10, + }), + ).toEqual({ + canMove: false, + canTrimStart: false, + canTrimEnd: false, + }); + }); +}); + describe("resolveTimelineAutoScroll", () => { it("does not scroll when the pointer stays away from the edges", () => { expect( @@ -273,6 +367,22 @@ describe("buildTimelineAgentPrompt", () => { }); }); +describe("buildTimelineElementAgentPrompt", () => { + it("includes the clip context and guidance for agent-based edits", () => { + expect( + buildTimelineElementAgentPrompt({ + id: "feature-card", + tag: "section", + start: 1.4, + duration: 1.6, + track: 1, + sourceFile: "index.html", + selector: "#feature-card", + }), + ).toContain("If this clip is animated with GSAP"); + }); +}); + describe("resolveTimelineResize", () => { it("shrinks clip duration from the right edge", () => { expect( diff --git a/packages/studio/src/player/components/timelineEditing.ts b/packages/studio/src/player/components/timelineEditing.ts index 8e592fe5e..ebff92530 100644 --- a/packages/studio/src/player/components/timelineEditing.ts +++ b/packages/studio/src/player/components/timelineEditing.ts @@ -169,6 +169,35 @@ export interface TimelinePromptElement { track: number; } +export interface TimelineEditCapabilities { + canMove: boolean; + canTrimStart: boolean; + canTrimEnd: boolean; +} + +function isDeterministicTimelineWindow(input: { + tag: string; + compositionSrc?: string; + playbackStartAttr?: "media-start" | "playback-start"; + sourceDuration?: number; +}): boolean { + if (input.compositionSrc) return true; + if (input.playbackStartAttr != null) return true; + if ( + input.sourceDuration != null && + Number.isFinite(input.sourceDuration) && + input.sourceDuration > 0 + ) { + return true; + } + const normalizedTag = input.tag.toLowerCase(); + return ["video", "audio", "img"].includes(normalizedTag); +} + +export function hasPatchableTimelineTarget(input: { domId?: string; selector?: string }): boolean { + return Boolean(input.domId || input.selector); +} + export function canOffsetTrimClipStart(input: { tag: string; playbackStart?: number; @@ -186,6 +215,26 @@ export function canOffsetTrimClipStart(input: { ); } +export function getTimelineEditCapabilities(input: { + tag: string; + duration: number; + domId?: string; + selector?: string; + compositionSrc?: string; + playbackStart?: number; + playbackStartAttr?: "media-start" | "playback-start"; + sourceDuration?: number; +}): TimelineEditCapabilities { + const canPatch = hasPatchableTimelineTarget(input); + const hasFiniteDuration = Number.isFinite(input.duration) && input.duration > 0; + const hasDeterministicWindow = isDeterministicTimelineWindow(input); + return { + canMove: canPatch && hasDeterministicWindow, + canTrimEnd: canPatch && hasFiniteDuration && hasDeterministicWindow, + canTrimStart: canPatch && hasFiniteDuration && canOffsetTrimClipStart(input), + }; +} + export function buildTimelineAgentPrompt({ rangeStart, rangeEnd, @@ -226,6 +275,40 @@ export function buildPromptCopyText(prompt: string): string { return prompt.trim(); } +export function buildTimelineElementAgentPrompt(element: { + id: string; + tag: string; + start: number; + duration: number; + track: number; + sourceFile?: string; + selector?: string; + compositionSrc?: string; +}): string { + const lines = [ + "Studio cannot directly move or resize this timeline clip because its visible timing is not fully controlled by patchable HTML timing attributes.", + "", + "Please update the source so the clip's actual visible timing stays consistent with the authored timeline.", + "", + "Clip:", + `- id: ${element.id}`, + `- tag: ${element.tag}`, + `- time: ${formatTime(element.start)} to ${formatTime(element.start + element.duration)}`, + `- track: ${element.track}`, + ]; + + if (element.sourceFile) lines.push(`- source file: ${element.sourceFile}`); + if (element.selector) lines.push(`- selector: ${element.selector}`); + if (element.compositionSrc) lines.push(`- composition src: ${element.compositionSrc}`); + + lines.push( + "", + "If this clip is animated with GSAP or another JS timeline, update the authored animation timing there as well instead of only changing data-start/data-duration.", + ); + + return lines.join("\n"); +} + export function formatTimelineAttributeNumber(value: number): string { return Number(roundToCentiseconds(value).toFixed(2)).toString(); }