fix: gate studio timeline actions by capability (#415)

## Summary - gate timeline actions to clips Studio can control deterministically - disable direct move/trim for generic GSAP-timed DOM clips - add an in-clip `Copy to Agent` fallback for unsupported edits ## Why Studio should only advertise timeline actions it can round-trip to source HTML with deterministic meaning. This PR now follows that stricter rule: - direct move/end-trim are only exposed for clips with a deterministic timeline window - start trim is only exposed for clips with a real content-offset model - unsupported motion clips now offer `Copy to Agent` so users still have a fast path to request source-level timing changes In practice this means generic GSAP-authored DOM clips no longer pretend Studio can rewrite their visible timing just by patching `data-start` / `data-duration`. ## What changed - added `hasPatchableTimelineTarget()` and `getTimelineEditCapabilities()` in `timelineEditing.ts` - tightened deterministic-window detection so only media, images, and composition hosts keep direct move/end-trim controls - kept wrapped media clips editable by recognizing real media metadata even when the host tag is a `div` - updated `TimelineClip` / `Timeline` to guard interactions with the shared capability model - added `buildTimelineElementAgentPrompt()` and a `Copy to Agent` fallback button for unsupported clips - added focused tests for capability derivation and the agent-prompt helper ## Verification ### Automated - `bun test packages/studio/src/player/components/timelineEditing.test.ts packages/studio/src/player/components/Timeline.test.ts packages/studio/src/player/store/playerStore.test.ts packages/studio/src/utils/sourcePatcher.test.ts` - `bun run --filter @hyperframes/studio typecheck` - `bunx oxlint packages/studio/src/player/components/Timeline.tsx packages/studio/src/player/components/timelineEditing.ts packages/studio/src/player/components/timelineEditing.test.ts packages/studio/src/player/components/TimelineClip.tsx` - `bunx oxfmt --check packages/studio/src/player/components/Timeline.tsx packages/studio/src/player/components/timelineEditing.ts packages/studio/src/player/components/timelineEditing.test.ts packages/studio/src/player/components/TimelineClip.tsx` ### Browser Verified in Studio with live browser automation against `http://127.0.0.1:4175/#project/timeline-edit-playground`: - generic GSAP-timed clips (`feature-card`, `title-card`, `prompt-card`) show `Copy to Agent` and no direct move/trim affordances - wrapped media (`media-card`) still exposes direct controls and remains draggable - the local playground timings were realigned to match the authored GSAP positions, so preview visibility now matches the timeline windows during manual testing Recording artifacts used during verification: - `/tmp/timeline-capabilities-proof/capabilities-flow.webm` - `/tmp/timeline-capabilities-proof/capabilities-agent-flow.webm`
2026-09-03 23:03:09 +00:00 · 2026-04-22 18:44:17 +02:00
parent c46abf9fa2
commit 5a4dd8bec1
4 changed files with 258 additions and 13 deletions
@@ -10,7 +10,8 @@ import { formatTime } from "../lib/time";
 import { TimelineClip } from "./TimelineClip";
 import { EditPopover } from "./EditModal";
 import {
-  canOffsetTrimClipStart,
+  buildTimelineElementAgentPrompt,
+  getTimelineEditCapabilities,
  resolveTimelineAutoScroll,
  resolveTimelineMove,
  resolveTimelineResize,
@@ -245,6 +246,7 @@ export const Timeline = memo(function Timeline({
  onResizeElementRef.current = onResizeElement;
  const suppressClickRef = useRef(false);
  const [showPopover, setShowPopover] = useState(false);
+  const [copiedAgentElementKey, setCopiedAgentElementKey] = useState<string | null>(null);
  const [viewportWidth, setViewportWidth] = useState(0);
  const roRef = useRef<ResizeObserver | null>(null);

@@ -896,6 +898,46 @@ export const Timeline = memo(function Timeline({
        }
      : null;
  const renderClipChildren = (element: TimelineElement, clipStyle: TrackVisualStyle) => {
+    const capabilities = getTimelineEditCapabilities(element);
+    const elementKey = element.key ?? element.id;
+    const needsAgentFallback =
+      !capabilities.canMove && !capabilities.canTrimStart && !capabilities.canTrimEnd;
+    const agentButton = needsAgentFallback ? (
+      <button
+        type="button"
+        onPointerDown={(e) => e.stopPropagation()}
+        onClick={async (e) => {
+          e.stopPropagation();
+          const text = buildTimelineElementAgentPrompt(element);
+          try {
+            await navigator.clipboard.writeText(text);
+          } catch {
+            const ta = document.createElement("textarea");
+            ta.value = text;
+            document.body.appendChild(ta);
+            ta.select();
+            document.execCommand("copy");
+            document.body.removeChild(ta);
+          }
+          setCopiedAgentElementKey(elementKey);
+          window.setTimeout(() => {
+            setCopiedAgentElementKey((current) => (current === elementKey ? null : current));
+          }, 900);
+        }}
+        className="absolute bottom-2 right-2 z-[5] rounded-md px-1.5 py-0.5 text-[10px] font-medium leading-none transition-colors"
+        style={{
+          color: copiedAgentElementKey === elementKey ? "#86efac" : clipStyle.label,
+          background:
+            copiedAgentElementKey === elementKey ? "rgba(34,197,94,0.16)" : `${clipStyle.accent}1e`,
+          boxShadow:
+            copiedAgentElementKey === elementKey
+              ? "inset 0 0 0 1px rgba(34,197,94,0.28)"
+              : `inset 0 0 0 1px ${clipStyle.accent}33`,
+        }}
+      >
+        {copiedAgentElementKey === elementKey ? "Copied!" : "Copy to Agent"}
+      </button>
+    ) : null;
    return (
      <>
        {renderClipOverlay?.(element)}
@@ -941,6 +983,7 @@ export const Timeline = memo(function Timeline({
            </div>
          )}
        </div>
+        {agentButton}
      </>
    );
  };
@@ -1082,6 +1125,7 @@ export const Timeline = memo(function Timeline({
                  {els.map((el, i) => {
                    const clipStyle = getStyle(el.tag);
                    const elementKey = el.key ?? el.id;
+                    const capabilities = getTimelineEditCapabilities(el);
                    const isSelected = selectedElementId === elementKey;
                    const isComposition = !!el.compositionSrc;
                    const clipKey = `${elementKey}-${i}`;
@@ -1110,7 +1154,8 @@ export const Timeline = memo(function Timeline({
                        onHoverEnd={() => setHoveredClip(null)}
                        onResizeStart={(edge, e) => {
                          if (e.button !== 0 || e.shiftKey || !onResizeElement) return;
-                          if (edge === "start" && !canOffsetTrimClipStart(el)) return;
+                          if (edge === "start" && !capabilities.canTrimStart) return;
+                          if (edge === "end" && !capabilities.canTrimEnd) return;
                          e.stopPropagation();
                          setShowPopover(false);
                          setRangeSelection(null);
@@ -1125,7 +1170,13 @@ export const Timeline = memo(function Timeline({
                          });
                        }}
                        onPointerDown={(e) => {
-                          if (e.button !== 0 || e.shiftKey || !onMoveElement) return;
+                          if (
+                            e.button !== 0 ||
+                            e.shiftKey ||
+                            !onMoveElement ||
+                            !capabilities.canMove
+                          )
+                            return;
                          setShowPopover(false);
                          setRangeSelection(null);
                          const rect = (e.currentTarget as HTMLElement).getBoundingClientRect();
@@ -4,7 +4,7 @@ import type { TimelineTrackStyle } from "./timelineTheme";
 import { memo, type ReactNode } from "react";
 import type { TimelineElement } from "../store/playerStore";
 import { defaultTimelineTheme, getClipHandleOpacity, type TimelineTheme } from "./timelineTheme";
-import { canOffsetTrimClipStart } from "./timelineEditing";
+import { getTimelineEditCapabilities } from "./timelineEditing";

 interface TimelineClipProps {
  el: TimelineElement;
@@ -60,7 +60,7 @@ export const TimelineClip = memo(function TimelineClip({
      : isHovered
        ? theme.clipShadowHover
        : theme.clipShadow;
-  const canTrimStart = canOffsetTrimClipStart(el);
+  const capabilities = getTimelineEditCapabilities(el);
  const showHandles = handleOpacity > 0.01;

  return (
@@ -87,7 +87,7 @@ export const TimelineClip = memo(function TimelineClip({
        transition:
          "border-color 120ms ease-out, box-shadow 140ms ease-out, background 140ms ease-out",
        zIndex: isDragging ? 20 : isSelected ? 10 : isHovered ? 5 : 1,
-        cursor: "grab",
+        cursor: capabilities.canMove ? "grab" : "default",
        transform: isDragging ? "translateY(-1px)" : undefined,
      }}
      title={
@@ -111,13 +111,13 @@ export const TimelineClip = memo(function TimelineClip({
          top: 0,
          bottom: 0,
          width: 18,
-          opacity: showHandles && canTrimStart ? 1 : 0,
-          pointerEvents: onResizeStart && canTrimStart ? "auto" : "none",
+          opacity: showHandles && capabilities.canTrimStart ? 1 : 0,
+          pointerEvents: onResizeStart && capabilities.canTrimStart ? "auto" : "none",
          zIndex: 4,
          transition: "opacity 120ms ease-out",
          cursor: "col-resize",
          background:
-            showHandles && canTrimStart
+            showHandles && capabilities.canTrimStart
              ? `linear-gradient(90deg, ${trackStyle.accent}4d 0%, ${trackStyle.accent}22 42%, transparent 100%)`
              : "transparent",
        }}
@@ -148,13 +148,14 @@ export const TimelineClip = memo(function TimelineClip({
          bottom: 0,
          width: 18,
          opacity: showHandles ? 1 : 0,
-          pointerEvents: onResizeStart ? "auto" : "none",
+          pointerEvents: onResizeStart && capabilities.canTrimEnd ? "auto" : "none",
          zIndex: 4,
          transition: "opacity 120ms ease-out",
          cursor: "col-resize",
-          background: showHandles
-            ? `linear-gradient(270deg, ${trackStyle.accent}4d 0%, ${trackStyle.accent}22 42%, transparent 100%)`
-            : "transparent",
+          background:
+            showHandles && capabilities.canTrimEnd
+              ? `linear-gradient(270deg, ${trackStyle.accent}4d 0%, ${trackStyle.accent}22 42%, transparent 100%)`
+              : "transparent",
        }}
      >
        <div
@@ -1,9 +1,12 @@
 import { describe, expect, it } from "vitest";
 import {
  buildPromptCopyText,
+  buildTimelineElementAgentPrompt,
  buildTimelineAgentPrompt,
  buildTrackZIndexMap,
  canOffsetTrimClipStart,
+  getTimelineEditCapabilities,
+  hasPatchableTimelineTarget,
  resolveTimelineAutoScroll,
  resolveTimelineMove,
  resolveTimelineResize,
@@ -205,6 +208,97 @@ describe("canOffsetTrimClipStart", () => {
  });
 });

+describe("hasPatchableTimelineTarget", () => {
+  it("returns true when the clip has a DOM id", () => {
+    expect(hasPatchableTimelineTarget({ domId: "hero-card" })).toBe(true);
+  });
+
+  it("returns true when the clip has a selector", () => {
+    expect(hasPatchableTimelineTarget({ selector: ".hero-card" })).toBe(true);
+  });
+
+  it("returns false when the clip has no stable patch target", () => {
+    expect(hasPatchableTimelineTarget({})).toBe(false);
+  });
+});
+
+describe("getTimelineEditCapabilities", () => {
+  it("disables move and trims for generic motion clips even when patchable", () => {
+    expect(
+      getTimelineEditCapabilities({
+        tag: "section",
+        duration: 2,
+        selector: ".feature-card",
+      }),
+    ).toEqual({
+      canMove: false,
+      canTrimStart: false,
+      canTrimEnd: false,
+    });
+  });
+
+  it("allows move and both trims for patchable media clips with offset support", () => {
+    expect(
+      getTimelineEditCapabilities({
+        tag: "video",
+        duration: 2,
+        selector: "#media-card",
+        playbackStartAttr: "media-start",
+        sourceDuration: 10,
+      }),
+    ).toEqual({
+      canMove: true,
+      canTrimStart: true,
+      canTrimEnd: true,
+    });
+  });
+
+  it("treats wrapped media clips with media metadata as deterministic", () => {
+    expect(
+      getTimelineEditCapabilities({
+        tag: "div",
+        duration: 2,
+        selector: "#media-card",
+        playbackStartAttr: "media-start",
+        sourceDuration: 10,
+      }),
+    ).toEqual({
+      canMove: true,
+      canTrimStart: true,
+      canTrimEnd: true,
+    });
+  });
+
+  it("allows move and end trim for patchable composition hosts", () => {
+    expect(
+      getTimelineEditCapabilities({
+        tag: "div",
+        duration: 3,
+        selector: '[data-composition-id="intro"]',
+        compositionSrc: "compositions/intro.html",
+      }),
+    ).toEqual({
+      canMove: true,
+      canTrimStart: false,
+      canTrimEnd: true,
+    });
+  });
+
+  it("disables all timeline edits for clips without a patchable target", () => {
+    expect(
+      getTimelineEditCapabilities({
+        tag: "video",
+        duration: 2,
+        sourceDuration: 10,
+      }),
+    ).toEqual({
+      canMove: false,
+      canTrimStart: false,
+      canTrimEnd: false,
+    });
+  });
+});
+
 describe("resolveTimelineAutoScroll", () => {
  it("does not scroll when the pointer stays away from the edges", () => {
    expect(
@@ -273,6 +367,22 @@ describe("buildTimelineAgentPrompt", () => {
  });
 });

+describe("buildTimelineElementAgentPrompt", () => {
+  it("includes the clip context and guidance for agent-based edits", () => {
+    expect(
+      buildTimelineElementAgentPrompt({
+        id: "feature-card",
+        tag: "section",
+        start: 1.4,
+        duration: 1.6,
+        track: 1,
+        sourceFile: "index.html",
+        selector: "#feature-card",
+      }),
+    ).toContain("If this clip is animated with GSAP");
+  });
+});
+
 describe("resolveTimelineResize", () => {
  it("shrinks clip duration from the right edge", () => {
    expect(
@@ -169,6 +169,35 @@ export interface TimelinePromptElement {
  track: number;
 }

+export interface TimelineEditCapabilities {
+  canMove: boolean;
+  canTrimStart: boolean;
+  canTrimEnd: boolean;
+}
+
+function isDeterministicTimelineWindow(input: {
+  tag: string;
+  compositionSrc?: string;
+  playbackStartAttr?: "media-start" | "playback-start";
+  sourceDuration?: number;
+}): boolean {
+  if (input.compositionSrc) return true;
+  if (input.playbackStartAttr != null) return true;
+  if (
+    input.sourceDuration != null &&
+    Number.isFinite(input.sourceDuration) &&
+    input.sourceDuration > 0
+  ) {
+    return true;
+  }
+  const normalizedTag = input.tag.toLowerCase();
+  return ["video", "audio", "img"].includes(normalizedTag);
+}
+
+export function hasPatchableTimelineTarget(input: { domId?: string; selector?: string }): boolean {
+  return Boolean(input.domId || input.selector);
+}
+
 export function canOffsetTrimClipStart(input: {
  tag: string;
  playbackStart?: number;
@@ -186,6 +215,26 @@ export function canOffsetTrimClipStart(input: {
  );
 }

+export function getTimelineEditCapabilities(input: {
+  tag: string;
+  duration: number;
+  domId?: string;
+  selector?: string;
+  compositionSrc?: string;
+  playbackStart?: number;
+  playbackStartAttr?: "media-start" | "playback-start";
+  sourceDuration?: number;
+}): TimelineEditCapabilities {
+  const canPatch = hasPatchableTimelineTarget(input);
+  const hasFiniteDuration = Number.isFinite(input.duration) && input.duration > 0;
+  const hasDeterministicWindow = isDeterministicTimelineWindow(input);
+  return {
+    canMove: canPatch && hasDeterministicWindow,
+    canTrimEnd: canPatch && hasFiniteDuration && hasDeterministicWindow,
+    canTrimStart: canPatch && hasFiniteDuration && canOffsetTrimClipStart(input),
+  };
+}
+
 export function buildTimelineAgentPrompt({
  rangeStart,
  rangeEnd,
@@ -226,6 +275,40 @@ export function buildPromptCopyText(prompt: string): string {
  return prompt.trim();
 }

+export function buildTimelineElementAgentPrompt(element: {
+  id: string;
+  tag: string;
+  start: number;
+  duration: number;
+  track: number;
+  sourceFile?: string;
+  selector?: string;
+  compositionSrc?: string;
+}): string {
+  const lines = [
+    "Studio cannot directly move or resize this timeline clip because its visible timing is not fully controlled by patchable HTML timing attributes.",
+    "",
+    "Please update the source so the clip's actual visible timing stays consistent with the authored timeline.",
+    "",
+    "Clip:",
+    `- id: ${element.id}`,
+    `- tag: ${element.tag}`,
+    `- time: ${formatTime(element.start)} to ${formatTime(element.start + element.duration)}`,
+    `- track: ${element.track}`,
+  ];
+
+  if (element.sourceFile) lines.push(`- source file: ${element.sourceFile}`);
+  if (element.selector) lines.push(`- selector: ${element.selector}`);
+  if (element.compositionSrc) lines.push(`- composition src: ${element.compositionSrc}`);
+
+  lines.push(
+    "",
+    "If this clip is animated with GSAP or another JS timeline, update the authored animation timing there as well instead of only changing data-start/data-duration.",
+  );
+
+  return lines.join("\n");
+}
+
 export function formatTimelineAttributeNumber(value: number): string {
  return Number(roundToCentiseconds(value).toFixed(2)).toString();
 }