Merge pull request #2230 from heygen-com/fix/studio-graded-element-editing

fix(studio): graded elements survive manual editing (disappear/resize/rotate/crop/panel)

packages/core/src/colorGrading.ts

@@ -1,5 +1,17 @@
 export const HF_COLOR_GRADING_ATTR = "data-color-grading";
 
+// Runtime <-> studio contract attributes. The runtime grading engine writes
+// them; studio editing/soft-reload code reads them. Single owner — never
+// re-declare these literals elsewhere.
+/** Set on a graded source while its pixels render on the grading canvas. */
+export const COLOR_GRADING_SOURCE_HIDDEN_ATTR = "data-hf-color-grading-source-hidden";
+/**
+ * The element's AUTHORED inline opacity, stamped at document parse time before
+ * any animation engine mutates it ("" = authored none; attribute absent =
+ * never captured). See installAuthoredOpacityCapture in the runtime.
+ */
+export const COLOR_GRADING_AUTHORED_OPACITY_ATTR = "data-hf-authored-opacity";
+
 export const HF_COLOR_GRADING_CANVAS_ID_PREFIX = "__hf_color_grading_";
 
 export const HF_COLOR_GRADING_COLOR_SPACE = "rec709";

packages/core/src/runtime/colorGrading.test.ts

@@ -1,7 +1,11 @@
 // fallow-ignore-file code-duplication
 import { afterEach, beforeEach, describe, expect, it, vi } from "vitest";
 import { HF_COLOR_GRADING_ATTR, serializeHfColorGrading } from "../colorGrading";
-import { createColorGradingRuntime, type RuntimeColorGradingApi } from "./colorGrading";
+import {
+  createColorGradingRuntime,
+  installAuthoredOpacityCapture,
+  type RuntimeColorGradingApi,
+} from "./colorGrading";
 
 let lastUniform1f: ReturnType<typeof vi.fn> | null = null;
 let lastUniform3f: ReturnType<typeof vi.fn> | null = null;
@@ -192,6 +196,60 @@ describe("createColorGradingRuntime", () => {
     runtime?.redraw();
   }
 
+  it("restores the authored inline opacity captured before animation transients", () => {
+    const video = makeDrawableVideo();
+    // Parse-time capture stamped the authored value; by hide time GSAP has
+    // already left a from()-tween transient (0) in the inline style.
+    video.setAttribute("data-hf-authored-opacity", "0.75");
+    video.style.opacity = "0";
+    startRuntimeWithVideo(video);
+
+    expect(video.style.getPropertyPriority("opacity")).toBe("important");
+
+    runtime?.destroy();
+    runtime = null;
+
+    // Restore must use the authored 0.75, not the GSAP transient 0.
+    expect(video.style.getPropertyValue("opacity")).toBe("0.75");
+    expect(video.style.getPropertyPriority("opacity")).toBe("");
+  });
+
+  it("restores no inline opacity when the authored capture recorded none", () => {
+    const video = makeDrawableVideo();
+    video.setAttribute("data-hf-authored-opacity", "");
+    video.style.opacity = "0";
+    startRuntimeWithVideo(video);
+
+    runtime?.destroy();
+    runtime = null;
+
+    expect(video.style.getPropertyValue("opacity")).toBe("");
+  });
+
+  it("re-syncs the graded canvas when the source's inline transform changes", async () => {
+    const { video } = startRuntimeWithVideo();
+    const drawsBefore = texImage2DCalls.length;
+
+    // Simulate a studio drag draft: only the inline transform moves.
+    video.style.transform = "translate(120px, 60px)";
+    await new Promise((resolve) => requestAnimationFrame(() => resolve(null)));
+
+    expect(texImage2DCalls.length).toBeGreaterThan(drawsBefore);
+  });
+
+  it("does not redraw-loop on its own hide writes (opacity/visibility only)", async () => {
+    const { video } = startRuntimeWithVideo();
+    await new Promise((resolve) => requestAnimationFrame(() => resolve(null)));
+    const drawsBefore = texImage2DCalls.length;
+
+    // drawEntry's own source-hide writes touch opacity — geometry unchanged.
+    video.style.opacity = "0.5";
+    await new Promise((resolve) => requestAnimationFrame(() => resolve(null)));
+    await new Promise((resolve) => requestAnimationFrame(() => resolve(null)));
+
+    expect(texImage2DCalls.length).toBe(drawsBefore);
+  });
+
   it("re-hides source media after timeline visibility sync", () => {
     const { video, canvas } = startRuntimeWithVideo();
 
@@ -555,3 +613,56 @@ describe("createColorGradingRuntime", () => {
     expect(video.style.getPropertyValue("opacity")).toBe("0");
   });
 });
+
+describe("installAuthoredOpacityCapture", () => {
+  it("stamps graded elements at insertion and never overwrites the stamp", async () => {
+    installAuthoredOpacityCapture();
+    const el = document.createElement("img");
+    el.setAttribute(HF_COLOR_GRADING_ATTR, serializeHfColorGrading({ adjust: { exposure: 0.5 } }));
+    el.style.opacity = "0.98";
+    document.body.appendChild(el);
+    await Promise.resolve();
+    expect(el.getAttribute("data-hf-authored-opacity")).toBe("0.98");
+
+    // A re-insert after an animation engine mutated the element keeps the
+    // original capture (has-attribute guard).
+    el.style.opacity = "0";
+    el.remove();
+    document.body.appendChild(el);
+    await Promise.resolve();
+    expect(el.getAttribute("data-hf-authored-opacity")).toBe("0.98");
+    el.remove();
+  });
+
+  it("stamps an empty value for graded elements without an authored inline opacity", async () => {
+    installAuthoredOpacityCapture();
+    const el = document.createElement("img");
+    el.setAttribute(HF_COLOR_GRADING_ATTR, serializeHfColorGrading({ adjust: { exposure: 0.5 } }));
+    document.body.appendChild(el);
+    await Promise.resolve();
+    expect(el.getAttribute("data-hf-authored-opacity")).toBe("");
+    el.remove();
+  });
+
+  it("stamps an already-inserted element the moment it GAINS grading at runtime", async () => {
+    installAuthoredOpacityCapture();
+    const el = document.createElement("img");
+    el.style.opacity = "0.9";
+    document.body.appendChild(el);
+    await Promise.resolve();
+    expect(el.hasAttribute("data-hf-authored-opacity")).toBe(false);
+
+    // Studio applies a preset to a previously ungraded element — no re-insert.
+    el.setAttribute(HF_COLOR_GRADING_ATTR, serializeHfColorGrading({ adjust: { exposure: 0.5 } }));
+    await Promise.resolve();
+    expect(el.getAttribute("data-hf-authored-opacity")).toBe("0.9");
+
+    // Later attribute rewrites (preset tweaks) never overwrite the stamp,
+    // even if a transient is live by then.
+    el.style.opacity = "0";
+    el.setAttribute(HF_COLOR_GRADING_ATTR, serializeHfColorGrading({ adjust: { exposure: 0.9 } }));
+    await Promise.resolve();
+    expect(el.getAttribute("data-hf-authored-opacity")).toBe("0.9");
+    el.remove();
+  });
+});

packages/core/src/runtime/colorGrading.ts

@@ -6,6 +6,8 @@ import {
   normalizeHfColorGradingWithVariables,
   type HfColorGradingTarget,
   type NormalizedHfColorGrading,
+  COLOR_GRADING_SOURCE_HIDDEN_ATTR,
+  COLOR_GRADING_AUTHORED_OPACITY_ATTR,
 } from "../colorGrading";
 import {
   DEFAULT_MAX_CUBE_LUT_SIZE,
@@ -196,8 +198,64 @@ type LutCacheEntry =
 
 const LUT_CACHE = new Map<string, LutCacheEntry>();
 const COLOR_GRADING_CANVAS_ATTR = "data-hf-color-grading-canvas";
-const COLOR_GRADING_SOURCE_HIDDEN_ATTR = "data-hf-color-grading-source-hidden";
 const COLOR_GRADING_CANVAS_CLASS = "__hf_color_grading_canvas__";
+
+/**
+ * Capture each color-graded element's AUTHORED inline opacity before any
+ * animation engine can mutate it.
+ *
+ * The grading engine hides its source elements with `opacity: 0 !important`
+ * and mirrors their pixels onto a canvas — so at runtime, a graded element's
+ * inline/computed opacity no longer represents authored state. Everything that
+ * later re-reads element state (GSAP from()-tween re-initialization after an
+ * invalidate or a studio soft reload, restoring the source when grading is
+ * removed, lint/selection tooling) needs the authored value, and by then it is
+ * unrecoverable from the DOM. Stamp it onto the element as
+ * `data-hf-authored-opacity` (empty string = no authored inline opacity).
+ *
+ * Must be installed at runtime-bundle evaluation, while the document is still
+ * parsing: the runtime `<script>` sits in `<head>`, and the HTML parser
+ * performs a microtask checkpoint before executing each parser-inserted
+ * script, so the observer stamps every composition element before the
+ * composition's own inline animation script runs. The observer stays alive so
+ * late insertions (inlined sub-compositions) are stamped at insert time, and
+ * the has-attribute guard makes re-inserted (already-stamped) elements a no-op.
+ */
+export function installAuthoredOpacityCapture(): void {
+  if (typeof MutationObserver === "undefined" || typeof document === "undefined") return;
+  const root = document.documentElement;
+  if (!root) return;
+  const stamp = (el: Element): void => {
+    if (!(el instanceof HTMLElement)) return;
+    if (el.hasAttribute(COLOR_GRADING_AUTHORED_OPACITY_ATTR)) return;
+    el.setAttribute(COLOR_GRADING_AUTHORED_OPACITY_ATTR, el.style.opacity);
+  };
+  const scan = (node: Node): void => {
+    if (!(node instanceof Element)) return;
+    if (node.hasAttribute(HF_COLOR_GRADING_ATTR)) stamp(node);
+    for (const el of node.querySelectorAll(`[${HF_COLOR_GRADING_ATTR}]`)) stamp(el);
+  };
+  scan(root);
+  new MutationObserver((mutations) => {
+    for (const mutation of mutations) {
+      for (const node of mutation.addedNodes) scan(node);
+      // An element can also GAIN grading at runtime (studio applies a preset to
+      // a previously ungraded element). Stamp at that moment — strictly earlier
+      // than the engine's hide, so the captured value can never be worse than
+      // the hide-time fallback, and after a soft reload's authored restore it
+      // IS the authored value. stamp() is idempotent: an existing stamp wins.
+      if (mutation.type === "attributes" && mutation.target instanceof Element) {
+        if (mutation.target.hasAttribute(HF_COLOR_GRADING_ATTR)) stamp(mutation.target);
+      }
+    }
+  }).observe(root, {
+    childList: true,
+    subtree: true,
+    attributes: true,
+    attributeFilter: [HF_COLOR_GRADING_ATTR],
+  });
+}
+
 // Map insertion order gives us simple FIFO eviction for authoring sessions that cycle LUTs.
 const MAX_LUT_CACHE_ENTRIES = 16;
 const DEFAULT_COMPARE: RuntimeColorGradingCompareState = {
@@ -1234,8 +1292,20 @@ function applyUniforms(
 
 function hideSourceElement(entry: ColorGradingEntry): void {
   if (!entry.sourceHidden) {
-    entry.sourceInlineOpacity = entry.element.style.getPropertyValue("opacity") || null;
-    entry.sourceInlineOpacityPriority = entry.element.style.getPropertyPriority("opacity");
+    // Prefer the parse-time authored capture: by the time the first hide runs,
+    // the inline opacity is usually an animation-engine transient (a from()
+    // tween's 0 at playhead 0), and restoring THAT when grading is removed
+    // would leave the element invisible. Fall back to the live inline value
+    // for documents loaded without the capture installed.
+    // `null` = never captured; "" = captured, authored none (store as null).
+    const authored = entry.element.getAttribute(COLOR_GRADING_AUTHORED_OPACITY_ATTR);
+    if (authored !== null) {
+      entry.sourceInlineOpacity = authored === "" ? null : authored;
+      entry.sourceInlineOpacityPriority = "";
+    } else {
+      entry.sourceInlineOpacity = entry.element.style.getPropertyValue("opacity") || null;
+      entry.sourceInlineOpacityPriority = entry.element.style.getPropertyPriority("opacity");
+    }
   }
   entry.element.setAttribute(COLOR_GRADING_SOURCE_HIDDEN_ATTR, "true");
   entry.element.style.setProperty("opacity", "0", "important");
@@ -1423,6 +1493,32 @@ function installEntryListeners(entry: ColorGradingEntry): void {
     entry.resizeObserver = new ResizeObserver(redraw);
     entry.resizeObserver.observe(entry.element);
   }
+  // A studio drag/nudge moves the source via its inline transform — no media
+  // event or ResizeObserver fires for that, so the graded canvas (the visible
+  // pixels) froze in place until the next seek while the invisible source
+  // followed the pointer. Track geometry-relevant inline style and re-sync the
+  // canvas, rAF-throttled. The signature guard keeps the opacity/visibility
+  // writes drawEntry itself makes (the source hide) from re-triggering a loop.
+  if (typeof MutationObserver !== "undefined") {
+    const geometrySignature = () => {
+      const s = entry.element.style;
+      return `${s.transform}|${s.translate}|${s.rotate}|${s.scale}|${s.left}|${s.top}|${s.width}|${s.height}`;
+    };
+    let lastGeometry = geometrySignature();
+    let framePending = false;
+    const styleObserver = new MutationObserver(() => {
+      if (framePending) return;
+      if (geometrySignature() === lastGeometry) return;
+      framePending = true;
+      requestAnimationFrame(() => {
+        framePending = false;
+        lastGeometry = geometrySignature();
+        drawEntry(entry);
+      });
+    });
+    styleObserver.observe(entry.element, { attributes: true, attributeFilter: ["style"] });
+    entry.cleanup.push(() => styleObserver.disconnect());
+  }
 }
 
 function destroyEntry(entry: ColorGradingEntry): void {

packages/core/src/runtime/entry.ts

@@ -1,4 +1,5 @@
 import { initSandboxRuntimeModular } from "./init";
+import { installAuthoredOpacityCapture } from "./colorGrading";
 import { fitTextFontSize } from "../text/fitTextFontSize";
 import { getVariables } from "./getVariables";
 
@@ -14,6 +15,11 @@ type HyperframeWindow = Window & {
 // Ensure timeline registry exists at script evaluation time.
 (window as HyperframeWindow).__timelines = (window as HyperframeWindow).__timelines || {};
 
+// Stamp color-graded elements with their authored inline opacity BEFORE the
+// composition's animation scripts (and the grading hide) mutate it — must run
+// at script evaluation time, while the document is still parsing.
+installAuthoredOpacityCapture();
+
 // Expose runtime helpers immediately so composition scripts can use them
 // before DOMContentLoaded (font sizing runs during script evaluation, and
 // getVariables is read by composition setup before the timeline is built).

packages/core/src/runtime/picker.ts

@@ -1,4 +1,5 @@
 import type { RuntimeJson, RuntimeOutboundMessage, RuntimePickerElementInfo } from "./types";
+import { COLOR_GRADING_SOURCE_HIDDEN_ATTR } from "../colorGrading";
 import { swallow } from "./diagnostics";
 
 type PickerModuleDeps = {
@@ -17,7 +18,6 @@ const PICKER_BLOCK_SELECTOR = [
   "[data-hyperframes-picker-block]",
   "[data-hyper-shader-loading]",
 ].join(",");
-const COLOR_GRADING_SOURCE_HIDDEN_ATTR = "data-hf-color-grading-source-hidden";
 
 export type PickerModule = {
   enablePickMode: () => void;