feat(studio): read overlay geometry from an opaque preview over its channel (#7628)

* feat(studio): read overlay geometry from an opaque preview over its channel

The runtime's editing channel gains a `geometry` request: per element
handle its client rect, the corners it paints at when rotated, whether it
renders, its clip-path, its source boundary's size, a group wrapper's
member rects, and optionally one element's descendant boxes. On a preview
frame Studio cannot script, the selection, child, group and hover boxes,
the root scale (from the mirrored stage size) and a press held across a
preview change read that geometry, with one read in flight at a time.
Same-origin frames keep reading the DOM directly; the measuring code they
use now lives in core so both paths share one rule.

Also from the review of #7613: opaque selections carry `hasAudio` and
`src` read off the description, a description with a cut attribute or
inline style builds no selection, a failed computed-styles or root read
builds none either, and the group refresh threads its frame and leaves a
group the user changed meanwhile. `findClosestByAttribute`, left without
a caller by #7613, is removed.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

* chore(producer): raise the packed size budget for the runtime's geometry read

The producer ships the preview runtime, which now carries the editing
channel's `geometry` request and the measuring code it shares with Studio.
The packed tarball measured 888,810 bytes in CI against an 880,000 budget.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

---------

Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
James Russo
2026-11-13 00:26:30 +00:00
committed by GitHub
co-authored by Claude Opus 5.5
parent aadd5bd6f2
commit faff4e16a8
24 changed files with 1394 additions and 256 deletions
@@ -145,6 +145,7 @@ describe("installEditingAgent", () => {
"describe",
"ancestors",
"query",
"geometry",
],
},
});
@@ -294,6 +295,24 @@ describe("installEditingAgent", () => {
90 + index,
"invalid-params",
]),
...[
{},
{ handles: [] },
{ handles: ["1"] },
{ handles: [1] },
{ handles: Array(257).fill("0123456789abcdef:1") },
{ handles: ["0123456789abcdef:1"], extra: 1 },
{ children: "0123456789abcdef:1" },
{ handles: ["0123456789abcdef:1"], children: undefined },
{ handles: ["0123456789abcdef:1"], children: null },
{ handles: ["0123456789abcdef:1"], children: true },
{ handles: ["0123456789abcdef:1"], children: ["0123456789abcdef:1"] },
{ handles: ["0123456789abcdef:1"], children: "0123456789abcdef:0" },
].map((params, index) => [
{ v: 1, id: 100 + index, kind: "geometry", params },
100 + index,
"invalid-params",
]),
...[["--"], ["--a b"], ["--a;b"], ["--a:b"], ["Color"], ["-x"], ["--" + "a".repeat(255)]].map(
(properties, index) => [
{ v: 1, id: 60 + index, kind: "computed-styles", params: { selector: "p", properties } },
@@ -27,6 +27,7 @@ import {
elementsAtPoint,
queryElements,
} from "./editingHandles";
import { elementGeometry } from "./editingGeometry";
import { getLogicalOpacity } from "../studioPreviewMark";
type Handlers = {
@@ -83,6 +84,7 @@ const HANDLERS: Handlers = {
describe: ({ handles }) => ({ elements: describeHandles(handles) }),
ancestors: ({ handles }) => ({ ancestors: ancestorHandles(handles) }),
query: ({ selector }) => queryElements(selector),
geometry: ({ handles, children }) => elementGeometry(handles, children),
};
function serve(port: MessagePort): void {
@@ -0,0 +1,171 @@
import { afterEach, beforeEach, describe, expect, it, vi } from "vitest";
import { installEditingAgent } from "./editingAgent";
import {
MAX_ELEMENT_VALUE,
MAX_GEOMETRY_CHILDREN,
type EditingElementBox,
type EditingElementGeometry,
type EditingElementHit,
} from "./editingPort";
const CONNECT = { source: "hf-parent", type: "editing-connect", v: 1 };
// jsdom has neither; the 2D slice of each is all the geometry reads.
class FakeDOMMatrix {
a = 1;
b = 0;
c = 0;
d = 1;
e = 0;
f = 0;
constructor(init?: string) {
const m = init ? /matrix\(([^)]+)\)/.exec(init) : null;
if (m) [this.a, this.b, this.c, this.d, this.e, this.f] = m[1]!.split(",").map(Number);
}
multiply(o: FakeDOMMatrix) {
const out = new FakeDOMMatrix();
out.a = this.a * o.a + this.c * o.b;
out.b = this.b * o.a + this.d * o.b;
out.c = this.a * o.c + this.c * o.d;
out.d = this.b * o.c + this.d * o.d;
out.e = this.a * o.e + this.c * o.f + this.e;
out.f = this.b * o.e + this.d * o.f + this.f;
return out;
}
rotateSelf() {
return this;
}
transformPoint(p: { x: number; y: number }) {
return { x: this.a * p.x + this.c * p.y + this.e, y: this.b * p.x + this.d * p.y + this.f };
}
}
class FakeDOMPoint {
constructor(
public x: number,
public y: number,
) {}
}
const QUARTER_TURN = "matrix(0, 1, -1, 0, 0, 0)";
type Geometry = {
elements: Array<EditingElementGeometry | null>;
children: EditingElementBox[] | null;
};
describe("overlay geometry over the editing port", () => {
let port: MessagePort;
let agent: { dispose: () => void };
let lastId = 0;
const ask = (kind: string, params: unknown): Promise<{ ok: boolean; result?: unknown }> =>
new Promise((resolve) => {
port.onmessage = (event) => resolve(event.data);
port.postMessage({ v: 1, id: ++lastId, kind, params });
});
const handleOf = async (element: Element) => {
Object.defineProperty(document, "elementsFromPoint", {
configurable: true,
value: () => [element],
});
const reply = await ask("elements-at-point", { x: 0, y: 0 });
return (reply.result as { elements: EditingElementHit[] }).elements[0]!.handle;
};
const geometry = async (params: unknown) => (await ask("geometry", params)).result as Geometry;
const box = (element: Element, [x, y, width, height]: number[], offset = true) => {
element.getBoundingClientRect = () =>
({ x, y, width, height, left: x, top: y, right: x + width, bottom: y + height }) as DOMRect;
if (!offset) return;
Object.defineProperty(element, "offsetWidth", { value: width, configurable: true });
Object.defineProperty(element, "offsetHeight", { value: height, configurable: true });
};
beforeEach(() => {
Object.assign(window, { DOMMatrix: FakeDOMMatrix, DOMPoint: FakeDOMPoint });
const iframe = document.createElement("iframe");
document.body.appendChild(iframe);
const parent = iframe.contentWindow!;
const handoff = vi.fn();
parent.postMessage = handoff as unknown as Window["postMessage"];
agent = installEditingAgent(parent);
window.dispatchEvent(new MessageEvent("message", { source: parent, data: CONNECT }));
port = (handoff.mock.calls[0]![2] as MessagePort[])[0]!;
});
afterEach(() => {
agent.dispose();
document.body.innerHTML = "";
});
it("measures rect, painted corners, visibility, crop and source boundary per handle", async () => {
document.body.innerHTML = `
<div data-composition-src="a.html" data-width="960" data-height="540" id="host">
<p id="turned" style="clip-path: inset(1px 2px 3px 4px)">T</p>
</div>
<div style="display: none"><span id="hidden"></span></div>`;
const host = document.querySelector("#host")!;
const turned = document.querySelector<HTMLElement>("#turned")!;
box(host, [0, 0, 480, 270]);
box(turned, [10, 20, 100, 200]);
turned.style.transform = QUARTER_TURN;
const handles = [await handleOf(turned), await handleOf(document.querySelector("#hidden")!)];
const { elements, children } = await geometry({ handles });
expect(children).toBeNull();
const [measured, hidden] = elements;
expect(measured).toMatchObject({
rect: { x: 10, y: 20, width: 100, height: 200 },
angle: 90,
visible: true,
members: null,
clipPath: "inset(1px 2px 3px 4px)",
source: { width: 480, height: 270, declaredWidth: 960, declaredHeight: 540 },
});
// A 100x200 box turned a quarter: its painted corners span 200 across and 100 down.
const { nw, ne, se, sw } = measured!.corners!;
expect([nw, ne, se, sw].map((p) => [p.x, p.y])).toEqual([
[210, 20],
[210, 120],
[10, 120],
[10, 20],
]);
expect(hidden).toMatchObject({ visible: false, angle: 0, corners: null, source: null });
expect(JSON.parse(JSON.stringify(elements))).toEqual(elements);
});
it("answers null for a removed element and leaves out a clip-path too long to send", async () => {
document.body.innerHTML = `<p id="a"></p><p id="b"></p>`;
const a = document.querySelector<HTMLElement>("#a")!;
a.style.setProperty("clip-path", `path("M0 0${" L1 1".repeat(MAX_ELEMENT_VALUE)}")`);
const b = document.querySelector("#b")!;
const handles = [await handleOf(a), await handleOf(b)];
b.remove();
const { elements } = await geometry({ handles });
expect(elements[0]!.clipPath).toBeNull();
expect(elements[1]).toBeNull();
});
it("gives a group wrapper its members' rects", async () => {
document.body.innerHTML = `<div data-hf-group="g"><i></i><b></b></div>`;
const group = document.querySelector("div")!;
box(group.children[0]!, [1, 2, 3, 4]);
box(group.children[1]!, [5, 6, 7, 8]);
const { elements } = await geometry({ handles: [await handleOf(group)] });
expect(elements[0]!.members).toEqual([
{ x: 1, y: 2, width: 3, height: 4 },
{ x: 5, y: 6, width: 7, height: 8 },
]);
});
it("adds the named element's descendant boxes, none past MAX_GEOMETRY_CHILDREN", async () => {
document.body.innerHTML = `<section><div><i></i></div></section>`;
const section = document.querySelector("section")!;
box(section.querySelector("i")!, [7, 8, 9, 10], false);
const handle = await handleOf(section);
const { children } = await geometry({ handles: [handle], children: handle });
expect(children).toHaveLength(2);
expect(children![1]).toMatchObject({ rect: { x: 7, y: 8, width: 9, height: 10 }, angle: 0 });
section.innerHTML = "<i></i>".repeat(MAX_GEOMETRY_CHILDREN + 1);
expect((await geometry({ handles: [handle], children: handle })).children).toEqual([]);
section.remove();
expect((await geometry({ handles: [handle], children: handle })).children).toBeNull();
});
});
@@ -0,0 +1,250 @@
/**
* Where an element sits on screen, read in the frame's CSS pixels: the overlay geometry the editing
* channel's `geometry` request answers with. Studio measures a frame it can script with these same
* functions, so both kinds of frame draw an element's selection chrome by one rule.
*/
import { COLOR_GRADING_SOURCE_HIDDEN_ATTR } from "../colorGrading";
import { getLogicalOpacity } from "../studioPreviewMark";
import {
MAX_ELEMENT_ENTRIES,
MAX_ELEMENT_VALUE,
type EditingElementHandle,
type EditingRect,
} from "./editingElement";
import { elementForHandle } from "./editingHandles";
import { composeElementTransform, type PlanarTransformOps } from "./elementTransform";
export type EditingCorner = "nw" | "ne" | "sw" | "se";
export type EditingCorners = Record<EditingCorner, { x: number; y: number }>;
export interface EditingElementBox {
rect: EditingRect;
/** The rotation it paints under, in degrees; 0 below ROTATION_GATE_EPSILON_DEG. */
angle: number;
/** Set for a rotated element only. */
corners: EditingCorners | null;
/** The nearest source-file boundary's rect and declared `data-width`/`data-height`. */
source: {
width: number;
height: number;
declaredWidth: number | null;
declaredHeight: number | null;
} | null;
/** Inline `clip-path`, else the computed one; null when longer than MAX_ELEMENT_VALUE. */
clipPath: string | null;
}
export interface EditingElementGeometry extends EditingElementBox {
/** It and every ancestor render: displayed, not hidden, not transparent. */
visible: boolean;
/** A `data-hf-group` wrapper's members' rects, up to MAX_ELEMENT_ENTRIES; null for other elements. */
members: EditingRect[] | null;
}
export const MAX_GEOMETRY_CHILDREN = 60;
/** Below this the element reads as unrotated: matrix noise, not a rotation. */
export const ROTATION_GATE_EPSILON_DEG = 1e-4;
/** Does this node render AT ALL, ignoring what it inherits? Sole owner of the
* rule; the walk below only decides which nodes to ask it about. */
function elementRendersItself(win: Window, el: HTMLElement): boolean {
const computed = win.getComputedStyle(el);
if (computed.display === "none" || computed.visibility === "hidden") return false;
const opacity = Number.parseFloat(getLogicalOpacity(el, computed));
return !(
Number.isFinite(opacity) &&
opacity <= 0.01 &&
!el.hasAttribute(COLOR_GRADING_SOURCE_HIDDEN_ATTR)
);
}
/**
* Does `el` render, given everything above it? `memo` serves one synchronous pass over many
* elements, where siblings share their chain; the caller drops it after the pass.
*/
export function isElementVisibleThroughAncestors(
el: HTMLElement,
memo?: Map<HTMLElement, boolean>,
): boolean {
const win = el.ownerDocument.defaultView;
if (!win) return true;
// Up to the first node already answered for, then back down: a node needs its ancestors' answer.
const pending: HTMLElement[] = [];
let inherited = true;
for (let node: HTMLElement | null = el; node; node = node.parentElement) {
const answered = memo?.get(node);
if (answered !== undefined) {
inherited = answered;
break;
}
pending.push(node);
}
for (let i = pending.length - 1; i >= 0; i -= 1) {
const node = pending[i]!;
// Once an ancestor is out, its descendants are too: no style read for them.
inherited = inherited && elementRendersItself(win, node);
memo?.set(node, inherited);
}
return inherited;
}
export interface ElementTransformSnapshot {
matrix: DOMMatrix;
cs: CSSStyleDeclaration;
}
export function readElementTransformSnapshot(
win: Window,
element: HTMLElement,
memo?: Map<HTMLElement, DOMMatrix | null>,
): ElementTransformSnapshot | null {
const DOMMatrixCtor = (win as Window & typeof globalThis).DOMMatrix;
if (!DOMMatrixCtor) return null;
const cs = win.getComputedStyle(element);
// The corner math transforms points, so this keeps the full matrix, translation included.
const ops: PlanarTransformOps<DOMMatrix> = {
identity: () => new DOMMatrixCtor(),
fromTransform: (value) => new DOMMatrixCtor(value),
fromRotate: (degrees) => new DOMMatrixCtor().rotateSelf(degrees),
compose: (outer, inner) => outer.multiply(inner),
};
try {
const matrix = composeElementTransform(
element,
ops,
(node) => (node === element ? cs : win.getComputedStyle(node)),
memo,
);
return matrix ? { matrix, cs } : null;
} catch {
return null;
}
}
/** The matrix's rotation in degrees, clockwise positive: atan2(b, a), whatever its scale or skew. */
export function rotationDegreesFromMatrix(matrix: Pick<DOMMatrix, "a" | "b" | "c" | "d">): number {
const a = Number.isFinite(matrix.a) ? matrix.a : 1;
const b = Number.isFinite(matrix.b) ? matrix.b : 0;
const c = Number.isFinite(matrix.c) ? matrix.c : 0;
const d = Number.isFinite(matrix.d) ? matrix.d : 1;
const fromX = (Math.atan2(b, a) * 180) / Math.PI;
const determinant = a * d - b * c;
// A reflection reads one basis direction 180° off; take the angle nearer zero, so a mirrored
// element's rotate handle stays on its own side.
const fromY = (Math.atan2(-c, d) * 180) / Math.PI;
const deg = determinant < 0 && Math.abs(fromY) < Math.abs(fromX) ? fromY : fromX;
return Number.isFinite(deg) ? deg : 0;
}
/**
* The border box's corners as painted, in viewport px: each local corner through the transform,
* then anchored by matching their bounding box to the element's real one. Null when unmeasurable.
*/
export function elementCornerPoints(
win: Window,
element: HTMLElement,
{ matrix, cs }: ElementTransformSnapshot,
): EditingCorners | null {
const DOMPointCtor = (win as Window & typeof globalThis).DOMPoint;
if (!DOMPointCtor) return null;
const w = element.offsetWidth;
const h = element.offsetHeight;
const originParts = cs.transformOrigin.split(" ").map((p) => Number.parseFloat(p));
const ox = Number.isFinite(originParts[0]!) ? originParts[0]! : w / 2;
const oy = Number.isFinite(originParts[1]!) ? originParts[1]! : h / 2;
const rel = (lx: number, ly: number): { x: number; y: number } => {
const p = matrix.transformPoint(new DOMPointCtor(lx - ox, ly - oy));
return { x: p.x, y: p.y };
};
const relCorners = { nw: rel(0, 0), ne: rel(w, 0), se: rel(w, h), sw: rel(0, h) };
const xs = [relCorners.nw.x, relCorners.ne.x, relCorners.se.x, relCorners.sw.x];
const ys = [relCorners.nw.y, relCorners.ne.y, relCorners.se.y, relCorners.sw.y];
const bcr = element.getBoundingClientRect();
const dx = bcr.left - Math.min(...xs);
const dy = bcr.top - Math.min(...ys);
const at = (pt: { x: number; y: number }) => ({ x: pt.x + dx, y: pt.y + dy });
return {
nw: at(relCorners.nw),
ne: at(relCorners.ne),
se: at(relCorners.se),
sw: at(relCorners.sw),
};
}
const positive = (value: string | null) => {
const parsed = Number.parseFloat(value ?? "");
return Number.isFinite(parsed) && parsed > 0 ? parsed : null;
};
const rectOf = (element: Element): EditingRect => {
const { x, y, width, height } = element.getBoundingClientRect();
return { x, y, width, height };
};
function sourceOf(element: Element): EditingElementBox["source"] {
const boundary = element.closest("[data-composition-file],[data-composition-src]");
if (!boundary) return null;
const { width, height } = boundary.getBoundingClientRect();
return {
width,
height,
declaredWidth: positive(boundary.getAttribute("data-width")),
declaredHeight: positive(boundary.getAttribute("data-height")),
};
}
function clipPathOf(element: HTMLElement): string | null {
const value =
element.style?.getPropertyValue("clip-path").trim() ||
window.getComputedStyle(element).getPropertyValue("clip-path").trim();
return value.length <= MAX_ELEMENT_VALUE ? value : null;
}
const finiteCorners = (corners: EditingCorners | null) =>
corners && Object.values(corners).every((p) => Number.isFinite(p.x) && Number.isFinite(p.y))
? corners
: null;
function boxOf(element: HTMLElement, memo: Map<HTMLElement, DOMMatrix | null>): EditingElementBox {
const transform = readElementTransformSnapshot(window, element, memo);
const turned = transform ? rotationDegreesFromMatrix(transform.matrix) : 0;
const angle = Math.abs(turned) < ROTATION_GATE_EPSILON_DEG ? 0 : turned;
const corners = angle && transform ? elementCornerPoints(window, element, transform) : null;
return {
rect: rectOf(element),
angle,
corners: finiteCorners(corners),
source: sourceOf(element),
clipPath: clipPathOf(element),
};
}
/** One entry per handle, null as in `describe`, and the boxes of `children`'s descendants. */
export function elementGeometry(
handles: readonly EditingElementHandle[],
children: EditingElementHandle | undefined,
): { elements: Array<EditingElementGeometry | null>; children: EditingElementBox[] | null } {
const memo = new Map<HTMLElement, DOMMatrix | null>();
const visible = new Map<HTMLElement, boolean>();
const elements = handles.map((handle) => {
const element = elementForHandle(handle) as HTMLElement | null;
if (!element) return null;
const members = element.hasAttribute("data-hf-group")
? Array.from(element.children).slice(0, MAX_ELEMENT_ENTRIES).map(rectOf)
: null;
return {
...boxOf(element, memo),
visible: isElementVisibleThroughAncestors(element, visible),
members,
};
});
const parent = children === undefined ? null : elementForHandle(children);
if (!parent) return { elements, children: null };
const descendants = parent.querySelectorAll<HTMLElement>("*");
const boxes =
descendants.length <= MAX_GEOMETRY_CHILDREN
? Array.from(descendants, (child) => boxOf(child, memo))
: [];
return { elements, children: boxes };
}
@@ -70,6 +70,7 @@ describe("createEditingPortClient", () => {
"describe",
"ancestors",
"query",
"geometry",
],
});
} finally {
+35
View File
@@ -17,6 +17,7 @@ import {
type EditingElementHit,
type EditingElementMatch,
} from "./editingElement";
import type { EditingElementBox, EditingElementGeometry } from "./editingGeometry";
import type { RuntimeTimelineMessage } from "./types";
import { wallClearTimeout, wallSetTimeout } from "./wallClock";
@@ -40,6 +41,24 @@ export {
type EditingElementSource,
type EditingRect,
} from "./editingElement";
export {
elementCornerPoints,
isElementVisibleThroughAncestors,
readElementTransformSnapshot,
rotationDegreesFromMatrix,
MAX_GEOMETRY_CHILDREN,
ROTATION_GATE_EPSILON_DEG,
type EditingCorner,
type EditingCorners,
type EditingElementBox,
type EditingElementGeometry,
type ElementTransformSnapshot,
} from "./editingGeometry";
export {
composeElementTransform,
individualRotateDegrees,
type PlanarTransformOps,
} from "./elementTransform";
export const EDITING_PROTOCOL_VERSION = 1 as const;
/** Parent to frame, on the window: asks the runtime for the port. */
@@ -57,6 +76,7 @@ export const EDITING_REQUEST_KINDS = [
"describe",
"ancestors",
"query",
"geometry",
] as const;
export type EditingRequestKind = (typeof EDITING_REQUEST_KINDS)[number];
@@ -107,6 +127,14 @@ export interface EditingRequests {
params: { selector: string };
result: { matches: EditingElementMatch[]; truncated: boolean };
};
/** Each handle's overlay geometry, null as in `describe`; `children` names whose descendant boxes to add. */
geometry: {
params: { handles: EditingElementHandle[]; children?: EditingElementHandle };
result: {
elements: Array<EditingElementGeometry | null>;
children: EditingElementBox[] | null;
};
};
}
export type EditingParams<K extends EditingRequestKind> = EditingRequests[K]["params"];
export type EditingResult<K extends EditingRequestKind> = EditingRequests[K]["result"];
@@ -209,6 +237,13 @@ const PARAM_PARSERS: {
hasKeys(params, "selector") && isSelector(params.selector)
? { selector: params.selector }
: null,
geometry: (params) => {
if (!isPlainRecord(params)) return null;
const { children, ...rest } = params;
const list = handleList(rest);
if (!list || !("children" in params)) return list;
return isHandle(children) ? { ...list, children } : null;
},
};
/** Validates one request off the wire. Never throws: anything unusable becomes a typed error. */
@@ -11,6 +11,8 @@
* Its own module because it is the one piece of that file every other piece
* depends on and nothing in it is about a single element's geometry.
*/
import { previewStageSize } from "../../player/lib/bridgePlayback";
import { previewFrameOpaque } from "../../player/lib/previewChannel";
/** iframe→overlay mapping basis shared by every overlay-geometry function. */
export interface OverlayRootScale {
@@ -52,7 +54,8 @@ function resolveRootDimensions(root: HTMLElement | null): { width: number; heigh
* instead of rootRect.width/height: when GSAP applies transforms (scale,
* translate) to the root, rootRect dimensions change but the composition's
* canonical size stays fixed, and using rootRect misaligns the overlay during
* animated playback. Returns null when the geometry is unmeasurable.
* animated playback. A frame Studio cannot script is scaled by the size its
* runtime posts. Returns null when the geometry is unmeasurable.
*/
export function computeOverlayRootScale(
overlayEl: HTMLDivElement,
@@ -61,7 +64,9 @@ export function computeOverlayRootScale(
): OverlayRootScale | null {
const iframeRect = iframe.getBoundingClientRect();
const overlayRect = overlayEl.getBoundingClientRect();
const dims = resolveRootDimensions(findOverlayRootElement(doc));
const dims = previewFrameOpaque(iframe)
? previewStageSize(iframe)
: resolveRootDimensions(findOverlayRootElement(doc));
if (!dims) return null;
return {
iframeRect,
@@ -8,7 +8,7 @@ import {
resolveCropInsetFromMoveDrag,
rotateDeltaIntoFrame,
} from "./domEditOverlayCrop";
import { individualRotateDegrees } from "./domEditOverlayTransform";
import { individualRotateDegrees } from "@hyperframes/core/runtime/editing-port";
describe("resolveCropInsetFromEdgeDrag", () => {
const startInsets = { top: 10, right: 20, bottom: 30, left: 40 };
@@ -1,4 +1,7 @@
import { composeElementTransform, type PlanarTransformOps } from "./domEditOverlayTransform";
import {
composeElementTransform,
type PlanarTransformOps,
} from "@hyperframes/core/runtime/editing-port";
import {
parseInsetClipPathSides,
type ClipPathInsetSides,
@@ -167,8 +170,19 @@ export function hugRectForElement(
rect: CropScreenRect & { editScaleX: number; editScaleY: number },
element: HTMLElement,
): CropScreenRect {
const insets = isElementCropLifted(element) ? null : readElementCropInsets(element);
// Uneditable or lifted clip — show the full element rect.
// A lifted clip shows the full element rect.
return isElementCropLifted(element)
? rect
: hugRectForClipPath(rect, readElementClipPath(element));
}
/** `hugRectForElement` for a clip-path read elsewhere; null is one that could not be read. */
export function hugRectForClipPath(
rect: CropScreenRect & { editScaleX: number; editScaleY: number },
clipPath: string | null,
): CropScreenRect {
const insets = clipPath === null ? null : parseCropClipPath(clipPath);
// Uneditable clip — show the full element rect.
if (!insets || (insets.top <= 0 && insets.right <= 0 && insets.bottom <= 0 && insets.left <= 0))
return rect;
return cropRectFromInsets(rect, insets, rect.editScaleX, rect.editScaleY);
@@ -6,7 +6,14 @@ import {
} from "./domEditOverlayBasis";
import { isElementVisibleThroughAncestors } from "./domEditingDom";
import { hugRectForElement } from "./domEditOverlayCrop";
import { composeElementTransform, type PlanarTransformOps } from "./domEditOverlayTransform";
import {
elementCornerPoints,
readElementTransformSnapshot,
rotationDegreesFromMatrix,
ROTATION_GATE_EPSILON_DEG,
type EditingCorners,
type ElementTransformSnapshot,
} from "@hyperframes/core/runtime/editing-port";
import { type OverlayMeasurePass, readThroughPass } from "./domEditOverlayMeasurePass";
export const RESIZE_HANDLE_HIT_PX = 16;
@@ -134,95 +141,6 @@ export function toVisibleOverlayRects(
});
}
/**
* getComputedStyle(element).transform decomposed into a DOMMatrix, read ONCE.
* Shared by orientedOverlayRect's rotation gate and elementCornerOverlayPoints
* so a single measurement pass serves both — constructing this twice per frame
* (one read per consumer) was redundant work; see orientedOverlayRect below.
*/
interface ElementTransformSnapshot {
matrix: DOMMatrix;
cs: CSSStyleDeclaration;
}
/**
* The transform from the element's own box to the composition's, ACCUMULATED
* over its ancestors rather than read from the element alone.
*
* What the user sees is the product of every transform between the element and
* the composition root, and an element is routinely a child of something
* scaled or rotated. Reading only its own transform drew the selection box at
* the element's untransformed size: a text layer inside a card carrying
* `scale(1.2)` got a box at 1/1.2 of the text, with the top-left correct (the
* caller anchors that to the real bounding rect) and the right and bottom
* edges falling short. The same read decides whether to draw the box rotated,
* so an element inside a rotated parent got an upright box too.
*
* Only the linear part matters here. Each transform's origin contributes
* translation, and the caller discards translation by matching the corners'
* bounding box to the element's real one, so composing the matrices alone is
* enough and there is no per-ancestor origin to unpick.
*
* The walk stops at the composition document's root. The canvas zoom lives on
* the iframe element in Studio's own document and is applied separately by
* `computeOverlayRootScale`; including it here would count it twice.
*/
function readElementTransformSnapshot(
win: Window,
element: HTMLElement,
pass?: OverlayMeasurePass,
): ElementTransformSnapshot | null {
const DOMMatrixCtor = (win as Window & typeof globalThis).DOMMatrix;
if (!DOMMatrixCtor) return null;
const cs = win.getComputedStyle(element);
// The corner math transforms points, so this algebra keeps the full matrix,
// translation included, where the crop frame's keeps only 2D components.
const ops: PlanarTransformOps<DOMMatrix> = {
identity: () => new DOMMatrixCtor(),
fromTransform: (value) => new DOMMatrixCtor(value),
fromRotate: (degrees) => new DOMMatrixCtor().rotateSelf(degrees),
compose: (outer, inner) => outer.multiply(inner),
};
try {
const matrix = composeElementTransform(
element,
ops,
(node) => (node === element ? cs : win.getComputedStyle(node)),
pass?.transform,
);
return matrix ? { matrix, cs } : null;
} catch {
return null;
}
}
/**
* The element's live transform rotation, in DEGREES (screen/CSS convention, CW
* positive), decomposed from its transform matrix (rotation = atan2(b, a)).
* GSAP folds rotation and scale into the same matrix; this reads rotation only.
* Skew is ignored (does not affect atan2(b, a)).
*/
function rotationDegreesFromMatrix(matrix: DOMMatrix): number {
const a = Number.isFinite(matrix.a) ? matrix.a : 1;
const b = Number.isFinite(matrix.b) ? matrix.b : 0;
const c = Number.isFinite(matrix.c) ? matrix.c : 0;
const d = Number.isFinite(matrix.d) ? matrix.d : 1;
const fromX = (Math.atan2(b, a) * 180) / Math.PI;
const determinant = a * d - b * c;
// A reflection makes one basis direction read 180° away from the authored
// rotation. For cursor/handle orientation those directions are equivalent;
// choose the representative nearest zero instead of drawing a pure mirror's
// rotate handle on the opposite side of the element.
const fromY = (Math.atan2(-c, d) * 180) / Math.PI;
const deg = determinant < 0 && Math.abs(fromY) < Math.abs(fromX) ? fromY : fromX;
return Number.isFinite(deg) ? deg : 0;
}
/** Below this, orientedOverlayRect treats the element as unrotated and returns
* the AABB directly (see its doc comment) — tight enough to only swallow
* matrix-decomposition floating-point noise, never an actual rotation. */
const ROTATION_GATE_EPSILON_DEG = 1e-4;
function toOverlayRect(
overlayEl: HTMLDivElement,
iframe: HTMLIFrameElement,
@@ -233,7 +151,6 @@ function toOverlayRect(
const scale =
precomputedScale ?? computeOverlayRootScale(overlayEl, iframe, iframe.contentDocument);
if (!scale) return null;
const { iframeRect, overlayRect, rootScaleX, rootScaleY } = scale;
const elementRect = element.getBoundingClientRect();
const sourceBoundary = findSourceBoundary(element, pass);
@@ -245,13 +162,35 @@ function toOverlayRect(
sourceBoundary.getBoundingClientRect(),
)
: sourceBoundary?.getBoundingClientRect();
return frameRectToOverlay(scale, elementRect, {
width: sourceBoundaryRect?.width,
height: sourceBoundaryRect?.height,
declaredWidth: readPositiveDimension(sourceBoundary?.getAttribute("data-width") ?? null),
declaredHeight: readPositiveDimension(sourceBoundary?.getAttribute("data-height") ?? null),
});
}
type FrameSource = {
width?: number;
height?: number;
declaredWidth: number | null;
declaredHeight: number | null;
} | null;
/** A client rect measured in the frame, and its source boundary's, in overlay coordinates. */
export function frameRectToOverlay(
scale: OverlayRootScale,
elementRect: { left: number; top: number; width: number; height: number },
source: FrameSource,
): OverlayRect {
const { iframeRect, overlayRect, rootScaleX, rootScaleY } = scale;
const editScale = resolveDomEditCoordinateScale({
rootScaleX,
rootScaleY,
sourceRectWidth: sourceBoundaryRect?.width,
sourceRectHeight: sourceBoundaryRect?.height,
sourceWidth: readPositiveDimension(sourceBoundary?.getAttribute("data-width") ?? null),
sourceHeight: readPositiveDimension(sourceBoundary?.getAttribute("data-height") ?? null),
sourceRectWidth: source?.width,
sourceRectHeight: source?.height,
sourceWidth: source?.declaredWidth,
sourceHeight: source?.declaredHeight,
});
return {
@@ -264,6 +203,23 @@ function toOverlayRect(
};
}
/** Corners measured in the frame's viewport px, in overlay coordinates. */
export function frameCornersToOverlay(
{ iframeRect, overlayRect, rootScaleX, rootScaleY }: OverlayRootScale,
corners: EditingCorners,
): Record<FixedCorner, { x: number; y: number }> {
const toOverlay = (pt: { x: number; y: number }): { x: number; y: number } => ({
x: iframeRect.left - overlayRect.left + pt.x * rootScaleX,
y: iframeRect.top - overlayRect.top + pt.y * rootScaleY,
});
return {
nw: toOverlay(corners.nw),
ne: toOverlay(corners.ne),
se: toOverlay(corners.se),
sw: toOverlay(corners.sw),
};
}
/** Which physical corner of the (possibly rotated) element a resize handle keeps
* fixed: NW grabs the top-left, so the bottom-right (se) is the anchor, etc. */
export type FixedCorner = "nw" | "ne" | "sw" | "se";
@@ -309,55 +265,15 @@ export function elementCornerOverlayPoints(
const win = iframe.contentWindow;
const doc = iframe.contentDocument;
if (!win || !doc) return null;
const DOMPointCtor = (win as Window & typeof globalThis).DOMPoint;
if (!DOMPointCtor) return null;
if (!(win as Window & typeof globalThis).DOMPoint) return null;
const scale = precomputedScale ?? computeOverlayRootScale(overlayEl, iframe, doc);
if (!scale) return null;
const { iframeRect, overlayRect, rootScaleX, rootScaleY } = scale;
// The element's local border box maps to viewport coords by the SAME transform
// matrix the browser used for its BCR. We recover the transform's screen-space
// action from the BCR: transformPoint(localCorner - origin) gives a corner
// RELATIVE to the transformed origin. We anchor those relative corners to the
// BCR by matching the AABB of the transformed corners to the real BCR — the
// constant offset cancels in the before/after difference the caller takes, but
// we resolve it fully here so callers can also read absolute overlay positions.
const transform = precomputedTransform ?? readElementTransformSnapshot(win, element);
if (!transform) return null;
const { matrix, cs } = transform;
const w = element.offsetWidth;
const h = element.offsetHeight;
const originParts = cs.transformOrigin.split(" ").map((p) => Number.parseFloat(p));
const ox = Number.isFinite(originParts[0]!) ? originParts[0]! : w / 2;
const oy = Number.isFinite(originParts[1]!) ? originParts[1]! : h / 2;
const rel = (lx: number, ly: number): { x: number; y: number } => {
const p = matrix.transformPoint(new DOMPointCtor(lx - ox, ly - oy));
return { x: p.x, y: p.y };
};
const relCorners = {
nw: rel(0, 0),
ne: rel(w, 0),
se: rel(w, h),
sw: rel(0, h),
};
// Recover the absolute viewport position by matching to the element's BCR:
// the relative corners' AABB min corresponds to the BCR's top-left.
const xs = [relCorners.nw.x, relCorners.ne.x, relCorners.se.x, relCorners.sw.x];
const ys = [relCorners.nw.y, relCorners.ne.y, relCorners.se.y, relCorners.sw.y];
const bcr = element.getBoundingClientRect();
const dx = bcr.left - Math.min(...xs);
const dy = bcr.top - Math.min(...ys);
const toOverlay = (pt: { x: number; y: number }): { x: number; y: number } => ({
x: iframeRect.left - overlayRect.left + (pt.x + dx) * rootScaleX,
y: iframeRect.top - overlayRect.top + (pt.y + dy) * rootScaleY,
});
return {
nw: toOverlay(relCorners.nw),
ne: toOverlay(relCorners.ne),
se: toOverlay(relCorners.se),
sw: toOverlay(relCorners.sw),
};
const corners = elementCornerPoints(win, element, transform);
return corners && frameCornersToOverlay(scale, corners);
}
/**
@@ -397,11 +313,20 @@ export function orientedOverlayRect(
if (!base) return null;
const win = iframe.contentWindow;
const transform = win ? readElementTransformSnapshot(win, element, pass) : null;
const transform = win ? readElementTransformSnapshot(win, element, pass?.transform) : null;
const angle = transform ? rotationDegreesFromMatrix(transform.matrix) : 0;
if (Math.abs(angle) < ROTATION_GATE_EPSILON_DEG) return base;
const corners = elementCornerOverlayPoints(overlayEl, iframe, element, scale, transform);
return orientOverlayRect(base, angle, corners);
}
/** The oriented box whose corners are `corners`, at `angle`; `base` when they cannot describe one. */
export function orientOverlayRect(
base: OverlayRect,
angle: number,
corners: Record<FixedCorner, { x: number; y: number }> | null,
): OverlayRect {
if (!corners) return base;
// Unrotated edge lengths (in overlay px): nw→ne is the width, nw→sw the height.
const width = cornerEdgeLength(corners.nw, corners.ne);
@@ -4,11 +4,11 @@
* No imports from other domEditing* modules — safe to import from anywhere.
*/
import { getLogicalOpacity } from "@hyperframes/core/studio-preview-mark";
import { COLOR_GRADING_SOURCE_HIDDEN_ATTR } from "@hyperframes/core/color-grading";
import {
buildStableSelector,
escapeCssString,
getPreferredClassSelector,
isElementVisibleThroughAncestors,
readElementSource,
type EditingElementSource,
} from "@hyperframes/core/runtime/editing-port";
@@ -45,65 +45,8 @@ export function isEditableTextLeaf(el: HTMLElement): boolean {
export { getLogicalOpacity };
// One addressing rule for Studio and the runtime that describes elements to it.
export { buildStableSelector, escapeCssString };
/** Does this node render AT ALL, ignoring what it inherits? Sole owner of the
* rule; the walk below only decides which nodes to ask it about. */
function elementRendersItself(win: Window, el: HTMLElement): boolean {
const computed = win.getComputedStyle(el);
if (computed.display === "none" || computed.visibility === "hidden") return false;
const opacity = Number.parseFloat(getLogicalOpacity(el, computed));
return !(
Number.isFinite(opacity) &&
opacity <= 0.01 &&
!el.hasAttribute(COLOR_GRADING_SOURCE_HIDDEN_ATTR)
);
}
/**
* Does `el` render, given everything above it?
*
* `memo` is for a caller asking this about MANY elements in one synchronous
* pass. Answers are a function of the node and its ancestors, and siblings
* share almost all of their chain, so memoizing per node turns a walk per
* element into one style read per node in the tree. It is only ever valid for
* the length of one pass — the DOM cannot change under a pass, and every read
* here is a read — so the caller creates it and drops it, and nothing survives
* to be invalidated. Omitted, every call walks the chain itself.
*
* The ANSWER is what it always was. Which nodes get a style read is not: this
* resolves top-down and stops at the first node that is out, where the previous
* version resolved bottom-up and stopped at the first one that is out going the
* other way. Same boolean for every input, a different (and, for a subtree
* hidden near the root, smaller) set of reads.
*/
export function isElementVisibleThroughAncestors(
el: HTMLElement,
memo?: Map<HTMLElement, boolean>,
): boolean {
const win = el.ownerDocument.defaultView;
if (!win) return true;
// Up to the first node already answered for, then back down: a node's answer
// needs its ancestors' first, and the topmost unanswered node is where the
// chain of unknowns starts.
const pending: HTMLElement[] = [];
let inherited = true;
for (let node: HTMLElement | null = el; node; node = node.parentElement) {
const answered = memo?.get(node);
if (answered !== undefined) {
inherited = answered;
break;
}
pending.push(node);
}
for (let i = pending.length - 1; i >= 0; i -= 1) {
const node = pending[i]!;
// Once an ancestor is out, its descendants are out with it, and asking the
// platform about them would be a style read for an answer already known.
inherited = inherited && elementRendersItself(win, node);
memo?.set(node, inherited);
}
return inherited;
}
// One visibility rule for Studio and the runtime that measures overlay geometry for it.
export { isElementVisibleThroughAncestors };
// ─── Style accessors ──────────────────────────────────────────────────────────
@@ -140,22 +83,6 @@ export function getDataAttributes(el: HTMLElement): Record<string, string> {
return attrs;
}
// ─── DOM traversal ────────────────────────────────────────────────────────────
export function findClosestByAttribute(
el: HTMLElement,
attributeNames: string[],
): HTMLElement | null {
let current: HTMLElement | null = el;
while (current) {
const candidate = current;
if (attributeNames.some((attribute) => candidate.hasAttribute(attribute))) {
return candidate;
}
current = current.parentElement;
}
return null;
}
// ─── Composition source resolution ───────────────────────────────────────────
// The runtime INLINES subcompositions and strips the source-file linkage from the
@@ -248,7 +248,7 @@ export function domEditSelectionToFacts(
hasEditableText: selection.textFields.length > 0,
hasTimingStart: selection.dataAttributes.start != null,
animationCount,
hasAudio: isAudibleVideoNode(selection.element),
hasAudio: selection.hasAudio ?? isAudibleVideoNode(selection.element),
};
}
@@ -80,6 +80,8 @@ export interface DomEditTextField {
export interface DomEditSelection extends PatchTarget {
element: HTMLElement;
handle?: EditingElementHandle;
hasAudio?: boolean;
src?: string | null;
label: string;
tagName: string;
sourceFile: string;
@@ -21,6 +21,17 @@ import { giveUpOnPreviewChange, isPreviewChanging } from "../../player/previewRe
import { setStudioWaitingPressCancel } from "../../utils/studioPendingEdits";
import { playheadMoment } from "../../hooks/editMoment";
import type { EditMoment } from "./manualEditsTypes";
import { previewFrameOpaque } from "../../player/lib/previewChannel";
import { findPreviewElementForSelection } from "./previewSelection";
import { measurePreviewRects } from "./previewOverlayRects";
type Shown = { selection: DomEditSelection; rect: OverlayRect };
type ShownStart = (
e: React.PointerEvent<HTMLElement>,
at: EditMoment,
waited: boolean,
overChannel?: Shown[],
) => boolean;
const MAX_SETTLED_WAIT_FRAMES = 60;
export const PRESS_WAIT_MAX_MS = 10_000;
@@ -128,12 +139,33 @@ export function createPreviewGestureStarts(
};
};
/** An opaque frame's `selections` on the page it shows now, read over its channel; null when any is gone. */
const shownOverChannel = async (
iframe: HTMLIFrameElement,
selections: DomEditSelection[],
): Promise<Shown[] | null> => {
const overlay = opts.overlayRef.current;
const path = opts.activeCompositionPathRef.current;
const found = await Promise.all(
selections.map((selection) => findPreviewElementForSelection(iframe, selection, path)),
);
const handles = found.map((element) => element?.handle);
if (!overlay || !handles.length || !handles.every(Boolean)) return null;
const rects = await measurePreviewRects(iframe, overlay, handles as string[]);
const shown = selections.flatMap((selection, i) => {
const rect = rects?.[i];
return rect ? [{ selection: { ...selection, handle: handles[i] }, rect }] : [];
});
return shown.length === selections.length ? shown : null;
};
// A press on the page a reload is replacing would edit it by its old rules: it starts on the new page instead.
const startOnShownPreview = (
e: React.PointerEvent<HTMLElement>,
pressed: () => HTMLElement[],
start: (e: React.PointerEvent<HTMLElement>, at: EditMoment, waited: boolean) => boolean,
start: ShownStart,
draw: WaitingPressState["draw"] = null,
targets: () => DomEditSelection[] = () => [],
): boolean => {
const at = playheadMoment();
if (!isPreviewChanging() && !opts.waitingPressRef.current) return start(e, at, false);
@@ -170,8 +202,25 @@ export function createPreviewGestureStarts(
settledFrames = settled ? settledFrames + 1 : 0;
return elements.length;
};
// An opaque frame's pressed elements are found over its channel, once the preview settles.
const iframe = opts.iframeRef.current;
const channel = iframe && previewFrameOpaque(iframe) ? iframe : null;
let overChannel: Shown[] | null | undefined;
let asked = false;
const channelStep = (frame: HTMLIFrameElement): "wait" | "lost" | "run" => {
const overdue = performance.now() - since > PRESS_WAIT_MAX_MS;
if (overdue) giveUpOnPreviewChange();
if (isPreviewChanging() && !overdue) return "wait";
if (!asked) {
asked = true;
void shownOverChannel(frame, targets()).then((shown) => (overChannel = shown));
}
if (overChannel === undefined) return "wait";
return overChannel ? "run" : "lost";
};
const waitStep = (): "wait" | "lost" | "run" => {
if (press.after && !press.after.ended) return "wait";
if (channel) return channelStep(channel);
const found = countFrame();
// A change that never ends (a save with no reply) would hold every press: run on the page shown.
const overdue = performance.now() - since > PRESS_WAIT_MAX_MS;
@@ -188,8 +237,8 @@ export function createPreviewGestureStarts(
return;
}
endWaitingPress(press);
if (step === "lost" || !start(down as unknown as React.PointerEvent<HTMLElement>, at, true))
return;
const pointerDown = down as unknown as React.PointerEvent<HTMLElement>;
if (step === "lost" || !start(pointerDown, at, true, overChannel ?? undefined)) return;
if (press.moved) moveActiveGesture(press.moved);
if (press.released) releaseActiveGesture(press.released);
};
@@ -208,6 +257,11 @@ export function createPreviewGestureStarts(
const iframe = opts.iframeRef.current;
return overlay && iframe && orientedGroupAwareOverlayRect(overlay, iframe, element);
};
const shownOnPage = (target: DomEditSelection | null): Shown | null => {
const selection = target && shownSelection(target);
const rect = selection && shownRect(selection.element);
return selection && rect ? { selection, rect } : null;
};
const startGroupDrag = (e: React.PointerEvent<HTMLElement>) => {
const items = opts.groupOverlayItemsRef.current;
@@ -218,8 +272,16 @@ export function createPreviewGestureStarts(
const selection = shownSelection(item.selection);
return selection ? [selection.element] : [];
}),
(pressed, at, waited) => {
(pressed, at, waited, overChannel) => {
if (!waited) return _startGroupDrag(pressed, opts, at);
if (overChannel) {
const measured = items.map((item, i) => ({ ...item, ...overChannel[i]! }));
return _startGroupDrag(
pressed,
{ ...opts, groupOverlayItemsRef: { current: measured } },
at,
);
}
const overlay = opts.overlayRef.current;
const iframe = opts.iframeRef.current;
if (!overlay || !iframe) return false;
@@ -243,6 +305,7 @@ export function createPreviewGestureStarts(
opts.setGroupOverlayItems(
items.map((item) => ({ ...item, rect: shiftedOverlayRect(item.rect, dx, dy) })),
),
() => items.map((item) => item.selection),
);
};
const startGesture = (
@@ -270,15 +333,14 @@ export function createPreviewGestureStarts(
const shown = target && shownSelection(target);
return shown ? [shown.element] : [];
},
(pressed, at, waited) => {
(pressed, at, waited, overChannel) => {
if (!waited) return _startGesture(kind, pressed, opts, { ...options, at });
const selection = target && shownSelection(target);
const rect = selection && shownRect(selection.element);
const shown = overChannel ? overChannel[0] : shownOnPage(target);
return (
!!(selection && rect) &&
!!shown &&
_startGesture(kind, pressed, opts, {
selection,
rect,
selection: shown.selection,
rect: shown.rect,
resizeHandle: options?.resizeHandle,
resizeCorner: options?.resizeCorner,
at,
@@ -286,6 +348,7 @@ export function createPreviewGestureStarts(
);
},
draw,
() => (target ? [target] : []),
);
};
@@ -0,0 +1,316 @@
// @vitest-environment happy-dom
import React, { act } from "react";
import { createRoot } from "react-dom/client";
import { afterEach, beforeEach, describe, expect, it, vi } from "vitest";
// The frame's real runtime agent, serving the opaque frame from this document.
import { installEditingAgent } from "../../../../core/src/runtime/editingAgent";
import { makeSelection, installReactActEnvironment } from "../../hooks/domSelectionTestHarness";
import { watchPreviewFrame } from "../../player/lib/bridgePlayback";
import { queryPreviewElements } from "../../player/lib/previewElements";
import { detachedFakeIframe } from "../../player/hooks/timelinePlayerTestHarness";
import type { DomEditSelection } from "./domEditing";
import { orientedGroupAwareOverlayRect, type OverlayRect } from "./domEditOverlayGeometry";
import type { UseDomEditOverlayGesturesOptions } from "./domEditOverlayGestures";
import { createPreviewGestureStarts } from "./previewGestureStarts";
import { useDomEditOverlayRects } from "./useDomEditOverlayRects";
const frames = vi.hoisted(() => new Set<() => void>());
vi.mock("./overlayFrameLoop", () => ({
subscribeOverlayFrame: (update: () => void) => {
frames.add(update);
return () => frames.delete(update);
},
}));
const press = vi.hoisted(() => ({ changing: true, start: vi.fn((..._args: unknown[]) => true) }));
vi.mock("./domEditOverlayStartGesture", () => ({
startGesture: press.start,
startGroupDrag: vi.fn(() => true),
}));
vi.mock("../../player/previewReloading", async (importOriginal) => ({
...(await importOriginal<typeof import("../../player/previewReloading")>()),
isPreviewChanging: () => press.changing,
giveUpOnPreviewChange: vi.fn(),
}));
installReactActEnvironment();
// The 2D slice of each that the overlay reads; one set for both frames, which share this window.
class FakeDOMMatrix {
a = 1;
b = 0;
c = 0;
d = 1;
e = 0;
f = 0;
constructor(init?: string) {
const m = init ? /matrix\(([^)]+)\)/.exec(init) : null;
if (m) [this.a, this.b, this.c, this.d, this.e, this.f] = m[1]!.split(",").map(Number);
}
multiply(o: FakeDOMMatrix) {
const out = new FakeDOMMatrix();
out.a = this.a * o.a + this.c * o.b;
out.b = this.b * o.a + this.d * o.b;
out.c = this.a * o.c + this.c * o.d;
out.d = this.b * o.c + this.d * o.d;
out.e = this.a * o.e + this.c * o.f + this.e;
out.f = this.b * o.e + this.d * o.f + this.f;
return out;
}
rotateSelf() {
return this;
}
transformPoint(p: { x: number; y: number }) {
return { x: this.a * p.x + this.c * p.y + this.e, y: this.b * p.x + this.d * p.y + this.f };
}
}
const FIXTURE = `
<div data-composition-id="main" data-width="1920" data-height="1080">
<div id="host" data-composition-src="a.html" data-width="960" data-height="540">
<p id="inner"></p><span id="speck"></span>
</div>
<div id="turned" style="transform: matrix(0.8, 0.6, -0.6, 0.8, 0, 0)"></div>
<div id="grp" data-hf-group="g"><i id="m1"></i><i id="m2"></i></div>
<div id="clipped" style="clip-path: inset(10px 20px 30px 40px)"></div>
<div style="display: none"><span id="gone"></span></div>
</div>`;
const RECTS: Record<string, number[]> = {
host: [200, 100, 960, 540],
inner: [220, 140, 300, 80],
speck: [230, 150, 2, 2],
turned: [600, 400, 300, 200],
m1: [50, 60, 100, 100],
m2: [400, 500, 120, 90],
grp: [0, 0, 10, 10],
clipped: [900, 700, 400, 300],
gone: [1, 1, 50, 50],
};
function sized(element: Element, [x, y, width, height]: number[]) {
element.getBoundingClientRect = () =>
({ x, y, width, height, left: x, top: y, right: x + width, bottom: y + height }) as DOMRect;
Object.defineProperty(element, "offsetWidth", { value: width, configurable: true });
Object.defineProperty(element, "offsetHeight", { value: height, configurable: true });
return element;
}
const closers: Array<() => void> = [];
beforeEach(() => {
Object.assign(window, { DOMMatrix: FakeDOMMatrix });
document.body.innerHTML = FIXTURE;
for (const [id, rect] of Object.entries(RECTS)) sized(document.getElementById(id)!, rect);
});
afterEach(() => {
closers.splice(0).forEach((close) => close());
frames.clear();
document.body.innerHTML = "";
});
/** A frame Studio scripts directly; its document and window are this one's. */
function liveFrame() {
const iframe = detachedFakeIframe(window as unknown as Record<string, unknown>);
Object.defineProperty(iframe, "contentDocument", { value: document });
return sized(iframe, [100, 50, 960, 540]) as HTMLIFrameElement;
}
/** A frame Studio cannot read, whose runtime is the real editing agent over this document. */
function opaqueFrame() {
const frame: Record<string, unknown> = {};
const iframe = sized(detachedFakeIframe(frame), [100, 50, 960, 540]) as HTMLIFrameElement;
Object.defineProperty(iframe, "contentDocument", { value: null });
Object.defineProperty(frame, "document", {
get: () => {
throw new DOMException("blocked", "SecurityError");
},
});
const parent = {
postMessage: (data: unknown, _origin: string, ports: MessagePort[]) =>
window.dispatchEvent(
new MessageEvent("message", { source: frame as unknown as Window, data, ports }),
),
} as unknown as Window;
const agent = installEditingAgent(parent);
frame.postMessage = (data: unknown) =>
window.dispatchEvent(new MessageEvent("message", { source: parent, data }));
closers.push(() => agent.dispose());
// The size the runtime posts for its composition, as Studio mirrors it.
watchPreviewFrame(iframe);
window.dispatchEvent(
new MessageEvent("message", {
source: frame as unknown as Window,
origin: window.location.origin,
data: { source: "hf-preview", type: "stage-size", width: 1920, height: 1080 },
}),
);
return iframe;
}
type Shown = ReturnType<typeof useDomEditOverlayRects>;
interface Picked {
selection: string | null;
group?: string[];
hover?: string | null;
}
/** The overlay hook over `iframe`, with the selections `picked` names; returns its latest output. */
async function overlayOn(iframe: HTMLIFrameElement, picked: Picked) {
const opaque = iframe.contentDocument === null;
const pick = async (id: string): Promise<DomEditSelection> => {
const selection = makeSelection(id, document.getElementById(id)!);
// The host is the root of its own source file.
if (id === "host") selection.sourceFile = "a.html";
if (!opaque) return selection;
const reply = await queryPreviewElements(iframe, `#${id}`);
return { ...selection, handle: reply!.matches[0]!.handle };
};
const overlay = sized(document.createElement("div"), [20, 10, 1200, 800]) as HTMLDivElement;
const refs = {
iframeRef: { current: iframe },
overlayRef: { current: overlay },
selectionRef: { current: picked.selection ? await pick(picked.selection) : null },
activeCompositionPathRef: { current: "index.html" },
groupSelectionsRef: { current: await Promise.all((picked.group ?? []).map(pick)) },
hoverSelectionRef: { current: picked.hover ? await pick(picked.hover) : null },
rafPausedRef: { current: false },
};
let shown: Shown | null = null;
function Harness() {
shown = useDomEditOverlayRects(refs);
return null;
}
const root = createRoot(document.createElement("div"));
act(() => root.render(React.createElement(Harness)));
closers.push(() => act(() => root.unmount()));
return { refs, read: () => shown!, frame: () => act(() => frames.forEach((update) => update())) };
}
const settled = async (overlay: Awaited<ReturnType<typeof overlayOn>>) => {
for (let i = 0; i < 3; i += 1) {
overlay.frame();
await act(() => new Promise((resolve) => setTimeout(resolve, 20)));
}
const { overlayRect, childRects, groupOverlayItems, hoverRect } = overlay.read();
return { overlayRect, childRects, group: groupOverlayItems.map((item) => item.rect), hoverRect };
};
const close = (rect: OverlayRect | null) =>
rect && Object.fromEntries(Object.entries(rect).map(([key, value]) => [key, +value.toFixed(9)]));
const closeAll = (shown: Awaited<ReturnType<typeof settled>>) => ({
overlayRect: close(shown.overlayRect),
childRects: shown.childRects.map(close),
group: shown.group.map(close),
hoverRect: close(shown.hoverRect),
});
describe("overlay rects on an opaque frame match a scriptable one", () => {
it.each<[string, Picked]>([
[
"a source host with child outlines, a group and a clipped member, a rotated hover",
{ selection: "host", group: ["grp", "clipped"], hover: "turned" },
],
["a rotated selection", { selection: "turned" }],
["a hidden selection draws nothing", { selection: "gone", hover: "grp" }],
])("%s", async (_name, picked) => {
const live = await settled(await overlayOn(liveFrame(), picked));
const remote = await settled(await overlayOn(opaqueFrame(), picked));
expect(closeAll(remote)).toEqual(closeAll(live));
if (picked.selection === "gone") expect(live.overlayRect).toBeNull();
else expect(live.overlayRect).not.toBeNull();
});
it("covers what each case decides", async () => {
const host = await settled(
await overlayOn(opaqueFrame(), {
selection: "host",
group: ["grp", "clipped"],
hover: "turned",
}),
);
// Scale 0.5 into an iframe 80,40 inside the overlay; the host is its own source boundary.
expect(host.overlayRect).toMatchObject({ left: 180, top: 90, width: 480, editScaleX: 0.5 });
// The 2px speck is too small to outline.
expect(host.childRects).toHaveLength(1);
expect(host.group[0]).toMatchObject({ left: 105, top: 70, width: 235, height: 265 });
expect(host.group[1]).toMatchObject({ left: 550, top: 395, width: 170, height: 130 });
expect(host.hoverRect!.angle).toBeCloseTo(36.87, 2);
});
it("keeps one geometry read in flight however many frames ask", async () => {
const { port1 } = new MessageChannel();
const send = vi.spyOn(Object.getPrototypeOf(port1) as MessagePort, "postMessage");
const geometryReads = () =>
send.mock.calls.filter(([message]) => (message as { kind?: string }).kind === "geometry");
const overlay = await overlayOn(opaqueFrame(), { selection: "turned" });
overlay.frame();
overlay.frame();
overlay.frame();
await vi.waitFor(() => expect(geometryReads()).toHaveLength(1));
await vi.waitFor(() => expect(overlay.read().overlayRect).not.toBeNull());
overlay.frame();
overlay.frame();
await vi.waitFor(() => expect(geometryReads()).toHaveLength(2));
send.mockRestore();
});
});
describe("a press held across a preview change on an opaque frame", () => {
async function pressDuringChange(selection: DomEditSelection) {
press.changing = true;
press.start.mockClear();
const iframe = opaqueFrame();
const overlay = sized(document.createElement("div"), [20, 10, 1200, 800]) as HTMLDivElement;
const opts = {
waitingPressRef: { current: null },
boxRef: { current: null },
rafPausedRef: { current: false },
iframeRef: { current: iframe },
overlayRef: { current: overlay },
activeCompositionPathRef: { current: "index.html" },
selectionRef: { current: selection },
overlayRectRef: { current: null },
setOverlayRect: vi.fn(),
} as unknown as UseDomEditOverlayGesturesOptions;
const { startGesture } = createPreviewGestureStarts(opts, vi.fn(), vi.fn());
const down = {
pointerId: 1,
clientX: 5,
clientY: 6,
currentTarget: { setPointerCapture: vi.fn() },
preventDefault: vi.fn(),
stopPropagation: vi.fn(),
};
startGesture("drag", down as unknown as React.PointerEvent<HTMLElement>);
await new Promise((resolve) => setTimeout(resolve, 50));
expect(press.start).not.toHaveBeenCalled();
press.changing = false;
return { opts, overlay };
}
it("re-finds the pressed element on the page shown and starts on its box, read over the channel", async () => {
const turned = document.getElementById("turned")!;
const stale = { ...makeSelection("turned", turned), handle: `${"f".repeat(16)}:1` };
const { overlay } = await pressDuringChange(stale);
await vi.waitFor(() => expect(press.start).toHaveBeenCalledOnce());
const [, , , started] = press.start.mock.calls[0] as [
unknown,
unknown,
unknown,
{ selection: DomEditSelection; rect: OverlayRect },
];
expect(started.selection.handle).not.toBe(stale.handle);
expect(started.selection.id).toBe("turned");
const live = orientedGroupAwareOverlayRect(overlay, liveFrame(), turned);
expect(close(started.rect)).toEqual(close(live));
expect(started.rect.angle).toBeCloseTo(36.87, 2);
});
it("drops the press when the page shown no longer has the element", async () => {
const turned = document.getElementById("turned")!;
const { opts } = await pressDuringChange(makeSelection("turned", turned));
turned.remove();
await vi.waitFor(() => expect(opts.waitingPressRef.current).toBeNull());
expect(press.start).not.toHaveBeenCalled();
});
});
@@ -0,0 +1,123 @@
/**
* Overlay rects for a preview frame Studio cannot script. The runtime measures the elements (see
* `editingGeometry.ts` in core); the arithmetic that places them on the overlay is
* `domEditOverlayGeometry`'s, so both kinds of frame draw an element's chrome in the same place.
*/
import {
ROTATION_GATE_EPSILON_DEG,
type EditingElementBox,
type EditingElementGeometry,
type EditingRect,
} from "@hyperframes/core/runtime/editing-port";
import { previewGeometry } from "../../player/lib/previewElements";
import { computeOverlayRootScale, type OverlayRootScale } from "./domEditOverlayBasis";
import { hugRectForClipPath } from "./domEditOverlayCrop";
import {
frameCornersToOverlay,
frameRectToOverlay,
orientOverlayRect,
resolveDomEditGroupOverlayRect,
type OverlayRect,
} from "./domEditOverlayGeometry";
const clientRect = ({ x, y, width, height }: EditingRect) => ({ left: x, top: y, width, height });
/** `toOverlayRect`. */
const baseRect = (scale: OverlayRootScale, box: EditingElementBox) =>
frameRectToOverlay(scale, clientRect(box.rect), box.source);
/** `orientedOverlayRect`. */
function orientedRect(scale: OverlayRootScale, box: EditingElementBox): OverlayRect {
const base = baseRect(scale, box);
if (Math.abs(box.angle) < ROTATION_GATE_EPSILON_DEG) return base;
return orientOverlayRect(
base,
box.angle,
box.corners && frameCornersToOverlay(scale, box.corners),
);
}
/** `groupAwareOverlayRect`, or with `oriented`, `orientedGroupAwareOverlayRect`. */
function groupAwareRect(
scale: OverlayRootScale,
geometry: EditingElementGeometry,
oriented: boolean,
): OverlayRect {
if (!geometry.members)
return oriented ? orientedRect(scale, geometry) : baseRect(scale, geometry);
const rect = baseRect(scale, geometry);
const members = geometry.members.map((member) =>
frameRectToOverlay(scale, clientRect(member), null),
);
const union = resolveDomEditGroupOverlayRect(members);
return union ? { ...union, editScaleX: rect.editScaleX, editScaleY: rect.editScaleY } : rect;
}
/** `orientedVisibleOverlayRect`. */
function visibleRect(scale: OverlayRootScale, box: EditingElementBox): OverlayRect {
const rect = orientedRect(scale, box);
return { ...rect, ...hugRectForClipPath(rect, box.clipPath) };
}
export interface PreviewOverlayTargets {
selection: string | null;
group: Array<string | null>;
hover: string | null;
}
export interface PreviewOverlayRects {
/** Null when the element is gone or does not render. */
selection: OverlayRect | null;
/** The selection's descendants' outlines, as `useDomEditOverlayRects` draws them. */
children: OverlayRect[];
group: Array<OverlayRect | null>;
hover: OverlayRect | null;
}
/** The rects `useDomEditOverlayRects` draws, in one geometry read; null when the frame cannot answer. */
export async function measurePreviewOverlay(
iframe: HTMLIFrameElement,
overlayEl: HTMLDivElement,
targets: PreviewOverlayTargets,
): Promise<PreviewOverlayRects | null> {
const handles = [...new Set([targets.selection, targets.hover, ...targets.group])].filter(
(handle): handle is string => Boolean(handle),
);
// A selection with no handle is not in this frame; it draws nothing, as one not found there.
const reply = handles.length
? await previewGeometry(iframe, handles, targets.selection ?? undefined)
: { elements: [], children: null };
const scale = reply && computeOverlayRootScale(overlayEl, iframe, null);
if (!reply || !scale) return null;
const read = (handle: string | null) =>
handle ? (reply.elements[handles.indexOf(handle)] ?? null) : null;
const selected = read(targets.selection);
const shown = selected?.visible ? selected : null;
const hovered = read(targets.hover);
return {
selection: shown && groupAwareRect(scale, shown, true),
children: shown
? (reply.children ?? [])
.map((box) => visibleRect(scale, box))
.filter((rect) => rect.width > 2 && rect.height > 2)
: [],
group: targets.group.map((handle) => {
const member = read(handle);
const rect = member && groupAwareRect(scale, member, false);
return rect && { ...rect, ...hugRectForClipPath(rect, member.clipPath) };
}),
hover: hovered && groupAwareRect(scale, hovered, true),
};
}
/** `orientedGroupAwareOverlayRect` for each handle; null when the frame cannot answer. */
export async function measurePreviewRects(
iframe: HTMLIFrameElement,
overlayEl: HTMLDivElement,
handles: string[],
): Promise<Array<OverlayRect | null> | null> {
const reply = await previewGeometry(iframe, handles);
const scale = reply && computeOverlayRootScale(overlayEl, iframe, null);
if (!reply || !scale) return null;
return reply.elements.map((geometry) => geometry && groupAwareRect(scale, geometry, true));
}
@@ -3,7 +3,7 @@
import React, { act } from "react";
import { createRoot } from "react-dom/client";
import { afterEach, describe, expect, it, vi } from "vitest";
import { EDITING_ELEMENT_HANDLE } from "@hyperframes/core/runtime/editing-port";
import { EDITING_ELEMENT_HANDLE, MAX_ELEMENT_TEXT } from "@hyperframes/core/runtime/editing-port";
// The frame's real runtime agent, serving the opaque frame from this document.
import { installEditingAgent } from "../../../../core/src/runtime/editingAgent";
import {
@@ -13,6 +13,7 @@ import {
type DomEditSelection,
} from "./domEditing";
import { setCompositionSourceMap } from "./domEditingDom";
import { domEditSelectionToFacts } from "./domEditingLayers";
import {
findPreviewElementForSelection,
findPreviewElementForTimelineElement,
@@ -31,6 +32,11 @@ import type { TimelineElement } from "../../player";
installReactActEnvironment();
const selectLog = vi.hoisted(() => ({ on: null as null | ((event: string) => void) }));
vi.mock("../../utils/selectDebug", () => ({
logSelect: (event: string) => selectLog.on?.(event),
}));
const closers: Array<() => void> = [];
afterEach(() => {
closers.splice(0).forEach((close) => close());
@@ -78,7 +84,9 @@ const CONTEXT: PreviewSelectionOptions = {
function shared(selection: DomEditSelection | null) {
if (!selection) return null;
const { element: _element, handle: _handle, textFields: _fields, ...rest } = selection;
return rest;
// Read off the description only when `element` is a stand-in; checked on their own below.
const { hasAudio: _audio, src: _src, ...compared } = rest;
return compared;
}
/** The opaque frame's description of `element`, as a hit-test would hand it over. */
@@ -300,6 +308,63 @@ describe("opaque frame selection handles", () => {
});
});
describe("opaque frame selections read only what arrived whole", () => {
it("reads audio and src off the description, not the tag-only stand-in", async () => {
document.body.innerHTML = `<div data-composition-id="main" data-width="1920" data-height="1080">
<video id="muted" muted src="a.mp4"></video><video id="silent" data-has-audio="false"></video>
<video id="loud" src="b.mp4"></video><img id="pic" src="pic.png"></div>`;
for (const [id, audible] of [
["muted", false],
["silent", false],
["loud", true],
] as const) {
const { live, remote } = await bothPaths(document.getElementById(id)!);
expect(domEditSelectionToFacts(live!).hasAudio).toBe(audible);
expect(domEditSelectionToFacts(remote!).hasAudio).toBe(audible);
}
const { remote } = await bothPaths(document.getElementById("pic")!);
expect(remote!.src).toBe("pic.png");
expect(remote!.element.getAttribute("src")).toBeNull();
});
it("builds nothing when an attribute or inline style was cut, and is not stopped by long text", async () => {
const chain = JSON.stringify(Array.from({ length: 200 }, (_, i) => ({ id: `node-${i}` })));
document.body.innerHTML = `<div data-composition-id="main" data-width="1920" data-height="1080">
<p id="fx" data-audio-fx='${chain}'>fx</p>
<p id="bg-img" style="background-image: url(data:image/png;base64,${"A".repeat(3000)})">i</p>
<p id="essay">${"y".repeat(MAX_ELEMENT_TEXT + 5)}</p></div>`;
for (const id of ["fx", "bg-img"]) {
const { live, remote } = await bothPaths(document.getElementById(id)!);
expect(live).not.toBeNull();
expect(remote).toBeNull();
}
const essay = await bothPaths(document.getElementById("essay")!);
expect(essay.remote).toMatchObject({ id: "essay", selector: "#essay" });
});
it("builds nothing when the frame cannot read computed styles or the composition root", async () => {
document.body.innerHTML = FIXTURE;
const opaque = opaqueFrame();
const locked = await describedOnOpaque(opaque, document.querySelector("#locked")!);
const bg = await describedOnOpaque(opaque, document.querySelector("#bg")!);
expect(await resolvePreviewSelection(opaque, bg, CONTEXT)).not.toBeNull();
const styles = vi.spyOn(window, "getComputedStyle").mockImplementation(() => {
throw new Error("parked");
});
expect(await resolvePreviewSelection(opaque, locked, CONTEXT)).toBeNull();
styles.mockRestore();
const query = document.querySelectorAll.bind(document);
const roots = vi.spyOn(document, "querySelectorAll").mockImplementation((selector: string) => {
if (selector === "[data-composition-id]") throw new Error("reloading");
return query(selector);
});
expect(await resolvePreviewSelection(opaque, bg, CONTEXT)).toBeNull();
roots.mockRestore();
});
});
describe("useDomSelection on an opaque frame", () => {
function renderSelection(iframe: HTMLIFrameElement) {
let hook: ReturnType<typeof useDomSelection> | null = null;
@@ -373,4 +438,31 @@ describe("useDomSelection on an opaque frame", () => {
expect(hook().domEditGroupSelections.map((s) => s.id)).toEqual(["plain"]);
expect(hook().domEditSelection!.handle).toBe(second.handle);
});
it("leaves a group the user changed while its refresh was reading the frame", async () => {
document.body.innerHTML = FIXTURE;
const hook = renderSelection(opaqueFrame());
await act(() => hook().handleTimelineElementSelect(clip("#plain")));
const plain = hook().domEditSelection!;
await act(() => hook().handleTimelineElementSelect(clip("#locked")));
const locked = hook().domEditSelection!;
act(() => hook().applyDomSelection(plain, { additive: true }));
await act(() => hook().handleTimelineElementSelect(clip("#scene")));
const scene = hook().domEditSelection!;
act(() => hook().applyDomSelection(locked, { additive: true }));
act(() => hook().applyDomSelection(plain, { additive: true }));
// The primary is gone, so the refresh re-reads the group; meanwhile the user picks one clip.
document.querySelector("#plain")!.remove();
selectLog.on = (event) => {
if (event === "refresh-lost") hook().applyDomSelection(scene, { revealPanel: false });
};
try {
await act(() => hook().refreshDomEditSelectionFromPreview(hook().domEditSelection!));
} finally {
selectLog.on = null;
}
expect(hook().domEditSelection!.id).toBe("scene");
expect(hook().domEditGroupSelections.map((s) => s.id)).toEqual(["scene"]);
});
});
@@ -9,7 +9,11 @@
* the selection's `handle`.
*/
import { resolveEditingAffordances } from "@hyperframes/core/editing";
import type { EditingPortClient } from "@hyperframes/core/runtime/editing-port";
import {
MAX_ELEMENT_ENTRIES,
MAX_ELEMENT_VALUE,
type EditingPortClient,
} from "@hyperframes/core/runtime/editing-port";
import {
previewComputedStyles,
previewEditingClient,
@@ -49,6 +53,7 @@ import {
type TimelineElementDomTargetOptions,
} from "./domEditingTypes";
import { existsInSourceFile } from "./probeSourceElement";
import { isVideoAudible } from "../../player/lib/timelineElementHelpers";
let inertDocument: Document | undefined;
const inert = () => (inertDocument ??= document.implementation.createHTMLDocument(""));
@@ -101,6 +106,23 @@ function readable(element: PreviewElement): HTMLElement {
const attr = (element: PreviewElement, name: string) =>
element.attributes.find(([key]) => key === name)?.[1] ?? null;
const mayBeCut = (value: string) =>
value.length >= MAX_ELEMENT_VALUE || (value.startsWith("data:") && value.endsWith(","));
/** A cut attribute or style (a long `data-audio-fx`, say) would be written back cut; text is not read back. */
function valuesWhole(element: PreviewElement): boolean {
if (!element.truncated) return true;
const styles = Object.values(element.inlineStyles);
if (element.attributes.length >= MAX_ELEMENT_ENTRIES || styles.length >= MAX_ELEMENT_ENTRIES)
return false;
return ![...element.attributes.map(([, value]) => value), ...styles].some(mayBeCut);
}
const hasAudio = (element: PreviewElement) =>
element.tag === "video" &&
attr(element, "muted") === null &&
isVideoAudible({ tag: "video", hasAudioAttr: attr(element, "data-has-audio"), muted: false });
/** Undefined when the frame cannot answer; null when nothing matches. */
type Found = PreviewElement | null | undefined;
@@ -184,8 +206,10 @@ function chainReader(chain: readonly PreviewElement[]): SelectionChainReader<Pre
async function compositionRoot(iframe: HTMLIFrameElement, documentElement: PreviewElement) {
const reply = await queryPreviewElements(iframe, "[data-composition-id]");
const first = reply?.matches[0];
const [root] = first ? ((await describePreviewElements(iframe, [first.handle])) ?? []) : [];
if (!reply) return null;
const first = reply.matches[0];
const [root] = first ? ((await describePreviewElements(iframe, [first.handle])) ?? [null]) : [];
if (root === null) return null;
const element = root ?? documentElement;
const { x, y, width, height } = element.rect;
return {
@@ -232,7 +256,7 @@ async function isPreviewCompositionRoot(
chain: readonly PreviewElement[],
index: number,
computedStyles: Record<string, string>,
): Promise<boolean> {
): Promise<boolean | null> {
const current = chain[index]!;
if (attr(current, "data-composition-id") !== null && !compositionSourceOf(current)) return true;
const parent = chain[index + 1];
@@ -240,10 +264,8 @@ async function isPreviewCompositionRoot(
if (!underBody || attr(current, "data-hf-allow-root-edit") !== null) return false;
const { x, y, width, height } = current.rect;
const rect = { left: x, top: y, width, height };
return coversCompositionRoot(
{ rect, computedStyles },
await compositionRoot(iframe, chain.at(-1)!),
);
const root = await compositionRoot(iframe, chain.at(-1)!);
return root && coversCompositionRoot({ rect, computedStyles }, root);
}
const compositionSourceOf = (element: PreviewElement) =>
@@ -262,14 +284,18 @@ export async function resolvePreviewSelection(
const index = chain && chainIndex(chain, chainReader(chain), options);
if (!chain || index === null) return null;
const current = chain[index]!;
if (!valuesWhole(current)) return null;
const element = standInFor(iframe, current);
const identity = previewIdentity(current, options.activeCompositionPath);
const compositionSrc = compositionSourceOf(current);
const probe = readable(current);
const inlineStyles = getInlineStyles(probe);
const computedStyles =
(await previewComputedStyles(iframe, current, [...CURATED_STYLE_PROPERTIES])) ?? {};
const computedStyles = await previewComputedStyles(iframe, current, [
...CURATED_STYLE_PROPERTIES,
]);
if (!computedStyles) return null;
const isCompositionRoot = await isPreviewCompositionRoot(iframe, chain, index, computedStyles);
if (isCompositionRoot === null) return null;
const isInsideLocked = chainLocked(chain, chainReader(chain), index);
const { sourceFile } = identity;
const existsInSource = await existsInSourceFile(
@@ -297,6 +323,8 @@ export async function resolvePreviewSelection(
return {
element,
handle: current.handle,
hasAudio: hasAudio(current),
src: attr(current, "src"),
...identity,
compositionPath: sourceFile,
compositionSrc,
@@ -101,7 +101,7 @@ export function ImageFillField({
const track = useTrackDesignInput();
const sourceFile = element.sourceFile;
const isImage = element.tagName === "img";
const imageUrl = isImage ? (element.element.getAttribute("src") ?? "") : value;
const imageUrl = isImage ? (selectionSrc(element) ?? "") : value;
const commitImage = (next: string, projectAsset = false) => {
if (isImage) {
if (next)
@@ -510,3 +510,7 @@ export function GradientField({
</div>
);
}
function selectionSrc(selection: DomEditSelection): string | null {
return selection.src === undefined ? selection.element.getAttribute("src") : selection.src;
}
@@ -1,6 +1,7 @@
/**
* RAF-driven hook that tracks overlay, hover, and group rects from the iframe DOM.
* Runs a requestAnimationFrame loop and writes React state only when rects change.
* A frame Studio cannot script is measured over its channel instead, one read at a time.
*/
import { useRef, useState, type RefObject } from "react";
import { useMountEffect } from "../../hooks/useMountEffect";
@@ -21,6 +22,8 @@ import {
} from "./domEditOverlayGeometry";
import { computeOverlayRootScale } from "./domEditOverlayBasis";
import { subscribeOverlayFrame } from "./overlayFrameLoop";
import { previewFrameOpaque } from "../../player/lib/previewChannel";
import { measurePreviewOverlay } from "./previewOverlayRects";
function childRectsEqual(a: OverlayRect[], b: OverlayRect[]): boolean {
if (a.length !== b.length) return false;
@@ -74,6 +77,7 @@ export function useDomEditOverlayRects({
const resolvedHoverElementRef = useRef<{ key: string; element: HTMLElement } | null>(null);
const resolvedGroupElementRef = useRef<Map<string, HTMLElement>>(new Map());
const childRectsRef = useRef<OverlayRect[]>([]);
const channelReadRef = useRef(false);
const setOverlayRect = (next: OverlayRect | null) => {
if (rectsEqual(overlayRectRef.current, next)) return;
@@ -107,6 +111,74 @@ export function useDomEditOverlayRects({
return next;
};
const setChildRects = (next: OverlayRect[]) => {
if (childRectsEqual(childRectsRef.current, next)) return;
childRectsRef.current = next;
setChildRectsState(next);
};
/** Members of the same group collapse to one selection under select-as-unit, so one key each. */
const distinctGroup = (group: DomEditSelection[]) => {
const keys = new Set<string>();
return group.flatMap((selection) => {
const key = selectionCacheKey(selection);
if (keys.has(key)) return [];
keys.add(key);
return [{ key, selection }];
});
};
const hoverShown = (sel: DomEditSelection | null, group: DomEditSelection[]) => {
const hoverSel = hoverSelectionRef.current;
const key = hoverSel && selectionCacheKey(hoverSel);
const covered =
(sel && selectionCacheKey(sel) === key) ||
group.some((entry) => selectionCacheKey(entry) === key);
return covered ? null : hoverSel;
};
/** The opaque frame's rects, one geometry read in flight at a time; a stale answer is dropped. */
const updateOverChannel = (iframe: HTMLIFrameElement, overlayEl: HTMLDivElement) => {
if (channelReadRef.current) return;
const sel = selectionRef.current;
const group = groupSelectionsRef.current;
const members = distinctGroup(group);
const hover = hoverShown(sel, group);
const targets = {
selection: sel?.handle ?? null,
group: members.map(({ selection }) => selection.handle ?? null),
hover: hover?.handle ?? null,
};
if (!sel && !members.length && !hover) {
setOverlayRect(null);
setChildRects([]);
setGroupOverlayItems([]);
setHoverRect(null);
return;
}
channelReadRef.current = true;
void measurePreviewOverlay(iframe, overlayEl, targets)
.then((rects) => {
const current =
selectionRef.current === sel &&
groupSelectionsRef.current === group &&
hoverShown(sel, group) === hover;
if (!rects || !current || rafPausedRef.current || iframeRef.current !== iframe) return;
setOverlayRect(rects.selection);
setChildRects(rects.children);
setGroupOverlayItems(
members.flatMap(({ key, selection }, index) => {
const rect = rects.group[index];
return rect ? [{ key, selection, element: selection.element, rect }] : [];
}),
);
setHoverRect(rects.hover);
})
.finally(() => {
channelReadRef.current = false;
});
};
useMountEffect(() => {
const clearAll = () => {
setOverlayRect(null);
@@ -261,7 +333,14 @@ export function useDomEditOverlayRects({
setHoverRect(orientedGroupAwareOverlayRect(overlayEl, iframe, hoverEl, scale));
};
return subscribeOverlayFrame(update);
const frame = () => {
const iframe = iframeRef.current;
const overlayEl = overlayRef.current;
const channel = !rafPausedRef.current && iframe && overlayEl && !iframe.contentDocument;
if (channel && previewFrameOpaque(iframe)) updateOverChannel(iframe, overlayEl);
else update();
};
return subscribeOverlayFrame(frame);
});
return {
+18 -6
View File
@@ -42,6 +42,15 @@ export type {
UseDomSelectionReturn,
} from "./useDomSelectionTypes";
/** A refreshed group lands only when it found someone and the user has not picked another meanwhile. */
function groupStillLive(
live: { current: DomEditSelection[] },
refreshed: DomEditSelection[],
next: DomEditSelection[],
): boolean {
return next.length > 0 && live.current === refreshed;
}
// ── Hook ──
export function useDomSelection({
@@ -366,7 +375,7 @@ export function useDomSelection({
// refresher falls back to it when the primary is gone, and a ref keeps that from
// forcing either callback to be declared in the other's dependency list.
const refreshDomEditGroupSelectionsFromPreviewRef = useRef<
(selections: DomEditSelection[]) => Promise<void>
(selections: DomEditSelection[], iframe: HTMLIFrameElement | null) => Promise<void>
>(async () => {});
const refreshDomEditSelectionFromPreview = useCallback(
@@ -386,7 +395,7 @@ export function useDomSelection({
group: group.length,
});
if (group.length > 1) {
await refreshDomEditGroupSelectionsFromPreviewRef.current(group);
await refreshDomEditGroupSelectionsFromPreviewRef.current(group, iframe);
return;
}
applyDomSelection(null, { revealPanel: false });
@@ -410,17 +419,20 @@ export function useDomSelection({
const refreshDomEditGroupSelectionsFromPreview = useCallback(
// fallow-ignore-next-line complexity
async (selections: DomEditSelection[]) => {
const iframe = previewIframeRef.current;
async (selections: DomEditSelection[], iframe = previewIframeRef.current) => {
const nextGroup: DomEditSelection[] = [];
for (const selection of selections) {
const element = await findSelectionElement(iframe, selection, activeCompPath);
if (element === undefined) return;
if (!element) continue;
const nextSelection = await buildDomSelectionFromTarget(element, { previous: selection });
const nextSelection = await buildDomSelectionFromTarget(
element,
{ previous: selection },
iframe,
);
if (nextSelection) nextGroup.push(nextSelection);
}
if (nextGroup.length === 0) return;
if (!groupStillLive(domEditGroupSelectionsRef, selections, nextGroup)) return;
const currentSelection = domEditSelectionRef.current;
const nextSelection =
@@ -5,7 +5,12 @@
import {
EDITING_ELEMENT_HANDLE,
MAX_DESCRIBE_HANDLES,
MAX_ELEMENT_ENTRIES,
MAX_ELEMENT_VALUE,
MAX_GEOMETRY_CHILDREN,
type EditingElementBox,
type EditingElementDescription,
type EditingElementGeometry,
type EditingElementHit,
type EditingElementMatch,
type EditingElementSource,
@@ -50,11 +55,12 @@ function hasPlainFields(value: Fields): boolean {
return groupsOk && attributesOk && childrenOk && typeof value.truncated === "boolean";
}
const isRect = (value: unknown) =>
isFields(value) && [value.x, value.y, value.width, value.height].every(Number.isFinite);
function readHit(value: unknown): PreviewElementHit | null {
if (!isFields(value) || !isHandle(value.handle) || !isText(value.tag)) return null;
const { rect } = value;
const rectOk = isFields(rect) && [rect.x, rect.y, rect.width, rect.height].every(Number.isFinite);
return rectOk ? (value as unknown as PreviewElementHit) : null;
return isRect(value.rect) ? (value as unknown as PreviewElementHit) : null;
}
/** Frame data checked against `EditingElementDescription`; null when any field is off. */
@@ -142,3 +148,67 @@ export function previewElementSelectorIndex(
.filter((group) => resolveElementSourceFile(group.source, activeCompositionPath) === file)
.reduce((sum, group) => sum + group.count, 0);
}
const isPoint = (value: unknown) =>
isFields(value) && Number.isFinite(value.x) && Number.isFinite(value.y);
const isSizeOrNull = (value: unknown) =>
value === null || (Number.isFinite(value) && (value as number) > 0);
const CORNERS = ["nw", "ne", "se", "sw"] as const;
const isCorners = (value: unknown) =>
value === null || (isFields(value) && CORNERS.every((corner) => isPoint(value[corner])));
const isSourceBox = (value: unknown) =>
value === null ||
(isFields(value) &&
Number.isFinite(value.width) &&
Number.isFinite(value.height) &&
isSizeOrNull(value.declaredWidth) &&
isSizeOrNull(value.declaredHeight));
const isClipPath = (value: unknown) =>
value === null || (isText(value) && (value as string).length <= MAX_ELEMENT_VALUE);
const isBox = (value: unknown): value is EditingElementBox =>
isFields(value) &&
isRect(value.rect) &&
Number.isFinite(value.angle) &&
isCorners(value.corners) &&
isSourceBox(value.source) &&
isClipPath(value.clipPath);
const readGeometry = (value: unknown): EditingElementGeometry | null => {
if (!isBox(value) || !isFields(value) || typeof value.visible !== "boolean") return null;
const { members } = value;
const membersOk =
members === null ||
(Array.isArray(members) && members.length <= MAX_ELEMENT_ENTRIES && members.every(isRect));
return membersOk ? (value as unknown as EditingElementGeometry) : null;
};
export interface PreviewGeometry {
/** One entry per handle, in order: null for an element the frame no longer has. */
elements: Array<EditingElementGeometry | null>;
/** The `children` element's descendants, when it has 1 to MAX_GEOMETRY_CHILDREN; else empty. */
children: EditingElementBox[] | null;
}
/** Overlay geometry for `handles`, and `children`'s descendant boxes; null when the runtime cannot answer. */
export async function previewGeometry(
iframe: HTMLIFrameElement,
handles: readonly string[],
children?: string,
): Promise<PreviewGeometry | null> {
if (!handles.length || handles.length > MAX_DESCRIBE_HANDLES) return null;
const params = { handles: [...handles], ...(children ? { children } : {}) };
const reply = await previewRequest(iframe, "geometry", params);
const elements: unknown = reply?.elements;
if (!Array.isArray(elements) || elements.length !== handles.length) return null;
const boxes: unknown = reply?.children;
const childrenOk =
Array.isArray(boxes) && boxes.length <= MAX_GEOMETRY_CHILDREN && boxes.every(isBox);
return {
elements: elements.map(readGeometry),
children: children && childrenOk ? (boxes as EditingElementBox[]) : null,
};
}
+1 -1
View File
@@ -36,7 +36,7 @@
"unpacked": 541601
},
"@hyperframes/producer": {
"packed": 880000,
"packed": 890000,
"unpacked": 3210000
},
"@hyperframes/sdk": {