mirror of
https://github.com/heygen-com/hyperframes.git
synced 2026-11-18 07:50:13 +00:00
feat(studio): layers panel and marquee read an opaque preview over its channel (#7639)
* feat(studio): layers panel and marquee read an opaque preview over its channel The runtime answers a new layers request with the Layers walk: each layer's handle, depth, child count, z-index and kind, by the rules Studio walks a scriptable frame with, which move to core. On an opaque frame the Layers panel lists, selects and drills in from it, and the marquee measures its candidates over the channel and selects from their descriptions. An opaque selection maps to a timeline clip known only by its clip id, so a marquee's picks survive the timeline sync. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> * fix(studio): opaque preview hover, layers drill-in, marquee paging and delete path Review follow-ups for the opaque-preview Layers panel and marquee: - Delete on an opaque selection runs the DOM delete again. The clip-id match that lets a selection find a row known only by its clip id is now opt-in (`clipIdScope`), passed only by selection sync and the Layers seek, never by delete or group commits, which need a patch target that row lacks. A row with no file reads as in the open composition, as the forward lookup does. - The runtime's `layers` reports a drill-in group it no longer has (`missingGroup`) and lists nothing, so the panel leaves the drill-in after a reload as the same-origin branch does, rather than list the whole page. - `layers` takes an `offset`, and the marquee pages through every layer, so elements past one reply's 2,048 cap still band-select. - A Layers refresh that lands while an opaque read is in flight runs once more when it settles, instead of being dropped. - A hover resolved for a pointer move that a leave or newer move superseded is dropped. - `pointer-targets` reads the hit stack once per request. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> * test(core): keep the layers paging test inside the Windows runner's timeout The paging test walked the 2,051-layer page three times with every row mapped, which ran past 5 s on windows-latest. It now maps only the last page's rows and has its own 20 s timeout. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> --------- Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
co-authored by
Claude Opus 5.5
parent
ff687fed99
commit
be67089a80
@@ -147,6 +147,7 @@ describe("installEditingAgent", () => {
|
||||
"ancestors",
|
||||
"query",
|
||||
"geometry",
|
||||
"layers",
|
||||
],
|
||||
},
|
||||
});
|
||||
@@ -316,6 +317,20 @@ describe("installEditingAgent", () => {
|
||||
100 + index,
|
||||
"invalid-params",
|
||||
]),
|
||||
...[
|
||||
{ group: null },
|
||||
{ group: "0123456789abcdef:0" },
|
||||
{ group: ["0123456789abcdef:1"] },
|
||||
{ group: "0123456789abcdef:1", max: 5 },
|
||||
{ scope: "0123456789abcdef:1" },
|
||||
{ offset: -1 },
|
||||
{ offset: 1.5 },
|
||||
{ offset: "2" },
|
||||
].map((params, index) => [
|
||||
{ v: 1, id: 130 + index, kind: "layers", params },
|
||||
130 + index,
|
||||
"invalid-params",
|
||||
]),
|
||||
...[["--"], ["--a b"], ["--a;b"], ["--a:b"], ["Color"], ["-x"], ["--" + "a".repeat(255)]].map(
|
||||
(properties, index) => [
|
||||
{ v: 1, id: 60 + index, kind: "computed-styles", params: { selector: "p", properties } },
|
||||
|
||||
@@ -29,6 +29,7 @@ import {
|
||||
} from "./editingHandles";
|
||||
import { elementGeometry } from "./editingGeometry";
|
||||
import { pointerTargetsAt } from "./editingTargets";
|
||||
import { layersOf } from "./editingLayers";
|
||||
import { getLogicalOpacity } from "../studioPreviewMark";
|
||||
|
||||
type Handlers = {
|
||||
@@ -87,6 +88,7 @@ const HANDLERS: Handlers = {
|
||||
ancestors: ({ handles }) => ({ ancestors: ancestorHandles(handles) }),
|
||||
query: ({ selector }) => queryElements(selector),
|
||||
geometry: ({ handles, children }) => elementGeometry(handles, children),
|
||||
layers: ({ group, offset }) => layersOf(group, offset),
|
||||
};
|
||||
|
||||
function serve(port: MessagePort): void {
|
||||
|
||||
@@ -34,7 +34,7 @@ const collected = new FinalizationRegistry<number>((n) => {
|
||||
});
|
||||
let lastNumber = 0;
|
||||
|
||||
function handleFor(element: Element): EditingElementHandle {
|
||||
export function handleFor(element: Element): EditingElementHandle {
|
||||
let n = numbers.get(element);
|
||||
if (n === undefined) {
|
||||
n = ++lastNumber;
|
||||
|
||||
@@ -0,0 +1,78 @@
|
||||
import { afterEach, describe, expect, it } from "vitest";
|
||||
import { MAX_QUERY_MATCHES } from "./editingElement";
|
||||
import { elementForHandle } from "./editingHandles";
|
||||
import { layersOf } from "./editingLayers";
|
||||
|
||||
afterEach(() => {
|
||||
document.body.innerHTML = "";
|
||||
});
|
||||
|
||||
const named = (handle: string) =>
|
||||
elementForHandle(handle)?.id || elementForHandle(handle)?.className;
|
||||
|
||||
describe("layers", () => {
|
||||
it("walks layers in document order with their depth, child count, z-index and kind", () => {
|
||||
document.body.innerHTML = `<main data-composition-id="m">
|
||||
<div id="card" style="z-index: 3"><p id="title">Hi <b>there</b></p><img id="pic"></div>
|
||||
<div id="pair" data-hf-group="g"><i id="a"></i></div>
|
||||
<div id="parked" data-hf-clip-hidden style="visibility: hidden"><p id="inside" style="visibility: visible">x</p></div>
|
||||
<div><span class="loose"></span></div>
|
||||
<script id="code"></script></main>`;
|
||||
const { layers, truncated } = layersOf();
|
||||
expect(truncated).toBe(false);
|
||||
expect(layers.map((layer) => [named(layer.handle), layer.depth, layer.childCount])).toEqual([
|
||||
["card", 0, 2],
|
||||
["title", 1, 0],
|
||||
["pic", 1, 0],
|
||||
["pair", 0, 1],
|
||||
["a", 1, 0],
|
||||
["loose", 0, 0],
|
||||
]);
|
||||
expect(layers.map((layer) => layer.kind)).toEqual([
|
||||
"group",
|
||||
"text",
|
||||
"image",
|
||||
"group",
|
||||
"shape",
|
||||
"shape",
|
||||
]);
|
||||
expect(layers[0]!.zIndex).toBe(3);
|
||||
});
|
||||
|
||||
it("scopes to a drilled-into group, and caps the walk", () => {
|
||||
const tiles = Array.from({ length: MAX_QUERY_MATCHES + 3 }, (_, i) => `<i id="t${i}"></i>`);
|
||||
document.body.innerHTML = `<main data-composition-id="m"><div id="pair" data-hf-group="g">
|
||||
<i id="a"></i><i id="b"></i></div>${tiles.join("")}</main>`;
|
||||
const all = layersOf();
|
||||
expect(all).toMatchObject({ truncated: true });
|
||||
expect(all.layers).toHaveLength(MAX_QUERY_MATCHES);
|
||||
const pair = all.layers[0]!.handle;
|
||||
expect(layersOf(pair).layers.map((layer) => [named(layer.handle), layer.depth])).toEqual([
|
||||
["a", 0],
|
||||
["b", 0],
|
||||
]);
|
||||
expect(layersOf(pair).missingGroup).toBe(false);
|
||||
});
|
||||
|
||||
it("pages past the cap from an offset, so a caller can read every layer", () => {
|
||||
const tiles = Array.from({ length: MAX_QUERY_MATCHES + 3 }, (_, i) => `<i id="t${i}"></i>`);
|
||||
document.body.innerHTML = `<main data-composition-id="m">${tiles.join("")}</main>`;
|
||||
const rest = layersOf(undefined, MAX_QUERY_MATCHES);
|
||||
expect(rest).toMatchObject({ truncated: false, missingGroup: false });
|
||||
expect(rest.layers.map((layer) => named(layer.handle))).toEqual([
|
||||
`t${MAX_QUERY_MATCHES}`,
|
||||
`t${MAX_QUERY_MATCHES + 1}`,
|
||||
`t${MAX_QUERY_MATCHES + 2}`,
|
||||
]);
|
||||
const last = layersOf(undefined, MAX_QUERY_MATCHES + 2).layers;
|
||||
expect(last.map((layer) => named(layer.handle))).toEqual([`t${MAX_QUERY_MATCHES + 2}`]);
|
||||
}, 20_000);
|
||||
|
||||
it("lists nothing for a group this page no longer has, and says so", () => {
|
||||
document.body.innerHTML = `<main data-composition-id="m"><div id="pair" data-hf-group="g"><i id="a"></i></div></main>`;
|
||||
const pair = layersOf().layers[0]!.handle;
|
||||
document.getElementById("pair")!.remove();
|
||||
expect(layersOf(pair)).toEqual({ layers: [], truncated: false, missingGroup: true });
|
||||
expect(layersOf("0123456789abcdef:1")).toMatchObject({ layers: [], missingGroup: true });
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,115 @@
|
||||
/** The Layers walk `layers` answers; Studio walks a frame it scripts by the same rules. */
|
||||
import { isRichTextFormattingTag } from "../utils/richTextSanitize";
|
||||
import { isClipHiddenByRuntime } from "../studioPreviewMark";
|
||||
import { MAX_QUERY_MATCHES, type EditingElementHandle } from "./editingElement";
|
||||
import { elementForHandle, handleFor } from "./editingHandles";
|
||||
import { isDomLayerElement } from "./editingTargets";
|
||||
import { isElementNode } from "./domRealm";
|
||||
|
||||
export type LayerKind = "image" | "video" | "audio" | "vector" | "group" | "text" | "shape";
|
||||
|
||||
export interface EditingLayer {
|
||||
handle: EditingElementHandle;
|
||||
depth: number;
|
||||
childCount: number;
|
||||
zIndex: number;
|
||||
kind: LayerKind;
|
||||
}
|
||||
|
||||
export function isCompositionHost(el: Element): boolean {
|
||||
return el.hasAttribute("data-composition-src") || el.hasAttribute("data-composition-file");
|
||||
}
|
||||
|
||||
export function hasOnlyFormattingChildren(element: HTMLElement): boolean {
|
||||
for (const child of Array.from(element.children)) {
|
||||
if (!isRichTextFormattingTag(child.tagName)) return false;
|
||||
if (child.namespaceURI !== "http://www.w3.org/1999/xhtml") return false;
|
||||
// Formatting nests, and a structural child hidden inside a span is still structural.
|
||||
if (!hasOnlyFormattingChildren(child as HTMLElement)) return false;
|
||||
}
|
||||
return true;
|
||||
}
|
||||
|
||||
/** What a Layers row tells a person an element is: its media tag, a group, text, or else a shape. */
|
||||
export function layerKindOf(el: Element, hasChildren: boolean): LayerKind {
|
||||
const tag = el.tagName.toLowerCase();
|
||||
if (tag === "img" || tag === "picture") return "image";
|
||||
if (tag === "video") return "video";
|
||||
if (tag === "audio") return "audio";
|
||||
if (tag === "svg") return "vector";
|
||||
if (el.hasAttribute("data-hf-group") || isCompositionHost(el)) return "group";
|
||||
if (hasOnlyFormattingChildren(el as HTMLElement) && el.textContent?.trim()) return "text";
|
||||
return hasChildren ? "group" : "shape";
|
||||
}
|
||||
|
||||
/** The inline `z-index`, else the computed one; 0 for `auto` or none. */
|
||||
export function elementZIndex(element: HTMLElement): number {
|
||||
const inline = element.style?.zIndex;
|
||||
if (inline && inline !== "auto") {
|
||||
const parsed = parseInt(inline, 10);
|
||||
if (Number.isFinite(parsed)) return parsed;
|
||||
}
|
||||
const value = element.ownerDocument?.defaultView?.getComputedStyle(element).zIndex;
|
||||
if (!value || value === "auto") return 0;
|
||||
const parsed = parseInt(value, 10);
|
||||
return Number.isFinite(parsed) ? parsed : 0;
|
||||
}
|
||||
|
||||
const layerChildren = (el: Element) =>
|
||||
Array.from(el.children).filter(
|
||||
(child) => isElementNode(child) && isDomLayerElement(child as HTMLElement),
|
||||
).length;
|
||||
|
||||
/** Every layer under `roots`, in document order, with its depth among layers. */
|
||||
function walkLayers(
|
||||
roots: readonly Element[],
|
||||
max = Number.POSITIVE_INFINITY,
|
||||
): {
|
||||
layers: Array<{ element: HTMLElement; depth: number; childCount: number }>;
|
||||
truncated: boolean;
|
||||
} {
|
||||
const layers: Array<{ element: HTMLElement; depth: number; childCount: number }> = [];
|
||||
let truncated = false;
|
||||
const visit = (el: HTMLElement, depth: number) => {
|
||||
if (truncated || isClipHiddenByRuntime(el)) return;
|
||||
const layer = isDomLayerElement(el);
|
||||
if (layer) {
|
||||
if (layers.length === max) {
|
||||
truncated = true;
|
||||
return;
|
||||
}
|
||||
layers.push({ element: el, depth, childCount: layerChildren(el) });
|
||||
}
|
||||
for (const child of Array.from(el.children)) {
|
||||
if (isElementNode(child)) visit(child as HTMLElement, layer ? depth + 1 : depth);
|
||||
}
|
||||
};
|
||||
for (const root of roots) if (isElementNode(root)) visit(root as HTMLElement, 0);
|
||||
return { layers, truncated };
|
||||
}
|
||||
|
||||
/**
|
||||
* `layers`: the composition's layers, or the drilled-into group's when `group` names one, from the
|
||||
* `offset`th on. A `group` this page no longer has lists nothing, so Studio can leave the drill-in.
|
||||
*/
|
||||
export function layersOf(
|
||||
group?: EditingElementHandle,
|
||||
offset = 0,
|
||||
): { layers: EditingLayer[]; truncated: boolean; missingGroup: boolean } {
|
||||
const scope = group ? elementForHandle(group) : null;
|
||||
if (group && !scope) return { layers: [], truncated: false, missingGroup: true };
|
||||
const root = document.querySelector("[data-composition-id]") ?? document.documentElement;
|
||||
const roots = scope ? Array.from(scope.children) : root ? [root] : [];
|
||||
const walk = walkLayers(roots, offset + MAX_QUERY_MATCHES);
|
||||
return {
|
||||
truncated: walk.truncated,
|
||||
missingGroup: false,
|
||||
layers: walk.layers.slice(offset).map(({ element, depth, childCount }) => ({
|
||||
handle: handleFor(element),
|
||||
depth,
|
||||
childCount,
|
||||
zIndex: elementZIndex(element),
|
||||
kind: layerKindOf(element, childCount > 0),
|
||||
})),
|
||||
};
|
||||
}
|
||||
@@ -72,6 +72,7 @@ describe("createEditingPortClient", () => {
|
||||
"ancestors",
|
||||
"query",
|
||||
"geometry",
|
||||
"layers",
|
||||
],
|
||||
});
|
||||
} finally {
|
||||
|
||||
@@ -18,6 +18,7 @@ import {
|
||||
type EditingElementMatch,
|
||||
} from "./editingElement";
|
||||
import type { EditingElementBox, EditingElementGeometry } from "./editingGeometry";
|
||||
import type { EditingLayer } from "./editingLayers";
|
||||
import type { RuntimeTimelineMessage } from "./types";
|
||||
import { wallClearTimeout, wallSetTimeout } from "./wallClock";
|
||||
|
||||
@@ -66,6 +67,14 @@ export {
|
||||
type EditingLocalPointer,
|
||||
type EditingViewport,
|
||||
} from "./editingTargets";
|
||||
export {
|
||||
elementZIndex,
|
||||
hasOnlyFormattingChildren,
|
||||
isCompositionHost,
|
||||
layerKindOf,
|
||||
type EditingLayer,
|
||||
type LayerKind,
|
||||
} from "./editingLayers";
|
||||
export {
|
||||
composeElementTransform,
|
||||
individualRotateDegrees,
|
||||
@@ -90,6 +99,7 @@ export const EDITING_REQUEST_KINDS = [
|
||||
"ancestors",
|
||||
"query",
|
||||
"geometry",
|
||||
"layers",
|
||||
] as const;
|
||||
export type EditingRequestKind = (typeof EDITING_REQUEST_KINDS)[number];
|
||||
|
||||
@@ -153,6 +163,12 @@ export interface EditingRequests {
|
||||
children: EditingElementBox[] | null;
|
||||
};
|
||||
};
|
||||
/** The Layers walk from `offset`, MAX_QUERY_MATCHES at a time, scoped to a drilled-into `group`;
|
||||
* `missingGroup` says this page has no such group, and lists nothing. */
|
||||
layers: {
|
||||
params: { group?: EditingElementHandle; offset?: number };
|
||||
result: { layers: EditingLayer[]; truncated: boolean; missingGroup: boolean };
|
||||
};
|
||||
}
|
||||
export type EditingParams<K extends EditingRequestKind> = EditingRequests[K]["params"];
|
||||
export type EditingResult<K extends EditingRequestKind> = EditingRequests[K]["result"];
|
||||
@@ -230,6 +246,21 @@ function point(params: unknown): { x: number; y: number } | null {
|
||||
return Number.isFinite(x) && Number.isFinite(y) ? { x: x as number, y: y as number } : null;
|
||||
}
|
||||
|
||||
const isOffset = (value: unknown): value is number =>
|
||||
Number.isSafeInteger(value) && (value as number) >= 0;
|
||||
|
||||
function layerParams(params: unknown): EditingParams<"layers"> | null {
|
||||
if (!isPlainRecord(params)) return null;
|
||||
const { group, offset, ...rest } = params;
|
||||
const valid = [
|
||||
Object.keys(rest).length === 0,
|
||||
group === undefined || isHandle(group),
|
||||
offset === undefined || isOffset(offset),
|
||||
];
|
||||
if (valid.includes(false)) return null;
|
||||
return { ...(isHandle(group) && { group }), ...(isOffset(offset) && { offset }) };
|
||||
}
|
||||
|
||||
const PARAM_PARSERS: {
|
||||
[K in EditingRequestKind]: (params: unknown) => EditingParams<K> | null;
|
||||
} = {
|
||||
@@ -265,6 +296,7 @@ const PARAM_PARSERS: {
|
||||
if (!list || !("children" in params)) return list;
|
||||
return isHandle(children) ? { ...list, children } : null;
|
||||
},
|
||||
layers: (params) => noParams(params) ?? layerParams(params),
|
||||
};
|
||||
|
||||
/** Validates one request off the wire. Never throws: anything unusable becomes a typed error. */
|
||||
|
||||
@@ -1,5 +1,6 @@
|
||||
import { afterEach, describe, expect, it } from "vitest";
|
||||
import { MAX_POINT_STACK } from "./editingElement";
|
||||
import { elementForHandle } from "./editingHandles";
|
||||
import { pointerTargetsAt } from "./editingTargets";
|
||||
|
||||
function sized(element: Element, [x, y, width, height]: number[]) {
|
||||
@@ -36,6 +37,19 @@ describe("pointer-targets", () => {
|
||||
expect(asked[0]).toEqual([480, 540]);
|
||||
expect(answer.target).toMatchObject({ tag: "p", rect: { x: 10, y: 20, width: 300 } });
|
||||
expect(answer.stack.map((hit) => hit.handle)).toEqual([answer.target!.handle]);
|
||||
expect(asked).toHaveLength(1);
|
||||
});
|
||||
|
||||
it("reads the stack once, and falls back to a group's gap only when it is empty", () => {
|
||||
document.body.innerHTML = `<main data-composition-id="m"><div id="g" data-hf-group="g"><i id="a"></i><i id="b"></i></div></main>`;
|
||||
sized(document.querySelector("main")!, [0, 0, 1920, 1080]);
|
||||
sized(document.getElementById("a")!, [0, 0, 100, 100]);
|
||||
sized(document.getElementById("b")!, [400, 400, 100, 100]);
|
||||
stackAt([]);
|
||||
const answer = pointerTargetsAt(0.1, 0.2);
|
||||
expect(asked).toHaveLength(1);
|
||||
expect(answer.stack).toEqual([]);
|
||||
expect(elementForHandle(answer.target!.handle)).toBe(document.getElementById("g"));
|
||||
});
|
||||
|
||||
it("caps the stack, and answers nothing when the composition has no size", () => {
|
||||
|
||||
@@ -269,7 +269,10 @@ export function pointerTarget(doc: Document, point: EditingLocalPointer): HTMLEl
|
||||
const visualTarget = candidates.find((el) => !isFullBleedTarget(el, point.viewport)) ?? null;
|
||||
if (visualTarget) return visualTarget;
|
||||
}
|
||||
return pointerFallbackTarget(doc, point);
|
||||
}
|
||||
|
||||
function pointerFallbackTarget(doc: Document, point: EditingLocalPointer): HTMLElement | null {
|
||||
const groupHit = findGroupAtPoint(doc, point.x, point.y);
|
||||
if (groupHit && !hasAuthorPointerEventsNone(groupHit) && isDomLayerElement(groupHit))
|
||||
return groupHit;
|
||||
@@ -316,7 +319,11 @@ export function pointerTargetsAt(
|
||||
): { target: EditingElementHit | null; stack: EditingElementHit[] } {
|
||||
const point = compositionPoint(document, x, y);
|
||||
if (!point) return { target: null, stack: [] };
|
||||
const target = pointerTarget(document, point);
|
||||
const stack = pointerTargets(document, point).slice(0, MAX_POINT_STACK);
|
||||
return { target: target && elementHit(target), stack: stack.map(elementHit) };
|
||||
const stack = pointerTargets(document, point);
|
||||
const visual = typeof document.elementsFromPoint === "function" ? stack[0] : undefined;
|
||||
const target = visual ?? pointerFallbackTarget(document, point);
|
||||
return {
|
||||
target: target && elementHit(target),
|
||||
stack: stack.slice(0, MAX_POINT_STACK).map(elementHit),
|
||||
};
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user