mirror of
https://github.com/heygen-com/hyperframes.git
synced 2026-11-18 16:08:56 +00:00
fix(studio): caption editing works in a sealed preview (#8002)
* test(studio): caption overlay shows and edits words in a sealed preview * fix(studio): caption editing reads and moves words through the editing port in a sealed preview * style(studio): format the caption port change and trim its comments * fix(studio): keep the caption port change inside its files and under the size checks * fix(studio): keep late caption frame answers from writing after Studio moves on A caption detection, nudge or drag still waiting on a sealed preview no longer writes after its effect is gone, and a frame-reported word transform past the frame's own write bound is refused instead of stored. Each has a test against the real editing agent that holds the frame's answer open. * refactor(studio): move the sealed caption nudge queue next to the frame word helpers Keeps the caption overlay under the studio file size limit and its comment share at the base. No behaviour change. * refactor(studio): keep the sealed caption nudge request inside its helper file
This commit is contained in:
@@ -0,0 +1,118 @@
|
||||
// `caption-transform`: reads and moves caption words, on a wrapper so the word's own tweens keep running.
|
||||
import { MAX_DESCRIBE_HANDLES, type EditingRect } from "../editingElement";
|
||||
import { captionWordSpans, isCaptionIndex, isRecord } from "./captionWords";
|
||||
|
||||
export interface CaptionTransform {
|
||||
x: number;
|
||||
y: number;
|
||||
scale: number;
|
||||
rotation: number;
|
||||
}
|
||||
|
||||
/** `set` writes those parts and keeps the rest; without it the word is only read. */
|
||||
export interface CaptionTransformEntry {
|
||||
group: number;
|
||||
word: number;
|
||||
set?: Partial<CaptionTransform>;
|
||||
}
|
||||
export type CaptionTransformParams = { entries: CaptionTransformEntry[] };
|
||||
|
||||
/** The wrapper's transform after any write, and the word's box; null for a word the frame lacks. */
|
||||
export type CaptionWordState = { transform: CaptionTransform; rect: EditingRect } | null;
|
||||
|
||||
const REST: CaptionTransform = { x: 0, y: 0, scale: 1, rotation: 0 };
|
||||
const PARTS = new Set(Object.keys(REST));
|
||||
/** Largest |x|, |y|, scale or rotation a caption word's transform may hold, in either direction. */
|
||||
export const MAX_CAPTION_TRANSFORM = 100_000;
|
||||
|
||||
const isBounded = (value: unknown): value is number =>
|
||||
Number.isFinite(value) && Math.abs(value as number) <= MAX_CAPTION_TRANSFORM;
|
||||
|
||||
function transformPart(set: unknown): Partial<CaptionTransform> | null {
|
||||
if (!isRecord(set)) return null;
|
||||
const valid = Object.entries(set).every(([key, v]) => PARTS.has(key) && isBounded(v));
|
||||
return valid ? (set as Partial<CaptionTransform>) : null;
|
||||
}
|
||||
|
||||
function entry(value: unknown): CaptionTransformEntry | null {
|
||||
if (!isRecord(value)) return null;
|
||||
const { group, word, set, ...rest } = value;
|
||||
if (Object.keys(rest).length || !isCaptionIndex(group) || !isCaptionIndex(word)) return null;
|
||||
if (!("set" in value)) return { group, word };
|
||||
const part = transformPart(set);
|
||||
return part ? { group, word, set: part } : null;
|
||||
}
|
||||
|
||||
export function captionTransformParams(params: unknown): CaptionTransformParams | null {
|
||||
if (!isRecord(params) || Object.keys(params).join() !== "entries") return null;
|
||||
const { entries } = params;
|
||||
if (!Array.isArray(entries) || !entries.length || entries.length > MAX_DESCRIBE_HANDLES)
|
||||
return null;
|
||||
const parsed = entries.map(entry);
|
||||
return parsed.includes(null) ? null : { entries: parsed as CaptionTransformEntry[] };
|
||||
}
|
||||
|
||||
type Gsap = {
|
||||
getProperty?: (el: Element, prop: string) => number;
|
||||
set?: (el: Element, props: Record<string, number>) => void;
|
||||
};
|
||||
const frameGsap = () => (window as unknown as { gsap?: Gsap }).gsap;
|
||||
|
||||
const CSS_PARTS: Array<[keyof CaptionTransform, RegExp]> = [
|
||||
["x", /translate\(([^,]+)px,\s*[^)]+px\)/],
|
||||
["y", /translate\([^,]+px,\s*([^)]+)px\)/],
|
||||
["scale", /scale\(([^)]+)\)/],
|
||||
["rotation", /rotate\(([^)]+)deg\)/],
|
||||
];
|
||||
|
||||
/** GSAP keeps transforms in its own cache, so read through it when the page has it. */
|
||||
function readTransform(el: HTMLElement): CaptionTransform {
|
||||
const gsap = frameGsap();
|
||||
const read = { ...REST };
|
||||
for (const [part, pattern] of CSS_PARTS) {
|
||||
if (gsap?.getProperty) read[part] = gsap.getProperty(el, part) || REST[part];
|
||||
else {
|
||||
const match = (el.style.transform || "").match(pattern);
|
||||
if (match) read[part] = parseFloat(match[1]!);
|
||||
}
|
||||
}
|
||||
return read;
|
||||
}
|
||||
|
||||
function writeTransform(el: HTMLElement, { x, y, scale, rotation }: CaptionTransform): void {
|
||||
const gsap = frameGsap();
|
||||
if (gsap?.set) gsap.set(el, { x, y, scale, rotation });
|
||||
else
|
||||
el.style.transform = `translate(${x.toFixed(1)}px, ${y.toFixed(1)}px) rotate(${rotation.toFixed(1)}deg) scale(${scale.toFixed(3)})`;
|
||||
}
|
||||
|
||||
function wrapperOf(word: HTMLElement): HTMLElement {
|
||||
if (word.dataset.captionWrapper === "true") return word;
|
||||
const parent = word.parentElement;
|
||||
if (parent?.dataset.captionWrapper === "true") return parent;
|
||||
const wrapper = document.createElement("span");
|
||||
wrapper.style.display = "inline-block";
|
||||
wrapper.dataset.captionWrapper = "true";
|
||||
word.parentNode?.insertBefore(wrapper, word);
|
||||
wrapper.appendChild(word);
|
||||
return wrapper;
|
||||
}
|
||||
|
||||
function wordAt(group: number, word: number): HTMLElement | null {
|
||||
const el = document.querySelectorAll<HTMLElement>(".caption-group")[group];
|
||||
return el ? (captionWordSpans(el)[word] ?? null) : null;
|
||||
}
|
||||
|
||||
export function captionTransform({ entries }: CaptionTransformParams): {
|
||||
words: CaptionWordState[];
|
||||
} {
|
||||
const words = entries.map(({ group, word, set }) => {
|
||||
const el = wordAt(group, word);
|
||||
if (!el) return null;
|
||||
const wrapper = wrapperOf(el);
|
||||
if (set) writeTransform(wrapper, { ...readTransform(wrapper), ...set });
|
||||
const { left, top, width, height } = el.getBoundingClientRect();
|
||||
return { transform: readTransform(wrapper), rect: { x: left, y: top, width, height } };
|
||||
});
|
||||
return { words };
|
||||
}
|
||||
@@ -0,0 +1,59 @@
|
||||
import { afterEach, expect, it } from "vitest";
|
||||
import { captionWords } from "./captionWords";
|
||||
import { captionTransform } from "./captionTransform";
|
||||
|
||||
afterEach(() => {
|
||||
document.body.innerHTML = "";
|
||||
Reflect.deleteProperty(window, "gsap");
|
||||
});
|
||||
|
||||
it("splits a plain-text group into word spans and boxes only visible groups' words", () => {
|
||||
document.body.innerHTML = `<div class="caption-group">Hello big world</div>
|
||||
<div class="caption-group" style="opacity: 0"><span>hidden</span></div>
|
||||
<div class="caption-group"><span>one</span><span>two</span></div>`;
|
||||
const { boxes } = captionWords({ words: [2, 1, 5] });
|
||||
expect(boxes.map(({ group, word }) => [group, word])).toEqual([
|
||||
[0, 0],
|
||||
[0, 1],
|
||||
[2, 0],
|
||||
[2, 1],
|
||||
]);
|
||||
const [first, , third] = document.querySelectorAll(".caption-group");
|
||||
expect([...first!.children].map((s) => s.textContent)).toEqual(["Hello ", "big ", "world "]);
|
||||
expect(third!.innerHTML).toBe("<span>one</span><span>two</span>");
|
||||
});
|
||||
|
||||
it("moves a word on a wrapper with a CSS transform, keeping the parts it was not given", () => {
|
||||
document.body.innerHTML = `<div class="caption-group"><span>a</span><span>b</span></div>`;
|
||||
const moved = captionTransform({ entries: [{ group: 0, word: 1, set: { x: 3, rotation: 9 } }] });
|
||||
expect(moved.words[0]!.transform).toEqual({ x: 3, y: 0, scale: 1, rotation: 9 });
|
||||
const b = document.querySelectorAll("span:not([data-caption-wrapper])")[1]!;
|
||||
expect((b.parentElement as HTMLElement).style.transform).toBe(
|
||||
"translate(3.0px, 0.0px) rotate(9.0deg) scale(1.000)",
|
||||
);
|
||||
const again = captionTransform({ entries: [{ group: 0, word: 1, set: { y: 4 } }] });
|
||||
expect(again.words[0]!.transform).toEqual({ x: 3, y: 4, scale: 1, rotation: 9 });
|
||||
expect(document.querySelectorAll("[data-caption-wrapper]")).toHaveLength(1);
|
||||
expect(
|
||||
captionTransform({
|
||||
entries: [
|
||||
{ group: 0, word: 2 },
|
||||
{ group: 1, word: 0 },
|
||||
],
|
||||
}).words,
|
||||
).toEqual([null, null]);
|
||||
});
|
||||
|
||||
it("moves a word through the page's GSAP when it has one", () => {
|
||||
const props = new WeakMap<Element, Record<string, number>>();
|
||||
Reflect.set(window, "gsap", {
|
||||
getProperty: (el: Element, prop: string) => props.get(el)?.[prop] ?? 0,
|
||||
set: (el: Element, values: Record<string, number>) => props.set(el, values),
|
||||
});
|
||||
document.body.innerHTML = `<div class="caption-group"><span>a</span></div>`;
|
||||
const { words } = captionTransform({ entries: [{ group: 0, word: 0, set: { scale: 2 } }] });
|
||||
expect(words[0]!.transform).toEqual({ x: 0, y: 0, scale: 2, rotation: 0 });
|
||||
const wrapper = document.querySelector<HTMLElement>("[data-caption-wrapper]")!;
|
||||
expect(props.get(wrapper)).toEqual({ x: 0, y: 0, scale: 2, rotation: 0 });
|
||||
expect(wrapper.style.transform).toBe("");
|
||||
});
|
||||
@@ -0,0 +1,89 @@
|
||||
// `caption-words`: the caption editor's word boxes, read where the words live.
|
||||
import { MAX_QUERY_MATCHES, type EditingRect } from "../editingElement";
|
||||
|
||||
const MAX_CAPTION_WORDS = 100_000;
|
||||
|
||||
export const isRecord = (value: unknown): value is Record<string, unknown> =>
|
||||
Object.prototype.toString.call(value) === "[object Object]";
|
||||
|
||||
/** Word `word` of the `group`-th `.caption-group`, both counted from 0. */
|
||||
export interface CaptionWordBox {
|
||||
group: number;
|
||||
word: number;
|
||||
/** `getBoundingClientRect()` of the word, in the frame's CSS pixels. */
|
||||
rect: EditingRect;
|
||||
}
|
||||
|
||||
/** `words[i]` is how many words Studio's model has in the i-th `.caption-group`. */
|
||||
export type CaptionWordsParams = { words: number[] };
|
||||
|
||||
export const isCaptionIndex = (value: unknown): value is number =>
|
||||
Number.isSafeInteger(value) && (value as number) >= 0 && (value as number) < MAX_CAPTION_WORDS;
|
||||
|
||||
export function captionWordsParams(params: unknown): CaptionWordsParams | null {
|
||||
if (!isRecord(params) || Object.keys(params).join() !== "words") return null;
|
||||
const { words } = params;
|
||||
if (!Array.isArray(words) || words.length > MAX_CAPTION_WORDS) return null;
|
||||
return words.every(isCaptionIndex) ? { words } : null;
|
||||
}
|
||||
|
||||
/** A group's word spans in order: a wrapped word is the span inside its wrapper. */
|
||||
export function captionWordSpans(group: HTMLElement): HTMLElement[] {
|
||||
const spans: HTMLElement[] = [];
|
||||
for (const child of group.children) {
|
||||
const el = child as HTMLElement;
|
||||
if (el.dataset.captionWrapper === "true") {
|
||||
const inner = el.querySelector<HTMLElement>(":scope > span");
|
||||
if (inner) spans.push(inner);
|
||||
} else if (el.tagName === "SPAN") {
|
||||
spans.push(el);
|
||||
}
|
||||
}
|
||||
return spans;
|
||||
}
|
||||
|
||||
function splitIntoSpans(text: string, replace: (fragment: DocumentFragment) => void) {
|
||||
const fragment = document.createDocumentFragment();
|
||||
const spans = text
|
||||
.split(/\s+/)
|
||||
.filter(Boolean)
|
||||
.map((word) => {
|
||||
const span = document.createElement("span");
|
||||
span.textContent = word + " ";
|
||||
span.style.display = "inline";
|
||||
return fragment.appendChild(span);
|
||||
});
|
||||
replace(fragment);
|
||||
return spans;
|
||||
}
|
||||
|
||||
/** A group written as plain text becomes one span per word, so each word can be picked. */
|
||||
function splitGroupText(group: HTMLElement): HTMLElement[] {
|
||||
const text = group.childNodes[0];
|
||||
if (text?.nodeType === Node.TEXT_NODE) {
|
||||
return splitIntoSpans(text.textContent || "", (f) => group.replaceChild(f, text));
|
||||
}
|
||||
const single = group.querySelector<HTMLElement>(":scope > span");
|
||||
if (!single?.textContent?.trim()) return [];
|
||||
return splitIntoSpans(single.textContent, (f) => single.replaceWith(f));
|
||||
}
|
||||
|
||||
/** Boxes of the words in visible groups, up to MAX_QUERY_MATCHES. */
|
||||
export function captionWords({ words }: CaptionWordsParams): { boxes: CaptionWordBox[] } {
|
||||
const groups = document.querySelectorAll<HTMLElement>(".caption-group");
|
||||
const boxes: CaptionWordBox[] = [];
|
||||
words.forEach((count, group) => {
|
||||
const el = groups[group];
|
||||
if (!el) return;
|
||||
const computed = window.getComputedStyle(el);
|
||||
if (parseFloat(computed.opacity) <= 0.01 || computed.visibility === "hidden") return;
|
||||
let spans = captionWordSpans(el);
|
||||
if (spans.length === 0 && el.textContent?.trim()) spans = splitGroupText(el);
|
||||
for (let word = 0; word < Math.min(count, spans.length); word++) {
|
||||
if (boxes.length >= MAX_QUERY_MATCHES) return;
|
||||
const { left, top, width, height } = spans[word]!.getBoundingClientRect();
|
||||
boxes.push({ group, word, rect: { x: left, y: top, width, height } });
|
||||
}
|
||||
});
|
||||
return { boxes };
|
||||
}
|
||||
@@ -170,6 +170,8 @@ describe("installEditingAgent", () => {
|
||||
"console-errors",
|
||||
"watch-keys",
|
||||
"restore-attributes",
|
||||
"caption-words",
|
||||
"caption-transform",
|
||||
],
|
||||
},
|
||||
});
|
||||
@@ -402,6 +404,28 @@ describe("installEditingAgent", () => {
|
||||
"invalid-params",
|
||||
],
|
||||
),
|
||||
...[{}, { words: 3 }, { words: [-1] }, { words: [1.5] }, { words: [100_000] }].map(
|
||||
(params, index) => [
|
||||
{ v: 1, id: 190 + index, kind: "caption-words", params },
|
||||
190 + index,
|
||||
"invalid-params",
|
||||
],
|
||||
),
|
||||
...[
|
||||
{ entries: [] },
|
||||
{ entries: Array(257).fill({ group: 0, word: 0 }) },
|
||||
{ entries: [{ group: -1, word: 0 }] },
|
||||
{ entries: [{ group: 0, word: 0, extra: 1 }] },
|
||||
{ entries: [{ group: 0, word: 0, set: { x: 100_001 } }] },
|
||||
{ entries: [{ group: 0, word: 0, set: { x: Infinity } }] },
|
||||
{ entries: [{ group: 0, word: 0, set: { x: undefined } }] },
|
||||
{ entries: [{ group: 0, word: 0, set: { skew: 1 } }] },
|
||||
{ entries: [{ group: 0, word: 0, set: [] }] },
|
||||
].map((params, index) => [
|
||||
{ v: 1, id: 200 + index, kind: "caption-transform", params },
|
||||
200 + index,
|
||||
"invalid-params",
|
||||
]),
|
||||
])("rejects %j with a typed error", async (request, id, code) => {
|
||||
const setLoopStart = vi.fn();
|
||||
window.__hf = { setLoopStart };
|
||||
|
||||
@@ -43,6 +43,8 @@ import {
|
||||
import { MAX_QUERY_MATCHES } from "./editingElement";
|
||||
import { matchCombo, parseCombo, type Combo } from "../keyCombo";
|
||||
import { shouldIgnorePlaybackShortcutTarget } from "../typingTarget";
|
||||
import { captionWords } from "./editing/captionWords";
|
||||
import { captionTransform } from "./editing/captionTransform";
|
||||
|
||||
type Handlers = {
|
||||
[K in EditingRequestKind]: (
|
||||
@@ -190,6 +192,8 @@ const HANDLERS: Handlers = {
|
||||
},
|
||||
"watch-keys": ({ combos }, port) => watchKeys(port, combos),
|
||||
"restore-attributes": (params) => restoreAttributes(params),
|
||||
"caption-words": captionWords,
|
||||
"caption-transform": captionTransform,
|
||||
};
|
||||
|
||||
function serve(port: MessagePort): void {
|
||||
|
||||
@@ -78,6 +78,8 @@ describe("createEditingPortClient", () => {
|
||||
"console-errors",
|
||||
"watch-keys",
|
||||
"restore-attributes",
|
||||
"caption-words",
|
||||
"caption-transform",
|
||||
],
|
||||
});
|
||||
} finally {
|
||||
|
||||
@@ -24,6 +24,16 @@ import type { EditingLayer } from "./editingLayers";
|
||||
import type { RuntimeTimelineMessage } from "./types";
|
||||
import { wallClearTimeout, wallSetTimeout } from "./wallClock";
|
||||
import { parseCombo } from "../keyCombo";
|
||||
import {
|
||||
captionWordsParams,
|
||||
type CaptionWordBox,
|
||||
type CaptionWordsParams,
|
||||
} from "./editing/captionWords";
|
||||
import {
|
||||
captionTransformParams,
|
||||
type CaptionTransformParams,
|
||||
type CaptionWordState,
|
||||
} from "./editing/captionTransform";
|
||||
|
||||
export {
|
||||
buildStableSelector,
|
||||
@@ -61,6 +71,7 @@ export {
|
||||
type ElementTransformSnapshot,
|
||||
} from "./editingGeometry";
|
||||
export type { EditingGestureStart } from "./editingGestureStart";
|
||||
export { MAX_CAPTION_TRANSFORM } from "./editing/captionTransform";
|
||||
export {
|
||||
compositionPoint,
|
||||
coversComposition,
|
||||
@@ -111,6 +122,8 @@ export const EDITING_REQUEST_KINDS = [
|
||||
"console-errors",
|
||||
"watch-keys",
|
||||
"restore-attributes",
|
||||
"caption-words",
|
||||
"caption-transform",
|
||||
] as const;
|
||||
export type EditingRequestKind = (typeof EDITING_REQUEST_KINDS)[number];
|
||||
|
||||
@@ -220,6 +233,9 @@ export interface EditingRequests {
|
||||
* keyups to this port; an empty list stops. */
|
||||
"watch-keys": { params: { combos: string[] }; result: { watching: number } };
|
||||
"restore-attributes": { params: RestoreAttributesParams; result: { applied: boolean } };
|
||||
/** Boxes of the words of visible caption groups; a group written as plain text is split into word spans. */
|
||||
"caption-words": { params: CaptionWordsParams; result: { boxes: CaptionWordBox[] } };
|
||||
"caption-transform": { params: CaptionTransformParams; result: { words: CaptionWordState[] } };
|
||||
}
|
||||
export type EditingParams<K extends EditingRequestKind> = EditingRequests[K]["params"];
|
||||
export type EditingResult<K extends EditingRequestKind> = EditingRequests[K]["result"];
|
||||
@@ -407,6 +423,8 @@ const PARAM_PARSERS: {
|
||||
hasKeys(params, "from") && isOffset(params.from) ? { from: params.from } : null,
|
||||
"watch-keys": comboList,
|
||||
"restore-attributes": parseRestoreAttributes,
|
||||
"caption-words": captionWordsParams,
|
||||
"caption-transform": captionTransformParams,
|
||||
};
|
||||
|
||||
/** Validates one request off the wire. Never throws: anything unusable becomes a typed error. */
|
||||
|
||||
@@ -0,0 +1,273 @@
|
||||
// @vitest-environment happy-dom
|
||||
import { act, createElement, Fragment, useRef } from "react";
|
||||
import { createRoot } from "react-dom/client";
|
||||
import { afterEach, expect, it, vi } from "vitest";
|
||||
// The frame's real runtime agent, serving the opaque frame from this document.
|
||||
import { installEditingAgent } from "../../../../core/src/runtime/editingAgent";
|
||||
import { installReactActEnvironment } from "../../hooks/domSelectionTestHarness";
|
||||
import { detachedFakeIframe } from "../../player/hooks/timelinePlayerTestHarness";
|
||||
import { previewRequest } from "../../player/lib/previewChannel";
|
||||
import { usePreviewIframeStore } from "../../player/store/previewIframeStore";
|
||||
import { useCaptionDetection } from "../../hooks/useCaptionDetection";
|
||||
import { studioApiFetch } from "../../utils/studioApiFetch";
|
||||
import type { useCaptionSync } from "../hooks/useCaptionSync";
|
||||
import { useCaptionStore } from "../store";
|
||||
import { CaptionOverlay } from "./CaptionOverlay";
|
||||
import { applyCaptionModelToIframe } from "./CaptionOverlayUtils";
|
||||
import { CaptionPropertyPanel } from "./CaptionPropertyPanel";
|
||||
|
||||
installReactActEnvironment();
|
||||
|
||||
const SOURCE = `const TRANSCRIPT = [
|
||||
{ text: "Hello", start: 0, end: 0.5 },
|
||||
{ text: "world", start: 0.5, end: 1 },
|
||||
];`;
|
||||
vi.mock("../../utils/studioApiFetch", () => ({
|
||||
studioApiFetch: vi.fn(async () => ({ json: async () => ({ content: SOURCE }) })),
|
||||
}));
|
||||
|
||||
const closers: Array<() => void> = [];
|
||||
afterEach(() => {
|
||||
closers.splice(0).forEach((close) => close());
|
||||
useCaptionStore.getState().reset();
|
||||
usePreviewIframeStore.setState({ iframe: null });
|
||||
document.body.innerHTML = "";
|
||||
vi.unstubAllGlobals();
|
||||
});
|
||||
|
||||
type PortRequest = { id?: unknown; kind?: unknown; params?: unknown };
|
||||
|
||||
/** A frame Studio cannot read, whose runtime is the real editing agent over this document. */
|
||||
function opaqueFrame() {
|
||||
const frame: Record<string, unknown> = {};
|
||||
const iframe = detachedFakeIframe(frame);
|
||||
Object.defineProperty(iframe, "contentDocument", { value: null });
|
||||
Object.defineProperty(frame, "document", {
|
||||
get: () => {
|
||||
throw new DOMException("blocked", "SecurityError");
|
||||
},
|
||||
});
|
||||
let pick: ((request: PortRequest) => boolean) | null = null;
|
||||
const heldIds = new Set<unknown>();
|
||||
const held: Array<() => void> = [];
|
||||
// Studio's end of the port, so a test can keep the frame's answer to a request until it releases it.
|
||||
const tap = (port: MessagePort) => {
|
||||
const end = {
|
||||
onmessage: null as ((event: MessageEvent) => void) | null,
|
||||
postMessage(request: PortRequest) {
|
||||
if (pick?.(request)) heldIds.add(request.id);
|
||||
port.postMessage(request);
|
||||
},
|
||||
close: () => port.close(),
|
||||
};
|
||||
port.onmessage = (event) => {
|
||||
const deliver = () => end.onmessage?.(event);
|
||||
if (heldIds.has(event.data?.id)) held.push(deliver);
|
||||
else deliver();
|
||||
};
|
||||
return end as unknown as MessagePort;
|
||||
};
|
||||
const parent = {
|
||||
postMessage: (data: unknown, _origin: string, ports: MessagePort[]) =>
|
||||
window.dispatchEvent(
|
||||
new MessageEvent("message", {
|
||||
source: frame as unknown as Window,
|
||||
data,
|
||||
ports: ports.map(tap),
|
||||
}),
|
||||
),
|
||||
} as unknown as Window;
|
||||
const agent = installEditingAgent(parent);
|
||||
frame.postMessage = (data: unknown) =>
|
||||
window.dispatchEvent(new MessageEvent("message", { source: parent, data }));
|
||||
closers.push(() => agent.dispose());
|
||||
iframe.style.width = "1920px";
|
||||
iframe.getBoundingClientRect = () => new DOMRect(0, 0, 1920, 1080);
|
||||
/** Keeps the answers to the requests `match` picks; the returned call hands them to Studio. */
|
||||
const hold = (match: (request: PortRequest) => boolean) => {
|
||||
pick = match;
|
||||
return () => {
|
||||
pick = null;
|
||||
heldIds.clear();
|
||||
held.splice(0).forEach((deliver) => deliver());
|
||||
};
|
||||
};
|
||||
return { iframe, hold, held };
|
||||
}
|
||||
|
||||
/** Settles after one frame round trip, once Studio has acted on every answer it was handed before. */
|
||||
const frameRoundTrip = (iframe: HTMLIFrameElement) =>
|
||||
previewRequest(iframe, "query", { selector: "body" });
|
||||
|
||||
const isWrite = ({ kind, params }: PortRequest) =>
|
||||
kind === "caption-transform" &&
|
||||
(params as { entries: Array<{ set?: unknown }> }).entries.some((entry) => entry.set);
|
||||
|
||||
const compIdToSrc = new Map([["captions", "compositions/captions.html"]]);
|
||||
const captionSync = { loadOverrides: vi.fn() } as unknown as ReturnType<typeof useCaptionSync>;
|
||||
|
||||
function CaptionEditor({ iframe }: { iframe: HTMLIFrameElement }) {
|
||||
const iframeRef = useRef(iframe);
|
||||
useCaptionDetection({
|
||||
projectId: "p",
|
||||
activeCompPath: "index.html",
|
||||
compIdToSrc,
|
||||
captionEditMode: false,
|
||||
captionHasSelection: false,
|
||||
previewIframeRef: iframeRef,
|
||||
captionSync,
|
||||
setRightCollapsed: () => {},
|
||||
});
|
||||
const editing = useCaptionStore((s) => s.isEditMode);
|
||||
// Studio mounts both only in caption edit mode.
|
||||
if (!editing) return null;
|
||||
return createElement(
|
||||
Fragment,
|
||||
null,
|
||||
createElement(CaptionOverlay, { iframeRef }),
|
||||
createElement(CaptionPropertyPanel, { iframeRef }),
|
||||
);
|
||||
}
|
||||
|
||||
const pointer = (type: string, x: number, y: number) =>
|
||||
new PointerEvent(type, { bubbles: true, clientX: x, clientY: y, pointerId: 1 });
|
||||
|
||||
const WORDS = `<span class="word">Hello</span> <span class="word">world</span>`;
|
||||
|
||||
/** Mounts Studio's caption editing over an opaque frame holding one caption group of `words`. */
|
||||
function mountEditor(words = WORDS) {
|
||||
vi.stubGlobal(
|
||||
"ResizeObserver",
|
||||
class {
|
||||
observe() {}
|
||||
disconnect() {}
|
||||
},
|
||||
);
|
||||
Element.prototype.setPointerCapture ??= () => {};
|
||||
document.body.innerHTML = `<div data-composition-id="captions" data-width="1920" data-height="1080" data-duration="1">
|
||||
<div class="caption-group">${words}</div></div>`;
|
||||
const frame = opaqueFrame();
|
||||
usePreviewIframeStore.setState({ iframe: frame.iframe });
|
||||
const host = document.createElement("div");
|
||||
document.body.append(host);
|
||||
const root = createRoot(host);
|
||||
act(() => root.render(createElement(CaptionEditor, { iframe: frame.iframe })));
|
||||
const unmount = () => act(() => root.unmount());
|
||||
closers.push(unmount);
|
||||
const boxes = () =>
|
||||
[...host.querySelectorAll<HTMLElement>("div.absolute")].filter((el) => el.style.cursor);
|
||||
return { ...frame, host, boxes, unmount };
|
||||
}
|
||||
|
||||
async function openEditor(words?: string) {
|
||||
const editor = mountEditor(words);
|
||||
await vi.waitFor(() => expect(editor.boxes()).toHaveLength(2), { timeout: 3_000 });
|
||||
return editor;
|
||||
}
|
||||
|
||||
const styleOf = (segmentId: string) =>
|
||||
useCaptionStore.getState().model!.segments.get(segmentId)!.style;
|
||||
|
||||
it("caption overlay shows + edits sealed", async () => {
|
||||
const { host, boxes } = await openEditor();
|
||||
const hello = document.querySelector<HTMLElement>(".word")!;
|
||||
|
||||
act(() => boxes()[0]!.click());
|
||||
const segmentId = [...useCaptionStore.getState().selectedSegmentIds][0]!;
|
||||
const overlay = boxes()[0]!.parentElement!;
|
||||
// The move lands before the frame answers the press's read; it must still draw.
|
||||
act(() => void boxes()[0]!.dispatchEvent(pointer("pointerdown", 100, 100)));
|
||||
act(() => void overlay.dispatchEvent(pointer("pointermove", 130, 110)));
|
||||
act(() => void overlay.dispatchEvent(pointer("pointerup", 130, 110)));
|
||||
const transform = () => hello.parentElement!.style.transform;
|
||||
await vi.waitFor(() =>
|
||||
expect(transform()).toBe("translate(30.0px, 10.0px) rotate(0.0deg) scale(1.000)"),
|
||||
);
|
||||
await vi.waitFor(() =>
|
||||
expect(useCaptionStore.getState().model!.segments.get(segmentId)!.style).toMatchObject({
|
||||
x: 30,
|
||||
y: 10,
|
||||
}),
|
||||
);
|
||||
|
||||
const xField = host.querySelector<HTMLInputElement>('input[aria-label="X position"]')!;
|
||||
const setValue = Object.getOwnPropertyDescriptor(HTMLInputElement.prototype, "value")!.set!;
|
||||
act(() => {
|
||||
setValue.call(xField, "50");
|
||||
xField.dispatchEvent(new Event("input", { bubbles: true }));
|
||||
});
|
||||
await vi.waitFor(() => expect(transform()).toContain("translate(50.0px, 10.0px)"));
|
||||
|
||||
act(() => void window.dispatchEvent(new KeyboardEvent("keydown", { key: "ArrowRight" })));
|
||||
await vi.waitFor(() => expect(transform()).toContain("translate(51.0px, 10.0px)"));
|
||||
|
||||
const model = useCaptionStore.getState().model!;
|
||||
const segments = new Map(model.segments);
|
||||
segments.set(segmentId, { ...segments.get(segmentId)!, style: { x: 5, y: 6 } });
|
||||
applyCaptionModelToIframe({ ...model, segments });
|
||||
await vi.waitFor(() => expect(transform()).toContain("translate(5.0px, 6.0px)"));
|
||||
});
|
||||
|
||||
it("a detection still waiting on the frame when Studio unmounts it loads no captions", async () => {
|
||||
vi.mocked(studioApiFetch).mockClear();
|
||||
const { iframe, hold, held, unmount } = mountEditor();
|
||||
// The detection's last question to the frame before it fetches the caption source.
|
||||
const release = hold(({ kind, params }) => {
|
||||
const selector = (params as { selector?: unknown } | undefined)?.selector;
|
||||
return kind === "query" && selector === '[data-composition-id*="caption"]';
|
||||
});
|
||||
await vi.waitFor(() => expect(held).toHaveLength(1));
|
||||
unmount();
|
||||
release();
|
||||
await frameRoundTrip(iframe);
|
||||
expect(studioApiFetch).not.toHaveBeenCalled();
|
||||
expect(useCaptionStore.getState()).toMatchObject({ isEditMode: false, model: null });
|
||||
});
|
||||
|
||||
it("a sealed drag the frame answers after the overlay unmounts stores no style", async () => {
|
||||
const { iframe, hold, held, boxes, unmount } = await openEditor();
|
||||
act(() => boxes()[0]!.click());
|
||||
const segmentId = [...useCaptionStore.getState().selectedSegmentIds][0]!;
|
||||
const overlay = boxes()[0]!.parentElement!;
|
||||
const release = hold(isWrite);
|
||||
act(() => void boxes()[0]!.dispatchEvent(pointer("pointerdown", 100, 100)));
|
||||
act(() => void overlay.dispatchEvent(pointer("pointermove", 130, 110)));
|
||||
act(() => void overlay.dispatchEvent(pointer("pointerup", 130, 110)));
|
||||
await vi.waitFor(() => expect(held).toHaveLength(1));
|
||||
unmount();
|
||||
release();
|
||||
await frameRoundTrip(iframe);
|
||||
expect(styleOf(segmentId)?.x).toBeUndefined();
|
||||
});
|
||||
|
||||
it("a sealed nudge the frame answers after the overlay unmounts stores no style", async () => {
|
||||
const { iframe, hold, held, boxes, unmount } = await openEditor();
|
||||
act(() => boxes()[0]!.click());
|
||||
const segmentId = [...useCaptionStore.getState().selectedSegmentIds][0]!;
|
||||
const release = hold(isWrite);
|
||||
act(() => void window.dispatchEvent(new KeyboardEvent("keydown", { key: "ArrowRight" })));
|
||||
await vi.waitFor(() => expect(held).toHaveLength(1));
|
||||
unmount();
|
||||
release();
|
||||
await frameRoundTrip(iframe);
|
||||
expect(styleOf(segmentId)?.x).toBeUndefined();
|
||||
});
|
||||
|
||||
it("a frame word past the write bound is not stored, and a word within it still is", async () => {
|
||||
const far = `<span data-caption-wrapper="true" style="display:inline-block; transform: translate(200000px, 0px)"><span class="word">world</span></span>`;
|
||||
const { boxes } = await openEditor(`<span class="word">Hello</span> ${far}`);
|
||||
const press = (box: HTMLElement, dx: number, dy: number) => {
|
||||
act(() => box.click());
|
||||
const overlay = box.parentElement!;
|
||||
act(() => void box.dispatchEvent(pointer("pointerdown", 100, 100)));
|
||||
if (dx || dy) act(() => void overlay.dispatchEvent(pointer("pointermove", 100 + dx, 100 + dy)));
|
||||
act(() => void overlay.dispatchEvent(pointer("pointerup", 100 + dx, 100 + dy)));
|
||||
return [...useCaptionStore.getState().selectedSegmentIds][0]!;
|
||||
};
|
||||
// A press and release with no move stores where the frame says the word is.
|
||||
const farWord = press(boxes()[1]!, 0, 0);
|
||||
const nearWord = press(boxes()[0]!, 30, 10);
|
||||
// The frame answers in order, so the far word's answer was handled before this one's.
|
||||
await vi.waitFor(() => expect(styleOf(nearWord)).toMatchObject({ x: 30, y: 10 }));
|
||||
expect(styleOf(farWord)?.x).toBeUndefined();
|
||||
});
|
||||
@@ -4,7 +4,17 @@ import { useCaptionStore } from "../store";
|
||||
import { usePlayerStore } from "../../player";
|
||||
import { resolveCaptionNudge, isEditableEventTarget } from "../keyboard";
|
||||
import { matchesKey } from "../../player/components/studioShortcuts";
|
||||
import { previewFrameOpaque } from "../../player/lib/previewChannel";
|
||||
import {
|
||||
dragTransform,
|
||||
moveSealedDrag,
|
||||
pressSealedDrag,
|
||||
readFrameWordBoxes,
|
||||
restoreSealedDrag,
|
||||
sealedNudger,
|
||||
transformStyle,
|
||||
type DragStart,
|
||||
type SealedDrag,
|
||||
readWordBoxes,
|
||||
getWordEl,
|
||||
readGsapTransform,
|
||||
@@ -25,7 +35,10 @@ const ROTATION_OFFSET = 20; // px above the selection box
|
||||
|
||||
/** Sync canvas state back to the Zustand store so the property panel reflects it. */
|
||||
function syncToStore(segmentId: string, el: HTMLElement, iframeWin: Window) {
|
||||
const style = computeTransformStyle(el, iframeWin);
|
||||
storeStyle(segmentId, computeTransformStyle(el, iframeWin));
|
||||
}
|
||||
|
||||
function storeStyle(segmentId: string, style: Record<string, number>) {
|
||||
if (Object.keys(style).length > 0) {
|
||||
useCaptionStore.getState().updateSegmentStyle(segmentId, style);
|
||||
}
|
||||
@@ -47,59 +60,57 @@ export const CaptionOverlay = memo(function CaptionOverlay({ iframeRef }: Captio
|
||||
const cancelDragRef = useRef<(() => boolean) | null>(null);
|
||||
|
||||
// Interaction mode — only one active at a time
|
||||
const interactionRef = useRef<
|
||||
| {
|
||||
type: "move";
|
||||
wordEl: HTMLElement;
|
||||
segmentId: string;
|
||||
startMX: number;
|
||||
startMY: number;
|
||||
origTX: number;
|
||||
origTY: number;
|
||||
origScale: number;
|
||||
origRotation: number;
|
||||
}
|
||||
| {
|
||||
type: "scale";
|
||||
wordEl: HTMLElement;
|
||||
segmentId: string;
|
||||
startMX: number;
|
||||
startDxFromCenter: number;
|
||||
origTX: number;
|
||||
origTY: number;
|
||||
origScale: number;
|
||||
origRotation: number;
|
||||
}
|
||||
| {
|
||||
type: "rotate";
|
||||
wordEl: HTMLElement;
|
||||
segmentId: string;
|
||||
startMX: number;
|
||||
origTX: number;
|
||||
origTY: number;
|
||||
origRotation: number;
|
||||
origScale: number;
|
||||
}
|
||||
| null
|
||||
>(null);
|
||||
const interactionRef = useRef<(DragStart & { wordEl: HTMLElement; segmentId: string }) | null>(
|
||||
null,
|
||||
);
|
||||
const sealedDragRef = useRef<SealedDrag | null>(null);
|
||||
const liveRef = useRef(false);
|
||||
|
||||
// The observer and message filter bind to the live iframe element, which a promoted reload replaces.
|
||||
// eslint-disable-next-line no-restricted-syntax
|
||||
useEffect(() => {
|
||||
if (!isEditMode) return;
|
||||
liveRef.current = true;
|
||||
|
||||
// Let undo/redo (useAppHotkeys) reapply restored models to this iframe.
|
||||
const unregisterIframe = registerCaptionIframe(iframeRef);
|
||||
|
||||
let prevBoxes: WordBox[] = [];
|
||||
let rafId: number | null = null;
|
||||
let disposed = false;
|
||||
let reading = false;
|
||||
let readAgain = false;
|
||||
const tick = () => {
|
||||
rafId = null;
|
||||
const iframe = iframeRef.current;
|
||||
const m = modelRef.current;
|
||||
const overlay = overlayRef.current;
|
||||
if (!iframe || !m || !overlay) return;
|
||||
const next = readWordBoxes(iframe, m, overlay);
|
||||
if (previewFrameOpaque(iframe)) return readSealed(iframe, m, overlay);
|
||||
show(readWordBoxes(iframe, m, overlay));
|
||||
};
|
||||
// One frame read in flight; triggers meanwhile collapse into one more read after it.
|
||||
const readSealed = (
|
||||
iframe: HTMLIFrameElement,
|
||||
m: NonNullable<typeof model>,
|
||||
overlay: HTMLElement,
|
||||
) => {
|
||||
if (reading) {
|
||||
readAgain = true;
|
||||
return;
|
||||
}
|
||||
reading = true;
|
||||
void readFrameWordBoxes(iframe, m, overlay).then((next) => {
|
||||
reading = false;
|
||||
if (disposed) return;
|
||||
if (next) show(next);
|
||||
if (readAgain) {
|
||||
readAgain = false;
|
||||
scheduleTick();
|
||||
}
|
||||
});
|
||||
};
|
||||
const show = (next: WordBox[]) => {
|
||||
if (
|
||||
next.length === prevBoxes.length &&
|
||||
next.every(
|
||||
@@ -155,6 +166,12 @@ export const CaptionOverlay = memo(function CaptionOverlay({ iframeRef }: Captio
|
||||
};
|
||||
|
||||
const cancelActiveDrag = (): boolean => {
|
||||
const sealed = sealedDragRef.current;
|
||||
if (sealed) {
|
||||
sealedDragRef.current = null;
|
||||
restoreSealedDrag(sealed);
|
||||
return true;
|
||||
}
|
||||
const i = interactionRef.current;
|
||||
if (!i) return false;
|
||||
const win = getWin();
|
||||
@@ -202,6 +219,10 @@ export const CaptionOverlay = memo(function CaptionOverlay({ iframeRef }: Captio
|
||||
const { dx, dy } = nudge;
|
||||
|
||||
const iframe = iframeRef.current;
|
||||
if (iframe && previewFrameOpaque(iframe)) {
|
||||
void nudgeSealed(iframe, m, sel, dx, dy).then(scheduleTick);
|
||||
return;
|
||||
}
|
||||
const win = iframe?.contentWindow;
|
||||
if (!iframe || !win) return;
|
||||
|
||||
@@ -223,8 +244,15 @@ export const CaptionOverlay = memo(function CaptionOverlay({ iframeRef }: Captio
|
||||
scheduleTick();
|
||||
};
|
||||
|
||||
const nudgeSealed = sealedNudger(
|
||||
() => !disposed,
|
||||
(segId, moved) => storeStyle(segId, transformStyle(moved)),
|
||||
);
|
||||
|
||||
window.addEventListener("keydown", handleKeyDown);
|
||||
return () => {
|
||||
disposed = true;
|
||||
liveRef.current = false;
|
||||
unregisterIframe();
|
||||
unsubPlayer();
|
||||
unsubPlayback();
|
||||
@@ -247,6 +275,27 @@ export const CaptionOverlay = memo(function CaptionOverlay({ iframeRef }: Captio
|
||||
return rect.width / nativeW;
|
||||
}, [iframeRef]);
|
||||
|
||||
/** A press on a frame Studio cannot read: the drag starts from the word the frame reports. */
|
||||
const startSealedDrag = useCallback(
|
||||
(
|
||||
type: DragStart["type"],
|
||||
word: { group: number; word: number },
|
||||
segmentId: string,
|
||||
e: React.PointerEvent,
|
||||
) => {
|
||||
const iframe = iframeRef.current;
|
||||
if (!iframe) return;
|
||||
const press = {
|
||||
startMX: e.clientX,
|
||||
startMY: e.clientY,
|
||||
cssScale: getCssScale(),
|
||||
left: iframe.getBoundingClientRect().left,
|
||||
};
|
||||
sealedDragRef.current = pressSealedDrag(iframe, type, word, segmentId, press);
|
||||
},
|
||||
[iframeRef, getCssScale],
|
||||
);
|
||||
|
||||
// --- Move ---
|
||||
const startMove = useCallback(
|
||||
(groupIndex: number, wordIndex: number, segmentId: string, e: React.PointerEvent) => {
|
||||
@@ -254,6 +303,8 @@ export const CaptionOverlay = memo(function CaptionOverlay({ iframeRef }: Captio
|
||||
(e.currentTarget as HTMLElement).setPointerCapture(e.pointerId);
|
||||
const iframe = iframeRef.current;
|
||||
if (!iframe) return;
|
||||
if (previewFrameOpaque(iframe))
|
||||
return startSealedDrag("move", { group: groupIndex, word: wordIndex }, segmentId, e);
|
||||
const wordEl = getWordEl(iframe, groupIndex, wordIndex);
|
||||
const win = iframe.contentWindow;
|
||||
if (!wordEl || !win) return;
|
||||
@@ -270,7 +321,7 @@ export const CaptionOverlay = memo(function CaptionOverlay({ iframeRef }: Captio
|
||||
origRotation: state.rotation,
|
||||
};
|
||||
},
|
||||
[iframeRef],
|
||||
[iframeRef, startSealedDrag],
|
||||
);
|
||||
|
||||
// --- Scale ---
|
||||
@@ -281,6 +332,8 @@ export const CaptionOverlay = memo(function CaptionOverlay({ iframeRef }: Captio
|
||||
(e.currentTarget as HTMLElement).setPointerCapture(e.pointerId);
|
||||
const iframe = iframeRef.current;
|
||||
if (!iframe) return;
|
||||
if (previewFrameOpaque(iframe))
|
||||
return startSealedDrag("scale", { group: groupIndex, word: wordIndex }, segmentId, e);
|
||||
const wordEl = getWordEl(iframe, groupIndex, wordIndex);
|
||||
const win = iframe.contentWindow;
|
||||
if (!wordEl || !win) return;
|
||||
@@ -303,7 +356,7 @@ export const CaptionOverlay = memo(function CaptionOverlay({ iframeRef }: Captio
|
||||
origRotation: state.rotation,
|
||||
};
|
||||
},
|
||||
[iframeRef, getCssScale],
|
||||
[iframeRef, getCssScale, startSealedDrag],
|
||||
);
|
||||
|
||||
// --- Rotate ---
|
||||
@@ -314,6 +367,13 @@ export const CaptionOverlay = memo(function CaptionOverlay({ iframeRef }: Captio
|
||||
(e.currentTarget as HTMLElement).setPointerCapture(e.pointerId);
|
||||
const iframe = iframeRef.current;
|
||||
if (!iframe) return;
|
||||
if (previewFrameOpaque(iframe))
|
||||
return startSealedDrag(
|
||||
"rotate",
|
||||
{ group: box.groupIndex, word: box.wordIndex },
|
||||
box.segmentId,
|
||||
e,
|
||||
);
|
||||
const wordEl = getWordEl(iframe, box.groupIndex, box.wordIndex);
|
||||
const win = iframe.contentWindow;
|
||||
if (!wordEl || !win) return;
|
||||
@@ -329,7 +389,7 @@ export const CaptionOverlay = memo(function CaptionOverlay({ iframeRef }: Captio
|
||||
origScale: state.scale,
|
||||
};
|
||||
},
|
||||
[iframeRef],
|
||||
[iframeRef, startSealedDrag],
|
||||
);
|
||||
|
||||
const getIframeWin = useCallback((): Window | null => {
|
||||
@@ -343,34 +403,29 @@ export const CaptionOverlay = memo(function CaptionOverlay({ iframeRef }: Captio
|
||||
// --- Unified pointer move ---
|
||||
const handlePointerMove = useCallback(
|
||||
(e: React.PointerEvent) => {
|
||||
const sealed = sealedDragRef.current;
|
||||
if (sealed) return moveSealedDrag(sealed, e.clientX, e.clientY, getCssScale());
|
||||
const i = interactionRef.current;
|
||||
if (!i) return;
|
||||
const win = getIframeWin();
|
||||
if (!win) return;
|
||||
|
||||
if (i.type === "move") {
|
||||
const cssScale = getCssScale();
|
||||
const dx = (e.clientX - i.startMX) / cssScale;
|
||||
const dy = (e.clientY - i.startMY) / cssScale;
|
||||
writeTransform(i.wordEl, win, i.origTX + dx, i.origTY + dy, i.origScale, i.origRotation);
|
||||
} else if (i.type === "scale") {
|
||||
const cx = i.startMX - i.startDxFromCenter;
|
||||
const startDist = Math.abs(i.startDxFromCenter);
|
||||
const currentDist = Math.abs(e.clientX - cx);
|
||||
const factor = startDist > 5 ? currentDist / startDist : 1;
|
||||
const newScale = Math.max(0.1, i.origScale * factor);
|
||||
writeTransform(i.wordEl, win, i.origTX, i.origTY, newScale, i.origRotation);
|
||||
} else if (i.type === "rotate") {
|
||||
const dx = e.clientX - i.startMX;
|
||||
const delta = (dx / 200) * 90;
|
||||
writeTransform(i.wordEl, win, i.origTX, i.origTY, i.origScale, i.origRotation + delta);
|
||||
}
|
||||
const t = dragTransform(i, e.clientX, e.clientY, getCssScale());
|
||||
writeTransform(i.wordEl, win, t.x, t.y, t.scale, t.rotation);
|
||||
},
|
||||
[getCssScale, getIframeWin],
|
||||
);
|
||||
|
||||
// --- Unified pointer up — sync back to store ---
|
||||
const handlePointerUp = useCallback(() => {
|
||||
const sealed = sealedDragRef.current;
|
||||
if (sealed) {
|
||||
sealedDragRef.current = null;
|
||||
void sealed.last.then(
|
||||
(t) => t && liveRef.current && storeStyle(sealed.segmentId, transformStyle(t)),
|
||||
);
|
||||
return;
|
||||
}
|
||||
const i = interactionRef.current;
|
||||
if (i) {
|
||||
const win = getIframeWin();
|
||||
|
||||
@@ -1,4 +1,11 @@
|
||||
// DOM helpers for CaptionOverlay — word box reading, transform I/O, wrapper management
|
||||
import {
|
||||
isPlainRecord,
|
||||
MAX_CAPTION_TRANSFORM,
|
||||
MAX_DESCRIBE_HANDLES,
|
||||
type EditingParams,
|
||||
} from "@hyperframes/core/runtime/editing-port";
|
||||
import { previewFrameOpaque, previewRequest } from "../../player/lib/previewChannel";
|
||||
|
||||
export interface WordBox {
|
||||
segmentId: string;
|
||||
@@ -11,6 +18,34 @@ export interface WordBox {
|
||||
height: number;
|
||||
}
|
||||
|
||||
/** Maps a box in the frame's CSS pixels into the overlay's. */
|
||||
function overlayPlacement(iframe: HTMLIFrameElement, overlayEl: HTMLElement) {
|
||||
const iframeDisplayRect = iframe.getBoundingClientRect();
|
||||
const overlayRect = overlayEl.getBoundingClientRect();
|
||||
const nativeW = parseFloat(iframe.style.width) || iframeDisplayRect.width;
|
||||
const cssScale = iframeDisplayRect.width / nativeW;
|
||||
const offsetX = iframeDisplayRect.left - overlayRect.left;
|
||||
const offsetY = iframeDisplayRect.top - overlayRect.top;
|
||||
return (x: number, y: number, width: number, height: number) => ({
|
||||
x: x * cssScale + offsetX,
|
||||
y: y * cssScale + offsetY,
|
||||
width: width * cssScale,
|
||||
height: height * cssScale,
|
||||
});
|
||||
}
|
||||
|
||||
function splitIntoSpans(doc: Document, text: string, into: HTMLElement[]): DocumentFragment {
|
||||
const frag = doc.createDocumentFragment();
|
||||
for (const word of text.split(/\s+/).filter(Boolean)) {
|
||||
const span = doc.createElement("span");
|
||||
span.textContent = word + " ";
|
||||
span.style.display = "inline";
|
||||
frag.appendChild(span);
|
||||
into.push(span);
|
||||
}
|
||||
return frag;
|
||||
}
|
||||
|
||||
export function readWordBoxes(
|
||||
iframe: HTMLIFrameElement,
|
||||
model: {
|
||||
@@ -29,13 +64,7 @@ export function readWordBoxes(
|
||||
}
|
||||
if (!doc || !win) return [];
|
||||
|
||||
const iframeDisplayRect = iframe.getBoundingClientRect();
|
||||
const overlayRect = overlayEl.getBoundingClientRect();
|
||||
const nativeW = parseFloat(iframe.style.width) || iframeDisplayRect.width;
|
||||
const cssScale = iframeDisplayRect.width / nativeW;
|
||||
const offsetX = iframeDisplayRect.left - overlayRect.left;
|
||||
const offsetY = iframeDisplayRect.top - overlayRect.top;
|
||||
|
||||
const place = overlayPlacement(iframe, overlayEl);
|
||||
const groupEls = doc.querySelectorAll<HTMLElement>(".caption-group");
|
||||
const boxes: WordBox[] = [];
|
||||
|
||||
@@ -60,29 +89,14 @@ export function readWordBoxes(
|
||||
if (resolvedWordEls.length === 0 && groupEl.textContent?.trim()) {
|
||||
const textNode = groupEl.childNodes[0];
|
||||
if (textNode && textNode.nodeType === Node.TEXT_NODE) {
|
||||
const words = (textNode.textContent || "").split(/\s+/).filter(Boolean);
|
||||
const frag = doc.createDocumentFragment();
|
||||
for (const word of words) {
|
||||
const span = doc.createElement("span");
|
||||
span.textContent = word + " ";
|
||||
span.style.display = "inline";
|
||||
frag.appendChild(span);
|
||||
resolvedWordEls.push(span);
|
||||
}
|
||||
groupEl.replaceChild(frag, textNode);
|
||||
groupEl.replaceChild(
|
||||
splitIntoSpans(doc, textNode.textContent || "", resolvedWordEls),
|
||||
textNode,
|
||||
);
|
||||
} else {
|
||||
const singleSpan = groupEl.querySelector<HTMLElement>(":scope > span");
|
||||
if (singleSpan && singleSpan.textContent?.trim()) {
|
||||
const words = singleSpan.textContent.split(/\s+/).filter(Boolean);
|
||||
const frag = doc.createDocumentFragment();
|
||||
for (const word of words) {
|
||||
const span = doc.createElement("span");
|
||||
span.textContent = word + " ";
|
||||
span.style.display = "inline";
|
||||
frag.appendChild(span);
|
||||
resolvedWordEls.push(span);
|
||||
}
|
||||
singleSpan.replaceWith(frag);
|
||||
singleSpan.replaceWith(splitIntoSpans(doc, singleSpan.textContent, resolvedWordEls));
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -96,10 +110,7 @@ export function readWordBoxes(
|
||||
groupId,
|
||||
groupIndex: gi,
|
||||
wordIndex: wi,
|
||||
x: rect.left * cssScale + offsetX,
|
||||
y: rect.top * cssScale + offsetY,
|
||||
width: rect.width * cssScale,
|
||||
height: rect.height * cssScale,
|
||||
...place(rect.left, rect.top, rect.width, rect.height),
|
||||
});
|
||||
}
|
||||
}
|
||||
@@ -237,6 +248,17 @@ export function applyCaptionModelToIframe(model: {
|
||||
}): void {
|
||||
const iframe = registeredIframe?.current;
|
||||
if (!iframe) return;
|
||||
if (previewFrameOpaque(iframe)) {
|
||||
model.groupOrder.forEach((groupId, group) =>
|
||||
model.groups.get(groupId)?.segmentIds.forEach((segmentId, word) => {
|
||||
const s = model.segments.get(segmentId)?.style;
|
||||
if (!s) return;
|
||||
const set = { x: s.x ?? 0, y: s.y ?? 0, scale: s.scaleX ?? 1, rotation: s.rotation ?? 0 };
|
||||
void frameWordTransform(iframe, { group, word, set });
|
||||
}),
|
||||
);
|
||||
return;
|
||||
}
|
||||
let win: Window | null = null;
|
||||
try {
|
||||
win = iframe.contentWindow;
|
||||
@@ -260,8 +282,10 @@ export function applyCaptionModelToIframe(model: {
|
||||
|
||||
/** Compute style deltas from the current wrapper transform — used by syncToStore in the overlay. */
|
||||
export function computeTransformStyle(el: HTMLElement, iframeWin: Window): Record<string, number> {
|
||||
const wrapper = getOrCreateWrapper(el);
|
||||
const { x, y, scale, rotation } = readGsapTransform(wrapper, iframeWin);
|
||||
return transformStyle(readGsapTransform(getOrCreateWrapper(el), iframeWin));
|
||||
}
|
||||
|
||||
export function transformStyle({ x, y, scale, rotation }: WordTransform): Record<string, number> {
|
||||
const style: Record<string, number> = {};
|
||||
if (Math.abs(x) > 0.5) style.x = x;
|
||||
if (Math.abs(y) > 0.5) style.y = y;
|
||||
@@ -272,3 +296,265 @@ export function computeTransformStyle(el: HTMLElement, iframeWin: Window): Recor
|
||||
if (Math.abs(rotation) > 0.1) style.rotation = rotation;
|
||||
return style;
|
||||
}
|
||||
|
||||
interface WordTransform {
|
||||
x: number;
|
||||
y: number;
|
||||
scale: number;
|
||||
rotation: number;
|
||||
}
|
||||
type WordEntry = EditingParams<"caption-transform">["entries"][number];
|
||||
interface FrameWord {
|
||||
transform: WordTransform;
|
||||
rect: { x: number; y: number; width: number; height: number };
|
||||
}
|
||||
|
||||
const allFinite = (value: unknown, keys: string[]) =>
|
||||
isPlainRecord(value) && keys.every((key) => Number.isFinite(value[key]));
|
||||
|
||||
// The frame's own write bound: a larger reply, once stored, would make every later write to the word refused.
|
||||
const allBounded = (value: unknown, keys: string[]) =>
|
||||
isPlainRecord(value) &&
|
||||
keys.every(
|
||||
(key) => Number.isFinite(value[key]) && Math.abs(value[key] as number) <= MAX_CAPTION_TRANSFORM,
|
||||
);
|
||||
|
||||
function readFrameWord(value: unknown): FrameWord | null {
|
||||
if (!isPlainRecord(value)) return null;
|
||||
const ok =
|
||||
allBounded(value.transform, ["x", "y", "scale", "rotation"]) &&
|
||||
allFinite(value.rect, ["x", "y", "width", "height"]);
|
||||
return ok ? (value as unknown as FrameWord) : null;
|
||||
}
|
||||
|
||||
export function findCaptionWord(
|
||||
model: { groupOrder: string[]; groups: Map<string, { segmentIds: string[] }> },
|
||||
segmentId: string,
|
||||
): { group: number; word: number } | null {
|
||||
for (let group = 0; group < model.groupOrder.length; group++) {
|
||||
const word = model.groups.get(model.groupOrder[group])?.segmentIds.indexOf(segmentId) ?? -1;
|
||||
if (word >= 0) return { group, word };
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
/** `readWordBoxes` for a frame Studio cannot read; null when its runtime does not answer. */
|
||||
export async function readFrameWordBoxes(
|
||||
iframe: HTMLIFrameElement,
|
||||
model: { groupOrder: string[]; groups: Map<string, { segmentIds: string[] }> },
|
||||
overlayEl: HTMLElement,
|
||||
): Promise<WordBox[] | null> {
|
||||
const words = model.groupOrder.map((id) => model.groups.get(id)?.segmentIds.length ?? 0);
|
||||
const reply = await previewRequest(iframe, "caption-words", { words });
|
||||
const boxes: unknown = reply?.boxes;
|
||||
if (!Array.isArray(boxes)) return null;
|
||||
const place = overlayPlacement(iframe, overlayEl);
|
||||
return boxes.flatMap((box: unknown): WordBox[] => {
|
||||
if (!isPlainRecord(box) || !allFinite(box.rect, ["x", "y", "width", "height"])) return [];
|
||||
const { group, word } = box;
|
||||
if (!Number.isSafeInteger(group) || !Number.isSafeInteger(word)) return [];
|
||||
const groupId = model.groupOrder[group as number];
|
||||
const segmentId = groupId && model.groups.get(groupId)?.segmentIds[word as number];
|
||||
if (!segmentId) return [];
|
||||
const { x, y, width, height } = box.rect as FrameWord["rect"];
|
||||
const at = place(x, y, width, height);
|
||||
return [{ segmentId, groupId, groupIndex: group as number, wordIndex: word as number, ...at }];
|
||||
});
|
||||
}
|
||||
|
||||
type Op = { entry: WordEntry; done: Array<(word: FrameWord | null) => void> };
|
||||
const transformQueues = new WeakMap<HTMLIFrameElement, { ops: Op[]; sending: boolean }>();
|
||||
|
||||
function sendTransforms(iframe: HTMLIFrameElement, queue: { ops: Op[]; sending: boolean }) {
|
||||
if (queue.sending || !queue.ops.length) return;
|
||||
const ops = queue.ops.splice(0, MAX_DESCRIBE_HANDLES);
|
||||
queue.sending = true;
|
||||
const entries = ops.map((op) => op.entry);
|
||||
void previewRequest(iframe, "caption-transform", { entries })
|
||||
.then((reply) => {
|
||||
const words: unknown = reply?.words;
|
||||
ops.forEach((op, i) => {
|
||||
const word = Array.isArray(words) ? readFrameWord(words[i]) : null;
|
||||
op.done.forEach((done) => done(word));
|
||||
});
|
||||
})
|
||||
.finally(() => {
|
||||
queue.sending = false;
|
||||
sendTransforms(iframe, queue);
|
||||
});
|
||||
}
|
||||
|
||||
/** One request in flight per frame; a waiting write to the same word takes the later one's parts. */
|
||||
export function frameWordTransform(
|
||||
iframe: HTMLIFrameElement,
|
||||
entry: WordEntry,
|
||||
): Promise<FrameWord | null> {
|
||||
let queue = transformQueues.get(iframe);
|
||||
if (!queue) transformQueues.set(iframe, (queue = { ops: [], sending: false }));
|
||||
const { ops } = queue;
|
||||
const promise = new Promise<FrameWord | null>((resolve) => {
|
||||
const last = ops[ops.length - 1];
|
||||
const sameWord = last?.entry.group === entry.group && last.entry.word === entry.word;
|
||||
if (last && sameWord && last.entry.set && entry.set) {
|
||||
last.entry = { ...entry, set: { ...last.entry.set, ...entry.set } };
|
||||
last.done.push(resolve);
|
||||
} else ops.push({ entry, done: [resolve] });
|
||||
});
|
||||
sendTransforms(iframe, queue);
|
||||
return promise;
|
||||
}
|
||||
|
||||
/** What a drag knows from its press: where it started and the word's transform then. */
|
||||
export type DragStart =
|
||||
| {
|
||||
type: "move";
|
||||
startMX: number;
|
||||
startMY: number;
|
||||
origTX: number;
|
||||
origTY: number;
|
||||
origScale: number;
|
||||
origRotation: number;
|
||||
}
|
||||
| {
|
||||
type: "scale";
|
||||
startMX: number;
|
||||
startDxFromCenter: number;
|
||||
origTX: number;
|
||||
origTY: number;
|
||||
origScale: number;
|
||||
origRotation: number;
|
||||
}
|
||||
| {
|
||||
type: "rotate";
|
||||
startMX: number;
|
||||
origTX: number;
|
||||
origTY: number;
|
||||
origRotation: number;
|
||||
origScale: number;
|
||||
};
|
||||
|
||||
/** The word's transform with the pointer at (`clientX`, `clientY`). */
|
||||
export function dragTransform(
|
||||
i: DragStart,
|
||||
clientX: number,
|
||||
clientY: number,
|
||||
cssScale: number,
|
||||
): WordTransform {
|
||||
const orig = { x: i.origTX, y: i.origTY, scale: i.origScale, rotation: i.origRotation };
|
||||
if (i.type === "move") {
|
||||
const dx = (clientX - i.startMX) / cssScale;
|
||||
const dy = (clientY - i.startMY) / cssScale;
|
||||
return { ...orig, x: i.origTX + dx, y: i.origTY + dy };
|
||||
}
|
||||
if (i.type === "scale") {
|
||||
const cx = i.startMX - i.startDxFromCenter;
|
||||
const startDist = Math.abs(i.startDxFromCenter);
|
||||
const currentDist = Math.abs(clientX - cx);
|
||||
const factor = startDist > 5 ? currentDist / startDist : 1;
|
||||
return { ...orig, scale: Math.max(0.1, i.origScale * factor) };
|
||||
}
|
||||
const dx = clientX - i.startMX;
|
||||
return { ...orig, rotation: i.origRotation + (dx / 200) * 90 };
|
||||
}
|
||||
|
||||
/** A drag on a frame Studio cannot read: its press read lands later, and moves wait for it. */
|
||||
export interface SealedDrag {
|
||||
iframe: HTMLIFrameElement;
|
||||
word: { group: number; word: number };
|
||||
segmentId: string;
|
||||
start: Promise<DragStart | null>;
|
||||
/** The word's transform once the latest write lands. */
|
||||
last: Promise<WordTransform | null>;
|
||||
}
|
||||
|
||||
const restOf = (drag: DragStart | null): WordTransform | null =>
|
||||
drag && { x: drag.origTX, y: drag.origTY, scale: drag.origScale, rotation: drag.origRotation };
|
||||
|
||||
/** The drag a press starts from the word the frame reports; `left` is the frame's left edge on screen. */
|
||||
function toDragStart(
|
||||
type: DragStart["type"],
|
||||
{ transform: t, rect }: FrameWord,
|
||||
press: { startMX: number; startMY: number; cssScale: number; left: number },
|
||||
): DragStart {
|
||||
const { startMX, startMY, cssScale, left } = press;
|
||||
const orig = { origTX: t.x, origTY: t.y, origScale: t.scale, origRotation: t.rotation };
|
||||
if (type === "move") return { type, startMX, startMY, ...orig };
|
||||
if (type === "rotate") return { type, startMX, ...orig };
|
||||
const boxCenterX = rect.x * cssScale + left + (rect.width * cssScale) / 2;
|
||||
return { type, startMX, startDxFromCenter: startMX - boxCenterX, ...orig };
|
||||
}
|
||||
|
||||
/** Moves a frame's word by (`dx`, `dy`) from where the frame has it now. */
|
||||
async function nudgeFrameWord(
|
||||
iframe: HTMLIFrameElement,
|
||||
word: { group: number; word: number },
|
||||
dx: number,
|
||||
dy: number,
|
||||
): Promise<FrameWord | null> {
|
||||
const now = await frameWordTransform(iframe, word);
|
||||
if (!now) return null;
|
||||
const { x, y } = now.transform;
|
||||
return frameWordTransform(iframe, { ...word, set: { x: x + dx, y: y + dy } });
|
||||
}
|
||||
|
||||
/** Nudges run one after another, so quick repeats all count; none starts or stores once `live` is false. */
|
||||
export function sealedNudger(
|
||||
live: () => boolean,
|
||||
onMoved: (segmentId: string, transform: WordTransform) => void,
|
||||
) {
|
||||
let nudging = Promise.resolve();
|
||||
return (
|
||||
iframe: HTMLIFrameElement,
|
||||
model: Parameters<typeof findCaptionWord>[0],
|
||||
segmentIds: Iterable<string>,
|
||||
dx: number,
|
||||
dy: number,
|
||||
): Promise<void> => {
|
||||
for (const segmentId of segmentIds) {
|
||||
const word = findCaptionWord(model, segmentId);
|
||||
if (!word) continue;
|
||||
nudging = nudging.then(async () => {
|
||||
if (!live()) return;
|
||||
const moved = await nudgeFrameWord(iframe, word, dx, dy);
|
||||
if (moved && live()) onMoved(segmentId, moved.transform);
|
||||
});
|
||||
}
|
||||
return nudging;
|
||||
};
|
||||
}
|
||||
|
||||
/** Draws the drag at the pointer once the press read lands; `last` follows the newest write. */
|
||||
export function moveSealedDrag(
|
||||
drag: SealedDrag,
|
||||
clientX: number,
|
||||
clientY: number,
|
||||
cssScale: number,
|
||||
) {
|
||||
drag.last = drag.start.then(async (start) => {
|
||||
if (!start) return null;
|
||||
const set = dragTransform(start, clientX, clientY, cssScale);
|
||||
return (await frameWordTransform(drag.iframe, { ...drag.word, set }))?.transform ?? null;
|
||||
});
|
||||
}
|
||||
|
||||
/** Puts the word back where the press found it. */
|
||||
export function restoreSealedDrag(drag: SealedDrag): void {
|
||||
void drag.start.then((start) => {
|
||||
const set = restOf(start);
|
||||
if (set) void frameWordTransform(drag.iframe, { ...drag.word, set });
|
||||
});
|
||||
}
|
||||
|
||||
/** A press on a word of a frame Studio cannot read; moves wait for the frame's answer. */
|
||||
export function pressSealedDrag(
|
||||
iframe: HTMLIFrameElement,
|
||||
type: DragStart["type"],
|
||||
word: { group: number; word: number },
|
||||
segmentId: string,
|
||||
press: { startMX: number; startMY: number; cssScale: number; left: number },
|
||||
): SealedDrag {
|
||||
const start = frameWordTransform(iframe, word).then(
|
||||
(read) => read && toDragStart(type, read, press),
|
||||
);
|
||||
return { iframe, word, segmentId, start, last: start.then(restOf) };
|
||||
}
|
||||
|
||||
@@ -1,8 +1,10 @@
|
||||
import { memo, useCallback, useState } from "react";
|
||||
import { previewFrameOpaque } from "../../player/lib/previewChannel";
|
||||
import { useCaptionStore } from "../store";
|
||||
import type { CaptionStyle } from "../types";
|
||||
import { CaptionAnimationPanel } from "./CaptionAnimationPanel";
|
||||
import { Section, Row, NumberField } from "./shared";
|
||||
import { findCaptionWord, frameWordTransform } from "./CaptionOverlayUtils";
|
||||
|
||||
/** True when the given style key differs across the selected segments. */
|
||||
function isMixedValue(
|
||||
@@ -78,6 +80,21 @@ export const CaptionPropertyPanel = memo(function CaptionPropertyPanel({
|
||||
(updates: Partial<CaptionStyle>) => {
|
||||
const iframe = iframeRef.current;
|
||||
if (!iframe || !model) return;
|
||||
if (previewFrameOpaque(iframe)) {
|
||||
const parts = {
|
||||
x: updates.x,
|
||||
y: updates.y,
|
||||
scale: updates.scaleX,
|
||||
rotation: updates.rotation,
|
||||
};
|
||||
const set = Object.fromEntries(Object.entries(parts).filter(([, v]) => v !== undefined));
|
||||
if (Object.keys(set).length === 0) return;
|
||||
for (const segId of selectedSegmentIds) {
|
||||
const word = findCaptionWord(model, segId);
|
||||
if (word) void frameWordTransform(iframe, { ...word, set });
|
||||
}
|
||||
return;
|
||||
}
|
||||
let doc: Document | null = null;
|
||||
try {
|
||||
doc = iframe.contentDocument;
|
||||
|
||||
@@ -5,7 +5,16 @@ import { useCaptionStore } from "../captions/store";
|
||||
import { acceptStudioRuntimeMessage } from "../player/lib/runtimeProtocol";
|
||||
import { useCaptionSync } from "../captions/hooks/useCaptionSync";
|
||||
import { parseCaptionComposition } from "../captions/parser";
|
||||
import type { CaptionModel } from "../captions/types";
|
||||
import { studioApiFetch } from "../utils/studioApiFetch";
|
||||
import { MAX_DESCRIBE_HANDLES } from "@hyperframes/core/runtime/editing-port";
|
||||
import { previewComputedStyles, previewFrameOpaque } from "../player/lib/previewChannel";
|
||||
import {
|
||||
describePreviewElements,
|
||||
previewAncestors,
|
||||
queryPreviewElements,
|
||||
type PreviewElement,
|
||||
} from "../player/lib/previewElements";
|
||||
|
||||
interface UseCaptionDetectionParams {
|
||||
projectId: string | null;
|
||||
@@ -18,6 +27,91 @@ interface UseCaptionDetectionParams {
|
||||
setRightCollapsed: (collapsed: boolean) => void;
|
||||
}
|
||||
|
||||
// The parser's GROUP_SELECTOR and WORD_SELECTOR, the elements it reads.
|
||||
const GROUP_SELECTOR = ".caption-group, .caption-line, .caption-block";
|
||||
const WORD_SELECTOR = ".word, .caption-word";
|
||||
|
||||
const attribute = (el: PreviewElement | null | undefined, name: string) =>
|
||||
el?.attributes.find(([key]) => key === name)?.[1] ?? null;
|
||||
|
||||
/** The styles `parseCaptionComposition` reads off its first word and group. */
|
||||
const CAPTION_STYLES = [
|
||||
"font-size",
|
||||
"font-weight",
|
||||
"font-family",
|
||||
"color",
|
||||
"text-transform",
|
||||
"letter-spacing",
|
||||
"background-color",
|
||||
"border-radius",
|
||||
"padding-top",
|
||||
"padding-right",
|
||||
"padding-bottom",
|
||||
"padding-left",
|
||||
];
|
||||
|
||||
async function frameStyles(iframe: HTMLIFrameElement, handle: string | undefined) {
|
||||
if (!handle) return null;
|
||||
const styles = await previewComputedStyles(iframe, { handle }, CAPTION_STYLES);
|
||||
if (!styles) return null;
|
||||
const camel = (name: string) => name.replace(/-(\w)/g, (_, c: string) => c.toUpperCase());
|
||||
return Object.fromEntries(CAPTION_STYLES.map((name) => [camel(name), styles[name] ?? ""]));
|
||||
}
|
||||
|
||||
/** A frame's caption groups and words rebuilt in order in a detached document, with a stand-in
|
||||
* `getComputedStyle` for the parser; null past one query's matches or with no answer. */
|
||||
async function frameCaptionDom(
|
||||
iframe: HTMLIFrameElement,
|
||||
): Promise<{ doc: Document; win: Window } | null> {
|
||||
const [all, groups, words] = await Promise.all(
|
||||
[`${GROUP_SELECTOR}, ${WORD_SELECTOR}`, GROUP_SELECTOR, WORD_SELECTOR].map((selector) =>
|
||||
queryPreviewElements(iframe, selector),
|
||||
),
|
||||
);
|
||||
if (!all || !groups || !words || all.truncated) return null;
|
||||
const handles = all.matches.map((match) => match.handle);
|
||||
const chains: Array<string[] | null> = [];
|
||||
for (let i = 0; i < handles.length; i += MAX_DESCRIBE_HANDLES) {
|
||||
const part = await previewAncestors(iframe, handles.slice(i, i + MAX_DESCRIBE_HANDLES));
|
||||
if (!part) return null;
|
||||
chains.push(...part);
|
||||
}
|
||||
const isGroup = new Set(groups.matches.map((match) => match.handle));
|
||||
const isWord = new Set(words.matches.map((match) => match.handle));
|
||||
const doc = document.implementation.createHTMLDocument("");
|
||||
const mirrored = new Map<string, Element>();
|
||||
handles.forEach((handle, i) => {
|
||||
const el = doc.createElement("span");
|
||||
if (isGroup.has(handle)) el.classList.add("caption-group");
|
||||
if (isWord.has(handle)) el.classList.add("word");
|
||||
const parent = chains[i]?.find((ancestor) => mirrored.has(ancestor));
|
||||
(parent ? mirrored.get(parent)! : doc.body).append(el);
|
||||
mirrored.set(handle, el);
|
||||
});
|
||||
const firstWord = words.matches[0]?.handle;
|
||||
const firstGroup = groups.matches[0]?.handle;
|
||||
const styles = new Map<Element | undefined, Record<string, string> | null>([
|
||||
[mirrored.get(firstWord ?? ""), await frameStyles(iframe, firstWord)],
|
||||
[mirrored.get(firstGroup ?? ""), await frameStyles(iframe, firstGroup)],
|
||||
]);
|
||||
const win = { getComputedStyle: (el: Element) => styles.get(el) ?? {} } as unknown as Window;
|
||||
return { doc, win };
|
||||
}
|
||||
|
||||
async function frameCaptionModel(
|
||||
iframe: HTMLIFrameElement,
|
||||
content: string,
|
||||
): Promise<CaptionModel | null> {
|
||||
const [root] = (await queryPreviewElements(iframe, "[data-composition-id]"))?.matches ?? [];
|
||||
const rootEl = root ? (await describePreviewElements(iframe, [root.handle]))?.[0] : null;
|
||||
const frame = await frameCaptionDom(iframe);
|
||||
if (!frame) return null;
|
||||
const w = parseInt(attribute(rootEl, "data-width") ?? "1920", 10);
|
||||
const h = parseInt(attribute(rootEl, "data-height") ?? "1080", 10);
|
||||
const dur = parseFloat(attribute(rootEl, "data-duration") ?? "0");
|
||||
return parseCaptionComposition(frame.doc, frame.win, content, w, h, dur);
|
||||
}
|
||||
|
||||
export function useCaptionDetection({
|
||||
projectId,
|
||||
activeCompPath,
|
||||
@@ -52,6 +146,51 @@ export function useCaptionDetection({
|
||||
if (!projectId) return;
|
||||
|
||||
let activating = false;
|
||||
// A detection still in flight when the composition changes must not enter edit mode for the old one.
|
||||
let disposed = false;
|
||||
|
||||
const pickCaptionSource = (hostSrcs: Array<string | null>, captionCompId: string | null) => {
|
||||
const host = hostSrcs.find((src) => src && src.includes("captions"));
|
||||
if (host) return host;
|
||||
for (const [id, src] of compIdToSrc) {
|
||||
if (id.includes("caption") || src.includes("caption")) return src;
|
||||
}
|
||||
if (activeCompPath?.includes("captions")) return activeCompPath;
|
||||
return captionCompId === null ? null : compIdToSrc.get(captionCompId) || null;
|
||||
};
|
||||
|
||||
const loadCaptions = (
|
||||
srcPath: string,
|
||||
parse: (content: string) => Promise<CaptionModel | null>,
|
||||
) =>
|
||||
studioApiFetch(buildProjectApiPath(projectId, `/files/${encodeURIComponent(srcPath)}`))
|
||||
.then((r) => r.json())
|
||||
.then(async (data: { content?: string }) => {
|
||||
if (disposed || !data.content || useCaptionStore.getState().isEditMode) return;
|
||||
const model = await parse(data.content);
|
||||
if (disposed || !model || useCaptionStore.getState().isEditMode) return;
|
||||
const store = useCaptionStore.getState();
|
||||
store.setModel(model);
|
||||
store.setSourceFilePath(srcPath);
|
||||
store.setEditMode(true);
|
||||
captionSync.loadOverrides();
|
||||
})
|
||||
.catch(() => {});
|
||||
|
||||
const frameCaptionSource = async (iframe: HTMLIFrameElement) => {
|
||||
if (!(await queryPreviewElements(iframe, ".caption-group"))?.matches.length) return null;
|
||||
const hosts = await queryPreviewElements(
|
||||
iframe,
|
||||
"[data-composition-src], [data-composition-file]",
|
||||
);
|
||||
const handles = (hosts?.matches ?? []).slice(0, MAX_DESCRIBE_HANDLES).map((m) => m.handle);
|
||||
const described = handles.length ? await describePreviewElements(iframe, handles) : [];
|
||||
const hostSrcs = (described ?? []).map(
|
||||
(el) => attribute(el, "data-composition-src") || attribute(el, "data-composition-file"),
|
||||
);
|
||||
const captionComp = await queryPreviewElements(iframe, '[data-composition-id*="caption"]');
|
||||
return pickCaptionSource(hostSrcs, captionComp?.matches[0]?.source.compositionId ?? null);
|
||||
};
|
||||
|
||||
const tryActivateCaptions = () => {
|
||||
const captionState = useCaptionStore.getState();
|
||||
@@ -61,6 +200,19 @@ export function useCaptionDetection({
|
||||
}
|
||||
|
||||
const iframe = previewIframeRef.current;
|
||||
if (iframe && previewFrameOpaque(iframe)) {
|
||||
activating = true;
|
||||
void frameCaptionSource(iframe)
|
||||
.then((src) =>
|
||||
src && !disposed
|
||||
? loadCaptions(src, (content) => frameCaptionModel(iframe, content))
|
||||
: null,
|
||||
)
|
||||
.finally(() => {
|
||||
activating = false;
|
||||
});
|
||||
return;
|
||||
}
|
||||
let doc: Document | null = null;
|
||||
let win: Window | null = null;
|
||||
try {
|
||||
@@ -74,63 +226,31 @@ export function useCaptionDetection({
|
||||
const groups = doc.querySelectorAll(".caption-group");
|
||||
if (groups.length === 0) return;
|
||||
|
||||
let captionSrcPath: string | null = null;
|
||||
|
||||
const compHosts = doc.querySelectorAll("[data-composition-src], [data-composition-file]");
|
||||
for (const host of compHosts) {
|
||||
const src =
|
||||
host.getAttribute("data-composition-src") || host.getAttribute("data-composition-file");
|
||||
if (src && src.includes("captions")) {
|
||||
captionSrcPath = src;
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
if (!captionSrcPath) {
|
||||
for (const [id, src] of compIdToSrc) {
|
||||
if (id.includes("caption") || src.includes("caption")) {
|
||||
captionSrcPath = src;
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if (!captionSrcPath && activeCompPath?.includes("captions")) {
|
||||
captionSrcPath = activeCompPath;
|
||||
}
|
||||
|
||||
if (!captionSrcPath) {
|
||||
const captionComp = doc.querySelector('[data-composition-id*="caption"]');
|
||||
if (captionComp) {
|
||||
const compId = captionComp.getAttribute("data-composition-id") || "";
|
||||
captionSrcPath = compIdToSrc.get(compId) || null;
|
||||
}
|
||||
}
|
||||
|
||||
const hostSrcs = [
|
||||
...doc.querySelectorAll("[data-composition-src], [data-composition-file]"),
|
||||
].map(
|
||||
(host) =>
|
||||
host.getAttribute("data-composition-src") || host.getAttribute("data-composition-file"),
|
||||
);
|
||||
const captionComp = doc.querySelector('[data-composition-id*="caption"]');
|
||||
const captionCompId = captionComp
|
||||
? captionComp.getAttribute("data-composition-id") || ""
|
||||
: null;
|
||||
const captionSrcPath = pickCaptionSource(hostSrcs, captionCompId);
|
||||
if (!captionSrcPath) return;
|
||||
|
||||
activating = true;
|
||||
const srcPath = captionSrcPath;
|
||||
studioApiFetch(buildProjectApiPath(projectId, `/files/${encodeURIComponent(srcPath)}`))
|
||||
.then((r) => r.json())
|
||||
.then((data: { content?: string }) => {
|
||||
if (!data.content || !doc || !win || useCaptionStore.getState().isEditMode) return;
|
||||
const root = doc.querySelector("[data-composition-id]");
|
||||
const w = parseInt(root?.getAttribute("data-width") ?? "1920", 10);
|
||||
const h = parseInt(root?.getAttribute("data-height") ?? "1080", 10);
|
||||
const dur = parseFloat(root?.getAttribute("data-duration") ?? "0");
|
||||
const model = parseCaptionComposition(doc, win, data.content, w, h, dur);
|
||||
if (!model) return;
|
||||
const store = useCaptionStore.getState();
|
||||
store.setModel(model);
|
||||
store.setSourceFilePath(srcPath);
|
||||
store.setEditMode(true);
|
||||
captionSync.loadOverrides();
|
||||
})
|
||||
.catch(() => {})
|
||||
.finally(() => {
|
||||
activating = false;
|
||||
});
|
||||
const frameDoc = doc;
|
||||
const frameWin = win;
|
||||
void loadCaptions(captionSrcPath, async (content) => {
|
||||
const root = frameDoc.querySelector("[data-composition-id]");
|
||||
const w = parseInt(root?.getAttribute("data-width") ?? "1920", 10);
|
||||
const h = parseInt(root?.getAttribute("data-height") ?? "1080", 10);
|
||||
const dur = parseFloat(root?.getAttribute("data-duration") ?? "0");
|
||||
return parseCaptionComposition(frameDoc, frameWin, content, w, h, dur);
|
||||
}).finally(() => {
|
||||
activating = false;
|
||||
});
|
||||
};
|
||||
|
||||
const handleMessage = (e: MessageEvent) => {
|
||||
@@ -146,6 +266,7 @@ export function useCaptionDetection({
|
||||
tryActivateCaptions();
|
||||
|
||||
return () => {
|
||||
disposed = true;
|
||||
window.removeEventListener("message", handleMessage);
|
||||
};
|
||||
}, [activeCompPath, projectId, compIdToSrc, captionSync, previewIframeRef, livePreviewIframe]);
|
||||
|
||||
Reference in New Issue
Block a user