diff --git a/packages/core/src/runtime/editing/captionTransform.ts b/packages/core/src/runtime/editing/captionTransform.ts new file mode 100644 index 0000000000..c7c633dfac --- /dev/null +++ b/packages/core/src/runtime/editing/captionTransform.ts @@ -0,0 +1,118 @@ +// `caption-transform`: reads and moves caption words, on a wrapper so the word's own tweens keep running. +import { MAX_DESCRIBE_HANDLES, type EditingRect } from "../editingElement"; +import { captionWordSpans, isCaptionIndex, isRecord } from "./captionWords"; + +export interface CaptionTransform { + x: number; + y: number; + scale: number; + rotation: number; +} + +/** `set` writes those parts and keeps the rest; without it the word is only read. */ +export interface CaptionTransformEntry { + group: number; + word: number; + set?: Partial; +} +export type CaptionTransformParams = { entries: CaptionTransformEntry[] }; + +/** The wrapper's transform after any write, and the word's box; null for a word the frame lacks. */ +export type CaptionWordState = { transform: CaptionTransform; rect: EditingRect } | null; + +const REST: CaptionTransform = { x: 0, y: 0, scale: 1, rotation: 0 }; +const PARTS = new Set(Object.keys(REST)); +/** Largest |x|, |y|, scale or rotation a caption word's transform may hold, in either direction. */ +export const MAX_CAPTION_TRANSFORM = 100_000; + +const isBounded = (value: unknown): value is number => + Number.isFinite(value) && Math.abs(value as number) <= MAX_CAPTION_TRANSFORM; + +function transformPart(set: unknown): Partial | null { + if (!isRecord(set)) return null; + const valid = Object.entries(set).every(([key, v]) => PARTS.has(key) && isBounded(v)); + return valid ? (set as Partial) : null; +} + +function entry(value: unknown): CaptionTransformEntry | null { + if (!isRecord(value)) return null; + const { group, word, set, ...rest } = value; + if (Object.keys(rest).length || !isCaptionIndex(group) || !isCaptionIndex(word)) return null; + if (!("set" in value)) return { group, word }; + const part = transformPart(set); + return part ? { group, word, set: part } : null; +} + +export function captionTransformParams(params: unknown): CaptionTransformParams | null { + if (!isRecord(params) || Object.keys(params).join() !== "entries") return null; + const { entries } = params; + if (!Array.isArray(entries) || !entries.length || entries.length > MAX_DESCRIBE_HANDLES) + return null; + const parsed = entries.map(entry); + return parsed.includes(null) ? null : { entries: parsed as CaptionTransformEntry[] }; +} + +type Gsap = { + getProperty?: (el: Element, prop: string) => number; + set?: (el: Element, props: Record) => void; +}; +const frameGsap = () => (window as unknown as { gsap?: Gsap }).gsap; + +const CSS_PARTS: Array<[keyof CaptionTransform, RegExp]> = [ + ["x", /translate\(([^,]+)px,\s*[^)]+px\)/], + ["y", /translate\([^,]+px,\s*([^)]+)px\)/], + ["scale", /scale\(([^)]+)\)/], + ["rotation", /rotate\(([^)]+)deg\)/], +]; + +/** GSAP keeps transforms in its own cache, so read through it when the page has it. */ +function readTransform(el: HTMLElement): CaptionTransform { + const gsap = frameGsap(); + const read = { ...REST }; + for (const [part, pattern] of CSS_PARTS) { + if (gsap?.getProperty) read[part] = gsap.getProperty(el, part) || REST[part]; + else { + const match = (el.style.transform || "").match(pattern); + if (match) read[part] = parseFloat(match[1]!); + } + } + return read; +} + +function writeTransform(el: HTMLElement, { x, y, scale, rotation }: CaptionTransform): void { + const gsap = frameGsap(); + if (gsap?.set) gsap.set(el, { x, y, scale, rotation }); + else + el.style.transform = `translate(${x.toFixed(1)}px, ${y.toFixed(1)}px) rotate(${rotation.toFixed(1)}deg) scale(${scale.toFixed(3)})`; +} + +function wrapperOf(word: HTMLElement): HTMLElement { + if (word.dataset.captionWrapper === "true") return word; + const parent = word.parentElement; + if (parent?.dataset.captionWrapper === "true") return parent; + const wrapper = document.createElement("span"); + wrapper.style.display = "inline-block"; + wrapper.dataset.captionWrapper = "true"; + word.parentNode?.insertBefore(wrapper, word); + wrapper.appendChild(word); + return wrapper; +} + +function wordAt(group: number, word: number): HTMLElement | null { + const el = document.querySelectorAll(".caption-group")[group]; + return el ? (captionWordSpans(el)[word] ?? null) : null; +} + +export function captionTransform({ entries }: CaptionTransformParams): { + words: CaptionWordState[]; +} { + const words = entries.map(({ group, word, set }) => { + const el = wordAt(group, word); + if (!el) return null; + const wrapper = wrapperOf(el); + if (set) writeTransform(wrapper, { ...readTransform(wrapper), ...set }); + const { left, top, width, height } = el.getBoundingClientRect(); + return { transform: readTransform(wrapper), rect: { x: left, y: top, width, height } }; + }); + return { words }; +} diff --git a/packages/core/src/runtime/editing/captionWords.test.ts b/packages/core/src/runtime/editing/captionWords.test.ts new file mode 100644 index 0000000000..353fceaa18 --- /dev/null +++ b/packages/core/src/runtime/editing/captionWords.test.ts @@ -0,0 +1,59 @@ +import { afterEach, expect, it } from "vitest"; +import { captionWords } from "./captionWords"; +import { captionTransform } from "./captionTransform"; + +afterEach(() => { + document.body.innerHTML = ""; + Reflect.deleteProperty(window, "gsap"); +}); + +it("splits a plain-text group into word spans and boxes only visible groups' words", () => { + document.body.innerHTML = `
Hello big world
+
hidden
+
onetwo
`; + const { boxes } = captionWords({ words: [2, 1, 5] }); + expect(boxes.map(({ group, word }) => [group, word])).toEqual([ + [0, 0], + [0, 1], + [2, 0], + [2, 1], + ]); + const [first, , third] = document.querySelectorAll(".caption-group"); + expect([...first!.children].map((s) => s.textContent)).toEqual(["Hello ", "big ", "world "]); + expect(third!.innerHTML).toBe("onetwo"); +}); + +it("moves a word on a wrapper with a CSS transform, keeping the parts it was not given", () => { + document.body.innerHTML = `
ab
`; + const moved = captionTransform({ entries: [{ group: 0, word: 1, set: { x: 3, rotation: 9 } }] }); + expect(moved.words[0]!.transform).toEqual({ x: 3, y: 0, scale: 1, rotation: 9 }); + const b = document.querySelectorAll("span:not([data-caption-wrapper])")[1]!; + expect((b.parentElement as HTMLElement).style.transform).toBe( + "translate(3.0px, 0.0px) rotate(9.0deg) scale(1.000)", + ); + const again = captionTransform({ entries: [{ group: 0, word: 1, set: { y: 4 } }] }); + expect(again.words[0]!.transform).toEqual({ x: 3, y: 4, scale: 1, rotation: 9 }); + expect(document.querySelectorAll("[data-caption-wrapper]")).toHaveLength(1); + expect( + captionTransform({ + entries: [ + { group: 0, word: 2 }, + { group: 1, word: 0 }, + ], + }).words, + ).toEqual([null, null]); +}); + +it("moves a word through the page's GSAP when it has one", () => { + const props = new WeakMap>(); + Reflect.set(window, "gsap", { + getProperty: (el: Element, prop: string) => props.get(el)?.[prop] ?? 0, + set: (el: Element, values: Record) => props.set(el, values), + }); + document.body.innerHTML = `
a
`; + const { words } = captionTransform({ entries: [{ group: 0, word: 0, set: { scale: 2 } }] }); + expect(words[0]!.transform).toEqual({ x: 0, y: 0, scale: 2, rotation: 0 }); + const wrapper = document.querySelector("[data-caption-wrapper]")!; + expect(props.get(wrapper)).toEqual({ x: 0, y: 0, scale: 2, rotation: 0 }); + expect(wrapper.style.transform).toBe(""); +}); diff --git a/packages/core/src/runtime/editing/captionWords.ts b/packages/core/src/runtime/editing/captionWords.ts new file mode 100644 index 0000000000..4598dd9fd2 --- /dev/null +++ b/packages/core/src/runtime/editing/captionWords.ts @@ -0,0 +1,89 @@ +// `caption-words`: the caption editor's word boxes, read where the words live. +import { MAX_QUERY_MATCHES, type EditingRect } from "../editingElement"; + +const MAX_CAPTION_WORDS = 100_000; + +export const isRecord = (value: unknown): value is Record => + Object.prototype.toString.call(value) === "[object Object]"; + +/** Word `word` of the `group`-th `.caption-group`, both counted from 0. */ +export interface CaptionWordBox { + group: number; + word: number; + /** `getBoundingClientRect()` of the word, in the frame's CSS pixels. */ + rect: EditingRect; +} + +/** `words[i]` is how many words Studio's model has in the i-th `.caption-group`. */ +export type CaptionWordsParams = { words: number[] }; + +export const isCaptionIndex = (value: unknown): value is number => + Number.isSafeInteger(value) && (value as number) >= 0 && (value as number) < MAX_CAPTION_WORDS; + +export function captionWordsParams(params: unknown): CaptionWordsParams | null { + if (!isRecord(params) || Object.keys(params).join() !== "words") return null; + const { words } = params; + if (!Array.isArray(words) || words.length > MAX_CAPTION_WORDS) return null; + return words.every(isCaptionIndex) ? { words } : null; +} + +/** A group's word spans in order: a wrapped word is the span inside its wrapper. */ +export function captionWordSpans(group: HTMLElement): HTMLElement[] { + const spans: HTMLElement[] = []; + for (const child of group.children) { + const el = child as HTMLElement; + if (el.dataset.captionWrapper === "true") { + const inner = el.querySelector(":scope > span"); + if (inner) spans.push(inner); + } else if (el.tagName === "SPAN") { + spans.push(el); + } + } + return spans; +} + +function splitIntoSpans(text: string, replace: (fragment: DocumentFragment) => void) { + const fragment = document.createDocumentFragment(); + const spans = text + .split(/\s+/) + .filter(Boolean) + .map((word) => { + const span = document.createElement("span"); + span.textContent = word + " "; + span.style.display = "inline"; + return fragment.appendChild(span); + }); + replace(fragment); + return spans; +} + +/** A group written as plain text becomes one span per word, so each word can be picked. */ +function splitGroupText(group: HTMLElement): HTMLElement[] { + const text = group.childNodes[0]; + if (text?.nodeType === Node.TEXT_NODE) { + return splitIntoSpans(text.textContent || "", (f) => group.replaceChild(f, text)); + } + const single = group.querySelector(":scope > span"); + if (!single?.textContent?.trim()) return []; + return splitIntoSpans(single.textContent, (f) => single.replaceWith(f)); +} + +/** Boxes of the words in visible groups, up to MAX_QUERY_MATCHES. */ +export function captionWords({ words }: CaptionWordsParams): { boxes: CaptionWordBox[] } { + const groups = document.querySelectorAll(".caption-group"); + const boxes: CaptionWordBox[] = []; + words.forEach((count, group) => { + const el = groups[group]; + if (!el) return; + const computed = window.getComputedStyle(el); + if (parseFloat(computed.opacity) <= 0.01 || computed.visibility === "hidden") return; + let spans = captionWordSpans(el); + if (spans.length === 0 && el.textContent?.trim()) spans = splitGroupText(el); + for (let word = 0; word < Math.min(count, spans.length); word++) { + if (boxes.length >= MAX_QUERY_MATCHES) return; + const { left, top, width, height } = spans[word]!.getBoundingClientRect(); + boxes.push({ group, word, rect: { x: left, y: top, width, height } }); + } + }); + return { boxes }; +} diff --git a/packages/core/src/runtime/editingAgent.test.ts b/packages/core/src/runtime/editingAgent.test.ts index baef4a8eda..7cf0a6a772 100644 --- a/packages/core/src/runtime/editingAgent.test.ts +++ b/packages/core/src/runtime/editingAgent.test.ts @@ -170,6 +170,8 @@ describe("installEditingAgent", () => { "console-errors", "watch-keys", "restore-attributes", + "caption-words", + "caption-transform", ], }, }); @@ -402,6 +404,28 @@ describe("installEditingAgent", () => { "invalid-params", ], ), + ...[{}, { words: 3 }, { words: [-1] }, { words: [1.5] }, { words: [100_000] }].map( + (params, index) => [ + { v: 1, id: 190 + index, kind: "caption-words", params }, + 190 + index, + "invalid-params", + ], + ), + ...[ + { entries: [] }, + { entries: Array(257).fill({ group: 0, word: 0 }) }, + { entries: [{ group: -1, word: 0 }] }, + { entries: [{ group: 0, word: 0, extra: 1 }] }, + { entries: [{ group: 0, word: 0, set: { x: 100_001 } }] }, + { entries: [{ group: 0, word: 0, set: { x: Infinity } }] }, + { entries: [{ group: 0, word: 0, set: { x: undefined } }] }, + { entries: [{ group: 0, word: 0, set: { skew: 1 } }] }, + { entries: [{ group: 0, word: 0, set: [] }] }, + ].map((params, index) => [ + { v: 1, id: 200 + index, kind: "caption-transform", params }, + 200 + index, + "invalid-params", + ]), ])("rejects %j with a typed error", async (request, id, code) => { const setLoopStart = vi.fn(); window.__hf = { setLoopStart }; diff --git a/packages/core/src/runtime/editingAgent.ts b/packages/core/src/runtime/editingAgent.ts index 69da49f23b..78a84d84cb 100644 --- a/packages/core/src/runtime/editingAgent.ts +++ b/packages/core/src/runtime/editingAgent.ts @@ -43,6 +43,8 @@ import { import { MAX_QUERY_MATCHES } from "./editingElement"; import { matchCombo, parseCombo, type Combo } from "../keyCombo"; import { shouldIgnorePlaybackShortcutTarget } from "../typingTarget"; +import { captionWords } from "./editing/captionWords"; +import { captionTransform } from "./editing/captionTransform"; type Handlers = { [K in EditingRequestKind]: ( @@ -190,6 +192,8 @@ const HANDLERS: Handlers = { }, "watch-keys": ({ combos }, port) => watchKeys(port, combos), "restore-attributes": (params) => restoreAttributes(params), + "caption-words": captionWords, + "caption-transform": captionTransform, }; function serve(port: MessagePort): void { diff --git a/packages/core/src/runtime/editingPort.test.ts b/packages/core/src/runtime/editingPort.test.ts index 1fef9c46b1..0ed526461d 100644 --- a/packages/core/src/runtime/editingPort.test.ts +++ b/packages/core/src/runtime/editingPort.test.ts @@ -78,6 +78,8 @@ describe("createEditingPortClient", () => { "console-errors", "watch-keys", "restore-attributes", + "caption-words", + "caption-transform", ], }); } finally { diff --git a/packages/core/src/runtime/editingPort.ts b/packages/core/src/runtime/editingPort.ts index 59cbcba70f..62b8aa25f1 100644 --- a/packages/core/src/runtime/editingPort.ts +++ b/packages/core/src/runtime/editingPort.ts @@ -24,6 +24,16 @@ import type { EditingLayer } from "./editingLayers"; import type { RuntimeTimelineMessage } from "./types"; import { wallClearTimeout, wallSetTimeout } from "./wallClock"; import { parseCombo } from "../keyCombo"; +import { + captionWordsParams, + type CaptionWordBox, + type CaptionWordsParams, +} from "./editing/captionWords"; +import { + captionTransformParams, + type CaptionTransformParams, + type CaptionWordState, +} from "./editing/captionTransform"; export { buildStableSelector, @@ -61,6 +71,7 @@ export { type ElementTransformSnapshot, } from "./editingGeometry"; export type { EditingGestureStart } from "./editingGestureStart"; +export { MAX_CAPTION_TRANSFORM } from "./editing/captionTransform"; export { compositionPoint, coversComposition, @@ -111,6 +122,8 @@ export const EDITING_REQUEST_KINDS = [ "console-errors", "watch-keys", "restore-attributes", + "caption-words", + "caption-transform", ] as const; export type EditingRequestKind = (typeof EDITING_REQUEST_KINDS)[number]; @@ -220,6 +233,9 @@ export interface EditingRequests { * keyups to this port; an empty list stops. */ "watch-keys": { params: { combos: string[] }; result: { watching: number } }; "restore-attributes": { params: RestoreAttributesParams; result: { applied: boolean } }; + /** Boxes of the words of visible caption groups; a group written as plain text is split into word spans. */ + "caption-words": { params: CaptionWordsParams; result: { boxes: CaptionWordBox[] } }; + "caption-transform": { params: CaptionTransformParams; result: { words: CaptionWordState[] } }; } export type EditingParams = EditingRequests[K]["params"]; export type EditingResult = EditingRequests[K]["result"]; @@ -407,6 +423,8 @@ const PARAM_PARSERS: { hasKeys(params, "from") && isOffset(params.from) ? { from: params.from } : null, "watch-keys": comboList, "restore-attributes": parseRestoreAttributes, + "caption-words": captionWordsParams, + "caption-transform": captionTransformParams, }; /** Validates one request off the wire. Never throws: anything unusable becomes a typed error. */ diff --git a/packages/studio/src/captions/components/CaptionOverlay.sealed.test.ts b/packages/studio/src/captions/components/CaptionOverlay.sealed.test.ts new file mode 100644 index 0000000000..c231f332fa --- /dev/null +++ b/packages/studio/src/captions/components/CaptionOverlay.sealed.test.ts @@ -0,0 +1,273 @@ +// @vitest-environment happy-dom +import { act, createElement, Fragment, useRef } from "react"; +import { createRoot } from "react-dom/client"; +import { afterEach, expect, it, vi } from "vitest"; +// The frame's real runtime agent, serving the opaque frame from this document. +import { installEditingAgent } from "../../../../core/src/runtime/editingAgent"; +import { installReactActEnvironment } from "../../hooks/domSelectionTestHarness"; +import { detachedFakeIframe } from "../../player/hooks/timelinePlayerTestHarness"; +import { previewRequest } from "../../player/lib/previewChannel"; +import { usePreviewIframeStore } from "../../player/store/previewIframeStore"; +import { useCaptionDetection } from "../../hooks/useCaptionDetection"; +import { studioApiFetch } from "../../utils/studioApiFetch"; +import type { useCaptionSync } from "../hooks/useCaptionSync"; +import { useCaptionStore } from "../store"; +import { CaptionOverlay } from "./CaptionOverlay"; +import { applyCaptionModelToIframe } from "./CaptionOverlayUtils"; +import { CaptionPropertyPanel } from "./CaptionPropertyPanel"; + +installReactActEnvironment(); + +const SOURCE = `const TRANSCRIPT = [ + { text: "Hello", start: 0, end: 0.5 }, + { text: "world", start: 0.5, end: 1 }, +];`; +vi.mock("../../utils/studioApiFetch", () => ({ + studioApiFetch: vi.fn(async () => ({ json: async () => ({ content: SOURCE }) })), +})); + +const closers: Array<() => void> = []; +afterEach(() => { + closers.splice(0).forEach((close) => close()); + useCaptionStore.getState().reset(); + usePreviewIframeStore.setState({ iframe: null }); + document.body.innerHTML = ""; + vi.unstubAllGlobals(); +}); + +type PortRequest = { id?: unknown; kind?: unknown; params?: unknown }; + +/** A frame Studio cannot read, whose runtime is the real editing agent over this document. */ +function opaqueFrame() { + const frame: Record = {}; + const iframe = detachedFakeIframe(frame); + Object.defineProperty(iframe, "contentDocument", { value: null }); + Object.defineProperty(frame, "document", { + get: () => { + throw new DOMException("blocked", "SecurityError"); + }, + }); + let pick: ((request: PortRequest) => boolean) | null = null; + const heldIds = new Set(); + const held: Array<() => void> = []; + // Studio's end of the port, so a test can keep the frame's answer to a request until it releases it. + const tap = (port: MessagePort) => { + const end = { + onmessage: null as ((event: MessageEvent) => void) | null, + postMessage(request: PortRequest) { + if (pick?.(request)) heldIds.add(request.id); + port.postMessage(request); + }, + close: () => port.close(), + }; + port.onmessage = (event) => { + const deliver = () => end.onmessage?.(event); + if (heldIds.has(event.data?.id)) held.push(deliver); + else deliver(); + }; + return end as unknown as MessagePort; + }; + const parent = { + postMessage: (data: unknown, _origin: string, ports: MessagePort[]) => + window.dispatchEvent( + new MessageEvent("message", { + source: frame as unknown as Window, + data, + ports: ports.map(tap), + }), + ), + } as unknown as Window; + const agent = installEditingAgent(parent); + frame.postMessage = (data: unknown) => + window.dispatchEvent(new MessageEvent("message", { source: parent, data })); + closers.push(() => agent.dispose()); + iframe.style.width = "1920px"; + iframe.getBoundingClientRect = () => new DOMRect(0, 0, 1920, 1080); + /** Keeps the answers to the requests `match` picks; the returned call hands them to Studio. */ + const hold = (match: (request: PortRequest) => boolean) => { + pick = match; + return () => { + pick = null; + heldIds.clear(); + held.splice(0).forEach((deliver) => deliver()); + }; + }; + return { iframe, hold, held }; +} + +/** Settles after one frame round trip, once Studio has acted on every answer it was handed before. */ +const frameRoundTrip = (iframe: HTMLIFrameElement) => + previewRequest(iframe, "query", { selector: "body" }); + +const isWrite = ({ kind, params }: PortRequest) => + kind === "caption-transform" && + (params as { entries: Array<{ set?: unknown }> }).entries.some((entry) => entry.set); + +const compIdToSrc = new Map([["captions", "compositions/captions.html"]]); +const captionSync = { loadOverrides: vi.fn() } as unknown as ReturnType; + +function CaptionEditor({ iframe }: { iframe: HTMLIFrameElement }) { + const iframeRef = useRef(iframe); + useCaptionDetection({ + projectId: "p", + activeCompPath: "index.html", + compIdToSrc, + captionEditMode: false, + captionHasSelection: false, + previewIframeRef: iframeRef, + captionSync, + setRightCollapsed: () => {}, + }); + const editing = useCaptionStore((s) => s.isEditMode); + // Studio mounts both only in caption edit mode. + if (!editing) return null; + return createElement( + Fragment, + null, + createElement(CaptionOverlay, { iframeRef }), + createElement(CaptionPropertyPanel, { iframeRef }), + ); +} + +const pointer = (type: string, x: number, y: number) => + new PointerEvent(type, { bubbles: true, clientX: x, clientY: y, pointerId: 1 }); + +const WORDS = `Hello world`; + +/** Mounts Studio's caption editing over an opaque frame holding one caption group of `words`. */ +function mountEditor(words = WORDS) { + vi.stubGlobal( + "ResizeObserver", + class { + observe() {} + disconnect() {} + }, + ); + Element.prototype.setPointerCapture ??= () => {}; + document.body.innerHTML = `
+
${words}
`; + const frame = opaqueFrame(); + usePreviewIframeStore.setState({ iframe: frame.iframe }); + const host = document.createElement("div"); + document.body.append(host); + const root = createRoot(host); + act(() => root.render(createElement(CaptionEditor, { iframe: frame.iframe }))); + const unmount = () => act(() => root.unmount()); + closers.push(unmount); + const boxes = () => + [...host.querySelectorAll("div.absolute")].filter((el) => el.style.cursor); + return { ...frame, host, boxes, unmount }; +} + +async function openEditor(words?: string) { + const editor = mountEditor(words); + await vi.waitFor(() => expect(editor.boxes()).toHaveLength(2), { timeout: 3_000 }); + return editor; +} + +const styleOf = (segmentId: string) => + useCaptionStore.getState().model!.segments.get(segmentId)!.style; + +it("caption overlay shows + edits sealed", async () => { + const { host, boxes } = await openEditor(); + const hello = document.querySelector(".word")!; + + act(() => boxes()[0]!.click()); + const segmentId = [...useCaptionStore.getState().selectedSegmentIds][0]!; + const overlay = boxes()[0]!.parentElement!; + // The move lands before the frame answers the press's read; it must still draw. + act(() => void boxes()[0]!.dispatchEvent(pointer("pointerdown", 100, 100))); + act(() => void overlay.dispatchEvent(pointer("pointermove", 130, 110))); + act(() => void overlay.dispatchEvent(pointer("pointerup", 130, 110))); + const transform = () => hello.parentElement!.style.transform; + await vi.waitFor(() => + expect(transform()).toBe("translate(30.0px, 10.0px) rotate(0.0deg) scale(1.000)"), + ); + await vi.waitFor(() => + expect(useCaptionStore.getState().model!.segments.get(segmentId)!.style).toMatchObject({ + x: 30, + y: 10, + }), + ); + + const xField = host.querySelector('input[aria-label="X position"]')!; + const setValue = Object.getOwnPropertyDescriptor(HTMLInputElement.prototype, "value")!.set!; + act(() => { + setValue.call(xField, "50"); + xField.dispatchEvent(new Event("input", { bubbles: true })); + }); + await vi.waitFor(() => expect(transform()).toContain("translate(50.0px, 10.0px)")); + + act(() => void window.dispatchEvent(new KeyboardEvent("keydown", { key: "ArrowRight" }))); + await vi.waitFor(() => expect(transform()).toContain("translate(51.0px, 10.0px)")); + + const model = useCaptionStore.getState().model!; + const segments = new Map(model.segments); + segments.set(segmentId, { ...segments.get(segmentId)!, style: { x: 5, y: 6 } }); + applyCaptionModelToIframe({ ...model, segments }); + await vi.waitFor(() => expect(transform()).toContain("translate(5.0px, 6.0px)")); +}); + +it("a detection still waiting on the frame when Studio unmounts it loads no captions", async () => { + vi.mocked(studioApiFetch).mockClear(); + const { iframe, hold, held, unmount } = mountEditor(); + // The detection's last question to the frame before it fetches the caption source. + const release = hold(({ kind, params }) => { + const selector = (params as { selector?: unknown } | undefined)?.selector; + return kind === "query" && selector === '[data-composition-id*="caption"]'; + }); + await vi.waitFor(() => expect(held).toHaveLength(1)); + unmount(); + release(); + await frameRoundTrip(iframe); + expect(studioApiFetch).not.toHaveBeenCalled(); + expect(useCaptionStore.getState()).toMatchObject({ isEditMode: false, model: null }); +}); + +it("a sealed drag the frame answers after the overlay unmounts stores no style", async () => { + const { iframe, hold, held, boxes, unmount } = await openEditor(); + act(() => boxes()[0]!.click()); + const segmentId = [...useCaptionStore.getState().selectedSegmentIds][0]!; + const overlay = boxes()[0]!.parentElement!; + const release = hold(isWrite); + act(() => void boxes()[0]!.dispatchEvent(pointer("pointerdown", 100, 100))); + act(() => void overlay.dispatchEvent(pointer("pointermove", 130, 110))); + act(() => void overlay.dispatchEvent(pointer("pointerup", 130, 110))); + await vi.waitFor(() => expect(held).toHaveLength(1)); + unmount(); + release(); + await frameRoundTrip(iframe); + expect(styleOf(segmentId)?.x).toBeUndefined(); +}); + +it("a sealed nudge the frame answers after the overlay unmounts stores no style", async () => { + const { iframe, hold, held, boxes, unmount } = await openEditor(); + act(() => boxes()[0]!.click()); + const segmentId = [...useCaptionStore.getState().selectedSegmentIds][0]!; + const release = hold(isWrite); + act(() => void window.dispatchEvent(new KeyboardEvent("keydown", { key: "ArrowRight" }))); + await vi.waitFor(() => expect(held).toHaveLength(1)); + unmount(); + release(); + await frameRoundTrip(iframe); + expect(styleOf(segmentId)?.x).toBeUndefined(); +}); + +it("a frame word past the write bound is not stored, and a word within it still is", async () => { + const far = `world`; + const { boxes } = await openEditor(`Hello ${far}`); + const press = (box: HTMLElement, dx: number, dy: number) => { + act(() => box.click()); + const overlay = box.parentElement!; + act(() => void box.dispatchEvent(pointer("pointerdown", 100, 100))); + if (dx || dy) act(() => void overlay.dispatchEvent(pointer("pointermove", 100 + dx, 100 + dy))); + act(() => void overlay.dispatchEvent(pointer("pointerup", 100 + dx, 100 + dy))); + return [...useCaptionStore.getState().selectedSegmentIds][0]!; + }; + // A press and release with no move stores where the frame says the word is. + const farWord = press(boxes()[1]!, 0, 0); + const nearWord = press(boxes()[0]!, 30, 10); + // The frame answers in order, so the far word's answer was handled before this one's. + await vi.waitFor(() => expect(styleOf(nearWord)).toMatchObject({ x: 30, y: 10 })); + expect(styleOf(farWord)?.x).toBeUndefined(); +}); diff --git a/packages/studio/src/captions/components/CaptionOverlay.tsx b/packages/studio/src/captions/components/CaptionOverlay.tsx index a1e6983f24..eaeb736b65 100644 --- a/packages/studio/src/captions/components/CaptionOverlay.tsx +++ b/packages/studio/src/captions/components/CaptionOverlay.tsx @@ -4,7 +4,17 @@ import { useCaptionStore } from "../store"; import { usePlayerStore } from "../../player"; import { resolveCaptionNudge, isEditableEventTarget } from "../keyboard"; import { matchesKey } from "../../player/components/studioShortcuts"; +import { previewFrameOpaque } from "../../player/lib/previewChannel"; import { + dragTransform, + moveSealedDrag, + pressSealedDrag, + readFrameWordBoxes, + restoreSealedDrag, + sealedNudger, + transformStyle, + type DragStart, + type SealedDrag, readWordBoxes, getWordEl, readGsapTransform, @@ -25,7 +35,10 @@ const ROTATION_OFFSET = 20; // px above the selection box /** Sync canvas state back to the Zustand store so the property panel reflects it. */ function syncToStore(segmentId: string, el: HTMLElement, iframeWin: Window) { - const style = computeTransformStyle(el, iframeWin); + storeStyle(segmentId, computeTransformStyle(el, iframeWin)); +} + +function storeStyle(segmentId: string, style: Record) { if (Object.keys(style).length > 0) { useCaptionStore.getState().updateSegmentStyle(segmentId, style); } @@ -47,59 +60,57 @@ export const CaptionOverlay = memo(function CaptionOverlay({ iframeRef }: Captio const cancelDragRef = useRef<(() => boolean) | null>(null); // Interaction mode — only one active at a time - const interactionRef = useRef< - | { - type: "move"; - wordEl: HTMLElement; - segmentId: string; - startMX: number; - startMY: number; - origTX: number; - origTY: number; - origScale: number; - origRotation: number; - } - | { - type: "scale"; - wordEl: HTMLElement; - segmentId: string; - startMX: number; - startDxFromCenter: number; - origTX: number; - origTY: number; - origScale: number; - origRotation: number; - } - | { - type: "rotate"; - wordEl: HTMLElement; - segmentId: string; - startMX: number; - origTX: number; - origTY: number; - origRotation: number; - origScale: number; - } - | null - >(null); + const interactionRef = useRef<(DragStart & { wordEl: HTMLElement; segmentId: string }) | null>( + null, + ); + const sealedDragRef = useRef(null); + const liveRef = useRef(false); // The observer and message filter bind to the live iframe element, which a promoted reload replaces. // eslint-disable-next-line no-restricted-syntax useEffect(() => { if (!isEditMode) return; + liveRef.current = true; // Let undo/redo (useAppHotkeys) reapply restored models to this iframe. const unregisterIframe = registerCaptionIframe(iframeRef); let prevBoxes: WordBox[] = []; let rafId: number | null = null; + let disposed = false; + let reading = false; + let readAgain = false; const tick = () => { rafId = null; const iframe = iframeRef.current; const m = modelRef.current; const overlay = overlayRef.current; if (!iframe || !m || !overlay) return; - const next = readWordBoxes(iframe, m, overlay); + if (previewFrameOpaque(iframe)) return readSealed(iframe, m, overlay); + show(readWordBoxes(iframe, m, overlay)); + }; + // One frame read in flight; triggers meanwhile collapse into one more read after it. + const readSealed = ( + iframe: HTMLIFrameElement, + m: NonNullable, + overlay: HTMLElement, + ) => { + if (reading) { + readAgain = true; + return; + } + reading = true; + void readFrameWordBoxes(iframe, m, overlay).then((next) => { + reading = false; + if (disposed) return; + if (next) show(next); + if (readAgain) { + readAgain = false; + scheduleTick(); + } + }); + }; + const show = (next: WordBox[]) => { if ( next.length === prevBoxes.length && next.every( @@ -155,6 +166,12 @@ export const CaptionOverlay = memo(function CaptionOverlay({ iframeRef }: Captio }; const cancelActiveDrag = (): boolean => { + const sealed = sealedDragRef.current; + if (sealed) { + sealedDragRef.current = null; + restoreSealedDrag(sealed); + return true; + } const i = interactionRef.current; if (!i) return false; const win = getWin(); @@ -202,6 +219,10 @@ export const CaptionOverlay = memo(function CaptionOverlay({ iframeRef }: Captio const { dx, dy } = nudge; const iframe = iframeRef.current; + if (iframe && previewFrameOpaque(iframe)) { + void nudgeSealed(iframe, m, sel, dx, dy).then(scheduleTick); + return; + } const win = iframe?.contentWindow; if (!iframe || !win) return; @@ -223,8 +244,15 @@ export const CaptionOverlay = memo(function CaptionOverlay({ iframeRef }: Captio scheduleTick(); }; + const nudgeSealed = sealedNudger( + () => !disposed, + (segId, moved) => storeStyle(segId, transformStyle(moved)), + ); + window.addEventListener("keydown", handleKeyDown); return () => { + disposed = true; + liveRef.current = false; unregisterIframe(); unsubPlayer(); unsubPlayback(); @@ -247,6 +275,27 @@ export const CaptionOverlay = memo(function CaptionOverlay({ iframeRef }: Captio return rect.width / nativeW; }, [iframeRef]); + /** A press on a frame Studio cannot read: the drag starts from the word the frame reports. */ + const startSealedDrag = useCallback( + ( + type: DragStart["type"], + word: { group: number; word: number }, + segmentId: string, + e: React.PointerEvent, + ) => { + const iframe = iframeRef.current; + if (!iframe) return; + const press = { + startMX: e.clientX, + startMY: e.clientY, + cssScale: getCssScale(), + left: iframe.getBoundingClientRect().left, + }; + sealedDragRef.current = pressSealedDrag(iframe, type, word, segmentId, press); + }, + [iframeRef, getCssScale], + ); + // --- Move --- const startMove = useCallback( (groupIndex: number, wordIndex: number, segmentId: string, e: React.PointerEvent) => { @@ -254,6 +303,8 @@ export const CaptionOverlay = memo(function CaptionOverlay({ iframeRef }: Captio (e.currentTarget as HTMLElement).setPointerCapture(e.pointerId); const iframe = iframeRef.current; if (!iframe) return; + if (previewFrameOpaque(iframe)) + return startSealedDrag("move", { group: groupIndex, word: wordIndex }, segmentId, e); const wordEl = getWordEl(iframe, groupIndex, wordIndex); const win = iframe.contentWindow; if (!wordEl || !win) return; @@ -270,7 +321,7 @@ export const CaptionOverlay = memo(function CaptionOverlay({ iframeRef }: Captio origRotation: state.rotation, }; }, - [iframeRef], + [iframeRef, startSealedDrag], ); // --- Scale --- @@ -281,6 +332,8 @@ export const CaptionOverlay = memo(function CaptionOverlay({ iframeRef }: Captio (e.currentTarget as HTMLElement).setPointerCapture(e.pointerId); const iframe = iframeRef.current; if (!iframe) return; + if (previewFrameOpaque(iframe)) + return startSealedDrag("scale", { group: groupIndex, word: wordIndex }, segmentId, e); const wordEl = getWordEl(iframe, groupIndex, wordIndex); const win = iframe.contentWindow; if (!wordEl || !win) return; @@ -303,7 +356,7 @@ export const CaptionOverlay = memo(function CaptionOverlay({ iframeRef }: Captio origRotation: state.rotation, }; }, - [iframeRef, getCssScale], + [iframeRef, getCssScale, startSealedDrag], ); // --- Rotate --- @@ -314,6 +367,13 @@ export const CaptionOverlay = memo(function CaptionOverlay({ iframeRef }: Captio (e.currentTarget as HTMLElement).setPointerCapture(e.pointerId); const iframe = iframeRef.current; if (!iframe) return; + if (previewFrameOpaque(iframe)) + return startSealedDrag( + "rotate", + { group: box.groupIndex, word: box.wordIndex }, + box.segmentId, + e, + ); const wordEl = getWordEl(iframe, box.groupIndex, box.wordIndex); const win = iframe.contentWindow; if (!wordEl || !win) return; @@ -329,7 +389,7 @@ export const CaptionOverlay = memo(function CaptionOverlay({ iframeRef }: Captio origScale: state.scale, }; }, - [iframeRef], + [iframeRef, startSealedDrag], ); const getIframeWin = useCallback((): Window | null => { @@ -343,34 +403,29 @@ export const CaptionOverlay = memo(function CaptionOverlay({ iframeRef }: Captio // --- Unified pointer move --- const handlePointerMove = useCallback( (e: React.PointerEvent) => { + const sealed = sealedDragRef.current; + if (sealed) return moveSealedDrag(sealed, e.clientX, e.clientY, getCssScale()); const i = interactionRef.current; if (!i) return; const win = getIframeWin(); if (!win) return; - if (i.type === "move") { - const cssScale = getCssScale(); - const dx = (e.clientX - i.startMX) / cssScale; - const dy = (e.clientY - i.startMY) / cssScale; - writeTransform(i.wordEl, win, i.origTX + dx, i.origTY + dy, i.origScale, i.origRotation); - } else if (i.type === "scale") { - const cx = i.startMX - i.startDxFromCenter; - const startDist = Math.abs(i.startDxFromCenter); - const currentDist = Math.abs(e.clientX - cx); - const factor = startDist > 5 ? currentDist / startDist : 1; - const newScale = Math.max(0.1, i.origScale * factor); - writeTransform(i.wordEl, win, i.origTX, i.origTY, newScale, i.origRotation); - } else if (i.type === "rotate") { - const dx = e.clientX - i.startMX; - const delta = (dx / 200) * 90; - writeTransform(i.wordEl, win, i.origTX, i.origTY, i.origScale, i.origRotation + delta); - } + const t = dragTransform(i, e.clientX, e.clientY, getCssScale()); + writeTransform(i.wordEl, win, t.x, t.y, t.scale, t.rotation); }, [getCssScale, getIframeWin], ); // --- Unified pointer up — sync back to store --- const handlePointerUp = useCallback(() => { + const sealed = sealedDragRef.current; + if (sealed) { + sealedDragRef.current = null; + void sealed.last.then( + (t) => t && liveRef.current && storeStyle(sealed.segmentId, transformStyle(t)), + ); + return; + } const i = interactionRef.current; if (i) { const win = getIframeWin(); diff --git a/packages/studio/src/captions/components/CaptionOverlayUtils.ts b/packages/studio/src/captions/components/CaptionOverlayUtils.ts index 214b351ac0..535976f3f3 100644 --- a/packages/studio/src/captions/components/CaptionOverlayUtils.ts +++ b/packages/studio/src/captions/components/CaptionOverlayUtils.ts @@ -1,4 +1,11 @@ // DOM helpers for CaptionOverlay — word box reading, transform I/O, wrapper management +import { + isPlainRecord, + MAX_CAPTION_TRANSFORM, + MAX_DESCRIBE_HANDLES, + type EditingParams, +} from "@hyperframes/core/runtime/editing-port"; +import { previewFrameOpaque, previewRequest } from "../../player/lib/previewChannel"; export interface WordBox { segmentId: string; @@ -11,6 +18,34 @@ export interface WordBox { height: number; } +/** Maps a box in the frame's CSS pixels into the overlay's. */ +function overlayPlacement(iframe: HTMLIFrameElement, overlayEl: HTMLElement) { + const iframeDisplayRect = iframe.getBoundingClientRect(); + const overlayRect = overlayEl.getBoundingClientRect(); + const nativeW = parseFloat(iframe.style.width) || iframeDisplayRect.width; + const cssScale = iframeDisplayRect.width / nativeW; + const offsetX = iframeDisplayRect.left - overlayRect.left; + const offsetY = iframeDisplayRect.top - overlayRect.top; + return (x: number, y: number, width: number, height: number) => ({ + x: x * cssScale + offsetX, + y: y * cssScale + offsetY, + width: width * cssScale, + height: height * cssScale, + }); +} + +function splitIntoSpans(doc: Document, text: string, into: HTMLElement[]): DocumentFragment { + const frag = doc.createDocumentFragment(); + for (const word of text.split(/\s+/).filter(Boolean)) { + const span = doc.createElement("span"); + span.textContent = word + " "; + span.style.display = "inline"; + frag.appendChild(span); + into.push(span); + } + return frag; +} + export function readWordBoxes( iframe: HTMLIFrameElement, model: { @@ -29,13 +64,7 @@ export function readWordBoxes( } if (!doc || !win) return []; - const iframeDisplayRect = iframe.getBoundingClientRect(); - const overlayRect = overlayEl.getBoundingClientRect(); - const nativeW = parseFloat(iframe.style.width) || iframeDisplayRect.width; - const cssScale = iframeDisplayRect.width / nativeW; - const offsetX = iframeDisplayRect.left - overlayRect.left; - const offsetY = iframeDisplayRect.top - overlayRect.top; - + const place = overlayPlacement(iframe, overlayEl); const groupEls = doc.querySelectorAll(".caption-group"); const boxes: WordBox[] = []; @@ -60,29 +89,14 @@ export function readWordBoxes( if (resolvedWordEls.length === 0 && groupEl.textContent?.trim()) { const textNode = groupEl.childNodes[0]; if (textNode && textNode.nodeType === Node.TEXT_NODE) { - const words = (textNode.textContent || "").split(/\s+/).filter(Boolean); - const frag = doc.createDocumentFragment(); - for (const word of words) { - const span = doc.createElement("span"); - span.textContent = word + " "; - span.style.display = "inline"; - frag.appendChild(span); - resolvedWordEls.push(span); - } - groupEl.replaceChild(frag, textNode); + groupEl.replaceChild( + splitIntoSpans(doc, textNode.textContent || "", resolvedWordEls), + textNode, + ); } else { const singleSpan = groupEl.querySelector(":scope > span"); if (singleSpan && singleSpan.textContent?.trim()) { - const words = singleSpan.textContent.split(/\s+/).filter(Boolean); - const frag = doc.createDocumentFragment(); - for (const word of words) { - const span = doc.createElement("span"); - span.textContent = word + " "; - span.style.display = "inline"; - frag.appendChild(span); - resolvedWordEls.push(span); - } - singleSpan.replaceWith(frag); + singleSpan.replaceWith(splitIntoSpans(doc, singleSpan.textContent, resolvedWordEls)); } } } @@ -96,10 +110,7 @@ export function readWordBoxes( groupId, groupIndex: gi, wordIndex: wi, - x: rect.left * cssScale + offsetX, - y: rect.top * cssScale + offsetY, - width: rect.width * cssScale, - height: rect.height * cssScale, + ...place(rect.left, rect.top, rect.width, rect.height), }); } } @@ -237,6 +248,17 @@ export function applyCaptionModelToIframe(model: { }): void { const iframe = registeredIframe?.current; if (!iframe) return; + if (previewFrameOpaque(iframe)) { + model.groupOrder.forEach((groupId, group) => + model.groups.get(groupId)?.segmentIds.forEach((segmentId, word) => { + const s = model.segments.get(segmentId)?.style; + if (!s) return; + const set = { x: s.x ?? 0, y: s.y ?? 0, scale: s.scaleX ?? 1, rotation: s.rotation ?? 0 }; + void frameWordTransform(iframe, { group, word, set }); + }), + ); + return; + } let win: Window | null = null; try { win = iframe.contentWindow; @@ -260,8 +282,10 @@ export function applyCaptionModelToIframe(model: { /** Compute style deltas from the current wrapper transform — used by syncToStore in the overlay. */ export function computeTransformStyle(el: HTMLElement, iframeWin: Window): Record { - const wrapper = getOrCreateWrapper(el); - const { x, y, scale, rotation } = readGsapTransform(wrapper, iframeWin); + return transformStyle(readGsapTransform(getOrCreateWrapper(el), iframeWin)); +} + +export function transformStyle({ x, y, scale, rotation }: WordTransform): Record { const style: Record = {}; if (Math.abs(x) > 0.5) style.x = x; if (Math.abs(y) > 0.5) style.y = y; @@ -272,3 +296,265 @@ export function computeTransformStyle(el: HTMLElement, iframeWin: Window): Recor if (Math.abs(rotation) > 0.1) style.rotation = rotation; return style; } + +interface WordTransform { + x: number; + y: number; + scale: number; + rotation: number; +} +type WordEntry = EditingParams<"caption-transform">["entries"][number]; +interface FrameWord { + transform: WordTransform; + rect: { x: number; y: number; width: number; height: number }; +} + +const allFinite = (value: unknown, keys: string[]) => + isPlainRecord(value) && keys.every((key) => Number.isFinite(value[key])); + +// The frame's own write bound: a larger reply, once stored, would make every later write to the word refused. +const allBounded = (value: unknown, keys: string[]) => + isPlainRecord(value) && + keys.every( + (key) => Number.isFinite(value[key]) && Math.abs(value[key] as number) <= MAX_CAPTION_TRANSFORM, + ); + +function readFrameWord(value: unknown): FrameWord | null { + if (!isPlainRecord(value)) return null; + const ok = + allBounded(value.transform, ["x", "y", "scale", "rotation"]) && + allFinite(value.rect, ["x", "y", "width", "height"]); + return ok ? (value as unknown as FrameWord) : null; +} + +export function findCaptionWord( + model: { groupOrder: string[]; groups: Map }, + segmentId: string, +): { group: number; word: number } | null { + for (let group = 0; group < model.groupOrder.length; group++) { + const word = model.groups.get(model.groupOrder[group])?.segmentIds.indexOf(segmentId) ?? -1; + if (word >= 0) return { group, word }; + } + return null; +} + +/** `readWordBoxes` for a frame Studio cannot read; null when its runtime does not answer. */ +export async function readFrameWordBoxes( + iframe: HTMLIFrameElement, + model: { groupOrder: string[]; groups: Map }, + overlayEl: HTMLElement, +): Promise { + const words = model.groupOrder.map((id) => model.groups.get(id)?.segmentIds.length ?? 0); + const reply = await previewRequest(iframe, "caption-words", { words }); + const boxes: unknown = reply?.boxes; + if (!Array.isArray(boxes)) return null; + const place = overlayPlacement(iframe, overlayEl); + return boxes.flatMap((box: unknown): WordBox[] => { + if (!isPlainRecord(box) || !allFinite(box.rect, ["x", "y", "width", "height"])) return []; + const { group, word } = box; + if (!Number.isSafeInteger(group) || !Number.isSafeInteger(word)) return []; + const groupId = model.groupOrder[group as number]; + const segmentId = groupId && model.groups.get(groupId)?.segmentIds[word as number]; + if (!segmentId) return []; + const { x, y, width, height } = box.rect as FrameWord["rect"]; + const at = place(x, y, width, height); + return [{ segmentId, groupId, groupIndex: group as number, wordIndex: word as number, ...at }]; + }); +} + +type Op = { entry: WordEntry; done: Array<(word: FrameWord | null) => void> }; +const transformQueues = new WeakMap(); + +function sendTransforms(iframe: HTMLIFrameElement, queue: { ops: Op[]; sending: boolean }) { + if (queue.sending || !queue.ops.length) return; + const ops = queue.ops.splice(0, MAX_DESCRIBE_HANDLES); + queue.sending = true; + const entries = ops.map((op) => op.entry); + void previewRequest(iframe, "caption-transform", { entries }) + .then((reply) => { + const words: unknown = reply?.words; + ops.forEach((op, i) => { + const word = Array.isArray(words) ? readFrameWord(words[i]) : null; + op.done.forEach((done) => done(word)); + }); + }) + .finally(() => { + queue.sending = false; + sendTransforms(iframe, queue); + }); +} + +/** One request in flight per frame; a waiting write to the same word takes the later one's parts. */ +export function frameWordTransform( + iframe: HTMLIFrameElement, + entry: WordEntry, +): Promise { + let queue = transformQueues.get(iframe); + if (!queue) transformQueues.set(iframe, (queue = { ops: [], sending: false })); + const { ops } = queue; + const promise = new Promise((resolve) => { + const last = ops[ops.length - 1]; + const sameWord = last?.entry.group === entry.group && last.entry.word === entry.word; + if (last && sameWord && last.entry.set && entry.set) { + last.entry = { ...entry, set: { ...last.entry.set, ...entry.set } }; + last.done.push(resolve); + } else ops.push({ entry, done: [resolve] }); + }); + sendTransforms(iframe, queue); + return promise; +} + +/** What a drag knows from its press: where it started and the word's transform then. */ +export type DragStart = + | { + type: "move"; + startMX: number; + startMY: number; + origTX: number; + origTY: number; + origScale: number; + origRotation: number; + } + | { + type: "scale"; + startMX: number; + startDxFromCenter: number; + origTX: number; + origTY: number; + origScale: number; + origRotation: number; + } + | { + type: "rotate"; + startMX: number; + origTX: number; + origTY: number; + origRotation: number; + origScale: number; + }; + +/** The word's transform with the pointer at (`clientX`, `clientY`). */ +export function dragTransform( + i: DragStart, + clientX: number, + clientY: number, + cssScale: number, +): WordTransform { + const orig = { x: i.origTX, y: i.origTY, scale: i.origScale, rotation: i.origRotation }; + if (i.type === "move") { + const dx = (clientX - i.startMX) / cssScale; + const dy = (clientY - i.startMY) / cssScale; + return { ...orig, x: i.origTX + dx, y: i.origTY + dy }; + } + if (i.type === "scale") { + const cx = i.startMX - i.startDxFromCenter; + const startDist = Math.abs(i.startDxFromCenter); + const currentDist = Math.abs(clientX - cx); + const factor = startDist > 5 ? currentDist / startDist : 1; + return { ...orig, scale: Math.max(0.1, i.origScale * factor) }; + } + const dx = clientX - i.startMX; + return { ...orig, rotation: i.origRotation + (dx / 200) * 90 }; +} + +/** A drag on a frame Studio cannot read: its press read lands later, and moves wait for it. */ +export interface SealedDrag { + iframe: HTMLIFrameElement; + word: { group: number; word: number }; + segmentId: string; + start: Promise; + /** The word's transform once the latest write lands. */ + last: Promise; +} + +const restOf = (drag: DragStart | null): WordTransform | null => + drag && { x: drag.origTX, y: drag.origTY, scale: drag.origScale, rotation: drag.origRotation }; + +/** The drag a press starts from the word the frame reports; `left` is the frame's left edge on screen. */ +function toDragStart( + type: DragStart["type"], + { transform: t, rect }: FrameWord, + press: { startMX: number; startMY: number; cssScale: number; left: number }, +): DragStart { + const { startMX, startMY, cssScale, left } = press; + const orig = { origTX: t.x, origTY: t.y, origScale: t.scale, origRotation: t.rotation }; + if (type === "move") return { type, startMX, startMY, ...orig }; + if (type === "rotate") return { type, startMX, ...orig }; + const boxCenterX = rect.x * cssScale + left + (rect.width * cssScale) / 2; + return { type, startMX, startDxFromCenter: startMX - boxCenterX, ...orig }; +} + +/** Moves a frame's word by (`dx`, `dy`) from where the frame has it now. */ +async function nudgeFrameWord( + iframe: HTMLIFrameElement, + word: { group: number; word: number }, + dx: number, + dy: number, +): Promise { + const now = await frameWordTransform(iframe, word); + if (!now) return null; + const { x, y } = now.transform; + return frameWordTransform(iframe, { ...word, set: { x: x + dx, y: y + dy } }); +} + +/** Nudges run one after another, so quick repeats all count; none starts or stores once `live` is false. */ +export function sealedNudger( + live: () => boolean, + onMoved: (segmentId: string, transform: WordTransform) => void, +) { + let nudging = Promise.resolve(); + return ( + iframe: HTMLIFrameElement, + model: Parameters[0], + segmentIds: Iterable, + dx: number, + dy: number, + ): Promise => { + for (const segmentId of segmentIds) { + const word = findCaptionWord(model, segmentId); + if (!word) continue; + nudging = nudging.then(async () => { + if (!live()) return; + const moved = await nudgeFrameWord(iframe, word, dx, dy); + if (moved && live()) onMoved(segmentId, moved.transform); + }); + } + return nudging; + }; +} + +/** Draws the drag at the pointer once the press read lands; `last` follows the newest write. */ +export function moveSealedDrag( + drag: SealedDrag, + clientX: number, + clientY: number, + cssScale: number, +) { + drag.last = drag.start.then(async (start) => { + if (!start) return null; + const set = dragTransform(start, clientX, clientY, cssScale); + return (await frameWordTransform(drag.iframe, { ...drag.word, set }))?.transform ?? null; + }); +} + +/** Puts the word back where the press found it. */ +export function restoreSealedDrag(drag: SealedDrag): void { + void drag.start.then((start) => { + const set = restOf(start); + if (set) void frameWordTransform(drag.iframe, { ...drag.word, set }); + }); +} + +/** A press on a word of a frame Studio cannot read; moves wait for the frame's answer. */ +export function pressSealedDrag( + iframe: HTMLIFrameElement, + type: DragStart["type"], + word: { group: number; word: number }, + segmentId: string, + press: { startMX: number; startMY: number; cssScale: number; left: number }, +): SealedDrag { + const start = frameWordTransform(iframe, word).then( + (read) => read && toDragStart(type, read, press), + ); + return { iframe, word, segmentId, start, last: start.then(restOf) }; +} diff --git a/packages/studio/src/captions/components/CaptionPropertyPanel.tsx b/packages/studio/src/captions/components/CaptionPropertyPanel.tsx index f1ea71f41c..f3be4df878 100644 --- a/packages/studio/src/captions/components/CaptionPropertyPanel.tsx +++ b/packages/studio/src/captions/components/CaptionPropertyPanel.tsx @@ -1,8 +1,10 @@ import { memo, useCallback, useState } from "react"; +import { previewFrameOpaque } from "../../player/lib/previewChannel"; import { useCaptionStore } from "../store"; import type { CaptionStyle } from "../types"; import { CaptionAnimationPanel } from "./CaptionAnimationPanel"; import { Section, Row, NumberField } from "./shared"; +import { findCaptionWord, frameWordTransform } from "./CaptionOverlayUtils"; /** True when the given style key differs across the selected segments. */ function isMixedValue( @@ -78,6 +80,21 @@ export const CaptionPropertyPanel = memo(function CaptionPropertyPanel({ (updates: Partial) => { const iframe = iframeRef.current; if (!iframe || !model) return; + if (previewFrameOpaque(iframe)) { + const parts = { + x: updates.x, + y: updates.y, + scale: updates.scaleX, + rotation: updates.rotation, + }; + const set = Object.fromEntries(Object.entries(parts).filter(([, v]) => v !== undefined)); + if (Object.keys(set).length === 0) return; + for (const segId of selectedSegmentIds) { + const word = findCaptionWord(model, segId); + if (word) void frameWordTransform(iframe, { ...word, set }); + } + return; + } let doc: Document | null = null; try { doc = iframe.contentDocument; diff --git a/packages/studio/src/hooks/useCaptionDetection.ts b/packages/studio/src/hooks/useCaptionDetection.ts index 1d188a390d..54721ec482 100644 --- a/packages/studio/src/hooks/useCaptionDetection.ts +++ b/packages/studio/src/hooks/useCaptionDetection.ts @@ -5,7 +5,16 @@ import { useCaptionStore } from "../captions/store"; import { acceptStudioRuntimeMessage } from "../player/lib/runtimeProtocol"; import { useCaptionSync } from "../captions/hooks/useCaptionSync"; import { parseCaptionComposition } from "../captions/parser"; +import type { CaptionModel } from "../captions/types"; import { studioApiFetch } from "../utils/studioApiFetch"; +import { MAX_DESCRIBE_HANDLES } from "@hyperframes/core/runtime/editing-port"; +import { previewComputedStyles, previewFrameOpaque } from "../player/lib/previewChannel"; +import { + describePreviewElements, + previewAncestors, + queryPreviewElements, + type PreviewElement, +} from "../player/lib/previewElements"; interface UseCaptionDetectionParams { projectId: string | null; @@ -18,6 +27,91 @@ interface UseCaptionDetectionParams { setRightCollapsed: (collapsed: boolean) => void; } +// The parser's GROUP_SELECTOR and WORD_SELECTOR, the elements it reads. +const GROUP_SELECTOR = ".caption-group, .caption-line, .caption-block"; +const WORD_SELECTOR = ".word, .caption-word"; + +const attribute = (el: PreviewElement | null | undefined, name: string) => + el?.attributes.find(([key]) => key === name)?.[1] ?? null; + +/** The styles `parseCaptionComposition` reads off its first word and group. */ +const CAPTION_STYLES = [ + "font-size", + "font-weight", + "font-family", + "color", + "text-transform", + "letter-spacing", + "background-color", + "border-radius", + "padding-top", + "padding-right", + "padding-bottom", + "padding-left", +]; + +async function frameStyles(iframe: HTMLIFrameElement, handle: string | undefined) { + if (!handle) return null; + const styles = await previewComputedStyles(iframe, { handle }, CAPTION_STYLES); + if (!styles) return null; + const camel = (name: string) => name.replace(/-(\w)/g, (_, c: string) => c.toUpperCase()); + return Object.fromEntries(CAPTION_STYLES.map((name) => [camel(name), styles[name] ?? ""])); +} + +/** A frame's caption groups and words rebuilt in order in a detached document, with a stand-in + * `getComputedStyle` for the parser; null past one query's matches or with no answer. */ +async function frameCaptionDom( + iframe: HTMLIFrameElement, +): Promise<{ doc: Document; win: Window } | null> { + const [all, groups, words] = await Promise.all( + [`${GROUP_SELECTOR}, ${WORD_SELECTOR}`, GROUP_SELECTOR, WORD_SELECTOR].map((selector) => + queryPreviewElements(iframe, selector), + ), + ); + if (!all || !groups || !words || all.truncated) return null; + const handles = all.matches.map((match) => match.handle); + const chains: Array = []; + for (let i = 0; i < handles.length; i += MAX_DESCRIBE_HANDLES) { + const part = await previewAncestors(iframe, handles.slice(i, i + MAX_DESCRIBE_HANDLES)); + if (!part) return null; + chains.push(...part); + } + const isGroup = new Set(groups.matches.map((match) => match.handle)); + const isWord = new Set(words.matches.map((match) => match.handle)); + const doc = document.implementation.createHTMLDocument(""); + const mirrored = new Map(); + handles.forEach((handle, i) => { + const el = doc.createElement("span"); + if (isGroup.has(handle)) el.classList.add("caption-group"); + if (isWord.has(handle)) el.classList.add("word"); + const parent = chains[i]?.find((ancestor) => mirrored.has(ancestor)); + (parent ? mirrored.get(parent)! : doc.body).append(el); + mirrored.set(handle, el); + }); + const firstWord = words.matches[0]?.handle; + const firstGroup = groups.matches[0]?.handle; + const styles = new Map | null>([ + [mirrored.get(firstWord ?? ""), await frameStyles(iframe, firstWord)], + [mirrored.get(firstGroup ?? ""), await frameStyles(iframe, firstGroup)], + ]); + const win = { getComputedStyle: (el: Element) => styles.get(el) ?? {} } as unknown as Window; + return { doc, win }; +} + +async function frameCaptionModel( + iframe: HTMLIFrameElement, + content: string, +): Promise { + const [root] = (await queryPreviewElements(iframe, "[data-composition-id]"))?.matches ?? []; + const rootEl = root ? (await describePreviewElements(iframe, [root.handle]))?.[0] : null; + const frame = await frameCaptionDom(iframe); + if (!frame) return null; + const w = parseInt(attribute(rootEl, "data-width") ?? "1920", 10); + const h = parseInt(attribute(rootEl, "data-height") ?? "1080", 10); + const dur = parseFloat(attribute(rootEl, "data-duration") ?? "0"); + return parseCaptionComposition(frame.doc, frame.win, content, w, h, dur); +} + export function useCaptionDetection({ projectId, activeCompPath, @@ -52,6 +146,51 @@ export function useCaptionDetection({ if (!projectId) return; let activating = false; + // A detection still in flight when the composition changes must not enter edit mode for the old one. + let disposed = false; + + const pickCaptionSource = (hostSrcs: Array, captionCompId: string | null) => { + const host = hostSrcs.find((src) => src && src.includes("captions")); + if (host) return host; + for (const [id, src] of compIdToSrc) { + if (id.includes("caption") || src.includes("caption")) return src; + } + if (activeCompPath?.includes("captions")) return activeCompPath; + return captionCompId === null ? null : compIdToSrc.get(captionCompId) || null; + }; + + const loadCaptions = ( + srcPath: string, + parse: (content: string) => Promise, + ) => + studioApiFetch(buildProjectApiPath(projectId, `/files/${encodeURIComponent(srcPath)}`)) + .then((r) => r.json()) + .then(async (data: { content?: string }) => { + if (disposed || !data.content || useCaptionStore.getState().isEditMode) return; + const model = await parse(data.content); + if (disposed || !model || useCaptionStore.getState().isEditMode) return; + const store = useCaptionStore.getState(); + store.setModel(model); + store.setSourceFilePath(srcPath); + store.setEditMode(true); + captionSync.loadOverrides(); + }) + .catch(() => {}); + + const frameCaptionSource = async (iframe: HTMLIFrameElement) => { + if (!(await queryPreviewElements(iframe, ".caption-group"))?.matches.length) return null; + const hosts = await queryPreviewElements( + iframe, + "[data-composition-src], [data-composition-file]", + ); + const handles = (hosts?.matches ?? []).slice(0, MAX_DESCRIBE_HANDLES).map((m) => m.handle); + const described = handles.length ? await describePreviewElements(iframe, handles) : []; + const hostSrcs = (described ?? []).map( + (el) => attribute(el, "data-composition-src") || attribute(el, "data-composition-file"), + ); + const captionComp = await queryPreviewElements(iframe, '[data-composition-id*="caption"]'); + return pickCaptionSource(hostSrcs, captionComp?.matches[0]?.source.compositionId ?? null); + }; const tryActivateCaptions = () => { const captionState = useCaptionStore.getState(); @@ -61,6 +200,19 @@ export function useCaptionDetection({ } const iframe = previewIframeRef.current; + if (iframe && previewFrameOpaque(iframe)) { + activating = true; + void frameCaptionSource(iframe) + .then((src) => + src && !disposed + ? loadCaptions(src, (content) => frameCaptionModel(iframe, content)) + : null, + ) + .finally(() => { + activating = false; + }); + return; + } let doc: Document | null = null; let win: Window | null = null; try { @@ -74,63 +226,31 @@ export function useCaptionDetection({ const groups = doc.querySelectorAll(".caption-group"); if (groups.length === 0) return; - let captionSrcPath: string | null = null; - - const compHosts = doc.querySelectorAll("[data-composition-src], [data-composition-file]"); - for (const host of compHosts) { - const src = - host.getAttribute("data-composition-src") || host.getAttribute("data-composition-file"); - if (src && src.includes("captions")) { - captionSrcPath = src; - break; - } - } - - if (!captionSrcPath) { - for (const [id, src] of compIdToSrc) { - if (id.includes("caption") || src.includes("caption")) { - captionSrcPath = src; - break; - } - } - } - - if (!captionSrcPath && activeCompPath?.includes("captions")) { - captionSrcPath = activeCompPath; - } - - if (!captionSrcPath) { - const captionComp = doc.querySelector('[data-composition-id*="caption"]'); - if (captionComp) { - const compId = captionComp.getAttribute("data-composition-id") || ""; - captionSrcPath = compIdToSrc.get(compId) || null; - } - } - + const hostSrcs = [ + ...doc.querySelectorAll("[data-composition-src], [data-composition-file]"), + ].map( + (host) => + host.getAttribute("data-composition-src") || host.getAttribute("data-composition-file"), + ); + const captionComp = doc.querySelector('[data-composition-id*="caption"]'); + const captionCompId = captionComp + ? captionComp.getAttribute("data-composition-id") || "" + : null; + const captionSrcPath = pickCaptionSource(hostSrcs, captionCompId); if (!captionSrcPath) return; activating = true; - const srcPath = captionSrcPath; - studioApiFetch(buildProjectApiPath(projectId, `/files/${encodeURIComponent(srcPath)}`)) - .then((r) => r.json()) - .then((data: { content?: string }) => { - if (!data.content || !doc || !win || useCaptionStore.getState().isEditMode) return; - const root = doc.querySelector("[data-composition-id]"); - const w = parseInt(root?.getAttribute("data-width") ?? "1920", 10); - const h = parseInt(root?.getAttribute("data-height") ?? "1080", 10); - const dur = parseFloat(root?.getAttribute("data-duration") ?? "0"); - const model = parseCaptionComposition(doc, win, data.content, w, h, dur); - if (!model) return; - const store = useCaptionStore.getState(); - store.setModel(model); - store.setSourceFilePath(srcPath); - store.setEditMode(true); - captionSync.loadOverrides(); - }) - .catch(() => {}) - .finally(() => { - activating = false; - }); + const frameDoc = doc; + const frameWin = win; + void loadCaptions(captionSrcPath, async (content) => { + const root = frameDoc.querySelector("[data-composition-id]"); + const w = parseInt(root?.getAttribute("data-width") ?? "1920", 10); + const h = parseInt(root?.getAttribute("data-height") ?? "1080", 10); + const dur = parseFloat(root?.getAttribute("data-duration") ?? "0"); + return parseCaptionComposition(frameDoc, frameWin, content, w, h, dur); + }).finally(() => { + activating = false; + }); }; const handleMessage = (e: MessageEvent) => { @@ -146,6 +266,7 @@ export function useCaptionDetection({ tryActivateCaptions(); return () => { + disposed = true; window.removeEventListener("message", handleMessage); }; }, [activeCompPath, projectId, compIdToSrc, captionSync, previewIframeRef, livePreviewIframe]);