From b741ecf1320ebad92f3a74b1e87a50a326c84db4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Miguel=20=C3=81ngel?= Date: Mon, 16 Nov 2026 08:09:03 +0000 Subject: [PATCH] feat(studio): a sealed preview frame forwards Studio's playback keys (#7844) * feat(studio): a sealed preview frame forwards Studio's playback keys * fix(core): a key held while Studio re-sends its watch list still forwards its keyup * fix(core): the key watch adds and removes its listeners by name, as the runtime typecheck needs * chore(producer): raise the unpacked size budget for the runtime's key forwarding --- .../core/src/runtime/editingAgent.test.ts | 110 ++++++++++++++++++ packages/core/src/runtime/editingAgent.ts | 68 ++++++++++- packages/core/src/runtime/editingPort.test.ts | 41 +++++++ packages/core/src/runtime/editingPort.ts | 63 +++++++++- .../src/player/components/studioShortcuts.ts | 5 +- .../player/hooks/timelinePlayerTestHarness.ts | 2 + .../hooks/timelineSyncHydration.test.ts | 30 ++++- .../src/player/hooks/timelineSyncHydration.ts | 10 +- .../player/hooks/usePlaybackKeyboard.test.ts | 109 ++++++++++++++++- .../src/player/hooks/usePlaybackKeyboard.ts | 39 ++++++- scripts/package-size-budgets.json | 2 +- 11 files changed, 461 insertions(+), 18 deletions(-) diff --git a/packages/core/src/runtime/editingAgent.test.ts b/packages/core/src/runtime/editingAgent.test.ts index 0bc8f6eb41..936382bec8 100644 --- a/packages/core/src/runtime/editingAgent.test.ts +++ b/packages/core/src/runtime/editingAgent.test.ts @@ -168,6 +168,7 @@ describe("installEditingAgent", () => { "draft", "gesture-start", "console-errors", + "watch-keys", ], }, }); @@ -380,6 +381,19 @@ describe("installEditingAgent", () => { "invalid-params", ], ), + ...[ + {}, + { combos: "Space" }, + { combos: [1] }, + { combos: ["ctrl+alt+x"] }, + { combos: ["k".repeat(65)] }, + { combos: Array(33).fill("k") }, + { combos: [], extra: 1 }, + ].map((params, index) => [ + { v: 1, id: 180 + index, kind: "watch-keys", params }, + 180 + index, + "invalid-params", + ]), ...[["--"], ["--a b"], ["--a;b"], ["--a:b"], ["Color"], ["-x"], ["--" + "a".repeat(255)]].map( (properties, index) => [ { v: 1, id: 60 + index, kind: "computed-styles", params: { selector: "p", properties } }, @@ -502,4 +516,100 @@ describe("installEditingAgent", () => { error: { code: "failed" }, }); }); + + describe("watch-keys", () => { + const press = (type: string, init: KeyboardEventInit, target: EventTarget = document.body) => { + const event = new KeyboardEvent(type, { bubbles: true, cancelable: true, ...init }); + target.dispatchEvent(event); + return event.defaultPrevented; + }; + const SPACE = { key: " ", code: "Space" }; + const forwarded = (type: string, key: string, code: string, repeat = false) => ({ + v: 1, + event: "key", + key: { + type, + key, + code, + altKey: false, + ctrlKey: false, + metaKey: false, + shiftKey: false, + repeat, + }, + }); + // Port messages arrive in order, so everything sent before this reply has arrived by then. + const received = async (port: MessagePort, seen: unknown[]) => { + port.postMessage({ v: 1, id: 99, kind: "hello" }); + await vi.waitFor(() => expect(seen.at(-1)).toMatchObject({ id: 99 })); + return seen.slice(0, -1); + }; + const watch = (port: MessagePort, combos: string[], id = 1) => + ask(port, { v: 1, id, kind: "watch-keys", params: { combos } }); + const collect = (port: MessagePort) => { + const seen: unknown[] = []; + port.onmessage = (event) => seen.push(event.data); + return seen; + }; + + it("cancels and forwards a watched combo outside the elements that own keys, and its keyup", async () => { + document.body.innerHTML = ``; + const port = connectedPort(); + expect(await watch(port, ["Space", "k"])).toMatchObject({ + ok: true, + result: { watching: 2 }, + }); + const seen = collect(port); + + expect(press("keydown", SPACE)).toBe(true); + expect(press("keydown", { ...SPACE, repeat: true })).toBe(true); + expect(press("keydown", { key: "j", code: "KeyJ" })).toBe(false); + expect(press("keydown", SPACE, document.getElementById("field")!)).toBe(false); + expect(press("keydown", SPACE, document.getElementById("go")!)).toBe(false); + expect(press("keyup", SPACE)).toBe(false); + press("keyup", { key: "j", code: "KeyJ" }); + + expect(await received(port, seen)).toEqual([ + forwarded("keydown", " ", "Space"), + forwarded("keydown", " ", "Space", true), + forwarded("keyup", " ", "Space"), + ]); + }); + + it("leaves a key the film cancelled before it", async () => { + const film = (event: Event) => event.preventDefault(); + window.addEventListener("keydown", film, true); + try { + const port = connectedPort(); + await watch(port, ["k"]); + const seen = collect(port); + press("keydown", { key: "k", code: "KeyK" }); + expect(await received(port, seen)).toEqual([]); + } finally { + window.removeEventListener("keydown", film, true); + } + }); + + it("still forwards the keyup of a key held while the watch list changes", async () => { + const port = connectedPort(); + await watch(port, ["k"]); + const seen = collect(port); + press("keydown", { key: "k", code: "KeyK" }); + await vi.waitFor(() => expect(seen).toEqual([forwarded("keydown", "k", "KeyK")])); + expect(await watch(port, ["k", "j"], 2)).toMatchObject({ result: { watching: 2 } }); + const after = collect(port); + press("keyup", { key: "k", code: "KeyK" }); + expect(await received(port, after)).toEqual([forwarded("keyup", "k", "KeyK")]); + }); + + it("stops on an empty list and when disposed", async () => { + const port = connectedPort(); + await watch(port, ["k"]); + expect(await watch(port, [], 2)).toMatchObject({ result: { watching: 0 } }); + expect(press("keydown", { key: "k", code: "KeyK" })).toBe(false); + await watch(port, ["k"], 3); + agent.dispose(); + expect(press("keydown", { key: "k", code: "KeyK" })).toBe(false); + }); + }); }); diff --git a/packages/core/src/runtime/editingAgent.ts b/packages/core/src/runtime/editingAgent.ts index 57310ecc11..851d774f0e 100644 --- a/packages/core/src/runtime/editingAgent.ts +++ b/packages/core/src/runtime/editingAgent.ts @@ -14,6 +14,7 @@ import { EDITING_REQUEST_KINDS, isPlainRecord, parseEditingRequest, + type EditingKeyMessage, type EditingParams, type EditingReply, type EditingRequestKind, @@ -39,10 +40,13 @@ import { STUDIO_PREVIEW_ERRORS, } from "../studioPreviewMark"; import { MAX_QUERY_MATCHES } from "./editingElement"; +import { matchCombo, parseCombo, type Combo } from "../keyCombo"; +import { shouldIgnorePlaybackShortcutTarget } from "../typingTarget"; type Handlers = { [K in EditingRequestKind]: ( params: EditingParams, + port: MessagePort, ) => EditingResult | Promise>; }; @@ -76,6 +80,63 @@ function watchConsoleErrors(): string[] { return seen; } +const keyWatches = new Map }>(); + +function forwardKey(port: MessagePort, e: KeyboardEvent): void { + const { type, key, code, altKey, ctrlKey, metaKey, shiftKey, repeat } = e; + const message: EditingKeyMessage = { + v: EDITING_PROTOCOL_VERSION, + event: "key", + key: { + type: type === "keyup" ? "keyup" : "keydown", + key, + code, + altKey, + ctrlKey, + metaKey, + shiftKey, + repeat, + }, + }; + port.postMessage(message); +} + +function onWatchedKeyDown(e: KeyboardEvent): void { + if (e.defaultPrevented || shouldIgnorePlaybackShortcutTarget(e.target)) return; + for (const [port, watch] of keyWatches) { + if (!watch.combos.some((combo) => matchCombo(e, combo))) continue; + e.preventDefault(); + watch.down.add(e.code || e.key); + forwardKey(port, e); + } +} + +function onWatchedKeyUp(e: KeyboardEvent): void { + for (const [port, watch] of keyWatches) { + if (watch.down.delete(e.code || e.key)) forwardKey(port, e); + } +} + +function listenForKeys(on: boolean): void { + if (on) { + window.addEventListener("keydown", onWatchedKeyDown, true); + window.addEventListener("keyup", onWatchedKeyUp, true); + } else { + window.removeEventListener("keydown", onWatchedKeyDown, true); + window.removeEventListener("keyup", onWatchedKeyUp, true); + } +} + +function watchKeys(port: MessagePort, combos: string[]): { watching: number } { + if (combos.length === 0) keyWatches.delete(port); + else { + const down = keyWatches.get(port)?.down ?? new Set(); + keyWatches.set(port, { combos: combos.map((text) => parseCombo(text)!), down }); + } + listenForKeys(keyWatches.size > 0); + return { watching: combos.length }; +} + const HANDLERS: Handlers = { hello: () => ({ version: EDITING_PROTOCOL_VERSION, kinds: [...EDITING_REQUEST_KINDS] }), "runtime-snapshot": () => ({ @@ -126,6 +187,7 @@ const HANDLERS: Handlers = { const errors = seen.slice(from, from + MAX_QUERY_MATCHES); return { errors, truncated: from + errors.length < seen.length }; }, + "watch-keys": ({ combos }, port) => watchKeys(port, combos), }; function serve(port: MessagePort): void { @@ -149,9 +211,9 @@ function serve(port: MessagePort): void { }); return; } - const handler = HANDLERS[request.kind] as (params: unknown) => unknown; + const handler = HANDLERS[request.kind] as (params: unknown, port: MessagePort) => unknown; Promise.resolve() - .then(() => handler(request.params)) + .then(() => handler(request.params, port)) .then( (result) => reply({ v: EDITING_PROTOCOL_VERSION, id: request.id, ok: true, result }), (err: unknown) => reply(failure(request.id, err)), @@ -203,6 +265,8 @@ export function installEditingAgent(parent: Window = window.parent): { dispose: ports.clear(); consoleWatch?.stop(); consoleWatch = null; + keyWatches.clear(); + listenForKeys(false); }, }; } diff --git a/packages/core/src/runtime/editingPort.test.ts b/packages/core/src/runtime/editingPort.test.ts index de6e9ccb45..cba273ceb9 100644 --- a/packages/core/src/runtime/editingPort.test.ts +++ b/packages/core/src/runtime/editingPort.test.ts @@ -76,6 +76,7 @@ describe("createEditingPortClient", () => { "draft", "gesture-start", "console-errors", + "watch-keys", ], }); } finally { @@ -83,6 +84,46 @@ describe("createEditingPortClient", () => { } }); + it("hands each key the frame forwards to its listeners and drops off-shape ones", async () => { + const { iframe, frame } = previewFrame(); + const { host, deliver } = fakeHost(); + const client = createEditingPortClient({ iframe, host }); + clients.push(client); + const { port1, port2 } = channel(); + deliver({ source: frame, data: PORT_MESSAGE, ports: [port1] }); + await client.ready; + const key = { + type: "keydown", + key: "k", + code: "KeyK", + altKey: false, + ctrlKey: false, + metaKey: false, + shiftKey: false, + repeat: false, + }; + const first = vi.fn(); + const second = vi.fn(); + const stopFirst = client.onKey(first); + client.onKey(second); + for (const bad of [ + { v: 2, event: "key", key }, + { v: 1, event: "keys", key }, + { v: 1, event: "key", key: { ...key, type: "keypress" } }, + { v: 1, event: "key", key: { ...key, repeat: "no" } }, + { v: 1, event: "key", key: { ...key, extra: 1 } }, + { v: 1, event: "key", key: { ...key, key: "k".repeat(65) } }, + ]) + port2.postMessage(bad); + port2.postMessage({ v: 1, event: "key", key }); + await vi.waitFor(() => expect(second).toHaveBeenCalledOnce()); + expect(first).toHaveBeenCalledExactlyOnceWith(key); + stopFirst(); + port2.postMessage({ v: 1, event: "key", key: { ...key, type: "keyup" } }); + await vi.waitFor(() => expect(second).toHaveBeenCalledTimes(2)); + expect(first).toHaveBeenCalledOnce(); + }); + it("serves a new client for the same frame after the first is disposed", async () => { const { iframe, frame } = previewFrame(); const { host, deliver } = fakeHost(); diff --git a/packages/core/src/runtime/editingPort.ts b/packages/core/src/runtime/editingPort.ts index 5cae0410dc..0206f79b70 100644 --- a/packages/core/src/runtime/editingPort.ts +++ b/packages/core/src/runtime/editingPort.ts @@ -22,6 +22,7 @@ import type { EditingGestureStart } from "./editingGestureStart"; import type { EditingLayer } from "./editingLayers"; import type { RuntimeTimelineMessage } from "./types"; import { wallClearTimeout, wallSetTimeout } from "./wallClock"; +import { parseCombo } from "../keyCombo"; export { buildStableSelector, @@ -107,6 +108,7 @@ export const EDITING_REQUEST_KINDS = [ "draft", "gesture-start", "console-errors", + "watch-keys", ] as const; export type EditingRequestKind = (typeof EDITING_REQUEST_KINDS)[number]; @@ -122,6 +124,24 @@ export interface EditingRuntimeSnapshot { } export const MAX_DRAFT_PX = 100_000; +export const MAX_WATCHED_KEYS = 32; + +/** A keystroke the frame forwards for `watch-keys`, as an unasked port message with no id. */ +export interface EditingKeyEvent { + type: "keydown" | "keyup"; + key: string; + code: string; + altKey: boolean; + ctrlKey: boolean; + metaKey: boolean; + shiftKey: boolean; + repeat: boolean; +} +export type EditingKeyMessage = { + v: typeof EDITING_PROTOCOL_VERSION; + event: "key"; + key: EditingKeyEvent; +}; /** A gesture's draft on one element: null puts that part back; `end` keeps what is drawn or restores. */ export type EditingDraft = @@ -194,6 +214,9 @@ export interface EditingRequests { result: { elements: Array; snap: EditingElementBox[] }; }; "console-errors": { params: { from: number }; result: { errors: string[]; truncated: boolean } }; + /** The frame cancels these combos outside the elements that own keys, and forwards them and their + * keyups to this port; an empty list stops. */ + "watch-keys": { params: { combos: string[] }; result: { watching: number } }; } export type EditingParams = EditingRequests[K]["params"]; export type EditingResult = EditingRequests[K]["result"]; @@ -313,6 +336,32 @@ function draftList(params: unknown): EditingParams<"draft"> | null { return parsed.includes(null) ? null : { drafts: parsed as EditingDraft[] }; } +const MAX_COMBO_LENGTH = 64; + +function comboList(params: unknown): EditingParams<"watch-keys"> | null { + if (!hasKeys(params, "combos") || !Array.isArray(params.combos)) return null; + const { combos } = params; + if (combos.length > MAX_WATCHED_KEYS) return null; + const valid = combos.every( + (combo) => typeof combo === "string" && combo.length <= MAX_COMBO_LENGTH && parseCombo(combo), + ); + return valid ? { combos } : null; +} + +const KEY_EVENT_FIELDS = "altKey,code,ctrlKey,key,metaKey,repeat,shiftKey,type"; +const KEY_FLAGS = ["altKey", "ctrlKey", "metaKey", "shiftKey", "repeat"] as const; + +function keyEvent(value: unknown): EditingKeyEvent | null { + if (!hasKeys(value, KEY_EVENT_FIELDS)) return null; + const { type, key, code } = value; + if (type !== "keydown" && type !== "keyup") return null; + const short = (text: unknown) => typeof text === "string" && text.length <= MAX_COMBO_LENGTH; + if (!short(key) || !short(code)) return null; + return KEY_FLAGS.every((flag) => typeof value[flag] === "boolean") + ? (value as unknown as EditingKeyEvent) + : null; +} + const PARAM_PARSERS: { [K in EditingRequestKind]: (params: unknown) => EditingParams | null; } = { @@ -353,6 +402,7 @@ const PARAM_PARSERS: { "gesture-start": handleList, "console-errors": (params) => hasKeys(params, "from") && isOffset(params.from) ? { from: params.from } : null, + "watch-keys": comboList, }; /** Validates one request off the wire. Never throws: anything unusable becomes a typed error. */ @@ -390,6 +440,8 @@ export interface EditingPortClient { params: EditingParams, options?: { timeoutMs?: number }, ): Promise>; + /** Calls `listener` with each key the frame forwards for `watch-keys`; returns the unsubscribe. */ + onKey(listener: (key: EditingKeyEvent) => void): () => void; /** Rejects every pending request with `disposed` and closes the port. */ dispose(): void; } @@ -446,6 +498,7 @@ export function createEditingPortClient(options: { }): EditingPortClient { const { iframe, timeoutMs = 5_000, host = window } = options; const pending = new Map(); + const keyListeners = new Set<(key: EditingKeyEvent) => void>(); let port: MessagePort | null = null; let disposed = false; let lastId = 0; @@ -467,7 +520,10 @@ export function createEditingPortClient(options: { }; const onReply = (event: MessageEvent) => { const data = event.data; - if (isPlainRecord(data) && typeof data.id === "number") settle(data.id, replyOutcome(data)); + if (!isPlainRecord(data)) return; + if (typeof data.id === "number") return settle(data.id, replyOutcome(data)); + const key = data.v === EDITING_PROTOCOL_VERSION && data.event === "key" && keyEvent(data.key); + if (key) for (const listener of keyListeners) listener(key); }; const connect = () => { try { @@ -516,9 +572,14 @@ export function createEditingPortClient(options: { ready.then(send, () => {}); return result as Promise>; }, + onKey(listener) { + keyListeners.add(listener); + return () => keyListeners.delete(listener); + }, dispose() { if (disposed) return; disposed = true; + keyListeners.clear(); host.removeEventListener("message", onWindowMessage); port?.close(); rejectReady(new EditingPortError("disposed", "client disposed")); diff --git a/packages/studio/src/player/components/studioShortcuts.ts b/packages/studio/src/player/components/studioShortcuts.ts index 1f1d39584e..f1376ef502 100644 --- a/packages/studio/src/player/components/studioShortcuts.ts +++ b/packages/studio/src/player/components/studioShortcuts.ts @@ -86,12 +86,13 @@ export function setStudioKeys(overrides: Partial export const getStudioKeys = (): StudioKeys => current; export const getStudioKey = (action: StudioKeyAction): string => current[action]; -function subscribe(listener: () => void) { +export function subscribeStudioKeys(listener: () => void) { listeners.add(listener); return () => listeners.delete(listener); } -export const useStudioKeys = (): StudioKeys => useSyncExternalStore(subscribe, getStudioKeys); +export const useStudioKeys = (): StudioKeys => + useSyncExternalStore(subscribeStudioKeys, getStudioKeys); export function useShortcutSections(host?: readonly ShortcutSection[]): readonly ShortcutSection[] { const keys = useStudioKeys(); diff --git a/packages/studio/src/player/hooks/timelinePlayerTestHarness.ts b/packages/studio/src/player/hooks/timelinePlayerTestHarness.ts index 4541464e1a..dad2cc5d6d 100644 --- a/packages/studio/src/player/hooks/timelinePlayerTestHarness.ts +++ b/packages/studio/src/player/hooks/timelinePlayerTestHarness.ts @@ -157,12 +157,14 @@ export async function paintShadow(getApi: () => TimelinePlayerApi, shadow: HTMLI export function serveEditingRequests( frame: Record, answer: (kind: string, params: unknown) => unknown, + onPort?: (port: MessagePort) => void, ): () => void { const ports: MessagePort[] = []; frame.postMessage = (message: { type?: string } | null) => { if (message?.type !== "editing-connect") return; const { port1, port2 } = new MessageChannel(); ports.push(port1); + onPort?.(port1); port1.onmessage = (event: MessageEvent<{ id: number; kind: string; params: unknown }>) => { const { id, kind, params } = event.data; port1.postMessage({ v: 1, id, ok: true, result: answer(kind, params) }); diff --git a/packages/studio/src/player/hooks/timelineSyncHydration.test.ts b/packages/studio/src/player/hooks/timelineSyncHydration.test.ts index 6b90e29382..23e6180739 100644 --- a/packages/studio/src/player/hooks/timelineSyncHydration.test.ts +++ b/packages/studio/src/player/hooks/timelineSyncHydration.test.ts @@ -1,11 +1,12 @@ // @vitest-environment happy-dom -import { describe, expect, it } from "vitest"; +import { describe, expect, it, vi } from "vitest"; import { collectSubCompositionDomChildren, collectSubCompositionHostState, + hydrateTimelineFromPreview, } from "./timelineSyncHydration"; -import type { ClipManifestClip } from "../lib/playbackTypes"; +import type { ClipManifestClip, PlaybackAdapter } from "../lib/playbackTypes"; const clip = (over: Partial): ClipManifestClip => ({ id: "x", @@ -138,3 +139,28 @@ describe("collectSubCompositionHostState", () => { expect(state.size).toBe(0); }); }); + +describe("hydrateTimelineFromPreview", () => { + it("attaches the shortcut listeners to a frame whose document it cannot read", () => { + const frame = {}; + Object.defineProperty(frame, "document", { + get: () => { + throw new DOMException("blocked", "SecurityError"); + }, + }); + const iframe = document.createElement("iframe"); + Object.defineProperty(iframe, "contentWindow", { value: frame }); + Object.defineProperty(iframe, "contentDocument", { value: null }); + const attachIframeShortcutListeners = vi.fn(); + hydrateTimelineFromPreview({ + iframe, + adapter: { getDuration: () => 1 } as unknown as PlaybackAdapter, + processTimelineMessage: vi.fn(), + enrichMissingCompositions: vi.fn(), + applyPreviewAudioState: vi.fn(), + attachIframeShortcutListeners, + syncTimelineElements: vi.fn(), + }); + expect(attachIframeShortcutListeners).toHaveBeenCalledOnce(); + }); +}); diff --git a/packages/studio/src/player/hooks/timelineSyncHydration.ts b/packages/studio/src/player/hooks/timelineSyncHydration.ts index 1b2e46312b..75cb76bb7f 100644 --- a/packages/studio/src/player/hooks/timelineSyncHydration.ts +++ b/packages/studio/src/player/hooks/timelineSyncHydration.ts @@ -419,15 +419,10 @@ export interface HydrateTimelineFromPreviewInput { * cross-origin or mid-navigation frame, and none of them is worth failing the * adapter's initialisation over. */ -function normalizePreviewDom( - doc: Document | null, - iframeWin: IframeWindow | null, - attachIframeShortcutListeners: () => void, -): void { +function normalizePreviewDom(doc: Document | null, iframeWin: IframeWindow | null): void { if (!doc || !iframeWin) return; normalizePreviewViewport(doc, iframeWin); autoHealMissingCompositionIds(doc); - attachIframeShortcutListeners(); } /** Hand the runtime's own clip manifest to the timeline, if it published one. */ @@ -444,7 +439,8 @@ export function hydrateTimelineFromPreview(input: HydrateTimelineFromPreviewInpu try { const doc = safeContentDocument(iframe); const iframeWin = (iframe?.contentWindow as IframeWindow | null) ?? null; - normalizePreviewDom(doc, iframeWin, input.attachIframeShortcutListeners); + normalizePreviewDom(doc, iframeWin); + input.attachIframeShortcutListeners(); applyRuntimeClipManifest(iframeWin, input.processTimelineMessage); input.enrichMissingCompositions(); input.applyPreviewAudioState(); diff --git a/packages/studio/src/player/hooks/usePlaybackKeyboard.test.ts b/packages/studio/src/player/hooks/usePlaybackKeyboard.test.ts index b66c49b158..b7b0e6d02c 100644 --- a/packages/studio/src/player/hooks/usePlaybackKeyboard.test.ts +++ b/packages/studio/src/player/hooks/usePlaybackKeyboard.test.ts @@ -8,6 +8,7 @@ import { usePlaybackKeyboard } from "./usePlaybackKeyboard"; import { usePlayerStore } from "../store/playerStore"; import { setStudioKeys } from "../components/studioShortcuts"; import { useLinkedClipPreferences } from "../../utils/linkedClipPreferences"; +import { detachedFakeIframe, serveEditingRequests } from "./timelinePlayerTestHarness"; globalThis.IS_REACT_ACT_ENVIRONMENT = true; @@ -28,10 +29,11 @@ interface Spies { interface HookHandle { dispatch: (event: KeyboardEvent) => void; release: (event: KeyboardEvent) => void; + attach: () => void; spies: Spies; } -function setupHook(): HookHandle { +function setupHook(iframe: HTMLIFrameElement | null = null): HookHandle { const spies: Spies = { seek: vi.fn(), play: vi.fn(), @@ -42,7 +44,7 @@ function setupHook(): HookHandle { let captured: ReturnType | null = null; function Harness() { - const iframeRef = React.useRef(null); + const iframeRef = React.useRef(iframe); const shuttleDirectionRef = React.useRef<"forward" | "backward" | null>(null); const shuttleSpeedIndexRef = React.useRef(0); const iframeShortcutCleanupRef = React.useRef<(() => void) | null>(null); @@ -72,6 +74,7 @@ function setupHook(): HookHandle { return { dispatch: (event) => captured!.playbackKeyDownRef.current(event), release: (event) => captured!.playbackKeyUpRef.current(event), + attach: () => captured!.attachIframeShortcutListeners(), spies, }; } @@ -483,3 +486,105 @@ describe("usePlaybackKeyboard: modifier combos and named keys", () => { expect(spies.playBackward).not.toHaveBeenCalled(); }); }); + +describe("usePlaybackKeyboard: a sealed preview frame", () => { + const closers: Array<() => void> = []; + afterEach(() => closers.splice(0).forEach((close) => close())); + + function sealedFrame() { + const frame: Record = {}; + const iframe = detachedFakeIframe(frame); + Object.defineProperty(iframe, "contentDocument", { value: null }); + Object.defineProperty(frame, "document", { + get: () => { + throw new DOMException("blocked", "SecurityError"); + }, + }); + const watched: string[][] = []; + let port: MessagePort | null = null; + closers.push( + serveEditingRequests( + frame, + (kind, params) => { + if (kind !== "watch-keys") return null; + const { combos } = params as { combos: string[] }; + watched.push(combos); + return { watching: combos.length }; + }, + (framePort) => (port = framePort), + ), + ); + const forward = (type: "keydown" | "keyup", key: string, code: string) => + port!.postMessage({ + v: 1, + event: "key", + key: { + type, + key, + code, + altKey: false, + ctrlKey: false, + metaKey: false, + shiftKey: false, + repeat: false, + }, + }); + return { iframe, watched, forward }; + } + + it("asks the frame for exactly the playback keys, and again when they are remapped", async () => { + const { iframe, watched } = sealedFrame(); + setupHook(iframe).attach(); + await vi.waitFor(() => expect(watched).toHaveLength(1)); + expect(new Set(watched[0])).toEqual( + new Set([ + "Space", + "ArrowLeft", + "ArrowRight", + "shift+ArrowLeft", + "shift+ArrowRight", + "m", + "shift+l", + "k", + "j", + "l", + "i", + "shift+i", + "o", + "shift+o", + "a", + "e", + ]), + ); + act(() => setStudioKeys({ playPause: "p" })); + await vi.waitFor(() => expect(watched).toHaveLength(2)); + expect(watched[1]).toContain("p"); + expect(watched[1]).not.toContain("Space"); + }); + + it("plays and steps from the keys the frame forwards, keyups included", async () => { + const { iframe, watched, forward } = sealedFrame(); + const { attach, spies } = setupHook(iframe); + attach(); + await vi.waitFor(() => expect(watched).toHaveLength(1)); + forward("keydown", " ", "Space"); + await vi.waitFor(() => expect(spies.play).toHaveBeenCalledOnce()); + forward("keydown", "k", "KeyK"); + forward("keydown", "l", "KeyL"); + await vi.waitFor(() => expect(spies.seek).toHaveBeenCalledOnce()); + forward("keyup", "k", "KeyK"); + forward("keydown", "l", "KeyL"); + await vi.waitFor(() => expect(spies.play).toHaveBeenCalledTimes(2)); + }); + + it("stops watching when its listeners are replaced", async () => { + const { iframe, watched } = sealedFrame(); + const { attach } = setupHook(iframe); + attach(); + await vi.waitFor(() => expect(watched).toHaveLength(1)); + attach(); + await vi.waitFor(() => expect(watched).toHaveLength(3)); + expect(watched[1]).toEqual([]); + expect(watched[2]).toEqual(watched[0]); + }); +}); diff --git a/packages/studio/src/player/hooks/usePlaybackKeyboard.ts b/packages/studio/src/player/hooks/usePlaybackKeyboard.ts index 39cda4b457..c8c9c8ff2a 100644 --- a/packages/studio/src/player/hooks/usePlaybackKeyboard.ts +++ b/packages/studio/src/player/hooks/usePlaybackKeyboard.ts @@ -15,7 +15,13 @@ import { } from "../lib/playbackShortcuts"; import { canvasNudgeKeysClaimed } from "../../utils/canvasNudgeGate"; import { usePlayerStore } from "../store/playerStore"; -import { matchesKey, type StudioKeyAction } from "../components/studioShortcuts"; +import { + getStudioKey, + matchesKey, + subscribeStudioKeys, + type StudioKeyAction, +} from "../components/studioShortcuts"; +import { previewEditingClient, previewFrameOpaque } from "../lib/previewChannel"; import { resolveCanvasNudgeDelta } from "../../components/editor/domEditNudge"; import { stepFrameTime } from "../lib/time"; import { useLinkedClipPreferences } from "../../utils/linkedClipPreferences"; @@ -67,6 +73,28 @@ function ignoresPlaybackKey(e: KeyboardEvent): boolean { }); } +type KeyHandlerRef = React.RefObject<(e: KeyboardEvent) => void>; + +function forwardSealedFrameKeys( + iframe: HTMLIFrameElement, + keyDown: KeyHandlerRef, + keyUp: KeyHandlerRef, +): () => void { + const client = previewEditingClient(iframe); + const watch = (combos: string[]) => void client.request("watch-keys", { combos }).catch(() => {}); + const watchPlayback = () => watch(PLAYBACK_ACTIONS.map(getStudioKey)); + watchPlayback(); + const stopRebinds = subscribeStudioKeys(watchPlayback); + const stopKeys = client.onKey((key) => + (key.type === "keydown" ? keyDown : keyUp).current(new KeyboardEvent(key.type, key)), + ); + return () => { + stopRebinds(); + stopKeys(); + watch([]); + }; +} + function handleWorkAreaKey( e: KeyboardEvent, action: PlaybackAction, @@ -198,6 +226,15 @@ export function usePlaybackKeyboard({ const attachIframeShortcutListeners = useCallback(() => { iframeShortcutCleanupRef.current?.(); iframeShortcutCleanupRef.current = null; + const iframe = iframeRef.current; + if (iframe && previewFrameOpaque(iframe)) { + iframeShortcutCleanupRef.current = forwardSealedFrameKeys( + iframe, + playbackKeyDownRef, + playbackKeyUpRef, + ); + return; + } let iframeWin: Window | null = null; let iframeDoc: Document | null = null; diff --git a/scripts/package-size-budgets.json b/scripts/package-size-budgets.json index af42ccd25b..c77afc3dfb 100644 --- a/scripts/package-size-budgets.json +++ b/scripts/package-size-budgets.json @@ -37,7 +37,7 @@ }, "@hyperframes/producer": { "packed": 921000, - "unpacked": 3290000 + "unpacked": 3491355 }, "@hyperframes/sdk": { "packed": 90662,